From 5081a1a19e5a1710a81ce7628e81acc71117ec29 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sun, 14 Jun 2026 02:20:26 +0530 Subject: [PATCH 01/57] feat: add the execution-observer foundation (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the **execution-observer foundation** β€” the seam the history and metrics plugins build on. - New `ExecutionObserver` contract in the SDK: six tagged lifecycle events (`ExecutionStarted`/`Finished`, `ToolCallStarted`/`Finished`, `InteractionStarted`/`Resolved`) + `composeExecutionObservers`, with a `plugin.runtime.executionObserver` hook. - The execution engine emits the events (re-derived onto the current Queue-based pause model), exposes `Executor.owner`, and tags the run span with the execution id + trigger. - `makeExecutionStack` composes plugin observers β€” one seam covers every HTTP host and the MCP Durable Objects; local's fixed-execution + in-process MCP are wired directly. **Behaviour-neutral / opt-in:** with no plugin observing, every path is a no-op. Gates: `format:check`, `lint`, full-monorepo `typecheck`, and observer unit tests all green. Live app verification still to come. ### [Stack](https://github.com/kitlangton/stack) 1. **#12** πŸ‘ˆ current 2. #13 3. #14 --- .changeset/execution-observer-foundation.md | 14 + apps/local/src/app.ts | 10 +- apps/local/src/main.ts | 5 +- .../core/api/src/server/execution-stack.ts | 13 +- .../execution/src/engine-observer.test.ts | 81 +++++ packages/core/execution/src/engine.ts | 282 ++++++++++++++++-- .../core/sdk/src/execution-observer.test.ts | 72 +++++ packages/core/sdk/src/execution-observer.ts | 143 +++++++++ packages/core/sdk/src/executor.ts | 6 + packages/core/sdk/src/index.ts | 22 ++ packages/core/sdk/src/plugin.ts | 8 + 11 files changed, 633 insertions(+), 23 deletions(-) create mode 100644 .changeset/execution-observer-foundation.md create mode 100644 packages/core/execution/src/engine-observer.test.ts create mode 100644 packages/core/sdk/src/execution-observer.test.ts create mode 100644 packages/core/sdk/src/execution-observer.ts diff --git a/.changeset/execution-observer-foundation.md b/.changeset/execution-observer-foundation.md new file mode 100644 index 0000000000..4d117c72b5 --- /dev/null +++ b/.changeset/execution-observer-foundation.md @@ -0,0 +1,14 @@ +--- +"@executor-js/sdk": minor +"@executor-js/execution": minor +"@executor-js/api": minor +--- + +Add the execution-observer foundation. The execution engine now emits a typed +lifecycle stream (`ExecutionStarted`/`Finished`, `ToolCallStarted`/`Finished`, +`InteractionStarted`/`Resolved`), plugins can subscribe via the new +`plugin.runtime.executionObserver` hook, and `makeExecutionStack` composes every +registered plugin's observer onto the engine. Behaviour is unchanged when no +plugin observes β€” this is the opt-in seam the execution-history and +execution-metrics plugins build on. Also exposes `Executor.owner` and enriches +the `mcp.execute` span with the run id and trigger. diff --git a/apps/local/src/app.ts b/apps/local/src/app.ts index f9bddc6e86..4510997e64 100644 --- a/apps/local/src/app.ts +++ b/apps/local/src/app.ts @@ -9,6 +9,7 @@ import { } from "@executor-js/api/server"; import { createExecutionEngine } from "@executor-js/execution"; import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; +import { composeExecutionObservers, type AnyPlugin } from "@executor-js/sdk"; import { getExecutorBundle, type LocalExecutor } from "./executor"; import { localIdentityLayer } from "./identity"; @@ -50,12 +51,17 @@ import { ErrorCaptureLive } from "./observability"; * `HostConfig`/`CodeExecutorProvider` seams β€” the fixed executor is the whole * execution model. */ -const localFixedExecutionLayer = (executor: LocalExecutor): Layer.Layer => +const localFixedExecutionLayer = ( + executor: LocalExecutor, + plugins: readonly AnyPlugin[], +): Layer.Layer => Layer.succeed(FixedExecutionProvider)({ executor, engine: createExecutionEngine({ executor, codeExecutor: makeQuickJsExecutor(), + // Local bypasses makeExecutionStack, so compose plugin observers here. + observer: composeExecutionObservers(plugins, executor), }), // The executor IS its own plugin-extension map (`executor[pluginId]`); the // fixed middleware reads `executor[id]` to satisfy each plugin's @@ -86,7 +92,7 @@ export const makeLocalApiHandler = async (): Promise => { // Layer is the `fixedExecution` seam declaration AND lives in `boot` so the // fixed middleware's residual `FixedExecutionProvider` resolves there β€” exactly // as self-host declares `db: SelfHostDbProvider` and puts the handle in `boot`. - const fixedExecution = localFixedExecutionLayer(executor); + const fixedExecution = localFixedExecutionLayer(executor, plugins); const { toWebHandler } = ExecutorApp.make({ plugins, diff --git a/apps/local/src/main.ts b/apps/local/src/main.ts index bce001a3a6..cf5122323e 100644 --- a/apps/local/src/main.ts +++ b/apps/local/src/main.ts @@ -2,6 +2,7 @@ import { Context, Effect, Layer, ManagedRuntime } from "effect"; import { createExecutionEngine } from "@executor-js/execution"; import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; +import { composeExecutionObservers } from "@executor-js/sdk"; import { makeLocalApiHandler } from "./app"; import { getExecutorBundle } from "./executor"; import { createMcpRequestHandler, type McpRequestHandler } from "./mcp"; @@ -61,10 +62,12 @@ export const createServerHandlers = async (): Promise => { // engine instance (the browser-approval + stdio surface is local-only and not // part of the shared API). Reuse the shared boot bundle so the MCP executor is // byte-identical to the one the API serves. - const { executor } = await getExecutorBundle(); + const { executor, plugins } = await getExecutorBundle(); const engine = createExecutionEngine({ executor, codeExecutor: makeQuickJsExecutor(), + // Local's in-process MCP also bypasses makeExecutionStack β€” compose here. + observer: composeExecutionObservers(plugins, executor), }); const mcp = createMcpRequestHandler({ engine }); diff --git a/packages/core/api/src/server/execution-stack.ts b/packages/core/api/src/server/execution-stack.ts index 6e58801fcf..ebfe63bf63 100644 --- a/packages/core/api/src/server/execution-stack.ts +++ b/packages/core/api/src/server/execution-stack.ts @@ -26,6 +26,7 @@ import { Context, Effect, Layer } from "effect"; import type * as Cause from "effect/Cause"; +import { composeExecutionObservers } from "@executor-js/sdk"; import type { AnyPlugin, Executor, StorageFailure } from "@executor-js/sdk"; import { createExecutionEngine, @@ -108,7 +109,17 @@ export const makeExecutionStack = < ); const codeExecutor = yield* CodeExecutorProvider; const { decorate } = yield* EngineDecorator; - const engine = decorate(createExecutionEngine({ executor, codeExecutor }), { + + // Compose every registered plugin's runtime.executionObserver, bound to the + // executor's own extensions. Resolves to a no-op when no plugin observes β€” + // so a stack with no history/metrics plugin is unaffected. `plugins()` is + // the erased AnyPlugin[]; the caller's TPlugins phantom recovers the tuple. + const { plugins } = yield* PluginsProvider; + // PluginsProvider erases the tuple to AnyPlugin[]; recover the caller's + // TPlugins phantom so the extensions arg (the executor) lines up. + const observer = composeExecutionObservers(plugins() as TPlugins, executor); + + const engine = decorate(createExecutionEngine({ executor, codeExecutor, observer }), { accountId, organizationId, organizationName, diff --git a/packages/core/execution/src/engine-observer.test.ts b/packages/core/execution/src/engine-observer.test.ts new file mode 100644 index 0000000000..4c46ab8667 --- /dev/null +++ b/packages/core/execution/src/engine-observer.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Predicate } from "effect"; + +import { createExecutor, definePlugin } from "@executor-js/sdk"; +import type { ExecutionEvent, ExecutionObserver } from "@executor-js/sdk"; +import { makeTestConfig } from "@executor-js/sdk/testing"; +import type { CodeExecutor, ExecuteResult } from "@executor-js/codemode-core"; + +import { createExecutionEngine } from "./engine"; + +const emptyPlugin = definePlugin(() => ({ + id: "observer-test" as const, + storage: () => ({}), + staticSources: () => [], +})); + +const makeExecutor = () => createExecutor(makeTestConfig({ plugins: [emptyPlugin()] as const })); + +// A code executor that issues one builtin tool call (tools.search) and then +// completes β€” enough to exercise the full event sequence. +const toolCallingExecutor: CodeExecutor = { + execute: (_code, invoker) => + invoker + .invoke({ path: "search", args: { query: "anything" } }) + .pipe(Effect.as({ result: "ok", logs: [] } satisfies ExecuteResult), Effect.orDie), +}; + +const collectingObserver = () => { + const events: ExecutionEvent[] = []; + const observer: ExecutionObserver = { + handle: (event) => Effect.sync(() => void events.push(event)), + }; + return { events, observer }; +}; + +describe("execution engine observer emission", () => { + it.effect("emits the full lifecycle for a completed run with a tool call", () => + Effect.gen(function* () { + const executor = yield* makeExecutor(); + const { events, observer } = collectingObserver(); + const engine = createExecutionEngine({ + executor, + codeExecutor: toolCallingExecutor, + observer, + }); + + const result = yield* engine.executeWithPause("noop", { trigger: { kind: "test" } }); + expect(result.status).toBe("completed"); + + // First event opens the run, last closes it; tool calls land in between. + // `.find` with isTagged narrows each result, so the assertions read the + // typed fields directly via optional chaining (no conditional blocks). + const started = events.find((e) => Predicate.isTagged(e, "ExecutionStarted")); + const finished = events.find((e) => Predicate.isTagged(e, "ExecutionFinished")); + const toolStarted = events.find((e) => Predicate.isTagged(e, "ToolCallStarted")); + const toolFinished = events.find((e) => Predicate.isTagged(e, "ToolCallFinished")); + + expect(Predicate.isTagged(events[0], "ExecutionStarted")).toBe(true); + expect(Predicate.isTagged(events[events.length - 1], "ExecutionFinished")).toBe(true); + + expect(started?.trigger?.kind).toBe("test"); + expect(started?.owner.tenant).toBeDefined(); + expect(toolStarted).toBeDefined(); + expect(finished?.status).toBe("completed"); + + // Tool-call events share the run's executionId and carry the path. + expect(toolFinished?.path).toBe("search"); + expect(toolFinished?.status).toBe("completed"); + expect(toolFinished?.executionId).toBe(started?.executionId); + }), + ); + + it.effect("does nothing observable when no observer is configured", () => + Effect.gen(function* () { + const executor = yield* makeExecutor(); + const engine = createExecutionEngine({ executor, codeExecutor: toolCallingExecutor }); + const result = yield* engine.executeWithPause("noop"); + expect(result.status).toBe("completed"); + }), + ); +}); diff --git a/packages/core/execution/src/engine.ts b/packages/core/execution/src/engine.ts index 5dee95e036..b67151118e 100644 --- a/packages/core/execution/src/engine.ts +++ b/packages/core/execution/src/engine.ts @@ -1,5 +1,5 @@ import { Deferred, Effect, Fiber, Predicate, Queue } from "effect"; -import type * as Cause from "effect/Cause"; +import * as Cause from "effect/Cause"; import type { Executor, @@ -7,6 +7,21 @@ import type { ElicitationResponse, ElicitationHandler, ElicitationContext, + ExecutionObserver, + ExecutionTrigger, +} from "@executor-js/sdk/core"; +import { + ExecutionId, + ExecutionInteractionId, + ExecutionToolCallId, + ExecutionFinished, + ExecutionStarted, + InteractionResolved, + InteractionStarted, + ToolCallFinished, + ToolCallStarted, + ignoreExecutionObserverErrors, + noopExecutionObserver, } from "@executor-js/sdk/core"; import { CodeExecutionError } from "@executor-js/codemode-core"; import type { CodeExecutor, ExecuteResult, SandboxToolInvoker } from "@executor-js/codemode-core"; @@ -29,6 +44,21 @@ export type ExecutionEngineConfig; readonly toolDiscoveryProvider?: ToolDiscoveryProvider; + /** Optional sink for execution lifecycle events. Defaults to a no-op, so a + * host that registers no observer pays only for constructing the events. */ + readonly observer?: ExecutionObserver; +}; + +/** Per-run options shared by both execute paths. */ +export type ExecutionRunOptions = { + /** What kicked off this run (e.g. `mcp.tool`, `api.http`); recorded on the + * `ExecutionStarted` event for downstream attribution. */ + readonly trigger?: ExecutionTrigger; +}; + +/** Options for the inline-elicitation execute path. */ +export type InlineExecutionOptions = ExecutionRunOptions & { + readonly onElicitation: ElicitationHandler; }; export type ExecutionResult = @@ -341,7 +371,7 @@ export type ExecutionEngine */ readonly execute: ( code: string, - options: { readonly onElicitation: ElicitationHandler }, + options: InlineExecutionOptions, ) => Effect.Effect; /** @@ -349,7 +379,10 @@ export type ExecutionEngine * Use this when the host doesn't support inline elicitation. * Returns either a completed result or a paused execution that can be resumed. */ - readonly executeWithPause: (code: string) => Effect.Effect; + readonly executeWithPause: ( + code: string, + options?: ExecutionRunOptions, + ) => Effect.Effect; /** * Resume a paused execution. Returns a completed result, a new pause, or @@ -379,6 +412,134 @@ export const createExecutionEngine = >(); let nextId = 0; + // Observer wiring. `emit` never fails (errors are swallowed per-observer) and + // resolves to a no-op when no plugin registered one β€” the opt-out default. + const observer = ignoreExecutionObserverErrors(config.observer ?? noopExecutionObserver); + const emit = observer.handle; + const owner = executor.owner; + + const makeExecutionId = (): ExecutionId => ExecutionId.make(`exec_${crypto.randomUUID()}`); + const makeToolCallId = (): ExecutionToolCallId => + ExecutionToolCallId.make(`tc_${crypto.randomUUID()}`); + const makeInteractionId = (): ExecutionInteractionId => + ExecutionInteractionId.make(`ix_${crypto.randomUUID()}`); + + const interactionStatusFromAction = (action: ResumeResponse["action"]) => + action === "accept" ? "accepted" : action === "decline" ? "declined" : "cancelled"; + + const finishFromResult = (executionId: ExecutionId, result: ExecuteResult): ExecutionFinished => + new ExecutionFinished({ + executionId, + owner, + status: result.error ? "failed" : "completed", + result: result.result, + error: result.error, + logs: result.logs, + completedAt: new Date(), + }); + + const finishFromCause = (executionId: ExecutionId, cause: Cause.Cause): ExecutionFinished => + new ExecutionFinished({ + executionId, + owner, + status: "failed", + error: Cause.pretty(cause), + completedAt: new Date(), + }); + + /** Wrap an invoker so each tool call brackets `ToolCallStarted`/`Finished`. */ + const observeToolCalls = ( + executionId: ExecutionId, + inner: SandboxToolInvoker, + ): SandboxToolInvoker => ({ + invoke: (call) => + Effect.gen(function* () { + const toolCallId = makeToolCallId(); + yield* emit( + new ToolCallStarted({ + executionId, + toolCallId, + owner, + path: call.path, + args: call.args, + startedAt: new Date(), + }), + ); + return yield* inner.invoke(call).pipe( + Effect.tap((result) => + emit( + new ToolCallFinished({ + executionId, + toolCallId, + owner, + path: call.path, + status: "completed", + result, + completedAt: new Date(), + }), + ), + ), + Effect.tapCause((cause) => + emit( + new ToolCallFinished({ + executionId, + toolCallId, + owner, + path: call.path, + status: "failed", + error: Cause.pretty(cause), + completedAt: new Date(), + }), + ), + ), + ); + }), + }); + + /** Wrap an inline elicitation handler so it brackets `InteractionStarted`/ + * `Resolved`. The pausable path emits these directly (see below). */ + const observeInlineElicitation = + (executionId: ExecutionId, handler: ElicitationHandler): ElicitationHandler => + (ctx) => + Effect.gen(function* () { + const interactionId = makeInteractionId(); + yield* emit( + new InteractionStarted({ + executionId, + interactionId, + owner, + context: ctx, + startedAt: new Date(), + }), + ); + return yield* handler(ctx).pipe( + Effect.tap((response) => + emit( + new InteractionResolved({ + executionId, + interactionId, + owner, + status: interactionStatusFromAction(response.action), + response, + completedAt: new Date(), + }), + ), + ), + Effect.tapCause((cause) => + emit( + new InteractionResolved({ + executionId, + interactionId, + owner, + status: "failed", + error: Cause.pretty(cause), + completedAt: new Date(), + }), + ), + ), + ); + }); + /** * Race a running fiber against the pause queue. Returns when either * the fiber completes or an elicitation handler fires (whichever @@ -411,12 +572,33 @@ export const createExecutionEngine = >(); @@ -428,6 +610,7 @@ export const createExecutionEngine = (); const id = `exec_${++nextId}`; + const interactionId = makeInteractionId(); const paused: InternalPausedExecution = { id, @@ -438,19 +621,59 @@ export const createExecutionEngine = + emit( + new InteractionResolved({ + executionId, + interactionId, + owner, + status: interactionStatusFromAction(response.action), + response, + completedAt: new Date(), + }), + ), + ), + Effect.tapCause((cause) => + emit( + new InteractionResolved({ + executionId, + interactionId, + owner, + status: "failed", + error: Cause.pretty(cause), + completedAt: new Date(), + }), + ), + ), + ); }); - const invoker = makeFullInvoker( - executor, - { onElicitation: elicitationHandler }, - toolDiscoveryProvider, + const invoker = observeToolCalls( + executionId, + makeFullInvoker(executor, { onElicitation: elicitationHandler }, toolDiscoveryProvider), ); fiber = yield* Effect.forkDetach( - codeExecutor.execute(code, invoker).pipe(Effect.withSpan("executor.code.exec")), + codeExecutor.execute(code, invoker).pipe( + Effect.withSpan("executor.code.exec"), + Effect.tap((result) => emit(finishFromResult(executionId, result))), + Effect.tapCause((cause) => emit(finishFromCause(executionId, cause))), + ), ); return (yield* awaitCompletionOrPause(fiber, pauseQueue)) as ExecutionResult; @@ -486,20 +709,41 @@ export const createExecutionEngine = emit(finishFromResult(executionId, result))), + Effect.tapCause((cause) => emit(finishFromCause(executionId, cause))), ); - return yield* codeExecutor.execute(code, invoker).pipe(Effect.withSpan("executor.code.exec")); }); return { diff --git a/packages/core/sdk/src/execution-observer.test.ts b/packages/core/sdk/src/execution-observer.test.ts new file mode 100644 index 0000000000..d8536f19fd --- /dev/null +++ b/packages/core/sdk/src/execution-observer.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { Subject, Tenant } from "./ids"; +import { ExecutionFinished, ExecutionId, composeExecutionObservers, definePlugin } from "./index"; + +const owner = { tenant: Tenant.make("tenant_test"), subject: Subject.make("subject_test") }; + +let calls: string[] = []; + +const observingPlugin = (id: string) => + definePlugin(() => ({ + id, + storage: () => ({}), + extension: () => ({ label: id }), + runtime: { + executionObserver: (self: { label: string }) => ({ + handle: () => Effect.sync(() => calls.push(self.label)), + }), + }, + })); + +const failingPlugin = definePlugin(() => ({ + id: "failing" as const, + storage: () => ({}), + extension: () => ({ label: "failing" }), + runtime: { + executionObserver: () => ({ + handle: () => Effect.die("observer failed"), + }), + }, +})); + +const finishedEvent = () => + new ExecutionFinished({ + executionId: ExecutionId.make("exec_test"), + owner, + status: "completed", + result: "ok", + completedAt: new Date(), + }); + +describe("composeExecutionObservers", () => { + it.effect("fans an event to every plugin observer and isolates failures", () => + Effect.gen(function* () { + calls = []; + const first = observingPlugin("first")(); + const failing = failingPlugin(); + const last = observingPlugin("last")(); + const observer = composeExecutionObservers([first, failing, last] as const, { + first: { label: "first" }, + failing: { label: "failing" }, + last: { label: "last" }, + }); + + // The failing plugin dies mid-fan; the others must still observe. + yield* observer.handle(finishedEvent()); + + expect(calls).toEqual(["first", "last"]); + }), + ); + + it.effect("returns a no-op observer when no plugin registers one", () => + Effect.gen(function* () { + const plain = definePlugin(() => ({ id: "plain", storage: () => ({}) }))(); + const observer = composeExecutionObservers([plain] as const, { plain: {} }); + + // No observer registered β†’ handling is a silent no-op, never throws. + yield* observer.handle(finishedEvent()); + }), + ); +}); diff --git a/packages/core/sdk/src/execution-observer.ts b/packages/core/sdk/src/execution-observer.ts new file mode 100644 index 0000000000..2c19bcb0f5 --- /dev/null +++ b/packages/core/sdk/src/execution-observer.ts @@ -0,0 +1,143 @@ +import { Data, Effect, Schema } from "effect"; + +import type { ElicitationContext, ElicitationResponse } from "./elicitation"; +import type { AnyPlugin, OwnerBinding, PluginExtensions } from "./plugin"; + +/* The execution-observer contract: a pull-model lifecycle stream the engine + * emits as it runs code. Plugins opt in via `plugin.runtime.executionObserver` + * and receive every event; sinks (history, metrics, tracing) are built on top. + * Emission is fanned to all registered observers with per-observer error + * suppression, so an observer can never break an execution. */ + +export const ExecutionId = Schema.String.pipe(Schema.brand("ExecutionId")); +export type ExecutionId = typeof ExecutionId.Type; + +export const ExecutionToolCallId = Schema.String.pipe(Schema.brand("ExecutionToolCallId")); +export type ExecutionToolCallId = typeof ExecutionToolCallId.Type; + +export const ExecutionInteractionId = Schema.String.pipe(Schema.brand("ExecutionInteractionId")); +export type ExecutionInteractionId = typeof ExecutionInteractionId.Type; + +export type ExecutionTrigger = { + readonly kind: string; + readonly metadata?: Record; +}; + +export type ToolCallStatus = "completed" | "failed"; +export type InteractionStatus = "accepted" | "declined" | "cancelled" | "failed"; +export type ExecutionStatus = "completed" | "failed"; + +export class ExecutionStarted extends Data.TaggedClass("ExecutionStarted")<{ + readonly executionId: ExecutionId; + readonly owner: OwnerBinding; + readonly code: string; + readonly trigger?: ExecutionTrigger; + readonly startedAt: Date; +}> {} + +export class ToolCallStarted extends Data.TaggedClass("ToolCallStarted")<{ + readonly executionId: ExecutionId; + readonly toolCallId: ExecutionToolCallId; + readonly owner: OwnerBinding; + readonly path: string; + readonly args: unknown; + readonly startedAt: Date; +}> {} + +export class ToolCallFinished extends Data.TaggedClass("ToolCallFinished")<{ + readonly executionId: ExecutionId; + readonly toolCallId: ExecutionToolCallId; + readonly owner: OwnerBinding; + readonly path: string; + readonly status: ToolCallStatus; + readonly result?: unknown; + readonly error?: string; + readonly completedAt: Date; +}> {} + +export class InteractionStarted extends Data.TaggedClass("InteractionStarted")<{ + readonly executionId: ExecutionId; + readonly interactionId: ExecutionInteractionId; + readonly owner: OwnerBinding; + readonly context: ElicitationContext; + readonly startedAt: Date; +}> {} + +export class InteractionResolved extends Data.TaggedClass("InteractionResolved")<{ + readonly executionId: ExecutionId; + readonly interactionId: ExecutionInteractionId; + readonly owner: OwnerBinding; + readonly status: InteractionStatus; + readonly response?: ElicitationResponse; + readonly error?: string; + readonly completedAt: Date; +}> {} + +export class ExecutionFinished extends Data.TaggedClass("ExecutionFinished")<{ + readonly executionId: ExecutionId; + readonly owner: OwnerBinding; + readonly status: ExecutionStatus; + readonly result?: unknown; + readonly error?: string; + readonly logs?: readonly string[]; + readonly completedAt: Date; +}> {} + +export type ExecutionEvent = + | ExecutionStarted + | ToolCallStarted + | ToolCallFinished + | InteractionStarted + | InteractionResolved + | ExecutionFinished; + +export interface ExecutionObserver { + readonly handle: (event: ExecutionEvent) => Effect.Effect; +} + +export const noopExecutionObserver: ExecutionObserver = { + handle: () => Effect.void, +}; + +/** Wrap an observer so any failure (defect or expected error) is swallowed β€” + * an observer must never propagate into the execution it observes. */ +export const ignoreExecutionObserverErrors = ( + observer: ExecutionObserver, +): ExecutionObserver => ({ + handle: (event) => observer.handle(event).pipe(Effect.catchCause(() => Effect.void)), +}); + +/** Collect every plugin's `runtime.executionObserver` and fan each event to + * all of them, suppressing per-observer errors. Returns the no-op observer + * when no plugin registers one β€” the common (opt-out) case. */ +export const composeExecutionObservers = ( + plugins: TPlugins, + extensions: PluginExtensions, +): ExecutionObserver => { + const observers: ExecutionObserver[] = []; + + for (const plugin of plugins) { + const observer = plugin.runtime?.executionObserver?.( + extensions[plugin.id as keyof PluginExtensions] as never, + ); + if (observer) { + observers.push(observer); + } + } + + if (observers.length === 0) { + return noopExecutionObserver; + } + + return { + handle: (event) => + Effect.forEach( + observers, + (observer) => observer.handle(event).pipe(Effect.catchCause(() => Effect.void)), + // Fan out in parallel β€” a slow sink (e.g. a DB-backed history observer) + // must not serialize behind another (e.g. a metrics push). Per-observer + // error isolation is preserved by the catchCause above. + { discard: true, concurrency: "unbounded" }, + ), + }; +}; diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index c8d578bf8b..a90003ef92 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -310,6 +310,11 @@ export type Executor = { ) => Effect.Effect; readonly close: () => Effect.Effect; + + /** The (tenant, subject) this executor acts as. Surfaced so engine-level + * machinery (e.g. execution observers) can attribute work to an owner + * without re-threading identity through every call site. */ + readonly owner: OwnerBinding; } & PluginExtensions; export interface ExecutorDb { @@ -3124,6 +3129,7 @@ export const createExecutor = => value as Executor; diff --git a/packages/core/sdk/src/index.ts b/packages/core/sdk/src/index.ts index 8d6a95258f..4b822b4310 100644 --- a/packages/core/sdk/src/index.ts +++ b/packages/core/sdk/src/index.ts @@ -167,6 +167,28 @@ export { type InvokeOptions, } from "./elicitation"; +// Execution observers β€” the engine lifecycle stream history/metrics/tracing build on. +export { + ExecutionId, + ExecutionToolCallId, + ExecutionInteractionId, + ExecutionStarted, + ToolCallStarted, + ToolCallFinished, + InteractionStarted, + InteractionResolved, + ExecutionFinished, + noopExecutionObserver, + ignoreExecutionObserverErrors, + composeExecutionObservers, + type ExecutionTrigger, + type ToolCallStatus, + type InteractionStatus, + type ExecutionStatus, + type ExecutionEvent, + type ExecutionObserver, +} from "./execution-observer"; + // Blob store β€” the plugin-facing contract (`BlobStore`/`PluginBlobStore`) // plus the platform-neutral backends (`makeFumaBlobStore` default, // `makeInMemoryBlobStore` for tests). Platform-specific backends live with diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index d48041f82b..6ab97f45ba 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -38,6 +38,7 @@ import type { IntegrationRemovalNotAllowedError, InvalidConnectionInputError, } from "./errors"; +import type { ExecutionObserver } from "./execution-observer"; import type { OAuthService } from "./oauth-client"; import type { CredentialProvider, ProviderEntry } from "./provider"; import type { PluginStorageConfig, PluginStorageFacade } from "./plugin-storage"; @@ -499,6 +500,13 @@ export interface PluginSpec< | ((ctx: PluginCtx) => readonly CredentialProvider[]) | ((ctx: PluginCtx) => Effect.Effect); + /** Runtime hooks invoked while the engine executes code. `executionObserver` + * receives this plugin's extension and returns an observer that is fanned + * every {@link ExecutionEvent} β€” the seam history/metrics sinks build on. */ + readonly runtime?: { + readonly executionObserver?: (self: NoInfer) => ExecutionObserver; + }; + readonly close?: () => Effect.Effect; } From f0cf727034a82b487ac0bb504dd21feb6814b659 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sun, 14 Jun 2026 02:20:57 +0530 Subject: [PATCH 02/57] feat: add the execution-history plugin (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds **`@executor-js/plugin-execution-history`** β€” an opt-in execution-history sink built on the observer foundation. - Records every run (plus its tool calls and interactions) into three owner-scoped `pluginStorage` collections via a buffered-batch writer β€” **no new tables, no migrations**; works on D1/libSQL/sqlite/postgres. - `./api`: a scoped read API (`list`/`get`/`listToolCalls`) over HttpApi (`executionHistoryHttpPlugin`). - `./client`: a Runs page at `/plugins/executionHistory/` (table, status filter, pagination, 4-tab detail drawer) β€” mounts via the existing plugins catch-all with zero router changes. The base `.` entry never loads the API or React layers. Fully opt-in (not registered in any default config). Stacked on the execution-observer foundation. Gates: `format:check`, `lint`, full-monorepo `typecheck`, unit + handler tests green. Runs-page live render still to verify. ### [Stack](https://github.com/kitlangton/stack) 1. #12 2. **#13** πŸ‘ˆ current 3. #14 --- .changeset/execution-history-plugin.md | 12 + bun.lock | 35 ++ .../plugins/execution-history/CHANGELOG.md | 1 + .../plugins/execution-history/package.json | 55 ++ .../execution-history/src/api/group.ts | 98 ++++ .../src/api/handlers.test.ts | 195 +++++++ .../execution-history/src/api/handlers.ts | 122 ++++ .../execution-history/src/api/index.ts | 29 + .../execution-history/src/react/RunsPage.tsx | 523 ++++++++++++++++++ .../execution-history/src/react/atoms.ts | 46 ++ .../execution-history/src/react/client.ts | 12 + .../execution-history/src/react/index.ts | 3 + .../src/react/plugin-client.tsx | 15 + .../execution-history/src/sdk/collections.ts | 97 ++++ .../execution-history/src/sdk/index.ts | 22 + .../execution-history/src/sdk/plugin.ts | 36 ++ .../execution-history/src/sdk/store.test.ts | 100 ++++ .../execution-history/src/sdk/store.ts | 447 +++++++++++++++ .../plugins/execution-history/tsconfig.json | 25 + .../plugins/execution-history/tsup.config.ts | 12 + .../execution-history/vitest.config.ts | 8 + 21 files changed, 1893 insertions(+) create mode 100644 .changeset/execution-history-plugin.md create mode 100644 packages/plugins/execution-history/CHANGELOG.md create mode 100644 packages/plugins/execution-history/package.json create mode 100644 packages/plugins/execution-history/src/api/group.ts create mode 100644 packages/plugins/execution-history/src/api/handlers.test.ts create mode 100644 packages/plugins/execution-history/src/api/handlers.ts create mode 100644 packages/plugins/execution-history/src/api/index.ts create mode 100644 packages/plugins/execution-history/src/react/RunsPage.tsx create mode 100644 packages/plugins/execution-history/src/react/atoms.ts create mode 100644 packages/plugins/execution-history/src/react/client.ts create mode 100644 packages/plugins/execution-history/src/react/index.ts create mode 100644 packages/plugins/execution-history/src/react/plugin-client.tsx create mode 100644 packages/plugins/execution-history/src/sdk/collections.ts create mode 100644 packages/plugins/execution-history/src/sdk/index.ts create mode 100644 packages/plugins/execution-history/src/sdk/plugin.ts create mode 100644 packages/plugins/execution-history/src/sdk/store.test.ts create mode 100644 packages/plugins/execution-history/src/sdk/store.ts create mode 100644 packages/plugins/execution-history/tsconfig.json create mode 100644 packages/plugins/execution-history/tsup.config.ts create mode 100644 packages/plugins/execution-history/vitest.config.ts diff --git a/.changeset/execution-history-plugin.md b/.changeset/execution-history-plugin.md new file mode 100644 index 0000000000..50308039ad --- /dev/null +++ b/.changeset/execution-history-plugin.md @@ -0,0 +1,12 @@ +--- +"@executor-js/plugin-execution-history": minor +--- + +Add the `@executor-js/plugin-execution-history` plugin β€” an opt-in execution +history sink built on the ExecutionObserver foundation. It records every run +(plus its tool calls and interactions) into three owner-scoped pluginStorage +collections via a buffered-batch writer (no new tables, no migrations; works on +D1/libSQL/sqlite/postgres), exposes a `list`/`get`/`listToolCalls` read API over +HttpApi (`./api`'s `executionHistoryHttpPlugin`), and ships a Runs page UI +(`./client`) that mounts at `/plugins/executionHistory/`. The base `.` entry is +SDK-only and never loads `@executor-js/api` or React. diff --git a/bun.lock b/bun.lock index c38a1d481a..b0e11e0592 100644 --- a/bun.lock +++ b/bun.lock @@ -806,6 +806,39 @@ "react", ], }, + "packages/plugins/execution-history": { + "name": "@executor-js/plugin-execution-history", + "version": "0.0.0", + "dependencies": { + "@executor-js/sdk": "workspace:*", + "effect": "catalog:", + }, + "devDependencies": { + "@effect/atom-react": "catalog:", + "@effect/vitest": "catalog:", + "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "@types/node": "catalog:", + "@types/react": "catalog:", + "bun-types": "catalog:", + "react": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:", + }, + "peerDependencies": { + "@effect/atom-react": "catalog:", + "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "@tanstack/react-router": "catalog:", + "react": "catalog:", + }, + "optionalPeers": [ + "@effect/atom-react", + "@executor-js/react", + "@tanstack/react-router", + "react", + ], + }, "packages/plugins/file-secrets": { "name": "@executor-js/plugin-file-secrets", "version": "1.5.10", @@ -1573,6 +1606,8 @@ "@executor-js/plugin-example": ["@executor-js/plugin-example@workspace:packages/plugins/example"], + "@executor-js/plugin-execution-history": ["@executor-js/plugin-execution-history@workspace:packages/plugins/execution-history"], + "@executor-js/plugin-file-secrets": ["@executor-js/plugin-file-secrets@workspace:packages/plugins/file-secrets"], "@executor-js/plugin-graphql": ["@executor-js/plugin-graphql@workspace:packages/plugins/graphql"], diff --git a/packages/plugins/execution-history/CHANGELOG.md b/packages/plugins/execution-history/CHANGELOG.md new file mode 100644 index 0000000000..45c8b65d04 --- /dev/null +++ b/packages/plugins/execution-history/CHANGELOG.md @@ -0,0 +1 @@ +# @executor-js/plugin-execution-history diff --git a/packages/plugins/execution-history/package.json b/packages/plugins/execution-history/package.json new file mode 100644 index 0000000000..dbc58080ed --- /dev/null +++ b/packages/plugins/execution-history/package.json @@ -0,0 +1,55 @@ +{ + "name": "@executor-js/plugin-execution-history", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/sdk/index.ts", + "./api": "./src/api/index.ts", + "./react": "./src/react/index.ts", + "./client": "./src/react/plugin-client.tsx" + }, + "scripts": { + "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src || true)", + "typecheck": "tsgo --noEmit", + "test": "bunx --bun vitest run", + "test:watch": "bunx --bun vitest" + }, + "dependencies": { + "@executor-js/sdk": "workspace:*", + "effect": "catalog:" + }, + "peerDependencies": { + "@effect/atom-react": "catalog:", + "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "@tanstack/react-router": "catalog:", + "react": "catalog:" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "@effect/atom-react": { + "optional": true + }, + "@tanstack/react-router": { + "optional": true + }, + "@executor-js/react": { + "optional": true + } + }, + "devDependencies": { + "@effect/atom-react": "catalog:", + "@effect/vitest": "catalog:", + "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "@types/node": "catalog:", + "@types/react": "catalog:", + "bun-types": "catalog:", + "react": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:" + } +} diff --git a/packages/plugins/execution-history/src/api/group.ts b/packages/plugins/execution-history/src/api/group.ts new file mode 100644 index 0000000000..84fdc4fb3c --- /dev/null +++ b/packages/plugins/execution-history/src/api/group.ts @@ -0,0 +1,98 @@ +import { Schema } from "effect"; +import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; + +import { InternalError } from "@executor-js/api"; + +import { InteractionRow, RunRow, ToolCallRow } from "../sdk/collections"; + +// --------------------------------------------------------------------------- +// HTTP surface for the execution-history read model. +// +// Routes are FLAT and plugin-id-prefixed (`/execution-history/...`), matching +// the onepassword/graphql convention: the per-request executor is already +// owner-scoped at the host edge, so there is no `:scopeId` path segment. +// +// Success schemas are real Effect Schemas built from the storage collection +// row schemas (`RunRow`/`ToolCallRow`/`InteractionRow`) β€” the store's TS +// interfaces (`ExecutionHistoryListResult`/`ExecutionHistoryDetail`) are not +// schemas and cannot encode on the wire, so the shapes are mirrored here. +// +// `InternalError` is the shared opaque 500 schema. Storage failures on the +// extension flow through the typed channel as `StorageFailure` and are +// captured + downgraded to `InternalError({ traceId })` by `capture` at the +// handler edge. +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Success schemas +// --------------------------------------------------------------------------- + +/** Mirrors `ExecutionHistoryListResult` from the store. */ +export const ListRunsResponse = Schema.Struct({ + runs: Schema.Array(RunRow), + total: Schema.Number, +}); + +/** Mirrors `ExecutionHistoryDetail` from the store. */ +export const RunDetailResponse = Schema.Struct({ + run: RunRow, + toolCalls: Schema.Array(ToolCallRow), + interactions: Schema.Array(InteractionRow), +}); + +export const ListToolCallsResponse = Schema.Struct({ + toolCalls: Schema.Array(ToolCallRow), +}); + +// --------------------------------------------------------------------------- +// Query / path params +// +// Numeric filters arrive as strings and decode via `NumberFromString`. The +// `status`/`trigger` CSV lists stay raw strings the handler splits, mirroring +// how the store's `list` takes `statusFilter`/`triggerFilter` arrays. The +// boolean `interaction` is carried as a string ("true"/"false") and +// interpreted in the handler β€” the same convention core uses for +// `includeAnnotations`/`includeBlocked`. +// --------------------------------------------------------------------------- + +const ListRunsQuery = Schema.Struct({ + status: Schema.optional(Schema.String), + trigger: Schema.optional(Schema.String), + from: Schema.optional(Schema.NumberFromString), + to: Schema.optional(Schema.NumberFromString), + interaction: Schema.optional(Schema.String), + limit: Schema.optional(Schema.NumberFromString), + offset: Schema.optional(Schema.NumberFromString), + sort: Schema.optional(Schema.Literals(["asc", "desc"])), +}); + +const RunParams = Schema.Struct({ + executionId: Schema.String, +}); + +// --------------------------------------------------------------------------- +// Group +// --------------------------------------------------------------------------- + +export const ExecutionHistoryGroup = HttpApiGroup.make("executionHistory") + .add( + HttpApiEndpoint.get("list", "/execution-history/runs", { + query: ListRunsQuery, + success: ListRunsResponse, + error: InternalError, + }), + ) + .add( + HttpApiEndpoint.get("get", "/execution-history/runs/:executionId", { + params: RunParams, + success: Schema.NullOr(RunDetailResponse), + error: InternalError, + }), + ) + .add( + HttpApiEndpoint.get("listToolCalls", "/execution-history/runs/:executionId/tool-calls", { + params: RunParams, + success: ListToolCallsResponse, + error: InternalError, + }), + ); diff --git a/packages/plugins/execution-history/src/api/handlers.test.ts b/packages/plugins/execution-history/src/api/handlers.test.ts new file mode 100644 index 0000000000..c805b4d6e9 --- /dev/null +++ b/packages/plugins/execution-history/src/api/handlers.test.ts @@ -0,0 +1,195 @@ +// --------------------------------------------------------------------------- +// Handler-level integration test for the execution-history HTTP group. +// +// Drives the three read endpoints end-to-end through the HttpApi layer: the +// handlers pull the read surface from the extension service, map query/path +// params onto the store options, and the wire schemas encode the row shapes. A +// stub extension stands in for the store so the test exercises the HTTP edge + +// handler wiring (param parsing, CSV split, time-range build) rather than a +// live server. +// --------------------------------------------------------------------------- + +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Layer } from "effect"; +import { HttpRouter, HttpServer } from "effect/unstable/http"; +import { HttpApiBuilder } from "effect/unstable/httpapi"; + +import { addGroup, observabilityMiddleware } from "@executor-js/api"; +import { CoreHandlers, ExecutionEngineService, ExecutorService } from "@executor-js/api/server"; + +import type { RunRow, ToolCallRow } from "../sdk/collections"; +import type { + ExecutionHistoryDetail, + ExecutionHistoryListOptions, + ExecutionHistoryListResult, +} from "../sdk/store"; +import { ExecutionHistoryGroup } from "./group"; +import { + ExecutionHistoryExtensionService, + ExecutionHistoryHandlers, + type ExecutionHistoryExtension, +} from "./handlers"; + +const Api = addGroup(ExecutionHistoryGroup); +const UnusedExecutor = Layer.succeed(ExecutorService)({} as ExecutorService["Service"]); +const UnusedExecutionEngine = Layer.succeed(ExecutionEngineService)( + {} as ExecutionEngineService["Service"], +); + +const runRow = (overrides: Partial): RunRow => ({ + executionId: "exec_1", + status: "completed", + code: "noop", + resultJson: null, + errorText: null, + logsJson: null, + triggerKind: "manual", + triggerMetaJson: null, + startedAt: 1000, + completedAt: 2000, + durationMs: 1000, + toolCallCount: 1, + hadInteraction: false, + ...overrides, +}); + +const toolCallRow = (overrides: Partial): ToolCallRow => ({ + executionId: "exec_1", + toolCallId: "call_1", + status: "completed", + path: "tools.shell.run", + namespace: "tools", + argsJson: null, + resultJson: null, + errorText: null, + startedAt: 1000, + completedAt: 1500, + durationMs: 500, + ...overrides, +}); + +// Records the options the handler maps so the test can assert query parsing. +const makeStubExtension = (captured: { + options?: ExecutionHistoryListOptions; +}): ExecutionHistoryExtension => ({ + list: (options): Effect.Effect => { + captured.options = options; + return Effect.succeed({ runs: [runRow({})], total: 1 }); + }, + get: (executionId): Effect.Effect => + Effect.succeed( + executionId === "exec_1" + ? { run: runRow({}), toolCalls: [toolCallRow({})], interactions: [] } + : null, + ), + listToolCalls: (): Effect.Effect => Effect.succeed([toolCallRow({})]), +}); + +const webHandlerFor = (extension: ExecutionHistoryExtension) => + Effect.acquireRelease( + Effect.sync(() => + HttpRouter.toWebHandler( + HttpApiBuilder.layer(Api).pipe( + Layer.provide(CoreHandlers), + Layer.provide(ExecutionHistoryHandlers), + Layer.provide(observabilityMiddleware(Api)), + Layer.provide(UnusedExecutor), + Layer.provide(UnusedExecutionEngine), + Layer.provide(Layer.succeed(ExecutionHistoryExtensionService, extension)), + Layer.provideMerge(HttpServer.layerServices), + Layer.provideMerge(Layer.succeed(HttpRouter.RouterConfig)({ maxParamLength: 1000 })), + ), + ), + ), + (web) => Effect.promise(() => web.dispose()), + ); + +const get = ( + web: { handler: (request: Request, ...rest: never[]) => Promise }, + url: string, +) => + Effect.promise(() => + (web.handler as (request: Request) => Promise)(new Request(url, { method: "GET" })), + ); + +describe("ExecutionHistoryHandlers", () => { + it.effect("list maps CSV/number/boolean query params onto the store options", () => + Effect.gen(function* () { + const captured: { options?: ExecutionHistoryListOptions } = {}; + const web = yield* webHandlerFor(makeStubExtension(captured)); + + const res = yield* get( + web, + "http://localhost/execution-history/runs?status=completed,failed&trigger=manual&from=100&to=900&interaction=true&limit=10&offset=5&sort=asc", + ); + expect(res.status).toBe(200); + const body = (yield* Effect.promise(() => res.json())) as { + runs: { executionId: string }[]; + total: number; + }; + expect(body.total).toBe(1); + expect(body.runs.map((r) => r.executionId)).toEqual(["exec_1"]); + + const options = captured.options; + expect(options).toBeDefined(); + expect(options?.statusFilter).toEqual(["completed", "failed"]); + expect(options?.triggerFilter).toEqual(["manual"]); + expect(options?.timeRange).toEqual({ from: 100, to: 900 }); + expect(options?.hadInteraction).toBe(true); + expect(options?.limit).toBe(10); + expect(options?.offset).toBe(5); + expect(options?.sort).toBe("asc"); + }), + ); + + it.effect("list omits filters when no query params are supplied", () => + Effect.gen(function* () { + const captured: { options?: ExecutionHistoryListOptions } = {}; + const web = yield* webHandlerFor(makeStubExtension(captured)); + + const res = yield* get(web, "http://localhost/execution-history/runs"); + expect(res.status).toBe(200); + yield* Effect.promise(() => res.json()); + + const options = captured.options; + expect(options?.statusFilter).toBeUndefined(); + expect(options?.triggerFilter).toBeUndefined(); + expect(options?.timeRange).toBeUndefined(); + expect(options?.hadInteraction).toBeUndefined(); + }), + ); + + it.effect("get returns the run detail and null for an unknown id", () => + Effect.gen(function* () { + const web = yield* webHandlerFor(makeStubExtension({})); + + const hit = yield* get(web, "http://localhost/execution-history/runs/exec_1"); + expect(hit.status).toBe(200); + const hitBody = (yield* Effect.promise(() => hit.json())) as { + run: { executionId: string }; + toolCalls: unknown[]; + interactions: unknown[]; + }; + expect(hitBody.run.executionId).toBe("exec_1"); + expect(hitBody.toolCalls).toHaveLength(1); + + const miss = yield* get(web, "http://localhost/execution-history/runs/nope"); + expect(miss.status).toBe(200); + const missBody = yield* Effect.promise(() => miss.json()); + expect(missBody).toBeNull(); + }), + ); + + it.effect("listToolCalls returns the run's tool calls", () => + Effect.gen(function* () { + const web = yield* webHandlerFor(makeStubExtension({})); + + const res = yield* get(web, "http://localhost/execution-history/runs/exec_1/tool-calls"); + expect(res.status).toBe(200); + const body = (yield* Effect.promise(() => res.json())) as { + toolCalls: { toolCallId: string }[]; + }; + expect(body.toolCalls.map((c) => c.toolCallId)).toEqual(["call_1"]); + }), + ); +}); diff --git a/packages/plugins/execution-history/src/api/handlers.ts b/packages/plugins/execution-history/src/api/handlers.ts new file mode 100644 index 0000000000..244e83e285 --- /dev/null +++ b/packages/plugins/execution-history/src/api/handlers.ts @@ -0,0 +1,122 @@ +import { Context, Effect, Schema } from "effect"; +import { HttpApiBuilder } from "effect/unstable/httpapi"; + +import { addGroup, capture } from "@executor-js/api"; + +import { RunStatus } from "../sdk/collections"; +import type { ExecutionHistoryListOptions, ExecutionHistoryStore } from "../sdk/store"; +import { ExecutionHistoryGroup } from "./group"; + +// --------------------------------------------------------------------------- +// Service tag +// +// Holds the `executor.executionHistory` read surface. The host app provides an +// already-wrapped extension via +// `Layer.succeed(ExecutionHistoryExtensionService, executor.executionHistory)`. +// Only the read methods are reached here; `handleEvent` is the runtime writer +// and never crosses the HTTP edge. Handlers see `StorageFailure` in the typed +// channel and `capture` downgrades it to `InternalError({ traceId })`, which +// matches the group's `error: InternalError`. +// --------------------------------------------------------------------------- + +export type ExecutionHistoryExtension = Pick< + ExecutionHistoryStore, + "list" | "get" | "listToolCalls" +>; + +export class ExecutionHistoryExtensionService extends Context.Service< + ExecutionHistoryExtensionService, + ExecutionHistoryExtension +>()("ExecutionHistoryExtensionService") {} + +// --------------------------------------------------------------------------- +// Composed API β€” core + executionHistory group +// --------------------------------------------------------------------------- + +const ExecutorApiWithExecutionHistory = addGroup(ExecutionHistoryGroup); + +// --------------------------------------------------------------------------- +// Query-param helpers +// --------------------------------------------------------------------------- + +/** Split a CSV query value into trimmed, non-empty parts. */ +const splitCsv = (value: string | undefined): string[] => + value === undefined + ? [] + : value + .split(",") + .map((part) => part.trim()) + .filter((part) => part.length > 0); + +/** Interpret a string query flag as a boolean, mirroring core's "true"/"false" + * convention. Any other value (including absent) is left undefined. */ +const parseBooleanFlag = (value: string | undefined): boolean | undefined => { + if (value === "true") return true; + if (value === "false") return false; + return undefined; +}; + +// --------------------------------------------------------------------------- +// Handlers +// +// Each handler yields the extension service, maps query/params onto the store's +// options, and returns. Storage failures flow through the typed channel and are +// captured + downgraded to `InternalError({ traceId })` by `capture`. +// --------------------------------------------------------------------------- + +export const ExecutionHistoryHandlers = HttpApiBuilder.group( + ExecutorApiWithExecutionHistory, + "executionHistory", + (handlers) => + handlers + .handle("list", ({ query }) => + capture( + Effect.gen(function* () { + const history = yield* ExecutionHistoryExtensionService; + // Keep only valid RunStatus literals β€” an unknown `?status=bogus` + // must not reach the storage `where` clause. + const statusFilter = splitCsv(query.status).filter(Schema.is(RunStatus)); + const triggerFilter = splitCsv(query.trigger); + const timeRange = + query.from !== undefined || query.to !== undefined + ? { from: query.from, to: query.to } + : undefined; + const options: ExecutionHistoryListOptions = { + statusFilter: statusFilter.length > 0 ? statusFilter : undefined, + triggerFilter: triggerFilter.length > 0 ? triggerFilter : undefined, + timeRange, + hadInteraction: parseBooleanFlag(query.interaction), + limit: query.limit, + offset: query.offset, + sort: query.sort, + }; + const result = yield* history.list(options); + return { runs: result.runs, total: result.total }; + }), + ), + ) + .handle("get", ({ params }) => + capture( + Effect.gen(function* () { + const history = yield* ExecutionHistoryExtensionService; + const detail = yield* history.get(params.executionId); + return detail === null + ? null + : { + run: detail.run, + toolCalls: detail.toolCalls, + interactions: detail.interactions, + }; + }), + ), + ) + .handle("listToolCalls", ({ params }) => + capture( + Effect.gen(function* () { + const history = yield* ExecutionHistoryExtensionService; + const toolCalls = yield* history.listToolCalls(params.executionId); + return { toolCalls }; + }), + ), + ), +); diff --git a/packages/plugins/execution-history/src/api/index.ts b/packages/plugins/execution-history/src/api/index.ts new file mode 100644 index 0000000000..34b30e066b --- /dev/null +++ b/packages/plugins/execution-history/src/api/index.ts @@ -0,0 +1,29 @@ +import { definePlugin } from "@executor-js/sdk/core"; + +import { executionHistoryPlugin } from "../sdk/plugin"; +import { ExecutionHistoryGroup } from "./group"; +import { ExecutionHistoryExtensionService, ExecutionHistoryHandlers } from "./handlers"; + +export { + ExecutionHistoryGroup, + ListRunsResponse, + RunDetailResponse, + ListToolCallsResponse, +} from "./group"; +export { + ExecutionHistoryHandlers, + ExecutionHistoryExtensionService, + type ExecutionHistoryExtension, +} from "./handlers"; + +// HTTP-augmented variant of `executionHistoryPlugin`. The returned plugin +// carries the HTTP `routes`, `handlers`, and `extensionService` so a host can +// mount the execution-history read API. Hosts that compose an `HttpApi` import +// this; SDK-only consumers stay on `@executor-js/plugin-execution-history` and +// never load `@executor-js/api`. +export const executionHistoryHttpPlugin = definePlugin(() => ({ + ...executionHistoryPlugin(), + routes: () => ExecutionHistoryGroup, + handlers: () => ExecutionHistoryHandlers, + extensionService: ExecutionHistoryExtensionService, +})); diff --git a/packages/plugins/execution-history/src/react/RunsPage.tsx b/packages/plugins/execution-history/src/react/RunsPage.tsx new file mode 100644 index 0000000000..4bb6fc147d --- /dev/null +++ b/packages/plugins/execution-history/src/react/RunsPage.tsx @@ -0,0 +1,523 @@ +import { useAtomValue } from "@effect/atom-react"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; +import { useMemo, useState } from "react"; +import { Badge } from "@executor-js/react/components/badge"; +import { Button } from "@executor-js/react/components/button"; +import { ScrollArea } from "@executor-js/react/components/scroll-area"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@executor-js/react/components/select"; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@executor-js/react/components/sheet"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@executor-js/react/components/table"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@executor-js/react/components/tabs"; +import { cn } from "@executor-js/react/lib/utils"; + +import type { InteractionRow, RunRow, ToolCallRow } from "../sdk/collections"; +import type { RunStatus, ToolCallStatus } from "../sdk/collections"; +import { runDetailAtom, runToolCallsAtom, runsAtom, type RunsQuery } from "./atoms"; + +// --------------------------------------------------------------------------- +// Execution-history runs page (v1). +// +// A runs table (status / trigger / started / duration / tool calls / +// interaction) with a status filter, limit/offset pagination, and a 4-tab +// detail drawer (Properties / Tool calls / Logs / Interaction) opened on row +// click. The richer command-palette filter, keyboard row navigation, live +// auto-refresh, and timeline chart from the original prototype are +// intentionally OUT OF SCOPE for v1. +// --------------------------------------------------------------------------- + +const STATUS_OPTIONS = [ + "all", + "running", + "waiting_for_interaction", + "completed", + "failed", +] as const; + +type StatusFilter = (typeof STATUS_OPTIONS)[number]; + +const PAGE_SIZE = 50; + +// --------------------------------------------------------------------------- +// Formatting. The stored `*Json` columns are already-serialized (compact) +// strings; decode them through Schema (only `JSON.parse` is lint-banned), then +// re-indent for display. `JSON.stringify` is allowed and won't throw on a value +// that just decoded from JSON. +// --------------------------------------------------------------------------- + +const decodeJson = Schema.decodeUnknownOption(Schema.UnknownFromJsonString); +const decodeLogLines = Schema.decodeUnknownOption( + Schema.fromJsonString(Schema.Array(Schema.String)), +); + +const prettyJson = (raw: string | null): string | null => { + if (!raw) return null; + return Option.match(decodeJson(raw), { + onNone: () => raw, + onSome: (value) => JSON.stringify(value, null, 2), + }); +}; + +const logLines = (raw: string | null): readonly string[] => + !raw + ? [] + : Option.match(decodeLogLines(raw), { + onNone: () => [raw], + onSome: (value) => value, + }); + +const statusLabel = (status: RunStatus | ToolCallStatus): string => + status === "waiting_for_interaction" ? "waiting" : status; + +const statusClass = (status: RunStatus | ToolCallStatus): string => + status === "completed" + ? "border-emerald-500/30 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300" + : status === "failed" + ? "border-destructive/30 bg-destructive/10 text-destructive" + : status === "waiting_for_interaction" + ? "border-amber-500/30 bg-amber-500/10 text-amber-700 dark:text-amber-300" + : "border-sky-500/30 bg-sky-500/10 text-sky-700 dark:text-sky-300"; + +const formatDateTime = (timestamp: number | null): string => + timestamp == null ? "Pending" : new Date(timestamp).toLocaleString(); + +const formatRelative = (timestamp: number): string => { + const delta = timestamp - Date.now(); + const abs = Math.abs(delta); + const formatter = new Intl.RelativeTimeFormat(undefined, { numeric: "auto" }); + if (abs < 60_000) return formatter.format(Math.round(delta / 1000), "second"); + if (abs < 3_600_000) return formatter.format(Math.round(delta / 60_000), "minute"); + if (abs < 86_400_000) return formatter.format(Math.round(delta / 3_600_000), "hour"); + return formatter.format(Math.round(delta / 86_400_000), "day"); +}; + +const formatDuration = (value: number | null): string => { + if (value == null) return "running"; + if (value < 1000) return `${value}ms`; + if (value < 60_000) return `${(value / 1000).toFixed(1)}s`; + return `${(value / 60_000).toFixed(1)}m`; +}; + +// --------------------------------------------------------------------------- +// Presentational blocks +// --------------------------------------------------------------------------- + +function StatusBadge(props: { readonly status: RunStatus | ToolCallStatus }) { + return ( + + {statusLabel(props.status)} + + ); +} + +function MetaCard(props: { readonly label: string; readonly children: React.ReactNode }) { + return ( +
+

{props.label}

+
{props.children}
+
+ ); +} + +function JsonBlock(props: { readonly title: string; readonly value: string | null }) { + const text = prettyJson(props.value); + if (!text) { + return ( +
+ No {props.title.toLowerCase()} recorded. +
+ ); + } + return ( +
+

{props.title}

+ 
+        {text}
+
+
+ ); +} + +function CodeBlock(props: { readonly code: string }) { + return ( +
+

Code

+ 
+        {props.code}
+
+
+ ); +} + +function LogsBlock(props: { readonly logsJson: string | null }) { + const lines = logLines(props.logsJson); + if (lines.length === 0) { + return ( +
+ No logs recorded. +
+ ); + } + return ( +
+ {lines.map((line, index) => ( + 
+          {line}
+
+ ))} +
+ ); +} + +function ToolCallItem(props: { readonly call: ToolCallRow }) { + const [expanded, setExpanded] = useState(false); + return ( +
+ + {expanded && ( +
+ + {props.call.status === "failed" && props.call.errorText ? ( +
+

Error

+ 
+                {props.call.errorText}
+
+
+ ) : ( + + )} +
+ )} +
+ ); +} + +function ToolCallsTab(props: { readonly executionId: string }) { + const toolCalls = useAtomValue(runToolCallsAtom(props.executionId)); + return AsyncResult.match(toolCalls, { + onInitial: () =>

Loading tool calls...

, + onFailure: () =>

Unable to load tool calls.

, + onSuccess: ({ value }) => + value.toolCalls.length === 0 ? ( +

No tool calls recorded.

+ ) : ( +
+ {value.toolCalls.map((call) => ( + + ))} +
+ ), + }); +} + +function InteractionTab(props: { readonly interactions: readonly InteractionRow[] }) { + if (props.interactions.length === 0) { + return ( +
+ No interactions recorded. +
+ ); + } + return ( +
+ {props.interactions.map((interaction) => ( +
+
+ {interaction.kind} + {interaction.status} + {interaction.purpose ?? "β€”"} +
+ + +
+ ))} +
+ ); +} + +function DetailContent(props: { + readonly run: RunRow; + readonly interactions: readonly InteractionRow[]; +}) { + const { run } = props; + return ( +
+ + {run.executionId} + + {statusLabel(run.status)} / {run.triggerKind ?? "unknown"} /{" "} + {formatDuration(run.durationMs)} + + + + + + Properties + Tool calls + Logs + Interaction + + + +
+ + {run.executionId} + + + + + {run.triggerKind ?? "unknown"} + {formatDuration(run.durationMs)} + {formatDateTime(run.startedAt)} + {formatDateTime(run.completedAt)} +
+ + + {run.errorText && ( +
+

Error

+ 
+                  {run.errorText}
+
+
+ )} + + + + + + + + + + + + +
+ ); +} + +function DetailDrawer(props: { + readonly executionId: string | null; + readonly onClose: () => void; +}) { + const detail = useAtomValue(runDetailAtom(props.executionId ?? "__none__")); + return ( + !open && props.onClose()}> + + {props.executionId == null + ? null + : AsyncResult.match(detail, { + onInitial: () => ( +
Loading run...
+ ), + onFailure: () => ( +
Unable to load run.
+ ), + onSuccess: ({ value }) => + value ? ( + + ) : ( +
Run not found.
+ ), + })} + + + ); +} + +function RunsTableRow(props: { readonly run: RunRow; readonly onSelect: () => void }) { + const { run } = props; + return ( + + + + + {run.triggerKind ?? "unknown"} + + {formatRelative(run.startedAt)} + + {formatDuration(run.durationMs)} + {run.toolCallCount} + + {run.hadInteraction ? ( + + yes + + ) : ( + β€” + )} + + + ); +} + +function RunsTable(props: { + readonly runs: readonly RunRow[]; + readonly onSelect: (executionId: string) => void; +}) { + if (props.runs.length === 0) { + return ( +
+ No runs match the current filters. +
+ ); + } + return ( + + + + Status + Trigger + Started + Duration + Tools + Interaction + + + + {props.runs.map((run) => ( + props.onSelect(run.executionId)} + /> + ))} + +
+ ); +} + +export function RunsPage() { + const [status, setStatus] = useState("all"); + const [offset, setOffset] = useState(0); + const [selected, setSelected] = useState(null); + + const query = useMemo( + () => ({ + limit: PAGE_SIZE, + offset, + sort: "desc", + ...(status === "all" ? {} : { status }), + }), + [offset, status], + ); + const runs = useAtomValue(runsAtom(query)); + + return ( +
+
+
+

Runs

+

+ Plugin-backed local execution history, newest first. +

+
+ +
+ + + {AsyncResult.match(runs, { + onInitial: () =>
Loading runs...
, + onFailure: () =>
Unable to load runs.
, + onSuccess: ({ value }) => ( +
+ +
+

+ {value.total.toLocaleString()} total + {value.runs.length > 0 + ? ` / showing ${offset + 1}–${offset + value.runs.length}` + : ""} +

+
+ + +
+
+
+ ), + })} + + + setSelected(null)} /> +
+ ); +} diff --git a/packages/plugins/execution-history/src/react/atoms.ts b/packages/plugins/execution-history/src/react/atoms.ts new file mode 100644 index 0000000000..40bd6f854a --- /dev/null +++ b/packages/plugins/execution-history/src/react/atoms.ts @@ -0,0 +1,46 @@ +import * as Atom from "effect/unstable/reactivity/Atom"; + +import { ExecutionHistoryClient } from "./client"; + +// --------------------------------------------------------------------------- +// Query atoms for the execution-history read model. +// +// The list atom is parameterized by the wire query shape of the `list` +// endpoint (status/trigger CSV strings, numeric from/to/limit/offset, the +// "true"/"false" interaction flag, and asc/desc sort). Detail + tool-call +// atoms key off the execution id. Each atom is `Atom.family`-keyed so a +// distinct filter/page/run gets its own cached result β€” matching the +// graphql plugin's per-input atom pattern. +// --------------------------------------------------------------------------- + +export interface RunsQuery { + readonly status?: string; + readonly trigger?: string; + readonly from?: number; + readonly to?: number; + readonly interaction?: string; + readonly limit?: number; + readonly offset?: number; + readonly sort?: "asc" | "desc"; +} + +export const runsAtom = Atom.family((query: RunsQuery) => + ExecutionHistoryClient.query("executionHistory", "list", { + query, + timeToLive: "10 seconds", + }), +); + +export const runDetailAtom = Atom.family((executionId: string) => + ExecutionHistoryClient.query("executionHistory", "get", { + params: { executionId }, + timeToLive: "10 seconds", + }), +); + +export const runToolCallsAtom = Atom.family((executionId: string) => + ExecutionHistoryClient.query("executionHistory", "listToolCalls", { + params: { executionId }, + timeToLive: "10 seconds", + }), +); diff --git a/packages/plugins/execution-history/src/react/client.ts b/packages/plugins/execution-history/src/react/client.ts new file mode 100644 index 0000000000..fd3dd4726e --- /dev/null +++ b/packages/plugins/execution-history/src/react/client.ts @@ -0,0 +1,12 @@ +import { createPluginAtomClient } from "@executor-js/sdk/client"; +import { + getExecutorApiBaseUrl, + getExecutorServerAuthorizationHeader, +} from "@executor-js/react/api/server-connection"; + +import { ExecutionHistoryGroup } from "../api/group"; + +export const ExecutionHistoryClient = createPluginAtomClient(ExecutionHistoryGroup, { + baseUrl: getExecutorApiBaseUrl, + authorizationHeader: getExecutorServerAuthorizationHeader, +}); diff --git a/packages/plugins/execution-history/src/react/index.ts b/packages/plugins/execution-history/src/react/index.ts new file mode 100644 index 0000000000..9bc0a07dca --- /dev/null +++ b/packages/plugins/execution-history/src/react/index.ts @@ -0,0 +1,3 @@ +export { RunsPage } from "./RunsPage"; +export { ExecutionHistoryClient } from "./client"; +export { runsAtom, runDetailAtom, runToolCallsAtom, type RunsQuery } from "./atoms"; diff --git a/packages/plugins/execution-history/src/react/plugin-client.tsx b/packages/plugins/execution-history/src/react/plugin-client.tsx new file mode 100644 index 0000000000..04dbe62e7c --- /dev/null +++ b/packages/plugins/execution-history/src/react/plugin-client.tsx @@ -0,0 +1,15 @@ +import { defineClientPlugin } from "@executor-js/sdk/client"; +import { lazy } from "react"; + +const RunsPage = lazy(() => import("./RunsPage").then((module) => ({ default: module.RunsPage }))); + +export default defineClientPlugin({ + id: "executionHistory" as const, + pages: [ + { + path: "/", + component: RunsPage, + nav: { label: "Runs" }, + }, + ], +}); diff --git a/packages/plugins/execution-history/src/sdk/collections.ts b/packages/plugins/execution-history/src/sdk/collections.ts new file mode 100644 index 0000000000..01cdb6277c --- /dev/null +++ b/packages/plugins/execution-history/src/sdk/collections.ts @@ -0,0 +1,97 @@ +import { Schema } from "effect"; + +import { definePluginStorageCollection } from "@executor-js/sdk/core"; + +// --------------------------------------------------------------------------- +// Execution-history storage collections. +// +// Three owner-scoped plugin-storage collections back the run history: one row +// per execution (`runs`), per tool call (`toolCalls`), and per interaction +// (`interactions`). Every payload that the engine hands us as `unknown` (tool +// args/results, interaction payloads/responses, execution results/logs) is +// stored as an already-serialized JSON string in a `*Json` column so the +// indexed columns stay primitive and query-friendly. Indexes are declared so +// the read surface can filter/sort on them (the facade type-enforces that only +// declared fields appear in `where`/`orderBy`). +// --------------------------------------------------------------------------- + +/** Terminal + transient lifecycle state of a single execution. */ +export const RunStatus = Schema.Literals([ + "running", + "waiting_for_interaction", + "completed", + "failed", +]); +export type RunStatus = typeof RunStatus.Type; + +/** Lifecycle state of a single tool call within an execution. */ +export const ToolCallStatus = Schema.Literals(["running", "completed", "failed"]); +export type ToolCallStatus = typeof ToolCallStatus.Type; + +/** Lifecycle state of a single interaction (elicitation) within an execution. */ +export const InteractionStatus = Schema.Literals([ + "pending", + "accepted", + "declined", + "cancelled", + "failed", +]); +export type InteractionStatus = typeof InteractionStatus.Type; + +export const RunRow = Schema.Struct({ + executionId: Schema.String, + status: RunStatus, + code: Schema.String, + resultJson: Schema.NullOr(Schema.String), + errorText: Schema.NullOr(Schema.String), + logsJson: Schema.NullOr(Schema.String), + triggerKind: Schema.NullOr(Schema.String), + triggerMetaJson: Schema.NullOr(Schema.String), + startedAt: Schema.Number, + completedAt: Schema.NullOr(Schema.Number), + durationMs: Schema.NullOr(Schema.Number), + toolCallCount: Schema.Number, + hadInteraction: Schema.Boolean, +}); +export type RunRow = typeof RunRow.Type; + +export const runs = definePluginStorageCollection("runs", RunRow, { + indexes: ["status", "triggerKind", "startedAt", "durationMs", "hadInteraction"], +}); + +export const ToolCallRow = Schema.Struct({ + executionId: Schema.String, + toolCallId: Schema.String, + status: ToolCallStatus, + path: Schema.String, + namespace: Schema.NullOr(Schema.String), + argsJson: Schema.NullOr(Schema.String), + resultJson: Schema.NullOr(Schema.String), + errorText: Schema.NullOr(Schema.String), + startedAt: Schema.Number, + completedAt: Schema.NullOr(Schema.Number), + durationMs: Schema.NullOr(Schema.Number), +}); +export type ToolCallRow = typeof ToolCallRow.Type; + +export const toolCalls = definePluginStorageCollection("toolCalls", ToolCallRow, { + indexes: ["executionId", "startedAt"], +}); + +export const InteractionRow = Schema.Struct({ + executionId: Schema.String, + interactionId: Schema.String, + status: InteractionStatus, + kind: Schema.String, + purpose: Schema.NullOr(Schema.String), + payloadJson: Schema.NullOr(Schema.String), + responseJson: Schema.NullOr(Schema.String), + errorText: Schema.NullOr(Schema.String), + startedAt: Schema.Number, + completedAt: Schema.NullOr(Schema.Number), +}); +export type InteractionRow = typeof InteractionRow.Type; + +export const interactions = definePluginStorageCollection("interactions", InteractionRow, { + indexes: ["executionId", "startedAt"], +}); diff --git a/packages/plugins/execution-history/src/sdk/index.ts b/packages/plugins/execution-history/src/sdk/index.ts new file mode 100644 index 0000000000..f562df4b9b --- /dev/null +++ b/packages/plugins/execution-history/src/sdk/index.ts @@ -0,0 +1,22 @@ +export { executionHistoryPlugin } from "./plugin"; + +export { + interactions, + runs, + toolCalls, + InteractionRow, + InteractionStatus, + RunRow, + RunStatus, + ToolCallRow, + ToolCallStatus, +} from "./collections"; + +export { + makeExecutionHistoryObserver, + makeExecutionHistoryStore, + type ExecutionHistoryDetail, + type ExecutionHistoryListOptions, + type ExecutionHistoryListResult, + type ExecutionHistoryStore, +} from "./store"; diff --git a/packages/plugins/execution-history/src/sdk/plugin.ts b/packages/plugins/execution-history/src/sdk/plugin.ts new file mode 100644 index 0000000000..b34857adc3 --- /dev/null +++ b/packages/plugins/execution-history/src/sdk/plugin.ts @@ -0,0 +1,36 @@ +import { definePlugin } from "@executor-js/sdk/core"; + +import { interactions, runs, toolCalls } from "./collections"; +import { makeExecutionHistoryObserver, makeExecutionHistoryStore } from "./store"; + +// --------------------------------------------------------------------------- +// Execution-history plugin (SDK surface). A pure sink: it contributes no tools +// or integrations, only the three storage collections, a read surface +// (`list`/`get`/`listToolCalls`) on `executor.executionHistory`, and a runtime +// ExecutionObserver that records the engine's event stream. +// +// One store instance is shared: `storage(deps)` builds it (read methods + +// buffered `handleEvent` writer), `extension(ctx)` surfaces the read methods +// AND `handleEvent` off `ctx.storage`, and `runtime.executionObserver(self)` +// (which receives the EXTENSION) wraps `self.handleEvent` into an observer. +// +// The HTTP transport (routes/handlers/extensionService) is layered on by the +// api-aware factory in `@executor-js/plugin-execution-history/api`, so SDK-only +// consumers never load `@executor-js/api`. +// --------------------------------------------------------------------------- + +export const executionHistoryPlugin = definePlugin(() => ({ + id: "executionHistory" as const, + packageName: "@executor-js/plugin-execution-history", + pluginStorage: { runs, toolCalls, interactions }, + storage: (deps) => makeExecutionHistoryStore(deps), + extension: (ctx) => ({ + list: ctx.storage.list, + get: ctx.storage.get, + listToolCalls: ctx.storage.listToolCalls, + handleEvent: ctx.storage.handleEvent, + }), + runtime: { + executionObserver: (self) => makeExecutionHistoryObserver(self), + }, +})); diff --git a/packages/plugins/execution-history/src/sdk/store.test.ts b/packages/plugins/execution-history/src/sdk/store.test.ts new file mode 100644 index 0000000000..bc55517a7a --- /dev/null +++ b/packages/plugins/execution-history/src/sdk/store.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { + ExecutionFinished, + ExecutionId, + ExecutionStarted, + ExecutionToolCallId, + Subject, + Tenant, + ToolCallFinished, + ToolCallStarted, +} from "@executor-js/sdk"; +import { makeTestExecutor } from "@executor-js/sdk/testing"; + +import { executionHistoryPlugin } from "./plugin"; + +const owner = { tenant: Tenant.make("tenant_test"), subject: Subject.make("subject_test") }; + +describe("execution-history store", () => { + it.effect("records a completed run with one tool call from the event stream", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + backend: "sqlite", + plugins: [executionHistoryPlugin()] as const, + }); + + const executionId = ExecutionId.make("exec_1"); + const toolCallId = ExecutionToolCallId.make("call_1"); + const startedAt = new Date("2026-05-29T10:00:00.000Z"); + const toolFinishedAt = new Date("2026-05-29T10:00:01.000Z"); + const completedAt = new Date("2026-05-29T10:00:02.000Z"); + + yield* executor.executionHistory.handleEvent( + new ExecutionStarted({ + executionId, + owner, + code: "await tools.shell({ command: 'ls' })", + trigger: { kind: "manual" }, + startedAt, + }), + ); + yield* executor.executionHistory.handleEvent( + new ToolCallStarted({ + executionId, + toolCallId, + owner, + path: "tools.shell.org.default.run", + args: { command: "ls" }, + startedAt, + }), + ); + yield* executor.executionHistory.handleEvent( + new ToolCallFinished({ + executionId, + toolCallId, + owner, + path: "tools.shell.org.default.run", + status: "completed", + result: { stdout: "a.txt" }, + completedAt: toolFinishedAt, + }), + ); + yield* executor.executionHistory.handleEvent( + new ExecutionFinished({ + executionId, + owner, + status: "completed", + result: { ok: true }, + logs: ["ran ls"], + completedAt, + }), + ); + + const listed = yield* executor.executionHistory.list(); + expect(listed.total).toBe(1); + const run = listed.runs[0]; + expect(run?.executionId).toBe("exec_1"); + expect(run?.status).toBe("completed"); + expect(run?.toolCallCount).toBe(1); + expect(run?.durationMs).toBe(2000); + expect(run?.hadInteraction).toBe(false); + // code + trigger from ExecutionStarted survive the terminal re-write. + expect(run?.code).toBe("await tools.shell({ command: 'ls' })"); + expect(run?.triggerKind).toBe("manual"); + + const detail = yield* executor.executionHistory.get("exec_1"); + expect(detail?.run.status).toBe("completed"); + expect(detail?.toolCalls).toHaveLength(1); + expect(detail?.toolCalls[0]?.toolCallId).toBe("call_1"); + expect(detail?.toolCalls[0]?.status).toBe("completed"); + expect(detail?.toolCalls[0]?.durationMs).toBe(1000); + expect(detail?.interactions).toHaveLength(0); + + const toolCallRows = yield* executor.executionHistory.listToolCalls("exec_1"); + expect(toolCallRows).toHaveLength(1); + expect(toolCallRows[0]?.path).toBe("tools.shell.org.default.run"); + }), + ); +}); diff --git a/packages/plugins/execution-history/src/sdk/store.ts b/packages/plugins/execution-history/src/sdk/store.ts new file mode 100644 index 0000000000..6fad9ed577 --- /dev/null +++ b/packages/plugins/execution-history/src/sdk/store.ts @@ -0,0 +1,447 @@ +import { Effect, Option, Predicate, Schema } from "effect"; + +import { + type ExecutionEvent, + type ExecutionInteractionId, + type ExecutionObserver, + type ExecutionToolCallId, + type Owner, + type OwnerBinding, + type PluginStorageCollectionFacade, + type PluginStorageFacade, + type StorageDeps, + type StorageFailure, +} from "@executor-js/sdk/core"; + +import { + type InteractionRow, + type InteractionStatus, + type RunRow, + type RunStatus, + type ToolCallRow, + type ToolCallStatus, + interactions, + runs, + toolCalls, +} from "./collections"; + +// --------------------------------------------------------------------------- +// Execution-history store. Translates the engine's ExecutionEvent stream into +// durable run/tool-call/interaction rows and exposes the read surface. +// +// Write model β€” buffered batch: tool-call and interaction detail is held in an +// in-memory buffer keyed by executionId and only flushed when the execution +// finishes, so a completed run lands as one batch of writes rather than a +// write-per-event. Two points are written eagerly for durability even before +// the buffer flushes: the `runs` row on ExecutionStarted (status "running") and +// again on InteractionStarted (status "waiting_for_interaction"), so a paused +// run survives a restart while it waits on the user. +// +// Every `unknown` payload (tool args/results, interaction payload/response, +// execution result/logs) is serialized to a JSON string via Effect Schema +// (`Schema.UnknownFromJsonString`) β€” no raw `JSON.stringify` in domain code. +// --------------------------------------------------------------------------- + +/** Serialize an arbitrary value to a JSON string, or null when absent or when + * the value isn't JSON-encodable (encoding never throws). */ +const encodeUnknownJson = Schema.encodeUnknownOption(Schema.UnknownFromJsonString); + +const toJson = (value: unknown): string | null => + value === undefined ? null : Option.getOrNull(encodeUnknownJson(value)); + +const ownerOf = (binding: OwnerBinding): Owner => (binding.subject != null ? "user" : "org"); + +/** First dot-delimited segment of a tool path (its namespace), or null. */ +const namespaceOf = (path: string): string | null => { + const index = path.indexOf("."); + return index > 0 ? path.slice(0, index) : null; +}; + +interface BufferedToolCall { + toolCallId: ExecutionToolCallId; + status: ToolCallStatus; + path: string; + namespace: string | null; + argsJson: string | null; + resultJson: string | null; + errorText: string | null; + startedAt: number; + completedAt: number | null; + durationMs: number | null; +} + +interface BufferedInteraction { + interactionId: ExecutionInteractionId; + status: InteractionStatus; + kind: string; + purpose: string | null; + payloadJson: string | null; + responseJson: string | null; + errorText: string | null; + startedAt: number; + completedAt: number | null; +} + +interface RunBuffer { + owner: Owner; + startedAt: number; + // Retained from ExecutionStarted so every re-write of the run row (waiting, + // terminal) keeps the code + trigger β€” later events don't carry them. + code: string; + triggerKind: string | null; + triggerMetaJson: string | null; + hadInteraction: boolean; + toolCalls: Map; + interactions: Map; +} + +// --------------------------------------------------------------------------- +// Read-surface option/result types. +// --------------------------------------------------------------------------- + +export interface ExecutionHistoryListOptions { + readonly statusFilter?: readonly RunStatus[]; + readonly triggerFilter?: readonly string[]; + readonly timeRange?: { readonly from?: number; readonly to?: number }; + readonly hadInteraction?: boolean; + readonly limit?: number; + readonly offset?: number; + readonly sort?: "asc" | "desc"; +} + +export interface ExecutionHistoryListResult { + readonly runs: readonly RunRow[]; + readonly total: number; +} + +export interface ExecutionHistoryDetail { + readonly run: RunRow; + readonly toolCalls: readonly ToolCallRow[]; + readonly interactions: readonly InteractionRow[]; +} + +export interface ExecutionHistoryStore { + readonly handleEvent: (event: ExecutionEvent) => Effect.Effect; + readonly list: ( + options?: ExecutionHistoryListOptions, + ) => Effect.Effect; + readonly get: ( + executionId: string, + ) => Effect.Effect; + readonly listToolCalls: ( + executionId: string, + ) => Effect.Effect; +} + +export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistoryStore => { + const pluginStorage: PluginStorageFacade = deps.pluginStorage; + const runsC: PluginStorageCollectionFacade = pluginStorage.collection(runs); + const toolCallsC: PluginStorageCollectionFacade = + pluginStorage.collection(toolCalls); + const interactionsC: PluginStorageCollectionFacade = + pluginStorage.collection(interactions); + + const buffers = new Map(); + + const putRun = (owner: Owner, row: RunRow): Effect.Effect => + runsC.put({ owner, key: row.executionId, data: row }).pipe(Effect.asVoid); + + const onExecutionStarted = (event: Extract) => { + const owner = ownerOf(event.owner); + const startedAt = event.startedAt.getTime(); + const triggerKind = event.trigger?.kind ?? null; + const triggerMetaJson = toJson(event.trigger?.metadata); + buffers.set(event.executionId, { + owner, + startedAt, + code: event.code, + triggerKind, + triggerMetaJson, + hadInteraction: false, + toolCalls: new Map(), + interactions: new Map(), + }); + return putRun(owner, { + executionId: event.executionId, + status: "running", + code: event.code, + resultJson: null, + errorText: null, + logsJson: null, + triggerKind, + triggerMetaJson, + startedAt, + completedAt: null, + durationMs: null, + toolCallCount: 0, + hadInteraction: false, + }); + }; + + const onToolCallStarted = (event: Extract) => { + const buffer = buffers.get(event.executionId); + if (buffer) { + buffer.toolCalls.set(event.toolCallId, { + toolCallId: event.toolCallId, + status: "running", + path: event.path, + namespace: namespaceOf(event.path), + argsJson: toJson(event.args), + resultJson: null, + errorText: null, + startedAt: event.startedAt.getTime(), + completedAt: null, + durationMs: null, + }); + } + return Effect.void; + }; + + const onToolCallFinished = (event: Extract) => { + const buffer = buffers.get(event.executionId); + if (buffer) { + const completedAt = event.completedAt.getTime(); + const existing = buffer.toolCalls.get(event.toolCallId); + const startedAt = existing?.startedAt ?? completedAt; + buffer.toolCalls.set(event.toolCallId, { + toolCallId: event.toolCallId, + status: event.status, + path: event.path, + namespace: existing?.namespace ?? namespaceOf(event.path), + argsJson: existing?.argsJson ?? null, + resultJson: toJson(event.result), + errorText: event.error ?? null, + startedAt, + completedAt, + durationMs: completedAt - startedAt, + }); + } + return Effect.void; + }; + + const onInteractionStarted = (event: Extract) => { + const buffer = buffers.get(event.executionId); + if (!buffer) return Effect.void; + const request = event.context.request; + const kind = Predicate.isTagged(request, "UrlElicitation") + ? "UrlElicitation" + : "FormElicitation"; + buffer.interactions.set(event.interactionId, { + interactionId: event.interactionId, + status: "pending", + kind, + purpose: request.message, + payloadJson: toJson(event.context), + responseJson: null, + errorText: null, + startedAt: event.startedAt.getTime(), + completedAt: null, + }); + buffer.hadInteraction = true; + return putRun(buffer.owner, { + executionId: event.executionId, + status: "waiting_for_interaction", + code: buffer.code, + resultJson: null, + errorText: null, + logsJson: null, + triggerKind: buffer.triggerKind, + triggerMetaJson: buffer.triggerMetaJson, + startedAt: buffer.startedAt, + completedAt: null, + durationMs: null, + toolCallCount: buffer.toolCalls.size, + hadInteraction: true, + }); + }; + + const onInteractionResolved = ( + event: Extract, + ) => { + const buffer = buffers.get(event.executionId); + if (buffer) { + const existing = buffer.interactions.get(event.interactionId); + buffer.interactions.set(event.interactionId, { + interactionId: event.interactionId, + status: event.status, + kind: existing?.kind ?? "unknown", + purpose: existing?.purpose ?? null, + payloadJson: existing?.payloadJson ?? null, + responseJson: toJson(event.response), + errorText: event.error ?? null, + startedAt: existing?.startedAt ?? event.completedAt.getTime(), + completedAt: event.completedAt.getTime(), + }); + } + return Effect.void; + }; + + const onExecutionFinished = (event: Extract) => { + const buffer = buffers.get(event.executionId); + const owner = buffer?.owner ?? ownerOf(event.owner); + const completedAt = event.completedAt.getTime(); + const toolCallEntries = buffer ? Array.from(buffer.toolCalls.values()) : []; + const interactionEntries = buffer ? Array.from(buffer.interactions.values()) : []; + + return Effect.gen(function* () { + // Preserve code/trigger/startedAt from the buffer, or from the persisted + // "running" row if the buffer was lost (e.g. a restart mid-run). + const existing = buffer ? null : yield* runsC.get({ key: event.executionId }); + const code = buffer?.code ?? existing?.data.code ?? ""; + const triggerKind = buffer?.triggerKind ?? existing?.data.triggerKind ?? null; + const triggerMetaJson = buffer?.triggerMetaJson ?? existing?.data.triggerMetaJson ?? null; + const startedAt = buffer?.startedAt ?? existing?.data.startedAt ?? completedAt; + const hadInteraction = + buffer?.hadInteraction ?? (existing?.data.hadInteraction || interactionEntries.length > 0); + + yield* putRun(owner, { + executionId: event.executionId, + status: event.status, + code, + resultJson: toJson(event.result), + errorText: event.error ?? null, + logsJson: toJson(event.logs), + triggerKind, + triggerMetaJson, + startedAt, + completedAt, + durationMs: completedAt - startedAt, + toolCallCount: toolCallEntries.length, + hadInteraction, + }); + + yield* Effect.forEach( + toolCallEntries, + (entry) => + toolCallsC.put({ + owner, + key: entry.toolCallId, + data: { + executionId: event.executionId, + toolCallId: entry.toolCallId, + status: entry.status, + path: entry.path, + namespace: entry.namespace, + argsJson: entry.argsJson, + resultJson: entry.resultJson, + errorText: entry.errorText, + startedAt: entry.startedAt, + completedAt: entry.completedAt, + durationMs: entry.durationMs, + }, + }), + { discard: true }, + ); + + yield* Effect.forEach( + interactionEntries, + (entry) => + interactionsC.put({ + owner, + key: entry.interactionId, + data: { + executionId: event.executionId, + interactionId: entry.interactionId, + status: entry.status, + kind: entry.kind, + purpose: entry.purpose, + payloadJson: entry.payloadJson, + responseJson: entry.responseJson, + errorText: entry.errorText, + startedAt: entry.startedAt, + completedAt: entry.completedAt, + }, + }), + { discard: true }, + ); + }).pipe( + // Always release the buffer, even if a write fails or is interrupted β€” + // otherwise a StorageFailure during the flush leaks the RunBuffer forever. + Effect.ensuring(Effect.sync(() => buffers.delete(event.executionId))), + ); + }; + + const handleEvent = (event: ExecutionEvent): Effect.Effect => { + if (Predicate.isTagged(event, "ExecutionStarted")) return onExecutionStarted(event); + if (Predicate.isTagged(event, "ToolCallStarted")) return onToolCallStarted(event); + if (Predicate.isTagged(event, "ToolCallFinished")) return onToolCallFinished(event); + if (Predicate.isTagged(event, "InteractionStarted")) return onInteractionStarted(event); + if (Predicate.isTagged(event, "InteractionResolved")) return onInteractionResolved(event); + // Explicit guard, not a fallthrough: a future ExecutionEvent variant must + // not be silently recorded as a finished run. + if (Predicate.isTagged(event, "ExecutionFinished")) return onExecutionFinished(event); + return Effect.void; + }; + + const list = ( + options?: ExecutionHistoryListOptions, + ): Effect.Effect => { + const where: { + status?: { in: readonly RunStatus[] }; + triggerKind?: { in: readonly string[] }; + startedAt?: { gte?: number; lte?: number }; + hadInteraction?: { eq: boolean }; + } = {}; + if (options?.statusFilter && options.statusFilter.length > 0) { + where.status = { in: options.statusFilter }; + } + if (options?.triggerFilter && options.triggerFilter.length > 0) { + where.triggerKind = { in: options.triggerFilter }; + } + if (options?.timeRange) { + where.startedAt = {}; + if (options.timeRange.from != null) where.startedAt.gte = options.timeRange.from; + if (options.timeRange.to != null) where.startedAt.lte = options.timeRange.to; + } + if (options?.hadInteraction != null) { + where.hadInteraction = { eq: options.hadInteraction }; + } + + return Effect.gen(function* () { + const rows = yield* runsC.query({ + where, + orderBy: [{ field: "startedAt", direction: options?.sort ?? "desc" }], + limit: options?.limit, + offset: options?.offset, + }); + const total = yield* runsC.count({ where }); + return { runs: rows.map((entry) => entry.data), total }; + }); + }; + + const get = (executionId: string): Effect.Effect => + Effect.gen(function* () { + const run = yield* runsC.get({ key: executionId }); + if (run === null) return null; + const toolCallRows = yield* toolCallsC.query({ + where: { executionId }, + orderBy: [{ field: "startedAt" }], + }); + const interactionRows = yield* interactionsC.query({ + where: { executionId }, + orderBy: [{ field: "startedAt" }], + }); + return { + run: run.data, + toolCalls: toolCallRows.map((entry) => entry.data), + interactions: interactionRows.map((entry) => entry.data), + }; + }); + + const listToolCalls = ( + executionId: string, + ): Effect.Effect => + toolCallsC + .query({ where: { executionId }, orderBy: [{ field: "startedAt" }] }) + .pipe(Effect.map((rows) => rows.map((entry) => entry.data))); + + return { handleEvent, list, get, listToolCalls }; +}; + +/** Build an ExecutionObserver over a store instance β€” every engine event is + * forwarded to the store's buffered-batch writer. */ +export const makeExecutionHistoryObserver = ( + store: Pick, +): ExecutionObserver => ({ + handle: (event) => store.handleEvent(event), +}); diff --git a/packages/plugins/execution-history/tsconfig.json b/packages/plugins/execution-history/tsconfig.json new file mode 100644 index 0000000000..c495442f10 --- /dev/null +++ b/packages/plugins/execution-history/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "skipLibCheck": true, + "lib": ["ES2022", "DOM"], + "types": ["bun-types", "node"], + "noUnusedLocals": true, + "noImplicitOverride": true, + "jsx": "react-jsx", + "plugins": [ + { + "name": "@effect/language-service", + "ignoreEffectSuggestionsInTscExitCode": true, + "ignoreEffectWarningsInTscExitCode": true, + "diagnosticSeverity": { + "preferSchemaOverJson": "off" + } + } + ] + }, + "include": ["src/**/*.ts", "src/**/*.tsx"] +} diff --git a/packages/plugins/execution-history/tsup.config.ts b/packages/plugins/execution-history/tsup.config.ts new file mode 100644 index 0000000000..800469114b --- /dev/null +++ b/packages/plugins/execution-history/tsup.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: { + index: "src/sdk/index.ts", + }, + format: ["esm"], + dts: false, + sourcemap: true, + clean: true, + external: [/^@executor-js\//, /^effect/, /^@effect\//], +}); diff --git a/packages/plugins/execution-history/vitest.config.ts b/packages/plugins/execution-history/vitest.config.ts new file mode 100644 index 0000000000..5bfa2d586e --- /dev/null +++ b/packages/plugins/execution-history/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + passWithNoTests: true, + }, +}); From 94b50a2b95fc0c742a7a4aaca51f9b4fb77f76ae Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sun, 14 Jun 2026 02:21:18 +0530 Subject: [PATCH 03/57] feat: add the execution-metrics plugin (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds **`@executor-js/plugin-execution-metrics`** β€” an opt-in metrics sink built on the observer foundation. - Base plugin drives Effect Metric counters + an execution-duration histogram from the event stream. - `./api`: an auth-gated `GET /execution-metrics/metrics` Prometheus scrape (`executionMetricsHttpPlugin`). - `./cloudflare`: a Workers Analytics Engine sink (`writeDataPoint`) β€” per-isolate Effect metrics being meaningless on a Worker fleet. Wired opt-in on host-cloudflare, gated on the `ANALYTICS` binding. Base `.` entry free of `@executor-js/api` and `@cloudflare/workers-types`. Stacked on the execution-observer foundation. Gates: `format:check`, `lint`, full-monorepo `typecheck`, and metrics-render tests green (an adversarial-verify pass caught + fixed a Prometheus histogram-bucket bug). ### [Stack](https://github.com/kitlangton/stack) 1. #12 2. #13 3. **#14** πŸ‘ˆ current --- .changeset/execution-metrics-plugin.md | 13 ++ apps/host-cloudflare/package.json | 1 + apps/host-cloudflare/src/app.ts | 2 +- apps/host-cloudflare/src/config.ts | 5 + apps/host-cloudflare/src/plugins.ts | 19 ++- apps/host-cloudflare/wrangler.jsonc | 7 + bun.lock | 26 ++++ .../plugins/execution-metrics/CHANGELOG.md | 1 + .../plugins/execution-metrics/package.json | 38 ++++++ .../execution-metrics/src/api/group.ts | 28 ++++ .../execution-metrics/src/api/handlers.ts | 27 ++++ .../execution-metrics/src/api/index.ts | 24 ++++ .../src/api/prometheus.test.ts | 109 +++++++++++++++ .../execution-metrics/src/api/prometheus.ts | 118 +++++++++++++++++ .../execution-metrics/src/cloudflare/index.ts | 91 +++++++++++++ .../execution-metrics/src/sdk/bounded-map.ts | 17 +++ .../execution-metrics/src/sdk/index.ts | 2 + .../execution-metrics/src/sdk/observer.ts | 124 ++++++++++++++++++ .../execution-metrics/src/sdk/plugin.ts | 26 ++++ .../plugins/execution-metrics/tsconfig.json | 24 ++++ .../plugins/execution-metrics/tsup.config.ts | 14 ++ .../execution-metrics/vitest.config.ts | 8 ++ 22 files changed, 722 insertions(+), 2 deletions(-) create mode 100644 .changeset/execution-metrics-plugin.md create mode 100644 packages/plugins/execution-metrics/CHANGELOG.md create mode 100644 packages/plugins/execution-metrics/package.json create mode 100644 packages/plugins/execution-metrics/src/api/group.ts create mode 100644 packages/plugins/execution-metrics/src/api/handlers.ts create mode 100644 packages/plugins/execution-metrics/src/api/index.ts create mode 100644 packages/plugins/execution-metrics/src/api/prometheus.test.ts create mode 100644 packages/plugins/execution-metrics/src/api/prometheus.ts create mode 100644 packages/plugins/execution-metrics/src/cloudflare/index.ts create mode 100644 packages/plugins/execution-metrics/src/sdk/bounded-map.ts create mode 100644 packages/plugins/execution-metrics/src/sdk/index.ts create mode 100644 packages/plugins/execution-metrics/src/sdk/observer.ts create mode 100644 packages/plugins/execution-metrics/src/sdk/plugin.ts create mode 100644 packages/plugins/execution-metrics/tsconfig.json create mode 100644 packages/plugins/execution-metrics/tsup.config.ts create mode 100644 packages/plugins/execution-metrics/vitest.config.ts diff --git a/.changeset/execution-metrics-plugin.md b/.changeset/execution-metrics-plugin.md new file mode 100644 index 0000000000..9728adf762 --- /dev/null +++ b/.changeset/execution-metrics-plugin.md @@ -0,0 +1,13 @@ +--- +"@executor-js/plugin-execution-metrics": minor +--- + +Add the `@executor-js/plugin-execution-metrics` plugin β€” an opt-in metrics sink +built on the ExecutionObserver foundation. The base plugin drives Effect Metric +counters and an execution-duration histogram from the event stream; `./api`'s +`executionMetricsHttpPlugin` exposes them as an auth-gated `GET +/execution-metrics/metrics` Prometheus scrape for long-lived hosts; and +`./cloudflare`'s `createWaeMetricsObserver` writes Workers Analytics Engine data +points on the Cloudflare host (per-isolate Effect metrics being meaningless on a +Worker fleet). The base `.` entry never loads `@executor-js/api` or +`@cloudflare/workers-types`. diff --git a/apps/host-cloudflare/package.json b/apps/host-cloudflare/package.json index eacf0335a1..a4b1bf9fc7 100644 --- a/apps/host-cloudflare/package.json +++ b/apps/host-cloudflare/package.json @@ -24,6 +24,7 @@ "@executor-js/fumadb": "workspace:*", "@executor-js/host-mcp": "workspace:*", "@executor-js/plugin-encrypted-secrets": "workspace:*", + "@executor-js/plugin-execution-metrics": "workspace:*", "@executor-js/plugin-graphql": "workspace:*", "@executor-js/plugin-mcp": "workspace:*", "@executor-js/plugin-openapi": "workspace:*", diff --git a/apps/host-cloudflare/src/app.ts b/apps/host-cloudflare/src/app.ts index b664e8a32d..7a5e50096f 100644 --- a/apps/host-cloudflare/src/app.ts +++ b/apps/host-cloudflare/src/app.ts @@ -33,7 +33,7 @@ import { preloadQuickJs } from "./quickjs"; export const makeCloudflareApp = async (env: CloudflareEnv) => { const config = loadConfig(env); - const plugins = makeCloudflarePlugins(config.secretKey); + const plugins = makeCloudflarePlugins(config.secretKey, env.ANALYTICS); // Load the Workers-compatible (WASM-inlined) QuickJS variant before any // executor is built β€” the default variant can't fetch its .wasm on Workers. diff --git a/apps/host-cloudflare/src/config.ts b/apps/host-cloudflare/src/config.ts index c372513792..f01968cd84 100644 --- a/apps/host-cloudflare/src/config.ts +++ b/apps/host-cloudflare/src/config.ts @@ -1,4 +1,5 @@ import type { D1Database, DurableObjectNamespace, R2Bucket } from "@cloudflare/workers-types"; +import type { AnalyticsEngineDataset } from "@executor-js/plugin-execution-metrics/cloudflare"; import { isValidOrgSlug } from "@executor-js/api"; @@ -18,6 +19,10 @@ export interface CloudflareEnv { readonly DB: D1Database; /** R2 bucket binding β€” holds values too large for a D1 row (~1-2MB cap). */ readonly BLOBS?: R2Bucket; + /** Workers Analytics Engine binding β€” opt-in sink for execution metrics. When + * bound (uncomment `analytics_engine_datasets` in wrangler.jsonc), each + * finished execution/tool call writes a data point; absent, metrics are off. */ + readonly ANALYTICS?: AnalyticsEngineDataset; /** MCP session Durable Object namespace β€” one addressable isolate per MCP * session (the DO id IS the session id), so a session survives across the * Worker's stateless isolates. */ diff --git a/apps/host-cloudflare/src/plugins.ts b/apps/host-cloudflare/src/plugins.ts index 941c807950..279f4ec728 100644 --- a/apps/host-cloudflare/src/plugins.ts +++ b/apps/host-cloudflare/src/plugins.ts @@ -2,6 +2,12 @@ import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api"; import { graphqlHttpPlugin } from "@executor-js/plugin-graphql/api"; import { encryptedSecretsPlugin } from "@executor-js/plugin-encrypted-secrets"; +import { executionMetricsPlugin } from "@executor-js/plugin-execution-metrics"; +import { + createWaeMetricsObserver, + type AnalyticsEngineDataset, +} from "@executor-js/plugin-execution-metrics/cloudflare"; +import { noopExecutionObserver } from "@executor-js/sdk"; // --------------------------------------------------------------------------- // The Cloudflare host's plugin list β€” the same protocol/provider plugins as @@ -12,14 +18,25 @@ import { encryptedSecretsPlugin } from "@executor-js/plugin-encrypted-secrets"; // // `dangerouslyAllowStdioMCP` is false: a multi-user instance must not let a user // spawn arbitrary stdio MCP processes. +// +// Execution metrics ship to Workers Analytics Engine β€” opt-in via the wrangler +// `ANALYTICS` binding. The plugin is always in the tuple (it has no tables/API, +// so the shape is stable), but its observer is a no-op until `env.ANALYTICS` is +// bound. Effect's Metric registry is per-isolate (meaningless on a Worker +// fleet), so the local Prometheus scrape is deliberately NOT mounted here; WAE +// is the durable sink. To enable: uncomment `analytics_engine_datasets` in +// wrangler.jsonc. // --------------------------------------------------------------------------- -export const makeCloudflarePlugins = (secretKey: string) => +export const makeCloudflarePlugins = (secretKey: string, analytics?: AnalyticsEngineDataset) => [ openApiHttpPlugin(), mcpHttpPlugin({ dangerouslyAllowStdioMCP: false }), graphqlHttpPlugin(), encryptedSecretsPlugin({ key: secretKey }), + executionMetricsPlugin({ + observer: () => (analytics ? createWaeMetricsObserver(analytics) : noopExecutionObserver), + }), ] as const; export type CloudflarePlugins = ReturnType; diff --git a/apps/host-cloudflare/wrangler.jsonc b/apps/host-cloudflare/wrangler.jsonc index aec20421ff..c7eee826a1 100644 --- a/apps/host-cloudflare/wrangler.jsonc +++ b/apps/host-cloudflare/wrangler.jsonc @@ -33,6 +33,13 @@ "bucket_name": "executor-blobs", }, ], + // Workers Analytics Engine β€” the opt-in sink for execution metrics. Uncomment + // to enable: each finished execution/tool call writes a data point (queryable + // via Cloudflare's GraphQL Analytics API). The plugin is inert until this + // binding is present. Requires a paid Workers plan. + // "analytics_engine_datasets": [ + // { "binding": "ANALYTICS", "dataset": "executor_executions" }, + // ], // The MCP session Durable Object: one addressable isolate per MCP session (the // DO id IS the session id) so a session survives across the Worker's stateless // isolates β€” without it, `tools/list` after `initialize` can land on a fresh diff --git a/bun.lock b/bun.lock index b0e11e0592..a50dbc31d5 100644 --- a/bun.lock +++ b/bun.lock @@ -167,6 +167,7 @@ "@executor-js/fumadb": "workspace:*", "@executor-js/host-mcp": "workspace:*", "@executor-js/plugin-encrypted-secrets": "workspace:*", + "@executor-js/plugin-execution-metrics": "workspace:*", "@executor-js/plugin-graphql": "workspace:*", "@executor-js/plugin-mcp": "workspace:*", "@executor-js/plugin-openapi": "workspace:*", @@ -839,6 +840,29 @@ "react", ], }, + "packages/plugins/execution-metrics": { + "name": "@executor-js/plugin-execution-metrics", + "version": "0.0.0", + "dependencies": { + "@executor-js/sdk": "workspace:*", + "effect": "catalog:", + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20250620.0", + "@effect/vitest": "catalog:", + "@executor-js/api": "workspace:*", + "@types/node": "catalog:", + "bun-types": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:", + }, + "peerDependencies": { + "@executor-js/api": "workspace:*", + }, + "optionalPeers": [ + "@executor-js/api", + ], + }, "packages/plugins/file-secrets": { "name": "@executor-js/plugin-file-secrets", "version": "1.5.10", @@ -1608,6 +1632,8 @@ "@executor-js/plugin-execution-history": ["@executor-js/plugin-execution-history@workspace:packages/plugins/execution-history"], + "@executor-js/plugin-execution-metrics": ["@executor-js/plugin-execution-metrics@workspace:packages/plugins/execution-metrics"], + "@executor-js/plugin-file-secrets": ["@executor-js/plugin-file-secrets@workspace:packages/plugins/file-secrets"], "@executor-js/plugin-graphql": ["@executor-js/plugin-graphql@workspace:packages/plugins/graphql"], diff --git a/packages/plugins/execution-metrics/CHANGELOG.md b/packages/plugins/execution-metrics/CHANGELOG.md new file mode 100644 index 0000000000..3d1d7847f1 --- /dev/null +++ b/packages/plugins/execution-metrics/CHANGELOG.md @@ -0,0 +1 @@ +# @executor-js/plugin-execution-metrics diff --git a/packages/plugins/execution-metrics/package.json b/packages/plugins/execution-metrics/package.json new file mode 100644 index 0000000000..a43bb47362 --- /dev/null +++ b/packages/plugins/execution-metrics/package.json @@ -0,0 +1,38 @@ +{ + "name": "@executor-js/plugin-execution-metrics", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/sdk/index.ts", + "./api": "./src/api/index.ts", + "./cloudflare": "./src/cloudflare/index.ts" + }, + "scripts": { + "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src || true)", + "typecheck": "tsgo --noEmit", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@executor-js/sdk": "workspace:*", + "effect": "catalog:" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20250620.0", + "@effect/vitest": "catalog:", + "@executor-js/api": "workspace:*", + "@types/node": "catalog:", + "bun-types": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:" + }, + "peerDependencies": { + "@executor-js/api": "workspace:*" + }, + "peerDependenciesMeta": { + "@executor-js/api": { + "optional": true + } + } +} diff --git a/packages/plugins/execution-metrics/src/api/group.ts b/packages/plugins/execution-metrics/src/api/group.ts new file mode 100644 index 0000000000..da2a259e4a --- /dev/null +++ b/packages/plugins/execution-metrics/src/api/group.ts @@ -0,0 +1,28 @@ +import { HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/unstable/httpapi"; +import { Schema } from "effect"; +import { InternalError } from "@executor-js/sdk/shared"; + +// --------------------------------------------------------------------------- +// Responses +// +// The scrape endpoint returns Prometheus text-exposition, not JSON. +// `HttpApiSchema.asText()` flips the success body to `text/plain` so the +// renderer's string is written verbatim. +// --------------------------------------------------------------------------- + +const PrometheusResponse = Schema.String.pipe(HttpApiSchema.asText()); + +// --------------------------------------------------------------------------- +// Group β€” the execution-metrics HTTP surface. +// +// One GET endpoint. Auth-gated automatically by the host's execution-stack +// middleware, like every other plugin route. `InternalError` is the shared +// opaque 500 surface translated at the HTTP edge by `capture`. +// --------------------------------------------------------------------------- + +export const ExecutionMetricsGroup = HttpApiGroup.make("execution-metrics").add( + HttpApiEndpoint.get("metrics", "/execution-metrics/metrics", { + success: PrometheusResponse, + error: InternalError, + }), +); diff --git a/packages/plugins/execution-metrics/src/api/handlers.ts b/packages/plugins/execution-metrics/src/api/handlers.ts new file mode 100644 index 0000000000..60b132f8e3 --- /dev/null +++ b/packages/plugins/execution-metrics/src/api/handlers.ts @@ -0,0 +1,27 @@ +import { HttpApiBuilder } from "effect/unstable/httpapi"; + +import { addGroup } from "@executor-js/api"; +import { ExecutionMetricsGroup } from "./group"; +import { renderPrometheus } from "./prometheus"; + +// --------------------------------------------------------------------------- +// Composed API β€” core + execution-metrics group +// --------------------------------------------------------------------------- + +const ExecutorApiWithMetrics = addGroup(ExecutionMetricsGroup); + +// --------------------------------------------------------------------------- +// Handlers +// +// The scrape reads the global Effect Metric registry directly (no plugin +// extension state), so there is no `extensionService` to require β€” the handler +// just renders the snapshot. `renderPrometheus` has no error channel, so no +// `capture` translation is needed; the group's `InternalError` covers defects +// the edge middleware captures. +// --------------------------------------------------------------------------- + +export const ExecutionMetricsHandlers = HttpApiBuilder.group( + ExecutorApiWithMetrics, + "execution-metrics", + (handlers) => handlers.handle("metrics", () => renderPrometheus), +); diff --git a/packages/plugins/execution-metrics/src/api/index.ts b/packages/plugins/execution-metrics/src/api/index.ts new file mode 100644 index 0000000000..298a0d6c30 --- /dev/null +++ b/packages/plugins/execution-metrics/src/api/index.ts @@ -0,0 +1,24 @@ +import { definePlugin } from "@executor-js/sdk/core"; + +import { executionMetricsPlugin, type ExecutionMetricsPluginOptions } from "../sdk/plugin"; +import { ExecutionMetricsGroup } from "./group"; +import { ExecutionMetricsHandlers } from "./handlers"; + +export { ExecutionMetricsGroup } from "./group"; +export { ExecutionMetricsHandlers } from "./handlers"; +export { renderPrometheus } from "./prometheus"; + +// HTTP-augmented variant of `executionMetricsPlugin`. The returned plugin +// carries the HTTP `routes` and `handlers` so a host can mount the Prometheus +// scrape (`GET /execution-metrics/metrics`). The scrape reads the global Effect +// Metric registry, so there is no plugin extension state and no +// `extensionService`. Hosts that compose an `HttpApi` import this; SDK-only +// consumers stay on `@executor-js/plugin-execution-metrics` and never load +// `@executor-js/api`. +export const executionMetricsHttpPlugin = definePlugin( + (options?: ExecutionMetricsPluginOptions) => ({ + ...executionMetricsPlugin(options), + routes: () => ExecutionMetricsGroup, + handlers: () => ExecutionMetricsHandlers, + }), +); diff --git a/packages/plugins/execution-metrics/src/api/prometheus.test.ts b/packages/plugins/execution-metrics/src/api/prometheus.test.ts new file mode 100644 index 0000000000..dee6aea730 --- /dev/null +++ b/packages/plugins/execution-metrics/src/api/prometheus.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { + ExecutionFinished, + ExecutionId, + ExecutionStarted, + ExecutionToolCallId, + Subject, + Tenant, + ToolCallFinished, + ToolCallStarted, +} from "@executor-js/sdk/core"; + +import { createExecutionMetricsObserver } from "../sdk/observer"; +import { renderPrometheus } from "./prometheus"; + +const owner = { tenant: Tenant.make("tenant_test"), subject: Subject.make("subject_test") }; + +const executionId = ExecutionId.make("exec_metrics_test"); +const toolCallId = ExecutionToolCallId.make("toolcall_metrics_test"); + +const startedAt = new Date("2026-06-13T00:00:00.000Z"); +const completedAt = new Date("2026-06-13T00:00:00.250Z"); + +describe("execution-metrics observer + renderPrometheus", () => { + it.effect("records counters/histogram a Prometheus scrape can expose", () => + Effect.gen(function* () { + const observer = createExecutionMetricsObserver(); + + yield* observer.handle( + new ExecutionStarted({ + executionId, + owner, + code: "console.log('hi')", + trigger: { kind: "manual" }, + startedAt, + }), + ); + yield* observer.handle( + new ToolCallStarted({ + executionId, + toolCallId, + owner, + path: "github.issues.create", + args: {}, + startedAt, + }), + ); + yield* observer.handle( + new ToolCallFinished({ + executionId, + toolCallId, + owner, + path: "github.issues.create", + status: "completed", + completedAt, + }), + ); + yield* observer.handle( + new ExecutionFinished({ + executionId, + owner, + status: "completed", + result: "ok", + completedAt, + }), + ); + + const text = yield* renderPrometheus; + + // Counter + histogram series names are exposed. + expect(text).toContain("# TYPE executor_execution_started_total counter"); + expect(text).toContain("# TYPE executor_execution_completed_total counter"); + expect(text).toContain("# TYPE executor_tool_call_started_total counter"); + expect(text).toContain("# TYPE executor_tool_call_completed_total counter"); + expect(text).toContain("# TYPE executor_execution_duration_ms histogram"); + expect(text).toContain("executor_execution_duration_ms_bucket"); + expect(text).toContain('le="+Inf"'); + + // Effect's histogram buckets already carry the terminal `+Inf` boundary, + // so the renderer must emit it exactly once per series β€” a duplicate + // `+Inf` bucket line is a duplicate label set and a scrape rejects it. + const infBuckets = text + .split("\n") + .filter( + (line) => + line.startsWith("executor_execution_duration_ms_bucket{") && line.includes('le="+Inf"'), + ); + expect(infBuckets).toHaveLength(1); + + // The trigger_kind attribute rides the tagged series. + expect(text).toContain('trigger_kind="manual"'); + + // At least one started execution was counted (global registry β€” value β‰₯ 1). + const startedLine = text + .split("\n") + .find( + (line) => + line.startsWith("executor_execution_started_total{") && + line.includes('trigger_kind="manual"'), + ); + const startedValue = startedLine + ? Number(startedLine.slice(startedLine.lastIndexOf(" ") + 1)) + : 0; + expect(startedValue).toBeGreaterThanOrEqual(1); + }), + ); +}); diff --git a/packages/plugins/execution-metrics/src/api/prometheus.ts b/packages/plugins/execution-metrics/src/api/prometheus.ts new file mode 100644 index 0000000000..1da67b9028 --- /dev/null +++ b/packages/plugins/execution-metrics/src/api/prometheus.ts @@ -0,0 +1,118 @@ +import { Effect, Match, Metric } from "effect"; + +// --------------------------------------------------------------------------- +// Prometheus text-exposition renderer. +// +// Snapshots the global Effect Metric registry (`Metric.snapshot`) and renders +// the v0.0.4 text format: one HELP/TYPE header pair per metric name, then a +// sample line per attribute-set. Histograms expand to cumulative `_bucket` +// series (with a terminal `le="+Inf"`), `_count`, and `_sum`. +// --------------------------------------------------------------------------- + +const sanitizeName = (raw: string): string => raw.replace(/[^a-zA-Z0-9_:]/g, "_"); + +const escapeLabelValue = (raw: string): string => + raw.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/"/g, '\\"'); + +const formatValue = (value: bigint | number): string => { + if (typeof value === "bigint") return value.toString(10); + if (value === Number.POSITIVE_INFINITY) return "+Inf"; + if (value === Number.NEGATIVE_INFINITY) return "-Inf"; + return String(value); +}; + +const formatLabels = ( + attributes: Metric.Metric.AttributeSet, + extra?: Readonly>, +): string => { + const pairs = Object.entries(attributes).map( + ([key, value]) => `${sanitizeName(key)}="${escapeLabelValue(String(value))}"`, + ); + if (extra) { + for (const [key, value] of Object.entries(extra)) { + pairs.push(`${sanitizeName(key)}="${escapeLabelValue(value)}"`); + } + } + return pairs.length === 0 ? "" : `{${pairs.join(",")}}`; +}; + +const emitHeader = ( + name: string, + type: "counter" | "gauge" | "histogram" | "summary", + description: string, + seen: Set, +): readonly string[] => { + if (seen.has(name)) return []; + seen.add(name); + const safeDescription = description.replace(/\\/g, "\\\\").replace(/\n/g, "\\n"); + return [`# HELP ${name} ${safeDescription || "(no description)"}`, `# TYPE ${name} ${type}`]; +}; + +const formatPair = (metric: Metric.Metric.Snapshot, seen: Set): readonly string[] => { + const name = sanitizeName(metric.id); + const description = metric.description ?? ""; + const attributes = metric.attributes ?? {}; + return Match.value(metric).pipe( + Match.when({ type: "Counter" }, (metric) => { + const state = metric.state; + return [ + ...emitHeader(name, "counter", description, seen), + `${name}${formatLabels(attributes)} ${formatValue(state.count)}`, + ]; + }), + Match.when({ type: "Gauge" }, (metric) => { + const state = metric.state; + return [ + ...emitHeader(name, "gauge", description, seen), + `${name}${formatLabels(attributes)} ${formatValue(state.value)}`, + ]; + }), + Match.when({ type: "Histogram" }, (metric) => { + const state = metric.state; + const lines = [...emitHeader(name, "histogram", description, seen)]; + // Effect's HistogramState.buckets already ends with the terminal + // `Infinity` boundary (formatted as `le="+Inf"`), whose cumulative count + // equals the total observation count. Emitting it from the loop covers + // Prometheus's required `+Inf` bucket β€” appending a second `+Inf` line + // would duplicate the series and the scrape would be rejected. + for (const [upperBound, count] of state.buckets) { + lines.push( + `${name}_bucket${formatLabels(attributes, { le: formatValue(upperBound) })} ${count}`, + ); + } + lines.push(`${name}_count${formatLabels(attributes)} ${state.count}`); + lines.push(`${name}_sum${formatLabels(attributes)} ${formatValue(state.sum)}`); + return lines; + }), + Match.when({ type: "Summary" }, (metric) => { + const state = metric.state; + const lines = [...emitHeader(name, "summary", description, seen)]; + for (const [quantile, value] of state.quantiles) { + lines.push( + `${name}${formatLabels(attributes, { quantile: formatValue(quantile) })} ${formatValue( + value ?? Number.NaN, + )}`, + ); + } + lines.push(`${name}_count${formatLabels(attributes)} ${state.count}`); + lines.push(`${name}_sum${formatLabels(attributes)} ${formatValue(state.sum)}`); + return lines; + }), + Match.when({ type: "Frequency" }, (metric) => { + const state = metric.state; + const lines = [...emitHeader(name, "counter", description, seen)]; + for (const [bucket, count] of state.occurrences) { + lines.push(`${name}${formatLabels(attributes, { bucket })} ${formatValue(count)}`); + } + return lines; + }), + Match.exhaustive, + ); +}; + +/** Snapshot the global Effect Metric registry and render Prometheus text. */ +export const renderPrometheus: Effect.Effect = Effect.map(Metric.snapshot, (snapshot) => { + const seen = new Set(); + const lines = snapshot.flatMap((pair) => formatPair(pair, seen)); + return lines.length === 0 ? "\n" : `${lines.join("\n")}\n`; +}); diff --git a/packages/plugins/execution-metrics/src/cloudflare/index.ts b/packages/plugins/execution-metrics/src/cloudflare/index.ts new file mode 100644 index 0000000000..399cdb80bf --- /dev/null +++ b/packages/plugins/execution-metrics/src/cloudflare/index.ts @@ -0,0 +1,91 @@ +import { Effect, Predicate } from "effect"; + +import { type ExecutionEvent, type ExecutionObserver } from "@executor-js/sdk/core"; + +import { rememberStart } from "../sdk/bounded-map"; + +/** + * Minimal structural view of a Workers Analytics Engine binding. The full type + * lives in `@cloudflare/workers-types` as a global ambient `interface + * AnalyticsEngineDataset`, which isn't a named export and would require pulling + * the Workers global lib into this package's `types`. A local interface keeps + * the Cloudflare entry self-contained; a real binding satisfies it structurally. + */ +export interface AnalyticsEngineDataset { + writeDataPoint(event: { + readonly blobs?: ReadonlyArray; + readonly doubles?: ReadonlyArray; + readonly indexes?: ReadonlyArray; + }): void; +} + +/** + * Build an {@link ExecutionObserver} that writes one Analytics Engine data + * point per finished execution / tool call. `writeDataPoint` is synchronous, + * fire-and-forget (no `await`, no `ctx.waitUntil`), so the observer wraps each + * call in `Effect.sync`. Errors are swallowed by the engine's per-observer + * suppression β€” an observer must never break the execution it watches. + * + * Layout: + * - `ExecutionFinished` β†’ blobs `[ "execution", status, trigger? ]`, doubles + * `[ durationMs ]`, indexes `[ executionId ]`. + * - `ToolCallFinished` β†’ blobs `[ "tool_call", status, path ]`, doubles + * `[ durationMs ]`, indexes `[ executionId ]`. + */ +export const createWaeMetricsObserver = (analytics: AnalyticsEngineDataset): ExecutionObserver => { + const executionStarts = new Map< + string, + { readonly startedAt: number; readonly trigger?: string } + >(); + const toolCallStarts = new Map(); + + return { + handle: (event: ExecutionEvent) => { + if (Predicate.isTagged(event, "ExecutionStarted")) { + return Effect.sync(() => { + rememberStart(executionStarts, event.executionId, { + startedAt: event.startedAt.getTime(), + trigger: event.trigger?.kind, + }); + }); + } + + if (Predicate.isTagged(event, "ToolCallStarted")) { + return Effect.sync(() => { + rememberStart(toolCallStarts, event.toolCallId, event.startedAt.getTime()); + }); + } + + if (Predicate.isTagged(event, "ExecutionFinished")) { + return Effect.sync(() => { + const started = executionStarts.get(event.executionId); + executionStarts.delete(event.executionId); + const durationMs = started + ? Math.max(0, event.completedAt.getTime() - started.startedAt) + : 0; + analytics.writeDataPoint({ + blobs: ["execution", event.status, started?.trigger ?? ""], + doubles: [durationMs], + indexes: [event.executionId], + }); + }); + } + + if (Predicate.isTagged(event, "ToolCallFinished")) { + return Effect.sync(() => { + const startedAt = toolCallStarts.get(event.toolCallId); + toolCallStarts.delete(event.toolCallId); + const durationMs = + startedAt !== undefined ? Math.max(0, event.completedAt.getTime() - startedAt) : 0; + analytics.writeDataPoint({ + blobs: ["tool_call", event.status, event.path], + doubles: [durationMs], + indexes: [event.executionId], + }); + }); + } + + return Effect.void; + }, + }; +}; diff --git a/packages/plugins/execution-metrics/src/sdk/bounded-map.ts b/packages/plugins/execution-metrics/src/sdk/bounded-map.ts new file mode 100644 index 0000000000..123cd3e618 --- /dev/null +++ b/packages/plugins/execution-metrics/src/sdk/bounded-map.ts @@ -0,0 +1,17 @@ +// Both observer sinks track per-execution start times in a Map so a Finished +// event can compute the elapsed duration. An execution killed before its +// Finished event (process signal, VM timeout, hard shutdown) would otherwise +// leak its entry forever in a long-lived host. `rememberStart` bounds the Map: +// when it's at capacity and the key is new, the oldest entry (insertion-order +// first) is evicted β€” so a steady stream of orphaned starts can't grow without +// limit. + +const MAX_TRACKED = 10_000; + +export const rememberStart = (map: Map, key: K, value: V): void => { + if (map.size >= MAX_TRACKED && !map.has(key)) { + const oldest = map.keys().next().value; + if (oldest !== undefined) map.delete(oldest); + } + map.set(key, value); +}; diff --git a/packages/plugins/execution-metrics/src/sdk/index.ts b/packages/plugins/execution-metrics/src/sdk/index.ts new file mode 100644 index 0000000000..b7e0e3732b --- /dev/null +++ b/packages/plugins/execution-metrics/src/sdk/index.ts @@ -0,0 +1,2 @@ +export { executionMetricsPlugin, type ExecutionMetricsPluginOptions } from "./plugin"; +export { createExecutionMetricsObserver } from "./observer"; diff --git a/packages/plugins/execution-metrics/src/sdk/observer.ts b/packages/plugins/execution-metrics/src/sdk/observer.ts new file mode 100644 index 0000000000..c95dddc1a3 --- /dev/null +++ b/packages/plugins/execution-metrics/src/sdk/observer.ts @@ -0,0 +1,124 @@ +import { Effect, Metric, Predicate } from "effect"; + +import { + type ExecutionEvent, + type ExecutionId, + type ExecutionObserver, +} from "@executor-js/sdk/core"; + +import { rememberStart } from "./bounded-map"; + +// --------------------------------------------------------------------------- +// Metric definitions β€” registered in the global Effect Metric registry the +// first time this module is loaded. The Prometheus scrape (`../api/prometheus`) +// and the Effect OTLP/Prometheus exporters read the same registry. +// --------------------------------------------------------------------------- + +const executionsStarted = Metric.counter("executor.execution.started_total", { + description: "Executor executions started.", + incremental: true, +}); + +const executionsCompleted = Metric.counter("executor.execution.completed_total", { + description: "Executor executions completed.", + incremental: true, +}); + +const executionsFailed = Metric.counter("executor.execution.failed_total", { + description: "Executor executions failed.", + incremental: true, +}); + +const executionDuration = Metric.histogram("executor.execution.duration_ms", { + description: "Executor execution duration in milliseconds.", + boundaries: Metric.exponentialBoundaries({ start: 1, factor: 2, count: 16 }), +}); + +const toolCallsStarted = Metric.counter("executor.tool_call.started_total", { + description: "Executor tool calls started.", + incremental: true, +}); + +const toolCallsCompleted = Metric.counter("executor.tool_call.completed_total", { + description: "Executor tool calls completed.", + incremental: true, +}); + +const toolCallsFailed = Metric.counter("executor.tool_call.failed_total", { + description: "Executor tool calls failed.", + incremental: true, +}); + +const triggerAttributes = (triggerKind: string | undefined): Metric.Metric.AttributeSet => + triggerKind ? { trigger_kind: triggerKind } : {}; + +/** + * Build an {@link ExecutionObserver} that drives the global Effect Metric + * registry from the execution lifecycle stream. Counters track executions and + * tool calls (started / completed / failed); a histogram records execution + * duration tagged by `trigger_kind`. Start timestamps are buffered per + * execution so {@link ExecutionFinished} can compute the elapsed duration. + */ +export const createExecutionMetricsObserver = (): ExecutionObserver => { + const executionStarts = new Map< + ExecutionId, + { readonly startedAt: Date; readonly trigger?: string } + >(); + + const updateCounter = ( + counter: Metric.Counter, + attributes: Metric.Metric.AttributeSet = {}, + ): Effect.Effect => Metric.update(Metric.withAttributes(counter, attributes), 1); + + const handleExecutionStarted = ( + event: Extract, + ) => + Effect.sync(() => { + rememberStart(executionStarts, event.executionId, { + startedAt: event.startedAt, + trigger: event.trigger?.kind, + }); + }).pipe( + Effect.andThen(updateCounter(executionsStarted, triggerAttributes(event.trigger?.kind))), + ); + + const handleExecutionFinished = ( + event: Extract, + ) => + Effect.gen(function* () { + const started = executionStarts.get(event.executionId); + if (started) { + executionStarts.delete(event.executionId); + const durationMs = Math.max(0, event.completedAt.getTime() - started.startedAt.getTime()); + yield* Metric.update( + Metric.withAttributes(executionDuration, triggerAttributes(started.trigger)), + durationMs, + ); + } + + const attributes = triggerAttributes(started?.trigger); + return yield* event.status === "completed" + ? updateCounter(executionsCompleted, attributes) + : updateCounter(executionsFailed, attributes); + }); + + return { + handle: (event) => { + if (Predicate.isTagged(event, "ExecutionStarted")) { + return handleExecutionStarted(event); + } + if (Predicate.isTagged(event, "ExecutionFinished")) { + return handleExecutionFinished(event); + } + if (Predicate.isTagged(event, "ToolCallStarted")) { + return updateCounter(toolCallsStarted); + } + if (Predicate.isTagged(event, "ToolCallFinished")) { + return event.status === "completed" + ? updateCounter(toolCallsCompleted) + : updateCounter(toolCallsFailed); + } + return Effect.void; + }, + }; +}; diff --git a/packages/plugins/execution-metrics/src/sdk/plugin.ts b/packages/plugins/execution-metrics/src/sdk/plugin.ts new file mode 100644 index 0000000000..1769bebca4 --- /dev/null +++ b/packages/plugins/execution-metrics/src/sdk/plugin.ts @@ -0,0 +1,26 @@ +import { definePlugin, type ExecutionObserver } from "@executor-js/sdk/core"; + +import { createExecutionMetricsObserver } from "./observer"; + +export interface ExecutionMetricsPluginOptions { + /** Override the observer factory. Defaults to the Effect Metric sink + * ({@link createExecutionMetricsObserver}). A host on Cloudflare can pass a + * Workers Analytics Engine observer here instead. */ + readonly observer?: () => ExecutionObserver; +} + +/** + * Opt-in execution-metrics sink. Registers an {@link ExecutionObserver} via + * `runtime.executionObserver` so the engine fans every {@link ExecutionEvent} + * to it. The base plugin carries NO HTTP surface β€” import + * `@executor-js/plugin-execution-metrics/api` for the Prometheus scrape, and + * never load `@executor-js/api` from here. + */ +export const executionMetricsPlugin = definePlugin((options?: ExecutionMetricsPluginOptions) => ({ + id: "execution-metrics" as const, + packageName: "@executor-js/plugin-execution-metrics", + storage: () => ({}), + runtime: { + executionObserver: options?.observer ?? createExecutionMetricsObserver, + }, +})); diff --git a/packages/plugins/execution-metrics/tsconfig.json b/packages/plugins/execution-metrics/tsconfig.json new file mode 100644 index 0000000000..eebc1e6f14 --- /dev/null +++ b/packages/plugins/execution-metrics/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "skipLibCheck": true, + "lib": ["ES2022"], + "types": ["bun-types", "node"], + "noUnusedLocals": true, + "noImplicitOverride": true, + "plugins": [ + { + "name": "@effect/language-service", + "ignoreEffectSuggestionsInTscExitCode": true, + "ignoreEffectWarningsInTscExitCode": true, + "diagnosticSeverity": { + "preferSchemaOverJson": "off" + } + } + ] + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/plugins/execution-metrics/tsup.config.ts b/packages/plugins/execution-metrics/tsup.config.ts new file mode 100644 index 0000000000..21cb020a31 --- /dev/null +++ b/packages/plugins/execution-metrics/tsup.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: { + index: "src/sdk/index.ts", + api: "src/api/index.ts", + cloudflare: "src/cloudflare/index.ts", + }, + format: ["esm"], + dts: false, + sourcemap: true, + clean: true, + external: [/^@executor-js\//, /^effect/, /^@effect\//], +}); diff --git a/packages/plugins/execution-metrics/vitest.config.ts b/packages/plugins/execution-metrics/vitest.config.ts new file mode 100644 index 0000000000..5bfa2d586e --- /dev/null +++ b/packages/plugins/execution-metrics/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + passWithNoTests: true, + }, +}); From e78112a7d7cb45fa46c2da0f74ffdd8705ee77ca Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sun, 14 Jun 2026 20:25:58 +0530 Subject: [PATCH 04/57] feat: JSON-document aggregation + keyset pagination pushdown (#15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pushes JSON-document aggregation and keyset pagination down to SQL. **FumaDB** (`packages/core/fumadb`): adds `jsonCount`, `jsonGroupCount`, `jsonTimeBuckets`, `jsonStats` (min/max + continuous percentiles), and `jsonPage` (cursor/keyset) over a JSON document column, on the memory and Drizzle (SQLite/Postgres) adapters; unsupported adapters fail loudly. A shared JS evaluator keeps memory + the SQLite percentile fallback aligned with Postgres `percentile_cont`. **Plugin storage** (`packages/core/sdk`): the facade exposes `collection.aggregate.{count,groupCount,timeBuckets,stats}` and `collection.queryKeyset(...)`, translating a collection's indexed-field `where` into JSON-path predicates while owner/tenant scoping stays enforced by the storage policy. Tests: fumadb 44 (incl. a 12-test suite run against both memory and real SQLite); sdk 341 (incl. a suite driving aggregate/keyset through the real Drizzle+SQLite executor). Repo-wide typecheck clean. Not included: per-collection expression indexes (declared indexes still full-scan the partition β€” a perf follow-up); the Postgres path uses standard SQL but isn't unit-tested locally (no pglite). First of a 3-PR stack: pushdown -> runs API -> runs UI. ### [Stack](https://github.com/kitlangton/stack) 1. **#15** πŸ‘ˆ current 2. #16 3. #17 --- .changeset/fumadb-json-pushdown.md | 16 + .../core/fumadb/src/adapters/drizzle/query.ts | 296 +++++++++++++++++ .../core/fumadb/src/adapters/memory/index.ts | 103 ++++++ .../core/fumadb/src/query/aggregate-eval.ts | 151 +++++++++ .../core/fumadb/src/query/aggregate.test.ts | 305 ++++++++++++++++++ packages/core/fumadb/src/query/aggregate.ts | 168 ++++++++++ packages/core/fumadb/src/query/index.ts | 46 +++ packages/core/fumadb/src/query/orm/index.ts | 119 +++++++ packages/core/sdk/src/executor.ts | 302 ++++++++++++++++- packages/core/sdk/src/fuma-runtime.ts | 5 + .../sdk/src/plugin-storage-aggregate.test.ts | 200 ++++++++++++ packages/core/sdk/src/plugin-storage.ts | 106 ++++++ packages/core/sdk/src/test-config.ts | 5 + 13 files changed, 1821 insertions(+), 1 deletion(-) create mode 100644 .changeset/fumadb-json-pushdown.md create mode 100644 packages/core/fumadb/src/query/aggregate-eval.ts create mode 100644 packages/core/fumadb/src/query/aggregate.test.ts create mode 100644 packages/core/fumadb/src/query/aggregate.ts create mode 100644 packages/core/sdk/src/plugin-storage-aggregate.test.ts diff --git a/.changeset/fumadb-json-pushdown.md b/.changeset/fumadb-json-pushdown.md new file mode 100644 index 0000000000..2f728d4919 --- /dev/null +++ b/.changeset/fumadb-json-pushdown.md @@ -0,0 +1,16 @@ +--- +"@executor-js/fumadb": minor +"@executor-js/sdk": minor +--- + +Push JSON-document aggregation and keyset pagination down to SQL. FumaDB's query +layer gains `jsonCount`, `jsonGroupCount`, `jsonTimeBuckets`, `jsonStats` +(min/max + continuous percentiles), and `jsonPage` (cursor pagination) over a +JSON document column, implemented natively for the memory and Drizzle +(SQLite/Postgres) adapters and failing loudly on adapters that don't support +them. Plugin storage exposes this as `collection.aggregate.{count,groupCount, +timeBuckets,stats}` and `collection.queryKeyset(...)`, translating a +collection's indexed-field `where` into JSON-path predicates while owner/tenant +scoping stays enforced by the storage policy. Aggregates and pages are computed +in the database instead of by fetching the whole collection and reducing in +memory. diff --git a/packages/core/fumadb/src/adapters/drizzle/query.ts b/packages/core/fumadb/src/adapters/drizzle/query.ts index 9471ecda0d..076b09c29b 100644 --- a/packages/core/fumadb/src/adapters/drizzle/query.ts +++ b/packages/core/fumadb/src/adapters/drizzle/query.ts @@ -1,6 +1,13 @@ import * as Drizzle from "drizzle-orm"; import type * as PostgreSQL from "drizzle-orm/pg-core"; import type { AbstractQuery, FindManyOptions } from "../../query"; +import type { + JsonFilter, + JsonPath, + JsonScalar, + JsonValueType, +} from "../../query/aggregate"; +import { coerceJsonValue, computePercentiles } from "../../query/aggregate-eval"; import { type Condition, ConditionType } from "../../query/condition-builder"; import { type SimplifyFindOptions, toORM } from "../../query/orm"; import { @@ -263,6 +270,119 @@ export function fromDrizzle( return out; } + // --- JSON-document aggregation helpers ---------------------------------- + // Extract a JSON path as a typed SQL expression. SQLite `json_extract` is + // natively typed; Postgres `#>>` returns text and needs explicit casts. + function jsonExtractSql( + jsonColumn: ColumnType, + path: JsonPath, + valueType: JsonValueType, + ): Drizzle.SQL { + if (provider === "postgresql") { + // Build a `text[]` array literal with each segment double-quoted so a + // segment containing a comma (the array element separator) descends the + // intended key instead of splitting into two. Inside a quoted element a + // backslash and a double-quote must be backslash-escaped. + const pgPath = `{${path + .map((segment) => `"${segment.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`) + .join(",")}}`; + const text = Drizzle.sql`(${jsonColumn} #>> ${pgPath})`; + if (valueType === "number") return Drizzle.sql`${text}::numeric`; + if (valueType === "boolean") return Drizzle.sql`${text}::boolean`; + return text; + } + // Build a quoted SQLite JSON path (`$."seg1"."seg2"`) so a segment + // containing a dot (the path separator) is treated as one key. Inside a + // quoted segment an embedded double-quote is doubled (`""`). + const jsonPath = `$${path + .map((segment) => `."${segment.replace(/"/g, '""')}"`) + .join("")}`; + return Drizzle.sql`json_extract(${jsonColumn}, ${jsonPath})`; + } + + // Escape LIKE wildcards (`%`, `_`) and the escape character itself so the + // value matches literally β€” parity with the memory adapter's + // includes/startsWith/endsWith. Paired with an explicit `escape '\'` clause. + function escapeLikePattern(value: string): string { + return value.replace(/[\\%_]/g, (char) => `\\${char}`); + } + + function jsonCompareSql( + operator: string, + expr: Drizzle.SQL, + value: JsonScalar, + ): Drizzle.SQL { + switch (operator) { + case "=": + return Drizzle.eq(expr, value); + case "!=": + return Drizzle.ne(expr, value); + case ">": + return Drizzle.gt(expr, value); + case ">=": + return Drizzle.gte(expr, value); + case "<": + return Drizzle.lt(expr, value); + case "<=": + return Drizzle.lte(expr, value); + case "contains": { + const escaped = escapeLikePattern(String(value)); + return Drizzle.sql`${expr} like ${`%${escaped}%`} escape '\\'`; + } + case "starts with": { + const escaped = escapeLikePattern(String(value)); + return Drizzle.sql`${expr} like ${`${escaped}%`} escape '\\'`; + } + case "ends with": { + const escaped = escapeLikePattern(String(value)); + return Drizzle.sql`${expr} like ${`%${escaped}`} escape '\\'`; + } + default: + throw new Error(`[FumaDB Drizzle] Unsupported JSON operator: ${operator}`); + } + } + + function buildJsonFilter( + jsonColumn: ColumnType, + filter: JsonFilter, + ): Drizzle.SQL | undefined { + if (filter.kind === "and") { + return Drizzle.and( + ...filter.items.map((item) => buildJsonFilter(jsonColumn, item)), + ); + } + if (filter.kind === "or") { + return Drizzle.or( + ...filter.items.map((item) => buildJsonFilter(jsonColumn, item)), + ); + } + const expr = jsonExtractSql(jsonColumn, filter.path, filter.valueType); + if (filter.kind === "array") { + return filter.operator === "in" + ? Drizzle.inArray(expr, filter.values as unknown[]) + : Drizzle.notInArray(expr, filter.values as unknown[]); + } + return jsonCompareSql(filter.operator, expr, filter.value); + } + + function buildScopedConditions( + jsonColumn: ColumnType, + where: Condition | undefined, + filter: JsonFilter | undefined, + ): Drizzle.SQL | undefined { + const parts: Drizzle.SQL[] = []; + if (where) { + const compiled = buildWhere(toDrizzleColumn, where); + if (compiled) parts.push(compiled); + } + if (filter) { + const compiled = buildJsonFilter(jsonColumn, filter); + if (compiled) parts.push(compiled); + } + if (parts.length === 0) return undefined; + return Drizzle.and(...parts); + } + return toORM({ tables: schema.tables, async count(table, v) { @@ -402,6 +522,182 @@ export function fromDrizzle( await query; }, + async jsonCount(table, { column, where, filter }) { + const conditions = buildScopedConditions(toDrizzleColumn(column), where, filter); + return await db.$count(toDrizzle(table), conditions); + }, + async jsonGroupCount(table, { column, where, filter, path, valueType }) { + const drizzleTable = toDrizzle(table); + const jsonColumn = toDrizzleColumn(column); + const groupExpr = jsonExtractSql(jsonColumn, path, valueType ?? "text"); + const conditions = buildScopedConditions(jsonColumn, where, filter); + const rows = await db + .select({ value: groupExpr, count: Drizzle.sql`count(*)` }) + .from(drizzleTable) + .where(conditions) + .groupBy(groupExpr); + return rows.map((row) => ({ + value: coerceJsonValue(row.value, valueType ?? "text"), + count: Number(row.count), + })); + }, + async jsonTimeBuckets(table, { column, where, filter, path, bucketMs }) { + const drizzleTable = toDrizzle(table); + const jsonColumn = toDrizzleColumn(column); + const valueExpr = jsonExtractSql(jsonColumn, path, "number"); + // `value - (value % bucket)` floors to the bucket start without relying + // on integer division (SQLite binds numeric params as REAL, so `/` would + // be float division). Matches `bucketFloor` for non-negative epochs. + const bucketExpr = Drizzle.sql`(${valueExpr} - (${valueExpr} % ${bucketMs}))`; + const conditions = buildScopedConditions(jsonColumn, where, filter); + const rows = await db + .select({ bucket: bucketExpr, count: Drizzle.sql`count(*)` }) + .from(drizzleTable) + .where(conditions) + .groupBy(bucketExpr) + .orderBy(bucketExpr); + return rows.map((row) => ({ bucket: Number(row.bucket), count: Number(row.count) })); + }, + async jsonStats(table, { column, where, filter, path, percentiles }) { + const drizzleTable = toDrizzle(table); + const jsonColumn = toDrizzleColumn(column); + const valueExpr = jsonExtractSql(jsonColumn, path, "number"); + const conditions = buildScopedConditions(jsonColumn, where, filter); + const aggregate = await db + .select({ + count: Drizzle.sql`count(${valueExpr})`, + min: Drizzle.sql`min(${valueExpr})`, + max: Drizzle.sql`max(${valueExpr})`, + }) + .from(drizzleTable) + .where(conditions); + const summary = aggregate[0]; + const count = Number(summary?.count ?? 0); + if (count === 0) return { count: 0, min: null, max: null, percentiles: [] }; + const min = summary?.min == null ? null : Number(summary.min); + const max = summary?.max == null ? null : Number(summary.max); + const fractions = percentiles ?? []; + if (fractions.length === 0) return { count, min, max, percentiles: [] }; + + if (provider === "postgresql") { + const pctExpr = Drizzle.sql`percentile_cont(array[${Drizzle.sql.join( + fractions.map((fraction) => Drizzle.sql`${fraction}`), + Drizzle.sql`, `, + )}]) within group (order by ${valueExpr})`; + const pctRows = await db + .select({ values: Drizzle.sql`${pctExpr}` }) + .from(drizzleTable) + .where(conditions); + const values = pctRows[0]?.values ?? []; + return { + count, + min, + max, + percentiles: fractions.map((fraction, index) => ({ + fraction, + value: Number(values[index]), + })), + }; + } + + // SQLite has no percentile_cont β€” compute over the projected values. + const valueRows = await db + .select({ value: valueExpr }) + .from(drizzleTable) + .where(conditions) + .orderBy(valueExpr); + const sorted = valueRows + .map((row) => row.value) + .filter((value): value is number | string => value != null) + .map((value) => Number(value)) + .filter((value) => !Number.isNaN(value)) + .sort((a, b) => a - b); + return { count, min, max, percentiles: computePercentiles(sorted, fractions) }; + }, + async jsonPage(table, { column, where, filter, orderBy, keyColumn, keyDirection, cursor, limit }) { + const drizzleTable = toDrizzle(table); + const jsonColumn = toDrizzleColumn(column); + const keyDrizzle = toDrizzleColumn(keyColumn); + const scoped = buildScopedConditions(jsonColumn, where, filter); + + const orderExprs: Drizzle.SQL[] = orderBy.map((entry) => { + const expr = jsonExtractSql(jsonColumn, entry.path, entry.valueType); + // Mirror the memory adapter's null ordering (null first in asc, last in + // desc β€” see compareNullableAscending). SQLite defaults to this; Postgres + // defaults to the opposite, so make it explicit on both dialects. + return entry.direction === "desc" + ? Drizzle.sql`${expr} desc nulls last` + : Drizzle.sql`${expr} asc nulls first`; + }); + orderExprs.push(keyDirection === "desc" ? Drizzle.desc(keyDrizzle) : Drizzle.asc(keyDrizzle)); + + // Keyset boundary terms with SQL three-valued-logic null handling. A naive + // `expr > NULL` / `expr < NULL` is always unknown, which silently empties + // every page once the cursor row's sort value is null. These mirror + // compareNullableAscending so a nullable sort column paginates correctly. + const eqTerm = (expr: Drizzle.SQL, cv: unknown): Drizzle.SQL => + cv == null ? Drizzle.isNull(expr) : Drizzle.eq(expr, cv); + const strictTerm = ( + expr: Drizzle.SQL, + cv: unknown, + dir: "asc" | "desc", + ): Drizzle.SQL | null => { + if (dir === "asc") { + // null is first: everything non-null is after a null cursor. + return cv == null ? Drizzle.isNotNull(expr) : Drizzle.gt(expr, cv); + } + // desc, null last: nothing is strictly after a null cursor on this field; + // and null rows fall after any non-null cursor value. + if (cv == null) return null; + return Drizzle.or(Drizzle.lt(expr, cv), Drizzle.isNull(expr)) ?? null; + }; + + let conditions = scoped; + if (cursor) { + const orTerms: Drizzle.SQL[] = []; + for (let boundary = 0; boundary <= orderBy.length; boundary += 1) { + const andTerms: Drizzle.SQL[] = []; + for (let prior = 0; prior < boundary; prior += 1) { + const entry = orderBy[prior]!; + andTerms.push( + eqTerm( + jsonExtractSql(jsonColumn, entry.path, entry.valueType), + cursor.values[prior] as unknown, + ), + ); + } + if (boundary < orderBy.length) { + const entry = orderBy[boundary]!; + const expr = jsonExtractSql(jsonColumn, entry.path, entry.valueType); + const strict = strictTerm(expr, cursor.values[boundary] as unknown, entry.direction); + if (strict === null) continue; + andTerms.push(strict); + } else { + andTerms.push( + keyDirection === "asc" + ? Drizzle.gt(keyDrizzle, cursor.key) + : Drizzle.lt(keyDrizzle, cursor.key), + ); + } + const combined = Drizzle.and(...andTerms); + if (combined) orTerms.push(combined); + } + const afterCursor = Drizzle.or(...orTerms); + conditions = scoped && afterCursor ? Drizzle.and(scoped, afterCursor) : (afterCursor ?? scoped); + } + + const projection: Record = {}; + for (const tableColumn of Object.values(table.columns)) { + projection[tableColumn.ormName] = drizzleTable[tableColumn.names.drizzle]; + } + + return await db + .select(projection) + .from(drizzleTable) + .where(conditions) + .orderBy(...orderExprs) + .limit(limit); + }, async transaction(run) { // Some SQLite-compatible engines (Cloudflare D1) reject interactive // transactions β€” both raw BEGIN/COMMIT and the driver's `.transaction()`. diff --git a/packages/core/fumadb/src/adapters/memory/index.ts b/packages/core/fumadb/src/adapters/memory/index.ts index a596373f5c..7b931407b9 100644 --- a/packages/core/fumadb/src/adapters/memory/index.ts +++ b/packages/core/fumadb/src/adapters/memory/index.ts @@ -1,5 +1,14 @@ import type { FumaDBAdapter } from "../"; import type { AbstractQuery } from "../../query"; +import type { JsonScalar } from "../../query/aggregate"; +import { + bucketFloor, + coerceJsonValue, + compareNullableAscending, + computePercentiles, + extractJsonPath, + matchesJsonFilter, +} from "../../query/aggregate-eval"; import { ConditionType, type Condition } from "../../query/condition-builder"; import { toORM, type SimplifyFindOptions } from "../../query/orm"; import type { AnyColumn, AnySchema, AnyTable } from "../../schema"; @@ -189,6 +198,100 @@ export function memoryAdapter(options: MemoryAdapterOptions = {}): FumaDBAdapter const rows = tableRows(db, table); db[table.ormName] = rows.filter((row) => !matchesCondition(row, v.where)); }, + async jsonCount(table, { column, where, filter }) { + return tableRows(db, table).filter( + (row) => + matchesCondition(row, where) && + (!filter || matchesJsonFilter(row[column.ormName], filter)), + ).length; + }, + async jsonGroupCount(table, { column, where, filter, path, valueType }) { + const counts = new Map(); + for (const row of tableRows(db, table)) { + if (!matchesCondition(row, where)) continue; + if (filter && !matchesJsonFilter(row[column.ormName], filter)) continue; + const value = coerceJsonValue( + extractJsonPath(row[column.ormName], path), + valueType ?? "text", + ); + counts.set(value, (counts.get(value) ?? 0) + 1); + } + return [...counts.entries()].map(([value, count]) => ({ value, count })); + }, + async jsonTimeBuckets(table, { column, where, filter, path, bucketMs }) { + const counts = new Map(); + for (const row of tableRows(db, table)) { + if (!matchesCondition(row, where)) continue; + if (filter && !matchesJsonFilter(row[column.ormName], filter)) continue; + const raw = coerceJsonValue(extractJsonPath(row[column.ormName], path), "number"); + if (typeof raw !== "number") continue; + const bucket = bucketFloor(raw, bucketMs); + counts.set(bucket, (counts.get(bucket) ?? 0) + 1); + } + return [...counts.entries()] + .map(([bucket, count]) => ({ bucket, count })) + .sort((a, b) => a.bucket - b.bucket); + }, + async jsonStats(table, { column, where, filter, path, percentiles }) { + const values: number[] = []; + for (const row of tableRows(db, table)) { + if (!matchesCondition(row, where)) continue; + if (filter && !matchesJsonFilter(row[column.ormName], filter)) continue; + const raw = coerceJsonValue(extractJsonPath(row[column.ormName], path), "number"); + if (typeof raw === "number") values.push(raw); + } + if (values.length === 0) return { count: 0, min: null, max: null, percentiles: [] }; + values.sort((a, b) => a - b); + return { + count: values.length, + min: values[0]!, + max: values[values.length - 1]!, + percentiles: computePercentiles(values, percentiles ?? []), + }; + }, + async jsonPage(table, { column, where, filter, orderBy, keyColumn, keyDirection, cursor, limit }) { + const sortValue = (row: Record, index: number): JsonScalar => + coerceJsonValue( + extractJsonPath(row[column.ormName], orderBy[index]!.path), + orderBy[index]!.valueType, + ); + const keyOf = (row: Record): string => String(row[keyColumn.ormName]); + const keyDir = keyDirection === "desc" ? -1 : 1; + + let rows = tableRows(db, table).filter( + (row) => + matchesCondition(row, where) && + (!filter || matchesJsonFilter(row[column.ormName], filter)), + ); + + rows = [...rows].sort((a, b) => { + for (let index = 0; index < orderBy.length; index += 1) { + const direction = orderBy[index]!.direction === "desc" ? -1 : 1; + const compared = + compareNullableAscending(sortValue(a, index), sortValue(b, index)) * direction; + if (compared !== 0) return compared; + } + return compareNullableAscending(keyOf(a), keyOf(b)) * keyDir; + }); + + if (cursor) { + const cursorValues = cursor.values.map((value, index) => + coerceJsonValue(value, orderBy[index]!.valueType), + ); + rows = rows.filter((row) => { + for (let index = 0; index < orderBy.length; index += 1) { + const direction = orderBy[index]!.direction === "desc" ? -1 : 1; + const compared = + compareNullableAscending(sortValue(row, index), cursorValues[index] ?? null) * + direction; + if (compared !== 0) return compared > 0; + } + return compareNullableAscending(keyOf(row), cursor.key) * keyDir > 0; + }); + } + + return rows.slice(0, limit).map((row) => selectRow(table, row, true)); + }, async transaction(run: (transactionInstance: AbstractQuery) => Promise) { const snapshot = cloneValue(db); try { diff --git a/packages/core/fumadb/src/query/aggregate-eval.ts b/packages/core/fumadb/src/query/aggregate-eval.ts new file mode 100644 index 0000000000..7556d92f79 --- /dev/null +++ b/packages/core/fumadb/src/query/aggregate-eval.ts @@ -0,0 +1,151 @@ +import type { + JsonFilter, + JsonPath, + JsonPercentile, + JsonScalar, + JsonValueType, +} from "./aggregate"; + +// --------------------------------------------------------------------------- +// Pure, in-memory evaluation of the JSON-document aggregation primitives. +// Shared by the memory adapter and used as the per-dialect fallback where a +// database can't express an operation natively (e.g. SQLite percentiles). The +// percentile definition matches Postgres `percentile_cont` (continuous, linear +// interpolation) so every backend agrees. +// --------------------------------------------------------------------------- + +export const extractJsonPath = (doc: unknown, path: JsonPath): unknown => { + let current: unknown = doc; + for (const segment of path) { + if (current == null || typeof current !== "object") return undefined; + current = (current as Record)[segment]; + } + return current; +}; + +/** Coerce a raw value to a comparable scalar per its declared extraction type. */ +export const coerceJsonValue = ( + value: unknown, + valueType: JsonValueType, +): JsonScalar => { + if (value == null) return null; + if (valueType === "number") { + const numeric = typeof value === "number" ? value : Number(value); + return Number.isNaN(numeric) ? null : numeric; + } + if (valueType === "boolean") { + if (typeof value === "boolean") return value; + if (value === "true" || value === 1) return true; + if (value === "false" || value === 0) return false; + return null; + } + return typeof value === "string" ? value : String(value); +}; + +const scalarsEqual = (left: JsonScalar, right: JsonScalar): boolean => left === right; + +/** Three-way compare; `null` when either side is null (SQL-like incomparable). */ +const compareScalars = (left: JsonScalar, right: JsonScalar): number | null => { + if (left == null || right == null) return null; + if (left < right) return -1; + if (left > right) return 1; + return 0; +}; + +const matchesCompareOperator = ( + operator: string, + left: JsonScalar, + right: JsonScalar, +): boolean => { + switch (operator) { + // SQL three-valued logic: any comparison with NULL is unknown and excluded + // from a WHERE clause, so `= NULL` and `!= NULL` both match nothing. Mirror + // that here so the memory adapter agrees with the SQL adapters. + case "=": + return left != null && right != null && scalarsEqual(left, right); + case "!=": + return left != null && right != null && !scalarsEqual(left, right); + case ">": { + const compared = compareScalars(left, right); + return compared != null && compared > 0; + } + case ">=": { + const compared = compareScalars(left, right); + return compared != null && compared >= 0; + } + case "<": { + const compared = compareScalars(left, right); + return compared != null && compared < 0; + } + case "<=": { + const compared = compareScalars(left, right); + return compared != null && compared <= 0; + } + case "contains": + return typeof left === "string" && typeof right === "string" && left.includes(right); + case "starts with": + return typeof left === "string" && typeof right === "string" && left.startsWith(right); + case "ends with": + return typeof left === "string" && typeof right === "string" && left.endsWith(right); + default: + return false; + } +}; + +export const matchesJsonFilter = (doc: unknown, filter: JsonFilter): boolean => { + switch (filter.kind) { + case "and": + return filter.items.every((item) => matchesJsonFilter(doc, item)); + case "or": + return filter.items.some((item) => matchesJsonFilter(doc, item)); + case "compare": { + const left = coerceJsonValue(extractJsonPath(doc, filter.path), filter.valueType); + const right = coerceJsonValue(filter.value, filter.valueType); + return matchesCompareOperator(filter.operator, left, right); + } + case "array": { + const left = coerceJsonValue(extractJsonPath(doc, filter.path), filter.valueType); + // SQL: `NULL IN (...)` and `NULL NOT IN (...)` are both unknown β†’ the row + // is excluded either way. Mirror that (the Drizzle adapter does this + // implicitly via NULL semantics). + if (left == null) return false; + const found = filter.values.some((candidate) => + scalarsEqual(left, coerceJsonValue(candidate, filter.valueType)), + ); + return filter.operator === "in" ? found : !found; + } + } +}; + +export const bucketFloor = (value: number, bucketMs: number): number => + Math.floor(value / bucketMs) * bucketMs; + +/** Percentiles over an ascending-sorted numeric array (Postgres `percentile_cont`). */ +export const computePercentiles = ( + sortedAscending: readonly number[], + fractions: readonly number[], +): JsonPercentile[] => { + const length = sortedAscending.length; + if (length === 0) return []; + return fractions.map((fraction) => { + const clamped = Math.min(1, Math.max(0, fraction)); + const rank = clamped * (length - 1); + const lower = Math.floor(rank); + const upper = Math.ceil(rank); + const value = + lower === upper + ? sortedAscending[lower]! + : sortedAscending[lower]! + (sortedAscending[upper]! - sortedAscending[lower]!) * (rank - lower); + return { fraction, value }; + }); +}; + +/** Null-aware ascending compare (null sorts first). */ +export const compareNullableAscending = (left: JsonScalar, right: JsonScalar): number => { + if (left == null && right == null) return 0; + if (left == null) return -1; + if (right == null) return 1; + if (left < right) return -1; + if (left > right) return 1; + return 0; +}; diff --git a/packages/core/fumadb/src/query/aggregate.test.ts b/packages/core/fumadb/src/query/aggregate.test.ts new file mode 100644 index 0000000000..cfd6b9eb37 --- /dev/null +++ b/packages/core/fumadb/src/query/aggregate.test.ts @@ -0,0 +1,305 @@ +import Database from "better-sqlite3"; +import { drizzle } from "drizzle-orm/better-sqlite3"; +import { afterEach, beforeEach, describe, expect, it } from "@effect/vitest"; +import { fumadb } from "@executor-js/fumadb"; +import { + createDrizzleRuntimeSchemaFromTables, + createDrizzleRuntimeSchemaSqlFromTables, + drizzleAdapter, +} from "@executor-js/fumadb/adapters/drizzle"; +import { memoryAdapter } from "@executor-js/fumadb/adapters/memory"; +import type { AbstractQuery, JsonFilter } from "@executor-js/fumadb/query"; +import { column, idColumn, schema, table } from "@executor-js/fumadb/schema"; + +const events = table("events", { + id: idColumn("id", "varchar(255)"), + tenant: column("tenant", "varchar(255)"), + data: column("data", "json"), +}); + +const v1 = schema({ version: "1.0.0", tables: { events } }); + +type EventsQuery = AbstractQuery; + +interface SeedRow { + readonly id: string; + readonly tenant: string; + readonly data: { + readonly status: string; + readonly trigger: string; + readonly startedAt: number; + readonly durationMs: number | null; + }; +} + +const seedRows: readonly SeedRow[] = [ + { id: "e1", tenant: "t1", data: { status: "completed", trigger: "cli", startedAt: 1000, durationMs: 100 } }, + { id: "e2", tenant: "t1", data: { status: "completed", trigger: "cli", startedAt: 2000, durationMs: 200 } }, + { id: "e3", tenant: "t1", data: { status: "failed", trigger: "http", startedAt: 3000, durationMs: 300 } }, + { id: "e4", tenant: "t1", data: { status: "running", trigger: "http", startedAt: 4000, durationMs: null } }, + { id: "e5", tenant: "t2", data: { status: "completed", trigger: "cli", startedAt: 5000, durationMs: 500 } }, +]; + +// Rows whose status carries literal LIKE wildcards (`_`, `%`). Kept out of the +// shared seed (existing assertions count exact t1 rows) and seeded only by the +// wildcard-parity test under a dedicated tenant. `a_b%c` must be matched +// literally; `axbyc` is the lookalike an unescaped LIKE would wrongly catch. +const wildcardRows: readonly SeedRow[] = [ + { id: "w1", tenant: "tw", data: { status: "a_b%c", trigger: "cli", startedAt: 6000, durationMs: 600 } }, + { id: "w2", tenant: "tw", data: { status: "axbyc", trigger: "cli", startedAt: 7000, durationMs: 700 } }, +]; + +const inT1 = (eb: Parameters[1]["where"]>>[0]) => + eb("tenant", "=", "t1"); + +const statusIn = (values: readonly string[]): JsonFilter => ({ + kind: "array", + path: ["status"], + valueType: "text", + operator: "in", + values, +}); + +const statusCompare = ( + operator: "=" | "contains" | "starts with" | "ends with", + value: string, +): JsonFilter => ({ + kind: "compare", + path: ["status"], + valueType: "text", + operator, + value, +}); + +const seed = async (orm: EventsQuery) => { + await orm.createMany("events", seedRows.map((row) => ({ ...row }))); +}; + +interface Harness { + readonly orm: EventsQuery; + readonly close: () => Promise; +} + +const makeMemoryHarness = async (): Promise => { + const client = fumadb({ namespace: "aggregate_test", schemas: [v1] }).client(memoryAdapter()); + return { orm: client.orm("1.0.0") as EventsQuery, close: async () => {} }; +}; + +const makeSqliteHarness = async (): Promise => { + const sqlite = new Database(":memory:"); + const schemaArgs = { + tables: v1.tables, + namespace: "aggregate_test", + version: "1.0.0", + provider: "sqlite", + } as const; + const drizzleDb = drizzle(sqlite, { schema: createDrizzleRuntimeSchemaFromTables(schemaArgs) }); + for (const statement of createDrizzleRuntimeSchemaSqlFromTables(schemaArgs)) { + sqlite.exec(statement); + } + const client = fumadb({ namespace: "aggregate_test", schemas: [v1] }).client( + drizzleAdapter({ db: drizzleDb, provider: "sqlite" }), + ); + return { + orm: client.orm("1.0.0") as EventsQuery, + close: async () => { + sqlite.close(); + }, + }; +}; + +const runSuite = (name: string, makeHarness: () => Promise) => { + describe(`json aggregation (${name})`, () => { + let harness: Harness; + beforeEach(async () => { + harness = await makeHarness(); + await seed(harness.orm); + }); + afterEach(async () => { + await harness.close(); + }); + + it("jsonCount scopes by real columns and JSON filter", async () => { + const { orm } = harness; + expect(await orm.jsonCount("events", { column: "data", where: inT1 })).toBe(4); + expect(await orm.jsonCount("events", { column: "data", where: (eb) => eb("tenant", "=", "t2") })).toBe(1); + expect( + await orm.jsonCount("events", { column: "data", where: inT1, filter: statusIn(["completed"]) }), + ).toBe(2); + }); + + it("jsonGroupCount counts distinct values", async () => { + const rows = await harness.orm.jsonGroupCount("events", { + column: "data", + where: inT1, + path: ["status"], + }); + const byValue = Object.fromEntries(rows.map((row) => [row.value, row.count])); + expect(byValue).toEqual({ completed: 2, failed: 1, running: 1 }); + }); + + it("jsonTimeBuckets buckets a numeric path", async () => { + const rows = await harness.orm.jsonTimeBuckets("events", { + column: "data", + where: inT1, + path: ["startedAt"], + bucketMs: 2000, + }); + expect(rows).toEqual([ + { bucket: 0, count: 1 }, + { bucket: 2000, count: 2 }, + { bucket: 4000, count: 1 }, + ]); + }); + + it("jsonStats computes count/min/max and continuous percentiles", async () => { + const stats = await harness.orm.jsonStats("events", { + column: "data", + where: inT1, + path: ["durationMs"], + percentiles: [0, 0.5, 1], + }); + expect(stats.count).toBe(3); + expect(stats.min).toBe(100); + expect(stats.max).toBe(300); + expect(stats.percentiles).toEqual([ + { fraction: 0, value: 100 }, + { fraction: 0.5, value: 200 }, + { fraction: 1, value: 300 }, + ]); + }); + + it("jsonPage keyset-paginates by a JSON path with a real-column tiebreak", async () => { + const { orm } = harness; + const page1 = await orm.jsonPage("events", { + column: "data", + where: inT1, + orderBy: [{ path: ["startedAt"], valueType: "number", direction: "desc" }], + keyColumn: "id", + keyDirection: "desc", + limit: 2, + }); + expect(page1.map((row) => row.id)).toEqual(["e4", "e3"]); + + const page2 = await orm.jsonPage("events", { + column: "data", + where: inT1, + orderBy: [{ path: ["startedAt"], valueType: "number", direction: "desc" }], + keyColumn: "id", + keyDirection: "desc", + limit: 2, + cursor: { values: [3000], key: "e3" }, + }); + expect(page2.map((row) => row.id)).toEqual(["e2", "e1"]); + }); + + it("jsonPage applies the JSON filter", async () => { + const rows = await harness.orm.jsonPage("events", { + column: "data", + where: inT1, + filter: statusIn(["completed", "failed"]), + orderBy: [{ path: ["startedAt"], valueType: "number", direction: "asc" }], + keyColumn: "id", + keyDirection: "asc", + limit: 10, + }); + expect(rows.map((row) => row.id)).toEqual(["e1", "e2", "e3"]); + }); + + it("excludes null-path rows from = / != filters (SQL three-valued logic)", async () => { + const { orm } = harness; + // e4 has durationMs: null. `!= 200` excludes e2 (it matches 200) AND e4 + // (NULL comparisons are unknown in SQL), leaving e1 + e3. + expect( + await orm.jsonCount("events", { + column: "data", + where: inT1, + filter: { kind: "compare", path: ["durationMs"], valueType: "number", operator: "!=", value: 200 }, + }), + ).toBe(2); + // `= 100` matches only e1; the null row never matches `=`. + expect( + await orm.jsonCount("events", { + column: "data", + where: inT1, + filter: { kind: "compare", path: ["durationMs"], valueType: "number", operator: "=", value: 100 }, + }), + ).toBe(1); + }); + + it("keyset-paginates a nullable sort column without truncating", async () => { + const { orm } = harness; + const collected: string[] = []; + let cursor: { readonly values: readonly (number | null)[]; readonly key: string } | undefined; + for (let i = 0; i < 10; i += 1) { + const rows = await orm.jsonPage("events", { + column: "data", + where: inT1, + orderBy: [{ path: ["durationMs"], valueType: "number", direction: "asc" }], + keyColumn: "id", + keyDirection: "asc", + limit: 1, + cursor, + }); + if (rows.length === 0) break; + const last = rows[rows.length - 1]!; + const key = last.id as string; + collected.push(key); + cursor = { values: [(last.data as { durationMs: number | null }).durationMs], key }; + } + // asc, nulls first: e4(null), then e1(100), e2(200), e3(300). The page + // whose cursor is the null row must still return the non-null rows β€” a + // naive `durationMs > NULL` predicate would truncate here. + expect(collected).toEqual(["e4", "e1", "e2", "e3"]); + }); + }); +}; + +runSuite("memory", makeMemoryHarness); +runSuite("sqlite", makeSqliteHarness); + +// LIKE-wildcard parity: a value carrying literal `_`/`%` must match the same +// rows under the drizzle (sqlite) adapter as under the literal memory adapter. +// Without ESCAPE these wildcards would make sqlite over-match (`a_b%c` would +// also catch `axbyc`). +describe("json filter LIKE-wildcard parity (memory vs sqlite)", () => { + let memory: Harness; + let sqlite: Harness; + beforeEach(async () => { + memory = await makeMemoryHarness(); + sqlite = await makeSqliteHarness(); + await memory.orm.createMany("events", wildcardRows.map((row) => ({ ...row }))); + await sqlite.orm.createMany("events", wildcardRows.map((row) => ({ ...row }))); + }); + afterEach(async () => { + await memory.close(); + await sqlite.close(); + }); + + const idsFor = async (orm: EventsQuery, filter: JsonFilter): Promise => { + const rows = await orm.jsonPage("events", { + column: "data", + where: (eb) => eb("tenant", "=", "tw"), + filter, + orderBy: [{ path: ["startedAt"], valueType: "number", direction: "asc" }], + keyColumn: "id", + keyDirection: "asc", + limit: 100, + }); + return rows.map((row) => row.id as string); + }; + + it.each([ + ["contains", statusCompare("contains", "_b%")], + ["eq", statusCompare("=", "a_b%c")], + ["starts with", statusCompare("starts with", "a_")], + ["ends with", statusCompare("ends with", "%c")], + ] as const)("matches identically for %s", async (_label, filter) => { + const fromMemory = await idsFor(memory.orm, filter); + const fromSqlite = await idsFor(sqlite.orm, filter); + expect(fromSqlite).toEqual(fromMemory); + // The wildcard row must be selected, the lookalike (`axbyc`) must not. + expect(fromMemory).toContain("w1"); + expect(fromMemory).not.toContain("w2"); + }); +}); diff --git a/packages/core/fumadb/src/query/aggregate.ts b/packages/core/fumadb/src/query/aggregate.ts new file mode 100644 index 0000000000..6d6e3fcbf8 --- /dev/null +++ b/packages/core/fumadb/src/query/aggregate.ts @@ -0,0 +1,168 @@ +import type { AnyColumn } from "../schema/create"; +import type { Condition, ConditionBuilder } from "./condition-builder"; + +// --------------------------------------------------------------------------- +// JSON-document aggregation + keyset pagination. +// +// These operate over values addressed *inside* a JSON document column (e.g. +// `plugin_storage.data`), not over real typed columns. The addressing model is +// therefore path-based and carries an explicit extraction type so adapters can +// emit correct per-dialect SQL (Postgres `->>` returns text and needs casts; +// SQLite `json_extract` is natively typed). Only adapters that implement the +// matching `ORMAdapter` hooks support these; others throw loudly. +// --------------------------------------------------------------------------- + +export type JsonValueType = "text" | "number" | "boolean"; + +export type JsonScalar = string | number | boolean | null; + +/** A non-empty path into a JSON document column, e.g. `["status"]`. */ +export type JsonPath = readonly [string, ...string[]]; + +export type JsonCompareOperator = + | "=" + | "!=" + | ">" + | ">=" + | "<" + | "<=" + | "contains" + | "starts with" + | "ends with"; + +export type JsonArrayOperator = "in" | "not in"; + +/** A predicate tree over JSON-document paths, ANDed into the real-column where. */ +export type JsonFilter = + | { + readonly kind: "compare"; + readonly path: JsonPath; + readonly valueType: JsonValueType; + readonly operator: JsonCompareOperator; + readonly value: JsonScalar; + } + | { + readonly kind: "array"; + readonly path: JsonPath; + readonly valueType: JsonValueType; + readonly operator: JsonArrayOperator; + readonly values: readonly JsonScalar[]; + } + | { readonly kind: "and"; readonly items: readonly JsonFilter[] } + | { readonly kind: "or"; readonly items: readonly JsonFilter[] }; + +// --- Adapter-facing options (column/keyColumn resolved to AnyColumn, where +// compiled to a Condition with read policies already applied) --------------- + +export interface JsonAdapterBase { + /** The JSON document column the paths address. */ + readonly column: AnyColumn; + /** Real-column scoping condition (read policies already applied). */ + readonly where?: Condition; + /** JSON-path predicates, conjoined with `where`. */ + readonly filter?: JsonFilter; +} + +export type JsonCountAdapterOptions = JsonAdapterBase; + +export interface JsonGroupCountAdapterOptions extends JsonAdapterBase { + readonly path: JsonPath; + /** Extraction type for the group key; defaults to "text". */ + readonly valueType?: JsonValueType; +} + +export interface JsonGroupCountRow { + readonly value: JsonScalar; + readonly count: number; +} + +export interface JsonTimeBucketAdapterOptions extends JsonAdapterBase { + /** Numeric (epoch-ms) path the buckets are computed from. */ + readonly path: JsonPath; + readonly bucketMs: number; +} + +export interface JsonTimeBucketRow { + /** Bucket floor: `floor(value / bucketMs) * bucketMs`. */ + readonly bucket: number; + readonly count: number; +} + +export interface JsonStatsAdapterOptions extends JsonAdapterBase { + /** Numeric path the stats are computed over. */ + readonly path: JsonPath; + /** Percentile fractions in [0, 1] (e.g. 0.5, 0.95). */ + readonly percentiles?: readonly number[]; +} + +export interface JsonPercentile { + readonly fraction: number; + readonly value: number; +} + +export interface JsonStats { + readonly count: number; + readonly min: number | null; + readonly max: number | null; + readonly percentiles: readonly JsonPercentile[]; +} + +export interface JsonKeysetOrder { + readonly path: JsonPath; + readonly valueType: JsonValueType; + readonly direction: "asc" | "desc"; +} + +export interface JsonKeysetCursor { + /** One value per `orderBy` entry, in order. */ + readonly values: readonly JsonScalar[]; + /** Tiebreak value of the `keyColumn` for the cursor row. */ + readonly key: string; +} + +export interface JsonPageAdapterOptions extends JsonAdapterBase { + readonly orderBy: readonly JsonKeysetOrder[]; + /** Real column used as a stable tiebreak (e.g. the storage key). */ + readonly keyColumn: AnyColumn; + readonly keyDirection: "asc" | "desc"; + readonly cursor?: JsonKeysetCursor; + readonly limit: number; +} + +// --- Public (AbstractQuery) options: name strings + a where builder -------- + +export interface JsonPublicBase> { + readonly column: keyof TColumns & string; + readonly where?: (eb: ConditionBuilder) => Condition | boolean; + readonly filter?: JsonFilter; +} + +export type JsonCountOptions> = + JsonPublicBase; + +export interface JsonGroupCountOptions> + extends JsonPublicBase { + readonly path: JsonPath; + readonly valueType?: JsonValueType; +} + +export interface JsonTimeBucketOptions> + extends JsonPublicBase { + readonly path: JsonPath; + readonly bucketMs: number; +} + +export interface JsonStatsOptions> + extends JsonPublicBase { + readonly path: JsonPath; + readonly percentiles?: readonly number[]; +} + +export interface JsonPageOptions> + extends JsonPublicBase { + readonly orderBy: readonly JsonKeysetOrder[]; + readonly keyColumn: keyof TColumns & string; + readonly keyDirection?: "asc" | "desc"; + readonly cursor?: JsonKeysetCursor; + readonly limit: number; +} diff --git a/packages/core/fumadb/src/query/index.ts b/packages/core/fumadb/src/query/index.ts index 2128adb0ef..a339b8b868 100644 --- a/packages/core/fumadb/src/query/index.ts +++ b/packages/core/fumadb/src/query/index.ts @@ -6,9 +6,20 @@ import type { TableInsertValues, TableUpdateValues, } from "../schema/create"; +import type { + JsonCountOptions, + JsonGroupCountOptions, + JsonGroupCountRow, + JsonPageOptions, + JsonStats, + JsonStatsOptions, + JsonTimeBucketOptions, + JsonTimeBucketRow, +} from "./aggregate"; import type { Condition, ConditionBuilder } from "./condition-builder"; import type { ORMAdapter } from "./orm"; +export type * from "./aggregate"; export type { Condition, ConditionBuilder } from "./condition-builder"; export { ConditionType } from "./condition-builder"; export { withQueryContext } from "./orm"; @@ -208,4 +219,39 @@ export interface AbstractQuery { eb: ConditionBuilder ) => Condition | boolean; }) => Promise; + + // --- JSON-document aggregation + keyset pagination ----------------------- + // Operate over values addressed inside a JSON document column. Only adapters + // that implement the matching `ORMAdapter` hooks support these; the rest + // throw `[FumaDB] is not supported by this adapter`. + + /** Count rows matching the real-column `where` and JSON-path `filter`. */ + jsonCount: ( + table: TableName, + options: JsonCountOptions + ) => Promise; + + /** Group by a JSON path and count rows per distinct value. */ + jsonGroupCount: ( + table: TableName, + options: JsonGroupCountOptions + ) => Promise; + + /** Bucket a numeric JSON path by `bucketMs` and count rows per bucket. */ + jsonTimeBuckets: ( + table: TableName, + options: JsonTimeBucketOptions + ) => Promise; + + /** Compute count/min/max and percentiles over a numeric JSON path. */ + jsonStats: ( + table: TableName, + options: JsonStatsOptions + ) => Promise; + + /** Keyset-paginate rows ordered by JSON paths with a real-column tiebreak. */ + jsonPage: ( + table: TableName, + options: JsonPageOptions + ) => Promise[]>; } diff --git a/packages/core/fumadb/src/query/orm/index.ts b/packages/core/fumadb/src/query/orm/index.ts index f0bac42e78..1d76a758fe 100644 --- a/packages/core/fumadb/src/query/orm/index.ts +++ b/packages/core/fumadb/src/query/orm/index.ts @@ -12,6 +12,16 @@ import type { JoinBuilder, OrderBy, } from ".."; +import type { + JsonCountAdapterOptions, + JsonGroupCountAdapterOptions, + JsonGroupCountRow, + JsonPageAdapterOptions, + JsonStats, + JsonStatsAdapterOptions, + JsonTimeBucketAdapterOptions, + JsonTimeBucketRow, +} from "../aggregate"; import { buildCondition, createBuilder, type Condition } from "../condition-builder"; export interface CompiledJoin { @@ -311,6 +321,25 @@ export interface ORMAdapter { transaction: ( run: (transactionInstance: AbstractQuery) => Promise, ) => Promise; + + // --- JSON-document aggregation + keyset pagination (optional) ------------ + // Adapters that can push these into the database implement them; `toORM` + // throws a clear error for adapters that don't. + + jsonCount?: (table: AnyTable, options: JsonCountAdapterOptions) => Promise; + jsonGroupCount?: ( + table: AnyTable, + options: JsonGroupCountAdapterOptions, + ) => Promise; + jsonTimeBuckets?: ( + table: AnyTable, + options: JsonTimeBucketAdapterOptions, + ) => Promise; + jsonStats?: (table: AnyTable, options: JsonStatsAdapterOptions) => Promise; + jsonPage?: ( + table: AnyTable, + options: JsonPageAdapterOptions, + ) => Promise[]>; } export interface ToORMOptions { @@ -334,6 +363,33 @@ export function toORM( return table; } + function toColumn(table: AnyTable, name: string): AnyColumn { + const column = table.columns[name]; + if (!column) + throw new Error(`[FumaDB] Invalid column name ${name} in ${table.ormName}.`); + return column; + } + + // Compiles a public where builder, applies read policies, and yields the + // scoping condition. Returns `false` when the query is statically empty. + async function compileScopedWhere( + table: AnyTable, + where: + | ((eb: ReturnType) => Condition | boolean) + | undefined, + ): Promise { + let conditions = where ? buildCondition(table.columns, where) : undefined; + if (conditions === true) conditions = undefined; + if (conditions === false) return false; + return applyReadPolicies(table, conditions, context); + } + + function requireJsonOp(op: T | undefined, name: string): T { + if (!op) + throw new Error(`[FumaDB] ${name} is not supported by this adapter.`); + return op; + } + const query = { internal, async count(name, { where } = {}) { @@ -428,6 +484,69 @@ export function toORM( if (constrainedWhere === false) return; return internal.updateMany(table, { set, where: constrainedWhere }); }, + async jsonCount(name, { column, where, filter }) { + const op = requireJsonOp(internal.jsonCount, "jsonCount"); + const table = toTable(name); + const scopedWhere = await compileScopedWhere(table, where); + if (scopedWhere === false) return 0; + return op(table, { column: toColumn(table, column), where: scopedWhere, filter }); + }, + async jsonGroupCount(name, { column, where, filter, path, valueType }) { + const op = requireJsonOp(internal.jsonGroupCount, "jsonGroupCount"); + const table = toTable(name); + const scopedWhere = await compileScopedWhere(table, where); + if (scopedWhere === false) return []; + return op(table, { + column: toColumn(table, column), + where: scopedWhere, + filter, + path, + valueType, + }); + }, + async jsonTimeBuckets(name, { column, where, filter, path, bucketMs }) { + const op = requireJsonOp(internal.jsonTimeBuckets, "jsonTimeBuckets"); + const table = toTable(name); + const scopedWhere = await compileScopedWhere(table, where); + if (scopedWhere === false) return []; + return op(table, { + column: toColumn(table, column), + where: scopedWhere, + filter, + path, + bucketMs, + }); + }, + async jsonStats(name, { column, where, filter, path, percentiles }) { + const op = requireJsonOp(internal.jsonStats, "jsonStats"); + const table = toTable(name); + const scopedWhere = await compileScopedWhere(table, where); + if (scopedWhere === false) + return { count: 0, min: null, max: null, percentiles: [] }; + return op(table, { + column: toColumn(table, column), + where: scopedWhere, + filter, + path, + percentiles, + }); + }, + async jsonPage(name, options) { + const op = requireJsonOp(internal.jsonPage, "jsonPage"); + const table = toTable(name); + const scopedWhere = await compileScopedWhere(table, options.where); + if (scopedWhere === false) return []; + return op(table, { + column: toColumn(table, options.column), + where: scopedWhere, + filter: options.filter, + orderBy: options.orderBy, + keyColumn: toColumn(table, options.keyColumn), + keyDirection: options.keyDirection ?? "asc", + cursor: options.cursor, + limit: options.limit, + }) as Promise[]>; + }, async transaction(run) { return internal.transaction((transactionInstance) => run(withQueryContext(transactionInstance, context)), diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index a90003ef92..bcf5dfebfd 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -2,7 +2,20 @@ import { Effect, Layer, Option, Predicate, Schema } from "effect"; import { FetchHttpClient, type HttpClient } from "effect/unstable/http"; import { fumadb } from "@executor-js/fumadb"; import { memoryAdapter } from "@executor-js/fumadb/adapters/memory"; -import { withQueryContext, type Condition, type ConditionBuilder } from "@executor-js/fumadb/query"; +import { + withQueryContext, + type Condition, + type ConditionBuilder, + type JsonFilter, + type JsonGroupCountRow, + type JsonKeysetCursor, + type JsonKeysetOrder, + type JsonPath, + type JsonScalar, + type JsonStats, + type JsonTimeBucketRow, + type JsonValueType, +} from "@executor-js/fumadb/query"; import { schema as fumaSchema, type RelationsMap } from "@executor-js/fumadb/schema"; import type { AnyColumn } from "@executor-js/fumadb/schema"; import { generateKeyBetween } from "fractional-indexing"; @@ -576,6 +589,31 @@ type CoreProjectedRow = TSelect extends re ? Pick, Extract>> : CoreRow; +type CoreJsonBase = { + readonly column: string; + readonly where?: CoreWhere; + readonly filter?: JsonFilter; +}; +type CoreJsonGroupCountOptions = CoreJsonBase & { + readonly path: JsonPath; + readonly valueType?: JsonValueType; +}; +type CoreJsonTimeBucketOptions = CoreJsonBase & { + readonly path: JsonPath; + readonly bucketMs: number; +}; +type CoreJsonStatsOptions = CoreJsonBase & { + readonly path: JsonPath; + readonly percentiles?: readonly number[]; +}; +type CoreJsonPageOptions = CoreJsonBase & { + readonly orderBy: readonly JsonKeysetOrder[]; + readonly keyColumn: string; + readonly keyDirection?: "asc" | "desc"; + readonly cursor?: JsonKeysetCursor; + readonly limit: number; +}; + type LooseStorageDb = { readonly count: (tableName: string, options?: unknown) => Promise; readonly create: ( @@ -596,6 +634,20 @@ type LooseStorageDb = { options?: unknown, ) => Promise[]>; readonly updateMany: (tableName: string, options: unknown) => Promise; + readonly jsonCount: (tableName: string, options: unknown) => Promise; + readonly jsonGroupCount: ( + tableName: string, + options: unknown, + ) => Promise; + readonly jsonTimeBuckets: ( + tableName: string, + options: unknown, + ) => Promise; + readonly jsonStats: (tableName: string, options: unknown) => Promise; + readonly jsonPage: ( + tableName: string, + options: unknown, + ) => Promise[]>; }; const asLooseStorageDb = (db: unknown): LooseStorageDb => db as LooseStorageDb; @@ -653,6 +705,37 @@ const makeCoreDb = (fuma: ReturnType) => ({ fuma.use(`${tableName}.updateMany`, (db) => asLooseStorageDb(db).updateMany(tableName, options), ), + jsonCount: ( + tableName: CoreTableName, + options: CoreJsonBase, + ): Effect.Effect => + fuma.use(`${tableName}.jsonCount`, (db) => asLooseStorageDb(db).jsonCount(tableName, options)), + jsonGroupCount: ( + tableName: CoreTableName, + options: CoreJsonGroupCountOptions, + ): Effect.Effect => + fuma.use(`${tableName}.jsonGroupCount`, (db) => + asLooseStorageDb(db).jsonGroupCount(tableName, options), + ), + jsonTimeBuckets: ( + tableName: CoreTableName, + options: CoreJsonTimeBucketOptions, + ): Effect.Effect => + fuma.use(`${tableName}.jsonTimeBuckets`, (db) => + asLooseStorageDb(db).jsonTimeBuckets(tableName, options), + ), + jsonStats: ( + tableName: CoreTableName, + options: CoreJsonStatsOptions, + ): Effect.Effect => + fuma.use(`${tableName}.jsonStats`, (db) => asLooseStorageDb(db).jsonStats(tableName, options)), + jsonPage: ( + tableName: TName, + options: CoreJsonPageOptions, + ): Effect.Effect[], StorageFailure> => + fuma.use(`${tableName}.jsonPage`, (db) => + asLooseStorageDb(db).jsonPage(tableName, options), + ) as Effect.Effect[], StorageFailure>, }); type CoreDb = ReturnType; @@ -788,6 +871,97 @@ const rowMatchesPluginStorageWhere = ( return true; }; +const inferJsonValueType = (value: unknown): JsonValueType => { + if (value instanceof Date) return "number"; + if (typeof value === "number") return "number"; + if (typeof value === "boolean") return "boolean"; + return "text"; +}; + +const pluginStorageJsonScalar = (value: unknown): JsonScalar => pluginStorageComparableValue(value); + +// Translate the facade's indexed-field `where` into a JSON-path filter the SQL +// pushdown understands. Field value types are inferred from the operands. +const pluginStorageWhereToJsonFilter = ( + where: Readonly> | undefined, +): JsonFilter | undefined => { + if (!where) return undefined; + const items: JsonFilter[] = []; + for (const [field, condition] of Object.entries(where)) { + const path: JsonPath = [field]; + if (isPluginStorageWhereFilter(condition)) { + for (const [operator, operand] of Object.entries(condition)) { + if (operator === "in") { + const values = Array.isArray(operand) ? operand : []; + items.push({ + kind: "array", + path, + valueType: inferJsonValueType(values[0]), + operator: "in", + values: values.map(pluginStorageJsonScalar), + }); + continue; + } + const compareOperator = + operator === "gt" + ? ">" + : operator === "gte" + ? ">=" + : operator === "lt" + ? "<" + : operator === "lte" + ? "<=" + : "="; + items.push({ + kind: "compare", + path, + valueType: inferJsonValueType(operand), + operator: compareOperator, + value: pluginStorageJsonScalar(operand), + }); + } + } else { + items.push({ + kind: "compare", + path, + valueType: inferJsonValueType(condition), + operator: "=", + value: pluginStorageJsonScalar(condition), + }); + } + } + if (items.length === 0) return undefined; + if (items.length === 1) return items[0]!; + return { kind: "and", items }; +}; + +const pluginStorageFieldsValidationError = ( + definition: PluginStorageRuntimeCollectionDefinition, + fields: Iterable, +): StorageError | null => { + const indexedFields = pluginStorageCollectionIndexedFields(definition); + for (const field of fields) { + if (!indexedFields.has(field)) { + return new StorageError({ + message: `Plugin storage collection "${definition.name}" cannot query field "${field}" because it is not declared as an index`, + cause: undefined, + }); + } + } + return null; +}; + +const pluginStorageInvalidLimitError = ( + definition: PluginStorageRuntimeCollectionDefinition, + limit: number, +): StorageError | null => + Number.isInteger(limit) && limit >= 0 + ? null + : new StorageError({ + message: `Plugin storage collection "${definition.name}" received an invalid query limit`, + cause: undefined, + }); + const makePluginStorageFacade = (input: { readonly core: CoreDb; readonly pluginId: string; @@ -813,6 +987,18 @@ const makePluginStorageFacade = (input: { key === undefined ? true : b("key", "=", key), ); + // Like `whereFor` but with an optional key-prefix match β€” used by the + // pushdown aggregate/keyset paths. Owner/tenant scoping is added by the + // table policy, exactly as for `whereFor`. + const whereForPrefix = + (collection: string, keyPrefix?: string): CoreWhere => + (b: AnyCb) => + b.and( + b("plugin_id", "=", input.pluginId), + b("collection", "=", collection), + keyPrefix === undefined ? true : b("key", "starts with", keyPrefix), + ); + const whereOwner = (owner: Owner, collection: string, key: string): CoreWhere => { const os = ownerSubject(owner); return (b: AnyCb) => @@ -1087,6 +1273,120 @@ const makePluginStorageFacade = (input: { query: (storageInput) => queryCollection(definition, storageInput), count: (storageInput) => queryCollection(definition, storageInput).pipe(Effect.map((rows) => rows.length)), + queryKeyset: (storageInput) => + Effect.gen(function* () { + const fields = new Set([ + ...Object.keys(storageInput.where ?? {}), + ...storageInput.orderBy.map((order) => order.field), + ]); + const validationError = pluginStorageFieldsValidationError(definition, fields); + if (validationError) return yield* validationError; + const limitError = pluginStorageInvalidLimitError(definition, storageInput.limit); + if (limitError) return yield* limitError; + + const orderBy: JsonKeysetOrder[] = storageInput.orderBy.map((order) => ({ + path: [order.field], + valueType: order.valueType ?? "text", + direction: order.direction ?? "asc", + })); + const keyDirection = orderBy[0]?.direction ?? "asc"; + + const rows = yield* input.core.jsonPage("plugin_storage", { + column: "data", + where: whereForPrefix(definition.name, storageInput.keyPrefix), + filter: pluginStorageWhereToJsonFilter( + storageInput.where as Readonly> | undefined, + ), + orderBy, + keyColumn: "key", + keyDirection, + cursor: storageInput.cursor, + limit: storageInput.limit, + }); + + const entries = rows.map((row) => + pluginStorageEntryFromRow>(row), + ); + const last = entries.at(-1); + const nextCursor = + last && entries.length >= storageInput.limit + ? { + values: storageInput.orderBy.map((order) => + pluginStorageJsonScalar(pluginStorageDataField(last.data, order.field)), + ), + key: last.key, + } + : null; + return { entries, nextCursor }; + }), + aggregate: { + count: (storageInput) => + Effect.gen(function* () { + const validationError = pluginStorageFieldsValidationError( + definition, + Object.keys(storageInput?.where ?? {}), + ); + if (validationError) return yield* validationError; + return yield* input.core.jsonCount("plugin_storage", { + column: "data", + where: whereForPrefix(definition.name, storageInput?.keyPrefix), + filter: pluginStorageWhereToJsonFilter( + storageInput?.where as Readonly> | undefined, + ), + }); + }), + groupCount: (storageInput) => + Effect.gen(function* () { + const validationError = pluginStorageFieldsValidationError(definition, [ + storageInput.field, + ...Object.keys(storageInput.where ?? {}), + ]); + if (validationError) return yield* validationError; + return yield* input.core.jsonGroupCount("plugin_storage", { + column: "data", + where: whereForPrefix(definition.name, storageInput.keyPrefix), + filter: pluginStorageWhereToJsonFilter( + storageInput.where as Readonly> | undefined, + ), + path: [storageInput.field], + valueType: storageInput.valueType ?? "text", + }); + }), + timeBuckets: (storageInput) => + Effect.gen(function* () { + const validationError = pluginStorageFieldsValidationError(definition, [ + storageInput.field, + ...Object.keys(storageInput.where ?? {}), + ]); + if (validationError) return yield* validationError; + return yield* input.core.jsonTimeBuckets("plugin_storage", { + column: "data", + where: whereForPrefix(definition.name, storageInput.keyPrefix), + filter: pluginStorageWhereToJsonFilter( + storageInput.where as Readonly> | undefined, + ), + path: [storageInput.field], + bucketMs: storageInput.bucketMs, + }); + }), + stats: (storageInput) => + Effect.gen(function* () { + const validationError = pluginStorageFieldsValidationError(definition, [ + storageInput.field, + ...Object.keys(storageInput.where ?? {}), + ]); + if (validationError) return yield* validationError; + return yield* input.core.jsonStats("plugin_storage", { + column: "data", + where: whereForPrefix(definition.name, storageInput.keyPrefix), + filter: pluginStorageWhereToJsonFilter( + storageInput.where as Readonly> | undefined, + ), + path: [storageInput.field], + percentiles: storageInput.percentiles, + }); + }), + }, remove: (storageInput) => removeImpl(storageInput.owner, definition.name, storageInput.key), }), get: (storageInput) => getVisible(storageInput.collection, storageInput.key), diff --git a/packages/core/sdk/src/fuma-runtime.ts b/packages/core/sdk/src/fuma-runtime.ts index e560b1b098..89f7b7e162 100644 --- a/packages/core/sdk/src/fuma-runtime.ts +++ b/packages/core/sdk/src/fuma-runtime.ts @@ -139,6 +139,11 @@ const makeSafeFumaQuery = ( deleteMany: (name, value) => db.deleteMany(table(name), value), findFirst: (name, value) => db.findFirst(table(name), value), findMany: (name, value) => db.findMany(table(name), value), + jsonCount: (name, value) => db.jsonCount(table(name), value), + jsonGroupCount: (name, value) => db.jsonGroupCount(table(name), value), + jsonTimeBuckets: (name, value) => db.jsonTimeBuckets(table(name), value), + jsonStats: (name, value) => db.jsonStats(table(name), value), + jsonPage: (name, value) => db.jsonPage(table(name), value), transaction: (run) => db.transaction((transactionDb) => run(makeSafeFumaQuery(transactionDb, options))), updateMany: (name, value) => db.updateMany(table(name), value), diff --git a/packages/core/sdk/src/plugin-storage-aggregate.test.ts b/packages/core/sdk/src/plugin-storage-aggregate.test.ts new file mode 100644 index 0000000000..3a2575e84c --- /dev/null +++ b/packages/core/sdk/src/plugin-storage-aggregate.test.ts @@ -0,0 +1,200 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Schema } from "effect"; + +import type { StorageFailure } from "./fuma-runtime"; +import { Owner } from "./ids"; +import { definePlugin } from "./plugin"; +import { + definePluginStorageCollection, + type PluginStorageAggregateFilter, + type PluginStorageGroupCountInput, + type PluginStorageQueryKeysetInput, + type PluginStorageStatsInput, + type PluginStorageTimeBucketInput, +} from "./plugin-storage"; +import { makeTestExecutor } from "./testing"; + +const Run = Schema.Struct({ + status: Schema.Literals(["completed", "failed", "running"]), + triggerKind: Schema.NullOr(Schema.String), + startedAt: Schema.Number, + durationMs: Schema.NullOr(Schema.Number), + hadInteraction: Schema.Boolean, +}); +type Run = typeof Run.Type; + +const runs = definePluginStorageCollection("runs", Run, { + indexes: ["status", "triggerKind", "startedAt", "durationMs", "hadInteraction"], +}); + +const runsPlugin = definePlugin(() => ({ + id: "runs" as const, + pluginStorage: { runs }, + storage: ({ pluginStorage }) => ({ runs: pluginStorage.collection(runs) }), + extension: (ctx) => ({ + record: (owner: Owner, key: string, data: Run) => ctx.storage.runs.put({ owner, key, data }), + count: (input?: PluginStorageAggregateFilter) => + ctx.storage.runs.aggregate.count(input), + groupCount: (input: PluginStorageGroupCountInput) => + ctx.storage.runs.aggregate.groupCount(input), + timeBuckets: (input: PluginStorageTimeBucketInput) => + ctx.storage.runs.aggregate.timeBuckets(input), + stats: (input: PluginStorageStatsInput) => ctx.storage.runs.aggregate.stats(input), + keyset: (input: PluginStorageQueryKeysetInput) => + ctx.storage.runs.queryKeyset(input), + }), +}))(); + +const seedRows: readonly (readonly [string, Run])[] = [ + [ + "r1", + { + status: "completed", + triggerKind: "cli", + startedAt: 1000, + durationMs: 100, + hadInteraction: false, + }, + ], + [ + "r2", + { + status: "completed", + triggerKind: "cli", + startedAt: 2000, + durationMs: 200, + hadInteraction: false, + }, + ], + [ + "r3", + { + status: "failed", + triggerKind: "http", + startedAt: 3000, + durationMs: 300, + hadInteraction: true, + }, + ], + [ + "r4", + { + status: "running", + triggerKind: "http", + startedAt: 4000, + durationMs: null, + hadInteraction: false, + }, + ], +]; + +const seed = (executor: { + runs: { + record: (owner: Owner, key: string, data: Run) => Effect.Effect; + }; +}) => + Effect.forEach(seedRows, ([key, data]) => executor.runs.record("org", key, data), { + discard: true, + }); + +describe("plugin storage aggregate + keyset (SQLite pushdown)", () => { + it.effect("counts with JSON filters and numeric ranges", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ plugins: [runsPlugin] as const }); + yield* seed(executor); + + expect(yield* executor.runs.count()).toBe(4); + expect(yield* executor.runs.count({ where: { status: "completed" } })).toBe(2); + expect(yield* executor.runs.count({ where: { startedAt: { gte: 3000 } } })).toBe(2); + expect(yield* executor.runs.count({ where: { durationMs: { gte: 200 } } })).toBe(2); + expect(yield* executor.runs.count({ where: { hadInteraction: true } })).toBe(1); + }), + ); + + it.effect("groups counts by a JSON field (facets)", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ plugins: [runsPlugin] as const }); + yield* seed(executor); + + const byStatus = yield* executor.runs.groupCount({ field: "status" }); + expect(Object.fromEntries(byStatus.map((row) => [row.value, row.count]))).toEqual({ + completed: 2, + failed: 1, + running: 1, + }); + + const byTrigger = yield* executor.runs.groupCount({ field: "triggerKind" }); + expect(Object.fromEntries(byTrigger.map((row) => [row.value, row.count]))).toEqual({ + cli: 2, + http: 2, + }); + }), + ); + + it.effect("buckets a numeric field over time", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ plugins: [runsPlugin] as const }); + yield* seed(executor); + + const buckets = yield* executor.runs.timeBuckets({ field: "startedAt", bucketMs: 2000 }); + expect(buckets).toEqual([ + { bucket: 0, count: 1 }, + { bucket: 2000, count: 2 }, + { bucket: 4000, count: 1 }, + ]); + }), + ); + + it.effect("computes duration stats and percentiles", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ plugins: [runsPlugin] as const }); + yield* seed(executor); + + const stats = yield* executor.runs.stats({ field: "durationMs", percentiles: [0, 0.5, 1] }); + expect(stats.count).toBe(3); + expect(stats.min).toBe(100); + expect(stats.max).toBe(300); + expect(stats.percentiles).toEqual([ + { fraction: 0, value: 100 }, + { fraction: 0.5, value: 200 }, + { fraction: 1, value: 300 }, + ]); + }), + ); + + it.effect("keyset-paginates with a cursor and JSON filter", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ plugins: [runsPlugin] as const }); + yield* seed(executor); + + const page1 = yield* executor.runs.keyset({ + orderBy: [{ field: "startedAt", direction: "desc", valueType: "number" }], + limit: 2, + }); + expect(page1.entries.map((entry) => entry.key)).toEqual(["r4", "r3"]); + expect(page1.nextCursor).not.toBeNull(); + + const page2 = yield* executor.runs.keyset({ + orderBy: [{ field: "startedAt", direction: "desc", valueType: "number" }], + limit: 2, + cursor: page1.nextCursor ?? undefined, + }); + expect(page2.entries.map((entry) => entry.key)).toEqual(["r2", "r1"]); + + const page3 = yield* executor.runs.keyset({ + orderBy: [{ field: "startedAt", direction: "desc", valueType: "number" }], + limit: 2, + cursor: page2.nextCursor ?? undefined, + }); + expect(page3.entries).toHaveLength(0); + expect(page3.nextCursor).toBeNull(); + + const completed = yield* executor.runs.keyset({ + where: { status: "completed" }, + orderBy: [{ field: "startedAt", direction: "asc", valueType: "number" }], + limit: 10, + }); + expect(completed.entries.map((entry) => entry.key)).toEqual(["r1", "r2"]); + }), + ); +}); diff --git a/packages/core/sdk/src/plugin-storage.ts b/packages/core/sdk/src/plugin-storage.ts index e854feb51b..b926965b65 100644 --- a/packages/core/sdk/src/plugin-storage.ts +++ b/packages/core/sdk/src/plugin-storage.ts @@ -164,6 +164,103 @@ export interface PluginStorageEntry { readonly updatedAt: Date; } +export type PluginStorageScalar = string | number | boolean | null; + +export type PluginStorageFieldType = "text" | "number" | "boolean"; + +export interface PluginStorageAggregateFilter { + readonly where?: PluginStorageCollectionWhere; + readonly keyPrefix?: string; +} + +export interface PluginStorageGroupCountInput< + TDefinition, +> extends PluginStorageAggregateFilter { + readonly field: PluginStorageCollectionIndexedField; + readonly valueType?: PluginStorageFieldType; +} + +export interface PluginStorageTimeBucketInput< + TDefinition, +> extends PluginStorageAggregateFilter { + /** Numeric (epoch-ms) field the buckets are computed from. */ + readonly field: PluginStorageCollectionIndexedField; + readonly bucketMs: number; +} + +export interface PluginStorageStatsInput< + TDefinition, +> extends PluginStorageAggregateFilter { + /** Numeric field the stats are computed over. */ + readonly field: PluginStorageCollectionIndexedField; + /** Percentile fractions in [0, 1] (e.g. 0.5, 0.95). */ + readonly percentiles?: readonly number[]; +} + +export interface PluginStorageGroupCount { + readonly value: PluginStorageScalar; + readonly count: number; +} + +export interface PluginStorageTimeBucket { + readonly bucket: number; + readonly count: number; +} + +export interface PluginStoragePercentile { + readonly fraction: number; + readonly value: number; +} + +export interface PluginStorageStats { + readonly count: number; + readonly min: number | null; + readonly max: number | null; + readonly percentiles: readonly PluginStoragePercentile[]; +} + +export interface PluginStorageKeysetOrderBy { + readonly field: PluginStorageCollectionIndexedField; + readonly direction?: "asc" | "desc"; + readonly valueType?: PluginStorageFieldType; +} + +export interface PluginStorageKeysetCursor { + readonly values: readonly PluginStorageScalar[]; + readonly key: string; +} + +export interface PluginStorageQueryKeysetInput< + TDefinition, +> extends PluginStorageAggregateFilter { + readonly orderBy: readonly PluginStorageKeysetOrderBy[]; + readonly limit: number; + readonly cursor?: PluginStorageKeysetCursor; +} + +export interface PluginStorageKeysetPage { + readonly entries: readonly PluginStorageEntry[]; + readonly nextCursor: PluginStorageKeysetCursor | null; +} + +/** SQL-pushed aggregation over a collection's JSON documents. */ +export interface PluginStorageAggregateFacade< + TDefinition extends PluginStorageCollectionDefinition, +> { + readonly count: ( + input?: PluginStorageAggregateFilter, + ) => Effect.Effect; + readonly groupCount: ( + input: PluginStorageGroupCountInput, + ) => Effect.Effect; + readonly timeBuckets: ( + input: PluginStorageTimeBucketInput, + ) => Effect.Effect; + readonly stats: ( + input: PluginStorageStatsInput, + ) => Effect.Effect; +} + export interface PluginStorageCollectionFacade< TDefinition extends PluginStorageCollectionDefinition = PluginStorageCollectionDefinition, > { @@ -197,6 +294,15 @@ export interface PluginStorageCollectionFacade< readonly count: ( input?: Omit, "orderBy" | "limit" | "offset">, ) => Effect.Effect; + /** SQL-pushed keyset pagination ordered by indexed JSON fields. */ + readonly queryKeyset: ( + input: PluginStorageQueryKeysetInput, + ) => Effect.Effect< + PluginStorageKeysetPage>, + StorageFailure + >; + /** SQL-pushed aggregation (counts, facets, time buckets, percentiles). */ + readonly aggregate: PluginStorageAggregateFacade; readonly remove: ( input: PluginStorageCollectionScopedKeyInput, ) => Effect.Effect; diff --git a/packages/core/sdk/src/test-config.ts b/packages/core/sdk/src/test-config.ts index 7aba38ff0c..66948f5c3d 100644 --- a/packages/core/sdk/src/test-config.ts +++ b/packages/core/sdk/src/test-config.ts @@ -62,6 +62,11 @@ const makeLazyTestFumaDb = (options: { "deleteMany", "findFirst", "findMany", + "jsonCount", + "jsonGroupCount", + "jsonTimeBuckets", + "jsonStats", + "jsonPage", "transaction", "updateMany", "upsert", From 9b5e223aaeea2f01e82908d6bbbfb1dac40ffd57 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sun, 14 Jun 2026 20:31:09 +0530 Subject: [PATCH 05/57] feat(execution-history): keyset runs API with aggregate meta (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds on the JSON pushdown. Reworks the runs list into a keyset-paginated, aggregate-aware read surface. `list` now takes an opaque keyset cursor, a sort field (`startedAt`/`durationMs`) + direction, and a live-tail `after` floor, and returns `{ runs, nextCursor, meta }`. The `meta` block (computed once per filter set on the first page) carries status/trigger facet counts (each ignoring its own filter), interaction counts, a stacked-by-status timeline with a server-chosen bucket width, total/filtered row counts, and duration min/max + P50-P99. All counts, buckets, percentiles, and pages are pushed to SQL. Tests: 9 plugin tests on real SQLite (facets, cursor pages, live `after`). Not included: tool-path facets + full-text code search (need a `contains` facade op). Stacked on the FumaDB pushdown PR. ### [Stack](https://github.com/kitlangton/stack) 1. #15 2. **#16** πŸ‘ˆ current 3. #17 --- .changeset/execution-history-rich-list.md | 15 + .../execution-history/src/api/group.ts | 57 +++- .../src/api/handlers.test.ts | 15 +- .../execution-history/src/api/handlers.ts | 34 +- .../execution-history/src/react/RunsPage.tsx | 33 +- .../execution-history/src/react/atoms.ts | 16 +- .../src/sdk/store-list.test.ts | 157 +++++++++ .../execution-history/src/sdk/store.test.ts | 7 +- .../execution-history/src/sdk/store.ts | 315 ++++++++++++++++-- 9 files changed, 573 insertions(+), 76 deletions(-) create mode 100644 .changeset/execution-history-rich-list.md create mode 100644 packages/plugins/execution-history/src/sdk/store-list.test.ts diff --git a/.changeset/execution-history-rich-list.md b/.changeset/execution-history-rich-list.md new file mode 100644 index 0000000000..8c0a9314ef --- /dev/null +++ b/.changeset/execution-history-rich-list.md @@ -0,0 +1,15 @@ +--- +"@executor-js/plugin-execution-history": minor +--- + +Rework the runs list into a keyset-paginated, aggregate-aware read surface +backed by the storage JSON pushdown. The `list` endpoint now takes an opaque +keyset `cursor` (instead of offset), a sort field (`startedAt`/`durationMs`) +with direction, and a live-tail `after` floor, and returns `{ runs, nextCursor, +meta }`. The `meta` block β€” computed once per filter set on the first page β€” +carries status/trigger facet counts (each ignoring its own filter), interaction +counts, a stacked-by-status timeline with a server-chosen bucket width, total +and filtered row counts, and duration min/max plus P50–P99 percentiles. All +counts, buckets, percentiles, and pages are pushed to SQL rather than reducing +the whole collection in memory. Tool-path facets and full-text code search are +intentionally not included yet. diff --git a/packages/plugins/execution-history/src/api/group.ts b/packages/plugins/execution-history/src/api/group.ts index 84fdc4fb3c..446388d07f 100644 --- a/packages/plugins/execution-history/src/api/group.ts +++ b/packages/plugins/execution-history/src/api/group.ts @@ -3,7 +3,7 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; import { InternalError } from "@executor-js/api"; -import { InteractionRow, RunRow, ToolCallRow } from "../sdk/collections"; +import { InteractionRow, RunRow, RunStatus, ToolCallRow } from "../sdk/collections"; // --------------------------------------------------------------------------- // HTTP surface for the execution-history read model. @@ -27,10 +27,55 @@ import { InteractionRow, RunRow, ToolCallRow } from "../sdk/collections"; // Success schemas // --------------------------------------------------------------------------- +// --- Aggregate `meta` block (facets, stacked timeline, duration percentiles) --- + +const RunStatusCount = Schema.Struct({ status: RunStatus, count: Schema.Number }); +const RunTriggerCount = Schema.Struct({ + triggerKind: Schema.NullOr(Schema.String), + count: Schema.Number, +}); +const RunInteractionCounts = Schema.Struct({ + withInteraction: Schema.Number, + withoutInteraction: Schema.Number, +}); +const RunChartBucket = Schema.Struct({ + bucketStart: Schema.Number, + // Partial status -> count map; absent statuses are omitted. + counts: Schema.Record(Schema.String, Schema.Number), +}); +const RunDurationStats = Schema.Struct({ + count: Schema.Number, + min: Schema.NullOr(Schema.Number), + max: Schema.NullOr(Schema.Number), + p50: Schema.NullOr(Schema.Number), + p75: Schema.NullOr(Schema.Number), + p90: Schema.NullOr(Schema.Number), + p95: Schema.NullOr(Schema.Number), + p99: Schema.NullOr(Schema.Number), +}); +export const ExecutionListMeta = Schema.Struct({ + totalRowCount: Schema.Number, + filterRowCount: Schema.Number, + statusCounts: Schema.Array(RunStatusCount), + triggerCounts: Schema.Array(RunTriggerCount), + interactionCounts: RunInteractionCounts, + chartBucketMs: Schema.Number, + chartData: Schema.Array(RunChartBucket), + durationStats: RunDurationStats, +}); + +/** Keyset cursor carried opaquely on the wire as a JSON string. */ +export const RunsCursor = Schema.Struct({ + sort: Schema.NullOr(Schema.Number), + key: Schema.String, +}); +export const RunsCursorFromString = Schema.fromJsonString(RunsCursor); + /** Mirrors `ExecutionHistoryListResult` from the store. */ export const ListRunsResponse = Schema.Struct({ runs: Schema.Array(RunRow), - total: Schema.Number, + nextCursor: Schema.NullOr(Schema.String), + meta: Schema.NullOr(ExecutionListMeta), }); /** Mirrors `ExecutionHistoryDetail` from the store. */ @@ -61,9 +106,13 @@ const ListRunsQuery = Schema.Struct({ from: Schema.optional(Schema.NumberFromString), to: Schema.optional(Schema.NumberFromString), interaction: Schema.optional(Schema.String), + // Live-tail floor: only runs newer than this `startedAt`. + after: Schema.optional(Schema.NumberFromString), + sort: Schema.optional(Schema.Literals(["startedAt", "durationMs"])), + dir: Schema.optional(Schema.Literals(["asc", "desc"])), limit: Schema.optional(Schema.NumberFromString), - offset: Schema.optional(Schema.NumberFromString), - sort: Schema.optional(Schema.Literals(["asc", "desc"])), + // Opaque keyset cursor returned as `nextCursor` from a prior page. + cursor: Schema.optional(Schema.String), }); const RunParams = Schema.Struct({ diff --git a/packages/plugins/execution-history/src/api/handlers.test.ts b/packages/plugins/execution-history/src/api/handlers.test.ts index c805b4d6e9..99d4c36543 100644 --- a/packages/plugins/execution-history/src/api/handlers.test.ts +++ b/packages/plugins/execution-history/src/api/handlers.test.ts @@ -74,7 +74,7 @@ const makeStubExtension = (captured: { }): ExecutionHistoryExtension => ({ list: (options): Effect.Effect => { captured.options = options; - return Effect.succeed({ runs: [runRow({})], total: 1 }); + return Effect.succeed({ runs: [runRow({})], nextCursor: null, meta: null }); }, get: (executionId): Effect.Effect => Effect.succeed( @@ -120,15 +120,17 @@ describe("ExecutionHistoryHandlers", () => { const res = yield* get( web, - "http://localhost/execution-history/runs?status=completed,failed&trigger=manual&from=100&to=900&interaction=true&limit=10&offset=5&sort=asc", + "http://localhost/execution-history/runs?status=completed,failed&trigger=manual&from=100&to=900&interaction=true&after=50&limit=10&sort=startedAt&dir=asc", ); expect(res.status).toBe(200); const body = (yield* Effect.promise(() => res.json())) as { runs: { executionId: string }[]; - total: number; + nextCursor: string | null; + meta: unknown; }; - expect(body.total).toBe(1); expect(body.runs.map((r) => r.executionId)).toEqual(["exec_1"]); + expect(body.nextCursor).toBeNull(); + expect(body.meta).toBeNull(); const options = captured.options; expect(options).toBeDefined(); @@ -136,9 +138,10 @@ describe("ExecutionHistoryHandlers", () => { expect(options?.triggerFilter).toEqual(["manual"]); expect(options?.timeRange).toEqual({ from: 100, to: 900 }); expect(options?.hadInteraction).toBe(true); + expect(options?.after).toBe(50); expect(options?.limit).toBe(10); - expect(options?.offset).toBe(5); - expect(options?.sort).toBe("asc"); + expect(options?.sortField).toBe("startedAt"); + expect(options?.sortDirection).toBe("asc"); }), ); diff --git a/packages/plugins/execution-history/src/api/handlers.ts b/packages/plugins/execution-history/src/api/handlers.ts index 244e83e285..519f5d84e2 100644 --- a/packages/plugins/execution-history/src/api/handlers.ts +++ b/packages/plugins/execution-history/src/api/handlers.ts @@ -1,11 +1,18 @@ -import { Context, Effect, Schema } from "effect"; +import { Context, Effect, Option, Schema } from "effect"; import { HttpApiBuilder } from "effect/unstable/httpapi"; import { addGroup, capture } from "@executor-js/api"; import { RunStatus } from "../sdk/collections"; import type { ExecutionHistoryListOptions, ExecutionHistoryStore } from "../sdk/store"; -import { ExecutionHistoryGroup } from "./group"; +import { ExecutionHistoryGroup, RunsCursorFromString } from "./group"; + +const DEFAULT_LIMIT = 50; +const MAX_LIMIT = 200; + +const isRunStatus = Schema.is(RunStatus); +const decodeCursor = Schema.decodeUnknownOption(RunsCursorFromString); +const encodeCursor = Schema.encodeUnknownOption(RunsCursorFromString); // --------------------------------------------------------------------------- // Service tag @@ -75,23 +82,36 @@ export const ExecutionHistoryHandlers = HttpApiBuilder.group( const history = yield* ExecutionHistoryExtensionService; // Keep only valid RunStatus literals β€” an unknown `?status=bogus` // must not reach the storage `where` clause. - const statusFilter = splitCsv(query.status).filter(Schema.is(RunStatus)); + const statusFilter = splitCsv(query.status).filter(isRunStatus); const triggerFilter = splitCsv(query.trigger); const timeRange = query.from !== undefined || query.to !== undefined ? { from: query.from, to: query.to } : undefined; + const cursor = + query.cursor !== undefined + ? Option.getOrUndefined(decodeCursor(query.cursor)) + : undefined; + const limit = Math.min(MAX_LIMIT, Math.max(1, query.limit ?? DEFAULT_LIMIT)); const options: ExecutionHistoryListOptions = { statusFilter: statusFilter.length > 0 ? statusFilter : undefined, triggerFilter: triggerFilter.length > 0 ? triggerFilter : undefined, timeRange, hadInteraction: parseBooleanFlag(query.interaction), - limit: query.limit, - offset: query.offset, - sort: query.sort, + after: query.after, + sortField: query.sort, + sortDirection: query.dir, + limit, + cursor, }; const result = yield* history.list(options); - return { runs: result.runs, total: result.total }; + return { + runs: result.runs, + nextCursor: result.nextCursor + ? Option.getOrNull(encodeCursor(result.nextCursor)) + : null, + meta: result.meta, + }; }), ), ) diff --git a/packages/plugins/execution-history/src/react/RunsPage.tsx b/packages/plugins/execution-history/src/react/RunsPage.tsx index 4bb6fc147d..ecbbf63f96 100644 --- a/packages/plugins/execution-history/src/react/RunsPage.tsx +++ b/packages/plugins/execution-history/src/react/RunsPage.tsx @@ -434,17 +434,21 @@ function RunsTable(props: { export function RunsPage() { const [status, setStatus] = useState("all"); - const [offset, setOffset] = useState(0); + // Forward keyset pagination: a stack of cursors, top = current page (page 1 + // is `undefined`). Next pushes `nextCursor`; Previous pops. + const [cursorStack, setCursorStack] = useState([undefined]); const [selected, setSelected] = useState(null); + const cursor = cursorStack[cursorStack.length - 1]; const query = useMemo( () => ({ limit: PAGE_SIZE, - offset, - sort: "desc", + sort: "startedAt", + dir: "desc", ...(status === "all" ? {} : { status }), + ...(cursor !== undefined ? { cursor } : {}), }), - [offset, status], + [cursor, status], ); const runs = useAtomValue(runsAtom(query)); @@ -461,7 +465,7 @@ export function RunsPage() { value={status} onValueChange={(value) => { setStatus(value as StatusFilter); - setOffset(0); + setCursorStack([undefined]); }} > @@ -486,18 +490,17 @@ export function RunsPage() {

- {value.total.toLocaleString()} total - {value.runs.length > 0 - ? ` / showing ${offset + 1}–${offset + value.runs.length}` - : ""} + {value.meta + ? `${value.meta.filterRowCount.toLocaleString()} total` + : `${value.runs.length.toLocaleString()} on this page`}

@@ -505,8 +508,12 @@ export function RunsPage() { type="button" variant="outline" size="sm" - disabled={offset + value.runs.length >= value.total} - onClick={() => setOffset((current) => current + PAGE_SIZE)} + disabled={value.nextCursor === null} + onClick={() => + setCursorStack((stack) => + value.nextCursor === null ? stack : [...stack, value.nextCursor], + ) + } > Next diff --git a/packages/plugins/execution-history/src/react/atoms.ts b/packages/plugins/execution-history/src/react/atoms.ts index 40bd6f854a..3589c1f710 100644 --- a/packages/plugins/execution-history/src/react/atoms.ts +++ b/packages/plugins/execution-history/src/react/atoms.ts @@ -6,11 +6,11 @@ import { ExecutionHistoryClient } from "./client"; // Query atoms for the execution-history read model. // // The list atom is parameterized by the wire query shape of the `list` -// endpoint (status/trigger CSV strings, numeric from/to/limit/offset, the -// "true"/"false" interaction flag, and asc/desc sort). Detail + tool-call -// atoms key off the execution id. Each atom is `Atom.family`-keyed so a -// distinct filter/page/run gets its own cached result β€” matching the -// graphql plugin's per-input atom pattern. +// endpoint (status/trigger CSV strings, numeric from/to/after/limit, the +// "true"/"false" interaction flag, sort field + direction, and an opaque +// keyset cursor). Detail + tool-call atoms key off the execution id. Each atom +// is `Atom.family`-keyed so a distinct filter/page/run gets its own cached +// result β€” matching the graphql plugin's per-input atom pattern. // --------------------------------------------------------------------------- export interface RunsQuery { @@ -19,9 +19,11 @@ export interface RunsQuery { readonly from?: number; readonly to?: number; readonly interaction?: string; + readonly after?: number; + readonly sort?: "startedAt" | "durationMs"; + readonly dir?: "asc" | "desc"; readonly limit?: number; - readonly offset?: number; - readonly sort?: "asc" | "desc"; + readonly cursor?: string; } export const runsAtom = Atom.family((query: RunsQuery) => diff --git a/packages/plugins/execution-history/src/sdk/store-list.test.ts b/packages/plugins/execution-history/src/sdk/store-list.test.ts new file mode 100644 index 0000000000..faf6a2c41b --- /dev/null +++ b/packages/plugins/execution-history/src/sdk/store-list.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { + ExecutionFinished, + ExecutionId, + ExecutionStarted, + Subject, + Tenant, +} from "@executor-js/sdk"; +import type { ExecutionEvent, StorageFailure } from "@executor-js/sdk/core"; +import { makeTestExecutor } from "@executor-js/sdk/testing"; + +import type { RunStatus } from "./collections"; +import { executionHistoryPlugin } from "./plugin"; + +const owner = { tenant: Tenant.make("tenant_test"), subject: Subject.make("subject_test") }; + +interface SeedRun { + readonly id: string; + readonly status: "completed" | "failed"; + readonly trigger: string; + readonly startedAt: number; + readonly durationMs: number; +} + +/** Only the writer surface seeding needs β€” the tests call `list` on the full + * inferred executor directly. */ +interface HistoryWriter { + readonly executionHistory: { + readonly handleEvent: (event: ExecutionEvent) => Effect.Effect; + }; +} + +const makeExecutor = () => + makeTestExecutor({ backend: "sqlite", plugins: [executionHistoryPlugin()] as const }); + +const seedRun = (executor: HistoryWriter, run: SeedRun) => + Effect.gen(function* () { + const executionId = ExecutionId.make(run.id); + yield* executor.executionHistory.handleEvent( + new ExecutionStarted({ + executionId, + owner, + code: "noop", + trigger: { kind: run.trigger }, + startedAt: new Date(run.startedAt), + }), + ); + yield* executor.executionHistory.handleEvent( + new ExecutionFinished({ + executionId, + owner, + status: run.status, + result: { ok: true }, + logs: [], + completedAt: new Date(run.startedAt + run.durationMs), + }), + ); + }); + +// Five completed (cli) + two failed (http), startedAt 1000..7000. +const seedRows: readonly SeedRun[] = [ + { id: "r1", status: "completed", trigger: "cli", startedAt: 1000, durationMs: 100 }, + { id: "r2", status: "completed", trigger: "cli", startedAt: 2000, durationMs: 200 }, + { id: "r3", status: "completed", trigger: "cli", startedAt: 3000, durationMs: 300 }, + { id: "r4", status: "completed", trigger: "cli", startedAt: 4000, durationMs: 400 }, + { id: "r5", status: "completed", trigger: "cli", startedAt: 5000, durationMs: 500 }, + { id: "r6", status: "failed", trigger: "http", startedAt: 6000, durationMs: 600 }, + { id: "r7", status: "failed", trigger: "http", startedAt: 7000, durationMs: 700 }, +]; + +const statusMap = (counts: readonly { status: RunStatus; count: number }[]) => + Object.fromEntries(counts.map((entry) => [entry.status, entry.count])); + +describe("execution-history list β€” meta, facets, keyset", () => { + it.effect("returns facet counts and a stacked timeline on the first page", () => + Effect.gen(function* () { + const executor = yield* makeExecutor(); + yield* Effect.forEach(seedRows, (row) => seedRun(executor, row), { discard: true }); + + const page = yield* executor.executionHistory.list({ limit: 3 }); + expect(page.runs.map((run) => run.executionId)).toEqual(["r7", "r6", "r5"]); + expect(page.nextCursor).not.toBeNull(); + + const meta = page.meta; + expect(meta?.totalRowCount).toBe(7); + expect(meta?.filterRowCount).toBe(7); + expect(statusMap(meta?.statusCounts ?? [])).toEqual({ completed: 5, failed: 2 }); + expect( + Object.fromEntries((meta?.triggerCounts ?? []).map((t) => [t.triggerKind, t.count])), + ).toEqual({ cli: 5, http: 2 }); + expect(meta?.durationStats.count).toBe(7); + expect(meta?.durationStats.min).toBe(100); + expect(meta?.durationStats.max).toBe(700); + expect(meta?.durationStats.p50).toBe(400); + // Stacked timeline: each bucket carries per-status counts. + const totalCharted = (meta?.chartData ?? []).reduce( + (sum, bucket) => sum + Object.values(bucket.counts).reduce((a, b) => a + b, 0), + 0, + ); + expect(totalCharted).toBe(7); + }), + ); + + it.effect("keyset-paginates forward with the cursor", () => + Effect.gen(function* () { + const executor = yield* makeExecutor(); + yield* Effect.forEach(seedRows, (row) => seedRun(executor, row), { discard: true }); + + const page1 = yield* executor.executionHistory.list({ limit: 3 }); + expect(page1.runs.map((r) => r.executionId)).toEqual(["r7", "r6", "r5"]); + + const page2 = yield* executor.executionHistory.list({ + limit: 3, + cursor: page1.nextCursor ?? undefined, + }); + expect(page2.runs.map((r) => r.executionId)).toEqual(["r4", "r3", "r2"]); + // Meta is computed only on the first page. + expect(page2.meta).toBeNull(); + + const page3 = yield* executor.executionHistory.list({ + limit: 3, + cursor: page2.nextCursor ?? undefined, + }); + expect(page3.runs.map((r) => r.executionId)).toEqual(["r1"]); + expect(page3.nextCursor).toBeNull(); + }), + ); + + it.effect("status facet ignores the status filter; filterRowCount honors it", () => + Effect.gen(function* () { + const executor = yield* makeExecutor(); + yield* Effect.forEach(seedRows, (row) => seedRun(executor, row), { discard: true }); + + const page = yield* executor.executionHistory.list({ + limit: 10, + statusFilter: ["failed"], + }); + expect(page.runs.map((r) => r.executionId)).toEqual(["r7", "r6"]); + expect(page.meta?.filterRowCount).toBe(2); + // The status facet still shows every option so the rail stays usable. + expect(statusMap(page.meta?.statusCounts ?? [])).toEqual({ completed: 5, failed: 2 }); + }), + ); + + it.effect("the live `after` floor returns only newer runs and skips meta", () => + Effect.gen(function* () { + const executor = yield* makeExecutor(); + yield* Effect.forEach(seedRows, (row) => seedRun(executor, row), { discard: true }); + + const page = yield* executor.executionHistory.list({ limit: 10, after: 5000 }); + expect(page.runs.map((r) => r.executionId)).toEqual(["r7", "r6"]); + expect(page.meta).toBeNull(); + }), + ); +}); diff --git a/packages/plugins/execution-history/src/sdk/store.test.ts b/packages/plugins/execution-history/src/sdk/store.test.ts index bc55517a7a..e2ddc820c2 100644 --- a/packages/plugins/execution-history/src/sdk/store.test.ts +++ b/packages/plugins/execution-history/src/sdk/store.test.ts @@ -72,8 +72,11 @@ describe("execution-history store", () => { }), ); - const listed = yield* executor.executionHistory.list(); - expect(listed.total).toBe(1); + const listed = yield* executor.executionHistory.list({ limit: 50 }); + expect(listed.meta?.totalRowCount).toBe(1); + expect(listed.meta?.filterRowCount).toBe(1); + expect(listed.meta?.statusCounts).toContainEqual({ status: "completed", count: 1 }); + expect(listed.nextCursor).toBeNull(); const run = listed.runs[0]; expect(run?.executionId).toBe("exec_1"); expect(run?.status).toBe("completed"); diff --git a/packages/plugins/execution-history/src/sdk/store.ts b/packages/plugins/execution-history/src/sdk/store.ts index 6fad9ed577..5bda6d307a 100644 --- a/packages/plugins/execution-history/src/sdk/store.ts +++ b/packages/plugins/execution-history/src/sdk/store.ts @@ -17,9 +17,9 @@ import { type InteractionRow, type InteractionStatus, type RunRow, - type RunStatus, type ToolCallRow, type ToolCallStatus, + RunStatus, interactions, runs, toolCalls, @@ -51,6 +51,9 @@ const toJson = (value: unknown): string | null => const ownerOf = (binding: OwnerBinding): Owner => (binding.subject != null ? "user" : "org"); +/** Hoisted: `Schema.is` compiles a guard, so it must not be rebuilt per row. */ +const isRunStatus = Schema.is(RunStatus); + /** First dot-delimited segment of a tool path (its namespace), or null. */ const namespaceOf = (path: string): string | null => { const index = path.indexOf("."); @@ -97,21 +100,84 @@ interface RunBuffer { // --------------------------------------------------------------------------- // Read-surface option/result types. +// +// The list surface is keyset-paginated and carries an aggregate `meta` block +// (facet counts, a stacked-by-status timeline, and duration percentiles) that +// the runs UI renders. All of it is pushed down to SQL through the plugin +// storage `aggregate`/`queryKeyset` facade β€” no whole-collection scans in JS. +// `meta` is computed only on the initial page (no cursor, no live `after`), +// matching how the UI fetches it once per filter set. // --------------------------------------------------------------------------- +export type RunsSortField = "startedAt" | "durationMs"; + +/** Opaque-to-the-client keyset cursor: the sort value + storage key of the last + * row on a page. The HTTP layer encodes/decodes it as a string. */ +export interface RunsCursor { + readonly sort: number | null; + readonly key: string; +} + export interface ExecutionHistoryListOptions { readonly statusFilter?: readonly RunStatus[]; readonly triggerFilter?: readonly string[]; readonly timeRange?: { readonly from?: number; readonly to?: number }; readonly hadInteraction?: boolean; - readonly limit?: number; - readonly offset?: number; - readonly sort?: "asc" | "desc"; + /** Live-tail floor: only runs whose `startedAt` is strictly greater. */ + readonly after?: number; + readonly sortField?: RunsSortField; + readonly sortDirection?: "asc" | "desc"; + readonly limit: number; + readonly cursor?: RunsCursor; +} + +export interface RunStatusCount { + readonly status: RunStatus; + readonly count: number; +} + +export interface RunTriggerCount { + readonly triggerKind: string | null; + readonly count: number; +} + +export interface RunInteractionCounts { + readonly withInteraction: number; + readonly withoutInteraction: number; +} + +export interface RunChartBucket { + readonly bucketStart: number; + /** Status -> run count for the bucket; absent statuses are omitted. */ + readonly counts: Readonly>; +} + +export interface RunDurationStats { + readonly count: number; + readonly min: number | null; + readonly max: number | null; + readonly p50: number | null; + readonly p75: number | null; + readonly p90: number | null; + readonly p95: number | null; + readonly p99: number | null; +} + +export interface ExecutionListMeta { + readonly totalRowCount: number; + readonly filterRowCount: number; + readonly statusCounts: readonly RunStatusCount[]; + readonly triggerCounts: readonly RunTriggerCount[]; + readonly interactionCounts: RunInteractionCounts; + readonly chartBucketMs: number; + readonly chartData: readonly RunChartBucket[]; + readonly durationStats: RunDurationStats; } export interface ExecutionHistoryListResult { readonly runs: readonly RunRow[]; - readonly total: number; + readonly nextCursor: RunsCursor | null; + readonly meta: ExecutionListMeta | null; } export interface ExecutionHistoryDetail { @@ -123,7 +189,7 @@ export interface ExecutionHistoryDetail { export interface ExecutionHistoryStore { readonly handleEvent: (event: ExecutionEvent) => Effect.Effect; readonly list: ( - options?: ExecutionHistoryListOptions, + options: ExecutionHistoryListOptions, ) => Effect.Effect; readonly get: ( executionId: string, @@ -133,6 +199,62 @@ export interface ExecutionHistoryStore { ) => Effect.Effect; } +// --------------------------------------------------------------------------- +// List helpers (pure). +// --------------------------------------------------------------------------- + +type RunsWhere = { + status?: { in: readonly RunStatus[] }; + triggerKind?: { in: readonly string[] }; + startedAt?: { gte?: number; lte?: number; gt?: number }; + hadInteraction?: { eq: boolean }; +}; + +/** Build the indexed-field `where` from list options. `omit` drops one facet's + * own field so a facet count reflects the *other* filters (a faceted rail still + * shows every option for the field you're filtering on). */ +const buildRunsWhere = ( + options: ExecutionHistoryListOptions, + omit?: "status" | "triggerKind" | "hadInteraction", +): RunsWhere => { + const where: RunsWhere = {}; + if (omit !== "status" && options.statusFilter && options.statusFilter.length > 0) { + where.status = { in: options.statusFilter }; + } + if (omit !== "triggerKind" && options.triggerFilter && options.triggerFilter.length > 0) { + where.triggerKind = { in: options.triggerFilter }; + } + const startedAt: { gte?: number; lte?: number; gt?: number } = {}; + if (options.timeRange?.from != null) startedAt.gte = options.timeRange.from; + if (options.timeRange?.to != null) startedAt.lte = options.timeRange.to; + if (options.after != null) startedAt.gt = options.after; + if (Object.keys(startedAt).length > 0) where.startedAt = startedAt; + if (omit !== "hadInteraction" && options.hadInteraction != null) { + where.hadInteraction = { eq: options.hadInteraction }; + } + return where; +}; + +const HOUR_MS = 3_600_000; +const BUCKET_STEPS_MS = [ + 60_000, + 5 * 60_000, + 15 * 60_000, + HOUR_MS, + 6 * HOUR_MS, + 24 * HOUR_MS, + 7 * 24 * HOUR_MS, +] as const; + +/** Pick a timeline bucket width (~48 buckets across the requested range). */ +const chooseBucketMs = (timeRange: { from?: number; to?: number } | undefined): number => { + if (timeRange?.from == null || timeRange.to == null) return HOUR_MS; + const target = Math.max(1, timeRange.to - timeRange.from) / 48; + return ( + BUCKET_STEPS_MS.find((step) => step >= target) ?? BUCKET_STEPS_MS[BUCKET_STEPS_MS.length - 1]! + ); +}; + export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistoryStore => { const pluginStorage: PluginStorageFacade = deps.pluginStorage; const runsC: PluginStorageCollectionFacade = pluginStorage.collection(runs); @@ -373,41 +495,160 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt return Effect.void; }; - const list = ( - options?: ExecutionHistoryListOptions, - ): Effect.Effect => { - const where: { - status?: { in: readonly RunStatus[] }; - triggerKind?: { in: readonly string[] }; - startedAt?: { gte?: number; lte?: number }; - hadInteraction?: { eq: boolean }; - } = {}; - if (options?.statusFilter && options.statusFilter.length > 0) { - where.status = { in: options.statusFilter }; - } - if (options?.triggerFilter && options.triggerFilter.length > 0) { - where.triggerKind = { in: options.triggerFilter }; - } - if (options?.timeRange) { - where.startedAt = {}; - if (options.timeRange.from != null) where.startedAt.gte = options.timeRange.from; - if (options.timeRange.to != null) where.startedAt.lte = options.timeRange.to; - } - if (options?.hadInteraction != null) { - where.hadInteraction = { eq: options.hadInteraction }; - } + const computeMeta = ( + options: ExecutionHistoryListOptions, + ): Effect.Effect => + Effect.gen(function* () { + const where = buildRunsWhere(options); + + // Wave 1: all independent aggregates concurrently. + const [ + totalRowCount, + filterRowCount, + statusGroups, + triggerGroups, + interactionGroups, + stats, + startedAtStats, + ] = yield* Effect.all( + [ + runsC.aggregate.count(), + runsC.aggregate.count({ where }), + runsC.aggregate.groupCount({ + field: "status", + where: buildRunsWhere(options, "status"), + }), + runsC.aggregate.groupCount({ + field: "triggerKind", + where: buildRunsWhere(options, "triggerKind"), + }), + runsC.aggregate.groupCount({ + field: "hadInteraction", + where: buildRunsWhere(options, "hadInteraction"), + valueType: "boolean", + }), + runsC.aggregate.stats({ + field: "durationMs", + where, + percentiles: [0.5, 0.75, 0.9, 0.95, 0.99], + }), + runsC.aggregate.stats({ + field: "startedAt", + where, + percentiles: [], + }), + ] as const, + { concurrency: "unbounded" }, + ); - return Effect.gen(function* () { - const rows = yield* runsC.query({ + const statusCounts: RunStatusCount[] = []; + for (const group of statusGroups) { + if (isRunStatus(group.value)) { + statusCounts.push({ status: group.value, count: group.count }); + } + } + + // Wave 2: stacked-by-status timeline β€” one bucketed count per present + // status, merged by bucket start. Respects every filter except `status` + // itself. Per-status queries are independent so run concurrently. + // + // When no explicit time range is set, derive the bucketing span from the + // data's actual startedAt min/max so the bucket count stays ~48 regardless + // of dataset age. Falls back to chooseBucketMs default only when the + // result set is empty (startedAtStats.min/max are null). + const effectiveBucketRange: { from?: number; to?: number } = + options.timeRange?.from != null && options.timeRange.to != null + ? options.timeRange + : { + from: startedAtStats.min ?? undefined, + to: startedAtStats.max ?? undefined, + }; + const bucketMs = chooseBucketMs(effectiveBucketRange); + const chartWhere = buildRunsWhere(options, "status"); + const perStatusBuckets = yield* Effect.forEach( + statusCounts, + (entry) => + runsC.aggregate + .timeBuckets({ + field: "startedAt", + bucketMs, + where: { ...chartWhere, status: { in: [entry.status] } }, + }) + .pipe(Effect.map((buckets) => ({ status: entry.status, buckets }))), + { concurrency: "unbounded" }, + ); + const chartByBucket = new Map>(); + for (const { status, buckets } of perStatusBuckets) { + for (const bucket of buckets) { + const counts = chartByBucket.get(bucket.bucket) ?? {}; + counts[status] = bucket.count; + chartByBucket.set(bucket.bucket, counts); + } + } + const chartData: RunChartBucket[] = [...chartByBucket.entries()] + .sort((left, right) => left[0] - right[0]) + .map(([bucketStart, counts]) => ({ bucketStart, counts })); + + const triggerCounts: RunTriggerCount[] = triggerGroups.map((group) => ({ + triggerKind: typeof group.value === "string" ? group.value : null, + count: group.count, + })); + const withInteraction = interactionGroups.find((group) => group.value === true)?.count ?? 0; + const withoutInteraction = + interactionGroups.find((group) => group.value === false)?.count ?? 0; + const percentileAt = (fraction: number): number | null => + stats.percentiles.find((entry) => entry.fraction === fraction)?.value ?? null; + + return { + totalRowCount, + filterRowCount, + statusCounts, + triggerCounts, + interactionCounts: { withInteraction, withoutInteraction }, + chartBucketMs: bucketMs, + chartData, + durationStats: { + count: stats.count, + min: stats.min, + max: stats.max, + p50: percentileAt(0.5), + p75: percentileAt(0.75), + p90: percentileAt(0.9), + p95: percentileAt(0.95), + p99: percentileAt(0.99), + }, + }; + }); + + const list = ( + options: ExecutionHistoryListOptions, + ): Effect.Effect => + Effect.gen(function* () { + const sortField = options.sortField ?? "startedAt"; + const sortDirection = options.sortDirection ?? "desc"; + const where = buildRunsWhere(options); + const page = yield* runsC.queryKeyset({ where, - orderBy: [{ field: "startedAt", direction: options?.sort ?? "desc" }], - limit: options?.limit, - offset: options?.offset, + orderBy: [{ field: sortField, direction: sortDirection, valueType: "number" }], + limit: options.limit, + cursor: options.cursor + ? { values: [options.cursor.sort], key: options.cursor.key } + : undefined, }); - const total = yield* runsC.count({ where }); - return { runs: rows.map((entry) => entry.data), total }; + const nextCursor: RunsCursor | null = page.nextCursor + ? { + sort: typeof page.nextCursor.values[0] === "number" ? page.nextCursor.values[0] : null, + key: page.nextCursor.key, + } + : null; + // Meta is computed once per filter set β€” only for the first page, and + // never during live polling (`after`). + const meta = + options.cursor === undefined && options.after === undefined + ? yield* computeMeta(options) + : null; + return { runs: page.entries.map((entry) => entry.data), nextCursor, meta }; }); - }; const get = (executionId: string): Effect.Effect => Effect.gen(function* () { From 58f7780c42a102da8cf77e1c549cbfc534a086c3 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sun, 14 Jun 2026 20:32:30 +0530 Subject: [PATCH 06/57] feat(execution-history): openstatus-style runs UI (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuilds the runs page as an openstatus-style observability table on the keyset + aggregate API, using the plugin's Effect-atoms idiom. - Faceted filter rail (status/trigger/interaction/time with live counts), cmdk filter palette, recharts stacked timeline with drag-to-zoom, live tailing, keyset infinite scroll, sortable columns, column visibility, keyboard shortcuts, and the existing 4-tab detail drawer. - List data flows through a `useRunsList` hook that accumulates cursor pages and polls the head for live mode; the accumulation/dedup logic is a pure, unit-tested reducer. - Adds `recharts` as an optional peer dependency (provided by `@executor-js/react`). Verified: lint/format/typecheck clean, 15 plugin tests, build success, repo-wide typecheck. An adversarial review pass caught and fixed 2 bugs (a refresh that blanked page 1; a recharts axis misconfig) plus 4 correctness issues (live-divider off-by-one, live-anchor timing, drag-cancel-on-leave, filter-merge clobbering). Coverage gap: no browser/render/e2e test this round β€” the reducer is unit-tested and the components were adversarially reviewed, but the rendered page has not been driven live. Worth exercising by hand: live divider, timeline drag-to-zoom, the cmdk parser, infinite scroll. Live mode is append-only (existing rows' status changes are not re-pulled; meta updates live). Filters are local state (not URL-persisted yet). Stacked on the runs API PR. ### [Stack](https://github.com/kitlangton/stack) 1. #15 2. #16 3. **#17** πŸ‘ˆ current --- .changeset/execution-history-runs-ui.md | 13 + bun.lock | 6 + .../plugins/execution-history/package.json | 36 +- .../execution-history/src/react/RunsPage.tsx | 708 ++++++------------ .../src/react/column-header.tsx | 111 +++ .../src/react/detail-drawer.tsx | 281 +++++++ .../src/react/filter-command-parser.ts | 96 +++ .../src/react/filter-command.tsx | 324 ++++++++ .../src/react/filter-rail.tsx | 366 +++++++++ .../execution-history/src/react/format.ts | 57 ++ .../src/react/keyboard-help.tsx | 56 ++ .../src/react/live-button.tsx | 36 + .../execution-history/src/react/live-row.tsx | 15 + .../src/react/refresh-button.tsx | 24 + .../execution-history/src/react/run-row.tsx | 125 ++++ .../src/react/runs-list-state.test.ts | 135 ++++ .../src/react/runs-list-state.ts | 113 +++ .../execution-history/src/react/shell.tsx | 134 ++++ .../execution-history/src/react/status.ts | 95 +++ .../src/react/timeline-chart.tsx | 197 +++++ .../src/react/use-runs-list.ts | 171 +++++ .../src/react/view-options.tsx | 45 ++ .../execution-history/src/react/view.ts | 28 + 23 files changed, 2660 insertions(+), 512 deletions(-) create mode 100644 .changeset/execution-history-runs-ui.md create mode 100644 packages/plugins/execution-history/src/react/column-header.tsx create mode 100644 packages/plugins/execution-history/src/react/detail-drawer.tsx create mode 100644 packages/plugins/execution-history/src/react/filter-command-parser.ts create mode 100644 packages/plugins/execution-history/src/react/filter-command.tsx create mode 100644 packages/plugins/execution-history/src/react/filter-rail.tsx create mode 100644 packages/plugins/execution-history/src/react/format.ts create mode 100644 packages/plugins/execution-history/src/react/keyboard-help.tsx create mode 100644 packages/plugins/execution-history/src/react/live-button.tsx create mode 100644 packages/plugins/execution-history/src/react/live-row.tsx create mode 100644 packages/plugins/execution-history/src/react/refresh-button.tsx create mode 100644 packages/plugins/execution-history/src/react/run-row.tsx create mode 100644 packages/plugins/execution-history/src/react/runs-list-state.test.ts create mode 100644 packages/plugins/execution-history/src/react/runs-list-state.ts create mode 100644 packages/plugins/execution-history/src/react/shell.tsx create mode 100644 packages/plugins/execution-history/src/react/status.ts create mode 100644 packages/plugins/execution-history/src/react/timeline-chart.tsx create mode 100644 packages/plugins/execution-history/src/react/use-runs-list.ts create mode 100644 packages/plugins/execution-history/src/react/view-options.tsx create mode 100644 packages/plugins/execution-history/src/react/view.ts diff --git a/.changeset/execution-history-runs-ui.md b/.changeset/execution-history-runs-ui.md new file mode 100644 index 0000000000..8669efc247 --- /dev/null +++ b/.changeset/execution-history-runs-ui.md @@ -0,0 +1,13 @@ +--- +"@executor-js/plugin-execution-history": minor +--- + +Rebuild the runs page as an openstatus-style observability table on the keyset + +aggregate API. The page now has a faceted filter rail (status / trigger / +interaction / time range with live counts), a cmdk filter command palette, a +stacked-by-status timeline chart with drag-to-zoom, live tailing, keyset +infinite scroll, sortable columns, column visibility, keyboard shortcuts, and +the existing 4-tab detail drawer. List data flows through an Effect-atoms hook +(`useRunsList`) that accumulates cursor pages and polls the head for live mode; +the accumulation logic is a pure, unit-tested reducer. Adds `recharts` as an +optional peer dependency (provided by `@executor-js/react`). diff --git a/bun.lock b/bun.lock index a50dbc31d5..040a65e3d0 100644 --- a/bun.lock +++ b/bun.lock @@ -822,7 +822,9 @@ "@types/node": "catalog:", "@types/react": "catalog:", "bun-types": "catalog:", + "lucide-react": "^1.7.0", "react": "catalog:", + "recharts": "3.8.0", "tsup": "catalog:", "vitest": "catalog:", }, @@ -831,13 +833,17 @@ "@executor-js/api": "workspace:*", "@executor-js/react": "workspace:*", "@tanstack/react-router": "catalog:", + "lucide-react": "^1.7.0", "react": "catalog:", + "recharts": "3.8.0", }, "optionalPeers": [ "@effect/atom-react", "@executor-js/react", "@tanstack/react-router", + "lucide-react", "react", + "recharts", ], }, "packages/plugins/execution-metrics": { diff --git a/packages/plugins/execution-history/package.json b/packages/plugins/execution-history/package.json index dbc58080ed..106d631e0e 100644 --- a/packages/plugins/execution-history/package.json +++ b/packages/plugins/execution-history/package.json @@ -19,12 +19,28 @@ "@executor-js/sdk": "workspace:*", "effect": "catalog:" }, + "devDependencies": { + "@effect/atom-react": "catalog:", + "@effect/vitest": "catalog:", + "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "@types/node": "catalog:", + "@types/react": "catalog:", + "bun-types": "catalog:", + "lucide-react": "^1.7.0", + "react": "catalog:", + "recharts": "3.8.0", + "tsup": "catalog:", + "vitest": "catalog:" + }, "peerDependencies": { "@effect/atom-react": "catalog:", "@executor-js/api": "workspace:*", "@executor-js/react": "workspace:*", "@tanstack/react-router": "catalog:", - "react": "catalog:" + "lucide-react": "^1.7.0", + "react": "catalog:", + "recharts": "3.8.0" }, "peerDependenciesMeta": { "react": { @@ -38,18 +54,12 @@ }, "@executor-js/react": { "optional": true + }, + "lucide-react": { + "optional": true + }, + "recharts": { + "optional": true } - }, - "devDependencies": { - "@effect/atom-react": "catalog:", - "@effect/vitest": "catalog:", - "@executor-js/api": "workspace:*", - "@executor-js/react": "workspace:*", - "@types/node": "catalog:", - "@types/react": "catalog:", - "bun-types": "catalog:", - "react": "catalog:", - "tsup": "catalog:", - "vitest": "catalog:" } } diff --git a/packages/plugins/execution-history/src/react/RunsPage.tsx b/packages/plugins/execution-history/src/react/RunsPage.tsx index ecbbf63f96..46f7201b69 100644 --- a/packages/plugins/execution-history/src/react/RunsPage.tsx +++ b/packages/plugins/execution-history/src/react/RunsPage.tsx @@ -1,528 +1,238 @@ -import { useAtomValue } from "@effect/atom-react"; -import * as Option from "effect/Option"; -import * as Schema from "effect/Schema"; -import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; -import { useMemo, useState } from "react"; -import { Badge } from "@executor-js/react/components/badge"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Button } from "@executor-js/react/components/button"; -import { ScrollArea } from "@executor-js/react/components/scroll-area"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@executor-js/react/components/select"; -import { - Sheet, - SheetContent, - SheetDescription, - SheetHeader, - SheetTitle, -} from "@executor-js/react/components/sheet"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@executor-js/react/components/table"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@executor-js/react/components/tabs"; -import { cn } from "@executor-js/react/lib/utils"; -import type { InteractionRow, RunRow, ToolCallRow } from "../sdk/collections"; -import type { RunStatus, ToolCallStatus } from "../sdk/collections"; -import { runDetailAtom, runToolCallsAtom, runsAtom, type RunsQuery } from "./atoms"; +import { RunsColumnHeader } from "./column-header"; +import { DetailDrawer } from "./detail-drawer"; +import { RunsFilterCommand } from "./filter-command"; +import { RunsFilterRail } from "./filter-rail"; +import { KeyboardHelp } from "./keyboard-help"; +import { LiveButton } from "./live-button"; +import { LiveDivider } from "./live-row"; +import { RefreshButton } from "./refresh-button"; +import { RunListRow } from "./run-row"; +import { RunsShell } from "./shell"; +import { RunsTimelineChart } from "./timeline-chart"; +import { + emptyRunsFilters, + useRunsList, + type RunsFilters, + type RunsListView, + type RunsSortField, +} from "./use-runs-list"; +import { DEFAULT_COLUMNS, type RunColumnKey, type RunColumns } from "./view"; +import { ViewOptions } from "./view-options"; // --------------------------------------------------------------------------- -// Execution-history runs page (v1). +// Execution-history runs page β€” openstatus-style. // -// A runs table (status / trigger / started / duration / tool calls / -// interaction) with a status filter, limit/offset pagination, and a 4-tab -// detail drawer (Properties / Tool calls / Logs / Interaction) opened on row -// click. The richer command-palette filter, keyboard row navigation, live -// auto-refresh, and timeline chart from the original prototype are -// intentionally OUT OF SCOPE for v1. -// --------------------------------------------------------------------------- - -const STATUS_OPTIONS = [ - "all", - "running", - "waiting_for_interaction", - "completed", - "failed", -] as const; - -type StatusFilter = (typeof STATUS_OPTIONS)[number]; - -const PAGE_SIZE = 50; - -// --------------------------------------------------------------------------- -// Formatting. The stored `*Json` columns are already-serialized (compact) -// strings; decode them through Schema (only `JSON.parse` is lint-banned), then -// re-indent for display. `JSON.stringify` is allowed and won't throw on a value -// that just decoded from JSON. -// --------------------------------------------------------------------------- - -const decodeJson = Schema.decodeUnknownOption(Schema.UnknownFromJsonString); -const decodeLogLines = Schema.decodeUnknownOption( - Schema.fromJsonString(Schema.Array(Schema.String)), -); - -const prettyJson = (raw: string | null): string | null => { - if (!raw) return null; - return Option.match(decodeJson(raw), { - onNone: () => raw, - onSome: (value) => JSON.stringify(value, null, 2), - }); -}; - -const logLines = (raw: string | null): readonly string[] => - !raw - ? [] - : Option.match(decodeLogLines(raw), { - onNone: () => [raw], - onSome: (value) => value, - }); - -const statusLabel = (status: RunStatus | ToolCallStatus): string => - status === "waiting_for_interaction" ? "waiting" : status; - -const statusClass = (status: RunStatus | ToolCallStatus): string => - status === "completed" - ? "border-emerald-500/30 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300" - : status === "failed" - ? "border-destructive/30 bg-destructive/10 text-destructive" - : status === "waiting_for_interaction" - ? "border-amber-500/30 bg-amber-500/10 text-amber-700 dark:text-amber-300" - : "border-sky-500/30 bg-sky-500/10 text-sky-700 dark:text-sky-300"; - -const formatDateTime = (timestamp: number | null): string => - timestamp == null ? "Pending" : new Date(timestamp).toLocaleString(); - -const formatRelative = (timestamp: number): string => { - const delta = timestamp - Date.now(); - const abs = Math.abs(delta); - const formatter = new Intl.RelativeTimeFormat(undefined, { numeric: "auto" }); - if (abs < 60_000) return formatter.format(Math.round(delta / 1000), "second"); - if (abs < 3_600_000) return formatter.format(Math.round(delta / 60_000), "minute"); - if (abs < 86_400_000) return formatter.format(Math.round(delta / 3_600_000), "hour"); - return formatter.format(Math.round(delta / 86_400_000), "day"); -}; - -const formatDuration = (value: number | null): string => { - if (value == null) return "running"; - if (value < 1000) return `${value}ms`; - if (value < 60_000) return `${(value / 1000).toFixed(1)}s`; - return `${(value / 60_000).toFixed(1)}m`; -}; - +// A faceted filter rail, a stacked-by-status timeline, a cmdk filter palette, +// live tailing, keyset infinite scroll, sortable columns, and a 4-tab detail +// drawer. All list data + accumulation flows through `useRunsList` (atoms); +// this component is filter/UI state + composition only. // --------------------------------------------------------------------------- -// Presentational blocks -// --------------------------------------------------------------------------- - -function StatusBadge(props: { readonly status: RunStatus | ToolCallStatus }) { - return ( - - {statusLabel(props.status)} - - ); -} -function MetaCard(props: { readonly label: string; readonly children: React.ReactNode }) { - return ( -
-

{props.label}

-
{props.children}
-
- ); +interface ShortcutContext { + readonly view: RunsListView; + readonly selected: string | null; } -function JsonBlock(props: { readonly title: string; readonly value: string | null }) { - const text = prettyJson(props.value); - if (!text) { - return ( -
- No {props.title.toLowerCase()} recorded. -
+export function RunsPage() { + const [filters, setFilters] = useState(emptyRunsFilters); + const [live, setLive] = useState(false); + const [selected, setSelected] = useState(null); + const [columns, setColumns] = useState(DEFAULT_COLUMNS); + const [commandOpen, setCommandOpen] = useState(false); + const [helpOpen, setHelpOpen] = useState(false); + const [railCollapsed, setRailCollapsed] = useState(false); + const commandInputRef = useRef(null); + + const view = useRunsList(filters, live); + + const onSort = useCallback((field: RunsSortField) => { + setFilters((current) => + current.sortField === field + ? { ...current, sortDirection: current.sortDirection === "asc" ? "desc" : "asc" } + : { ...current, sortField: field, sortDirection: "desc" }, ); - } - return ( -
-

{props.title}

- 
-        {text}
-
-
+ }, []); + + const toggleColumn = useCallback((key: RunColumnKey) => { + setColumns((current) => ({ ...current, [key]: !current[key] })); + }, []); + + // Keep the latest view/selected reachable from a stable keydown listener. + const shortcutRef = useRef({ view, selected }); + shortcutRef.current = { view, selected }; + + useEffect(() => { + const onKeyDown = (event: KeyboardEvent) => { + const target = event.target as HTMLElement | null; + const typing = + target != null && + (target.tagName === "INPUT" || target.tagName === "TEXTAREA" || target.isContentEditable); + + if (event.key === "/" && !typing) { + event.preventDefault(); + setCommandOpen(true); + requestAnimationFrame(() => commandInputRef.current?.focus()); + return; + } + if (event.key === "Escape") { + setCommandOpen(false); + return; + } + if (typing) return; + + const { view: latestView, selected: latestSelected } = shortcutRef.current; + if (event.key === "j") { + setLive((value) => !value); + } else if (event.key === "r") { + latestView.refresh(); + } else if (event.key === "?") { + setHelpOpen((value) => !value); + } else if (event.key === "b") { + setRailCollapsed((value) => !value); + } else if (latestSelected != null && (event.key === "ArrowUp" || event.key === "ArrowDown")) { + event.preventDefault(); + const ids = latestView.rows.map((run) => run.executionId); + const index = ids.indexOf(latestSelected); + if (index < 0) return; + const nextId = event.key === "ArrowUp" ? ids[index - 1] : ids[index + 1]; + if (nextId != null) setSelected(nextId); + } + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, []); + + // The live divider + isPast styling assume newest-first ordering. For + // ascending sort live rows append at the bottom with no divider, so only + // compute the cutoff for descending sort. + const isDescending = filters.sortDirection !== "asc"; + const cutoffIndex = useMemo( + () => + !isDescending || view.liveCutoffId == null + ? -1 + : view.rows.findIndex((run) => run.executionId === view.liveCutoffId), + [isDescending, view.rows, view.liveCutoffId], ); -} -function CodeBlock(props: { readonly code: string }) { - return ( -
-

Code

- 
-        {props.code}
-
-
- ); -} + const rowItems: React.ReactNode[] = []; + view.rows.forEach((run, index) => { + if (index === cutoffIndex && cutoffIndex > 0) { + rowItems.push(); + } + rowItems.push( + 0 && index >= cutoffIndex} + columns={columns} + onSelect={() => setSelected(run.executionId)} + />, + ); + }); -function LogsBlock(props: { readonly logsJson: string | null }) { - const lines = logLines(props.logsJson); - if (lines.length === 0) { - return ( -
- No logs recorded. -
+ if (view.isLoadMoreError) { + rowItems.push( +
+

Failed to load more

+ +
, ); } - return ( -
- {lines.map((line, index) => ( - 
-          {line}
-
- ))} -
- ); -} -function ToolCallItem(props: { readonly call: ToolCallRow }) { - const [expanded, setExpanded] = useState(false); - return ( -
- - {expanded && ( -
- - {props.call.status === "failed" && props.call.errorText ? ( -
-

Error

- 
-                {props.call.errorText}
-
-
- ) : ( - - )} + const toolbar = ( +
+
+
+

Runs

+
- )} -
- ); -} - -function ToolCallsTab(props: { readonly executionId: string }) { - const toolCalls = useAtomValue(runToolCallsAtom(props.executionId)); - return AsyncResult.match(toolCalls, { - onInitial: () =>

Loading tool calls...

, - onFailure: () =>

Unable to load tool calls.

, - onSuccess: ({ value }) => - value.toolCalls.length === 0 ? ( -

No tool calls recorded.

- ) : ( -
- {value.toolCalls.map((call) => ( - - ))} +
+ setLive((value) => !value)} /> + + +
- ), - }); -} - -function InteractionTab(props: { readonly interactions: readonly InteractionRow[] }) { - if (props.interactions.length === 0) { - return ( -
- No interactions recorded.
- ); - } - return ( -
- {props.interactions.map((interaction) => ( -
-
- {interaction.kind} - {interaction.status} - {interaction.purpose ?? "β€”"} -
- - -
- ))} -
- ); -} - -function DetailContent(props: { - readonly run: RunRow; - readonly interactions: readonly InteractionRow[]; -}) { - const { run } = props; - return ( -
- - {run.executionId} - - {statusLabel(run.status)} / {run.triggerKind ?? "unknown"} /{" "} - {formatDuration(run.durationMs)} - - - - - - Properties - Tool calls - Logs - Interaction - - - -
- - {run.executionId} - - - - - {run.triggerKind ?? "unknown"} - {formatDuration(run.durationMs)} - {formatDateTime(run.startedAt)} - {formatDateTime(run.completedAt)} -
- - - {run.errorText && ( -
-

Error

- 
-                  {run.errorText}
-
-
- )} - - - - - - - - - - - - +
); -} -function DetailDrawer(props: { - readonly executionId: string | null; - readonly onClose: () => void; -}) { - const detail = useAtomValue(runDetailAtom(props.executionId ?? "__none__")); - return ( - !open && props.onClose()}> - - {props.executionId == null - ? null - : AsyncResult.match(detail, { - onInitial: () => ( -
Loading run...
- ), - onFailure: () => ( -
Unable to load run.
- ), - onSuccess: ({ value }) => - value ? ( - - ) : ( -
Run not found.
- ), - })} - - + const chart = + view.meta != null && view.meta.chartData.length > 0 ? ( + setFilters((current) => ({ ...current, from, to }))} + /> + ) : undefined; + + const body = view.isLoading ? ( +
Loading runs…
+ ) : view.isError ? ( +
Unable to load runs.
+ ) : ( + rowItems ); -} - -function RunsTableRow(props: { readonly run: RunRow; readonly onSelect: () => void }) { - const { run } = props; - return ( - - - - - {run.triggerKind ?? "unknown"} - - {formatRelative(run.startedAt)} - - {formatDuration(run.durationMs)} - {run.toolCallCount} - - {run.hadInteraction ? ( - - yes - - ) : ( - β€” - )} - - - ); -} - -function RunsTable(props: { - readonly runs: readonly RunRow[]; - readonly onSelect: (executionId: string) => void; -}) { - if (props.runs.length === 0) { - return ( -
- No runs match the current filters. -
- ); - } - return ( - - - - Status - Trigger - Started - Duration - Tools - Interaction - - - - {props.runs.map((run) => ( - props.onSelect(run.executionId)} - /> - ))} - -
- ); -} - -export function RunsPage() { - const [status, setStatus] = useState("all"); - // Forward keyset pagination: a stack of cursors, top = current page (page 1 - // is `undefined`). Next pushes `nextCursor`; Previous pops. - const [cursorStack, setCursorStack] = useState([undefined]); - const [selected, setSelected] = useState(null); - const cursor = cursorStack[cursorStack.length - 1]; - - const query = useMemo( - () => ({ - limit: PAGE_SIZE, - sort: "startedAt", - dir: "desc", - ...(status === "all" ? {} : { status }), - ...(cursor !== undefined ? { cursor } : {}), - }), - [cursor, status], - ); - const runs = useAtomValue(runsAtom(query)); return (
-
-
-

Runs

-

- Plugin-backed local execution history, newest first. -

-
- -
- - - {AsyncResult.match(runs, { - onInitial: () =>
Loading runs...
, - onFailure: () =>
Unable to load runs.
, - onSuccess: ({ value }) => ( -
- -
-

- {value.meta - ? `${value.meta.filterRowCount.toLocaleString()} total` - : `${value.runs.length.toLocaleString()} on this page`} -

-
- - -
-
-
- ), - })} - + setFilters(emptyRunsFilters)} + /> + } + toolbar={toolbar} + chart={chart} + columnHeader={ + + } + onLoadMore={view.loadMore} + hasMore={view.hasMore} + isLoadingMore={view.isLoadingMore} + isEmpty={!view.isLoading && !view.isError && view.rows.length === 0} + emptyState={ +
+ No runs match the current filters. +
+ } + railCollapsed={railCollapsed} + > + {body} + setSelected(null)} />
diff --git a/packages/plugins/execution-history/src/react/column-header.tsx b/packages/plugins/execution-history/src/react/column-header.tsx new file mode 100644 index 0000000000..64df6ff758 --- /dev/null +++ b/packages/plugins/execution-history/src/react/column-header.tsx @@ -0,0 +1,111 @@ +import { ArrowDown, ArrowUp, ArrowUpDown } from "lucide-react"; +import { Button } from "@executor-js/react/components/button"; +import { cn } from "@executor-js/react/lib/utils"; + +import type { RunsSortField } from "./use-runs-list"; +import type { RunColumns } from "./view"; +import { COL_DURATION, COL_INTERACTION, COL_TOOLS, COL_TRIGGER } from "./run-row"; + +// --------------------------------------------------------------------------- +// RunsColumnHeader β€” sticky header row aligned to RunListRow's layout. +// Imports the shared column slot classes from run-row so that header and rows +// use the same widths and breakpoints. +// --------------------------------------------------------------------------- + +interface SortButtonProps { + readonly label: string; + readonly field: RunsSortField; + readonly sortField: RunsSortField; + readonly sortDirection: "asc" | "desc"; + readonly onSort: (field: RunsSortField) => void; + readonly className?: string; +} + +function SortButton({ + label, + field, + sortField, + sortDirection, + onSort, + className, +}: SortButtonProps) { + const isActive = sortField === field; + const Icon = isActive ? (sortDirection === "desc" ? ArrowDown : ArrowUp) : ArrowUpDown; + + return ( + + ); +} + +export interface RunsColumnHeaderProps { + readonly sortField: RunsSortField; + readonly sortDirection: "asc" | "desc"; + readonly onSort: (field: RunsSortField) => void; + readonly columns: RunColumns; +} + +export function RunsColumnHeader({ + sortField, + sortDirection, + onSort, + columns, +}: RunsColumnHeaderProps) { + return ( +
+ {/* dot spacer */} + + + {/* Started (sortable) */} + + + {/* Status */} + status + + {/* Trigger */} + {columns.trigger ? trigger : null} + + {/* Duration (sortable) */} + {columns.duration ? ( + + ) : null} + + {/* Tools */} + {columns.tools ? tools : null} + + {/* Interaction */} + {columns.interaction ? interaction : null} + + {/* Code */} + code +
+ ); +} diff --git a/packages/plugins/execution-history/src/react/detail-drawer.tsx b/packages/plugins/execution-history/src/react/detail-drawer.tsx new file mode 100644 index 0000000000..1db05e318e --- /dev/null +++ b/packages/plugins/execution-history/src/react/detail-drawer.tsx @@ -0,0 +1,281 @@ +import { useAtomValue } from "@effect/atom-react"; +import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; +import { useState } from "react"; +import { Badge } from "@executor-js/react/components/badge"; +import { Button } from "@executor-js/react/components/button"; +import { ScrollArea } from "@executor-js/react/components/scroll-area"; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@executor-js/react/components/sheet"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@executor-js/react/components/tabs"; +import { cn } from "@executor-js/react/lib/utils"; + +import type { + InteractionRow, + RunRow, + RunStatus, + ToolCallRow, + ToolCallStatus, +} from "../sdk/collections"; +import { runDetailAtom, runToolCallsAtom } from "./atoms"; +import { formatDateTime, formatDuration, logLines, prettyJson, statusLabel } from "./format"; +import { STATUS_TONES } from "./status"; + +// --------------------------------------------------------------------------- +// Right-side run detail drawer: a 4-tab Sheet (Properties / Tool calls / Logs / +// Interaction) opened on row click. Reads detail + tool calls through the +// plugin's atoms. Kept as a self-contained module so the runs page and any +// other surface can mount it. +// --------------------------------------------------------------------------- + +export function StatusBadge(props: { readonly status: RunStatus | ToolCallStatus }) { + return ( + + {statusLabel(props.status)} + + ); +} + +function MetaCard(props: { readonly label: string; readonly children: React.ReactNode }) { + return ( +
+

{props.label}

+
{props.children}
+
+ ); +} + +function JsonBlock(props: { readonly title: string; readonly value: string | null }) { + const text = prettyJson(props.value); + if (!text) { + return ( +
+ No {props.title.toLowerCase()} recorded. +
+ ); + } + return ( +
+

{props.title}

+ 
+        {text}
+
+
+ ); +} + +function CodeBlock(props: { readonly code: string }) { + return ( +
+

Code

+ 
+        {props.code}
+
+
+ ); +} + +function LogsBlock(props: { readonly logsJson: string | null }) { + const lines = logLines(props.logsJson); + if (lines.length === 0) { + return ( +
+ No logs recorded. +
+ ); + } + return ( +
+ {lines.map((line, index) => ( + 
+          {line}
+
+ ))} +
+ ); +} + +function ToolCallItem(props: { readonly call: ToolCallRow }) { + const [expanded, setExpanded] = useState(false); + return ( +
+ + {expanded && ( +
+ + {props.call.status === "failed" && props.call.errorText ? ( +
+

Error

+ 
+                {props.call.errorText}
+
+
+ ) : ( + + )} +
+ )} +
+ ); +} + +function ToolCallsTab(props: { readonly executionId: string }) { + const toolCalls = useAtomValue(runToolCallsAtom(props.executionId)); + return AsyncResult.match(toolCalls, { + onInitial: () =>

Loading tool calls...

, + onFailure: () =>

Unable to load tool calls.

, + onSuccess: ({ value }) => + value.toolCalls.length === 0 ? ( +

No tool calls recorded.

+ ) : ( +
+ {value.toolCalls.map((call) => ( + + ))} +
+ ), + }); +} + +function InteractionTab(props: { readonly interactions: readonly InteractionRow[] }) { + if (props.interactions.length === 0) { + return ( +
+ No interactions recorded. +
+ ); + } + return ( +
+ {props.interactions.map((interaction) => ( +
+
+ {interaction.kind} + {interaction.status} + {interaction.purpose ?? "β€”"} +
+ + +
+ ))} +
+ ); +} + +function DetailContent(props: { + readonly run: RunRow; + readonly interactions: readonly InteractionRow[]; +}) { + const { run } = props; + return ( +
+ + {run.executionId} + + {statusLabel(run.status)} / {run.triggerKind ?? "unknown"} /{" "} + {formatDuration(run.durationMs)} + + + + + + Properties + Tool calls + Logs + Interaction + + + +
+ + {run.executionId} + + + + + {run.triggerKind ?? "unknown"} + {formatDuration(run.durationMs)} + {formatDateTime(run.startedAt)} + {formatDateTime(run.completedAt)} +
+ + + {run.errorText && ( +
+

Error

+ 
+                  {run.errorText}
+
+
+ )} + + + + + + + + + + + + +
+ ); +} + +export function DetailDrawer(props: { + readonly executionId: string | null; + readonly onClose: () => void; +}) { + const detail = useAtomValue(runDetailAtom(props.executionId ?? "__none__")); + return ( + !open && props.onClose()}> + + {props.executionId == null + ? null + : AsyncResult.match(detail, { + onInitial: () => ( +
Loading run...
+ ), + onFailure: () => ( +
Unable to load run.
+ ), + onSuccess: ({ value }) => + value ? ( + + ) : ( +
Run not found.
+ ), + })} + + + ); +} diff --git a/packages/plugins/execution-history/src/react/filter-command-parser.ts b/packages/plugins/execution-history/src/react/filter-command-parser.ts new file mode 100644 index 0000000000..df16a071b5 --- /dev/null +++ b/packages/plugins/execution-history/src/react/filter-command-parser.ts @@ -0,0 +1,96 @@ +// --------------------------------------------------------------------------- +// Pure parser for the runs filter command palette. Splits a free-text query +// into `key:value` tokens and produces a partial filter set. Adapted to the +// plugin's contract: only the 4 statuses, no tool/code/duration facets β€” the +// keys are status, trigger, interaction, after, and before. +// +// Parsing is forgiving by design: unknown keys, empty values, and unparsable +// dates are silently dropped (never throws). Relative durations like "1h" / +// "30m" / "7d" are resolved against `Date.now()`; absolute values fall back to +// `Date.parse` (the only allowed JSON-free epoch decode). +// --------------------------------------------------------------------------- + +export interface RunsFilterTokens { + readonly status: string[]; + readonly trigger: string[]; + readonly interaction: "true" | "false" | null; + readonly from: number | null; + readonly to: number | null; +} + +export interface FilterKey { + readonly key: string; + readonly hint: string; +} + +export const FILTER_KEYS: readonly FilterKey[] = [ + { key: "status", hint: "completed,failed,running,waiting" }, + { key: "trigger", hint: "mcp, http, cli, manual" }, + { key: "interaction", hint: "true | false" }, + { key: "after", hint: "1h Β· 30m Β· 7d Β· ISO date" }, + { key: "before", hint: "1h Β· 7d Β· ISO date" }, +]; + +const RELATIVE_UNIT_MS: Record = { + m: 60_000, + h: 3_600_000, + d: 86_400_000, + w: 604_800_000, +}; + +const RELATIVE_PATTERN = /^(\d+)([mhdw])$/; + +/** Convert a relative duration token ("1h", "30m", "7d") to milliseconds, or + * null when it isn't a recognized relative literal. */ +export const relativeToMs = (token: string): number | null => { + const match = RELATIVE_PATTERN.exec(token.trim()); + if (!match) return null; + const amount = Number(match[1]); + const unit = RELATIVE_UNIT_MS[match[2]!]; + if (unit === undefined || !Number.isFinite(amount)) return null; + return amount * unit; +}; + +/** Resolve a time token to an epoch-ms instant. Relative tokens map to + * `now - duration`; otherwise fall back to an absolute ISO/date parse. */ +const tokenToEpochMs = (value: string): number | null => { + const relative = relativeToMs(value); + if (relative !== null) return Date.now() - relative; + const absolute = Date.parse(value); + return Number.isNaN(absolute) ? null : absolute; +}; + +export const parseRunsFilter = (text: string): RunsFilterTokens => { + const status: string[] = []; + const trigger: string[] = []; + let interaction: "true" | "false" | null = null; + let from: number | null = null; + let to: number | null = null; + + const parts = text.trim().split(/\s+/).filter(Boolean); + + for (const part of parts) { + const colon = part.indexOf(":"); + if (colon <= 0) continue; + const key = part.slice(0, colon); + const value = part.slice(colon + 1); + if (value.length === 0) continue; + + if (key === "status") { + status.push(...value.split(",").filter(Boolean)); + } else if (key === "trigger") { + trigger.push(...value.split(",").filter(Boolean)); + } else if (key === "interaction") { + if (value === "true" || value === "false") interaction = value; + } else if (key === "after") { + const epoch = tokenToEpochMs(value); + if (epoch !== null) from = epoch; + } else if (key === "before") { + const epoch = tokenToEpochMs(value); + if (epoch !== null) to = epoch; + } + // Unknown keys are silently dropped so the input stays forgiving. + } + + return { status, trigger, interaction, from, to }; +}; diff --git a/packages/plugins/execution-history/src/react/filter-command.tsx b/packages/plugins/execution-history/src/react/filter-command.tsx new file mode 100644 index 0000000000..9e3aadfc80 --- /dev/null +++ b/packages/plugins/execution-history/src/react/filter-command.tsx @@ -0,0 +1,324 @@ +import { forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react"; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList, + CommandSeparator, + CommandShortcut, +} from "@executor-js/react/components/command"; +import { cn } from "@executor-js/react/lib/utils"; + +import type { RunStatus } from "../sdk/collections"; +import type { ExecutionListMeta } from "../sdk/store"; +import { FILTER_KEYS, parseRunsFilter, type RunsFilterTokens } from "./filter-command-parser"; +import type { RunsFilters } from "./use-runs-list"; +import { STATUS_LABELS, STATUS_ORDER, statusTone, triggerTone } from "./status"; + +// --------------------------------------------------------------------------- +// Runs filter command palette. A controlled cmdk input that surfaces live facet +// suggestions (status + trigger counts from `meta`) while typing a token, plus +// an "Add filter" group of key hints. Selecting a suggestion toggles that facet +// immediately; pressing Enter parses the whole input as `key:value` tokens and +// merges them onto the current filters. +// +// Contract adaptations vs the inline prototype: only the 4 statuses, no tool or +// code facets, and `meta.statusCounts`/`triggerCounts` are arrays (iterated, +// never `Object.entries`). `status:` accepts either the canonical status value +// or its display label (e.g. `waiting` -> `waiting_for_interaction`). +// --------------------------------------------------------------------------- + +export interface RunsFilterCommandProps { + readonly filters: RunsFilters; + readonly meta: ExecutionListMeta | null; + readonly onApply: (filters: RunsFilters) => void; + readonly open: boolean; + readonly onOpenChange: (open: boolean) => void; +} + +/** Map a typed status token (canonical value or display label) back to the + * canonical RunStatus, or null when it matches neither. */ +const canonicalStatus = (token: string): RunStatus | null => { + for (const status of STATUS_ORDER) { + if (status === token || STATUS_LABELS[status] === token) return status; + } + return null; +}; + +/** Toggle one value within a readonly facet list (add when absent, drop when + * present), returning a new array. */ +const toggleFacet = (current: readonly string[], value: string): string[] => + current.includes(value) ? current.filter((entry) => entry !== value) : [...current, value]; + +/** Merge parsed tokens onto the current filters: status/trigger REPLACE (only + * when present and valid), interaction/from/to are set only when the token was + * actually typed β€” so applying a command never wipes a time range or + * interaction filter set elsewhere (the rail, the timeline drag). The palette + * sets filters; clearing is done from the rail. Status tokens are normalized to + * canonical RunStatus values; an all-invalid status token leaves status + * untouched. */ +const mergeTokens = (filters: RunsFilters, tokens: RunsFilterTokens): RunsFilters => { + const statuses = tokens.status + .map(canonicalStatus) + .filter((status): status is RunStatus => status !== null); + return { + ...filters, + status: statuses.length > 0 ? statuses : filters.status, + trigger: tokens.trigger.length > 0 ? tokens.trigger : filters.trigger, + interaction: tokens.interaction !== null ? tokens.interaction : filters.interaction, + from: tokens.from !== null ? tokens.from : filters.from, + to: tokens.to !== null ? tokens.to : filters.to, + }; +}; + +interface FacetSuggestion { + readonly value: string; + readonly label: string; + readonly hint: string; +} + +export const RunsFilterCommand = forwardRef( + function RunsFilterCommand({ filters, meta, onApply, open, onOpenChange }, forwardedRef) { + const [value, setValue] = useState(""); + const containerRef = useRef(null); + const inputRef = useRef(null); + + useImperativeHandle(forwardedRef, () => inputRef.current as HTMLInputElement, []); + + // Close on an outside pointer press while open. + useEffect(() => { + if (!open) return; + const handlePointer = (event: PointerEvent) => { + const target = event.target as Node | null; + if (!containerRef.current || !target) return; + if (!containerRef.current.contains(target)) onOpenChange(false); + }; + window.addEventListener("pointerdown", handlePointer); + return () => window.removeEventListener("pointerdown", handlePointer); + }, [open, onOpenChange]); + + const statusCounts = useMemo(() => { + const counts = new Map(); + for (const entry of meta?.statusCounts ?? []) counts.set(entry.status, entry.count); + return counts; + }, [meta]); + + const triggerSuggestions = useMemo(() => { + const entries = meta?.triggerCounts ?? []; + return entries + .filter( + (entry): entry is { triggerKind: string; count: number } => entry.triggerKind !== null, + ) + .map((entry) => ({ + value: entry.triggerKind, + label: triggerTone(entry.triggerKind).label, + hint: `${entry.count}`, + })); + }, [meta]); + + const statusSuggestions = useMemo( + () => + STATUS_ORDER.map((status) => ({ + value: status, + label: STATUS_LABELS[status], + hint: `${statusCounts.get(status) ?? 0}`, + })), + [statusCounts], + ); + + const applyFilters = (next: RunsFilters) => { + onApply(next); + }; + + const applyTokens = () => { + const merged = mergeTokens(filters, parseRunsFilter(value)); + applyFilters(merged); + setValue(""); + onOpenChange(false); + inputRef.current?.blur(); + }; + + const toggleStatus = (status: string) => { + applyFilters({ ...filters, status: toggleFacet(filters.status, status) }); + }; + + const toggleTrigger = (trigger: string) => { + applyFilters({ ...filters, trigger: toggleFacet(filters.trigger, trigger) }); + }; + + const insertKey = (key: string) => { + const trimmed = value.trimEnd(); + setValue(trimmed.length === 0 ? `${key}:` : `${trimmed} ${key}:`); + inputRef.current?.focus(); + }; + + if (!open) { + return ( +
+ + onOpenChange(true)} + placeholder="Filter runs β€” status:… trigger:… interaction:… after:…" + className="font-mono text-xs" + /> + +
+ ); + } + + return ( +
+ { + if (event.key === "Enter") { + event.preventDefault(); + applyTokens(); + } else if (event.key === "Escape") { + event.preventDefault(); + onOpenChange(false); + inputRef.current?.blur(); + } + }} + > + onOpenChange(true)} + placeholder="Filter runs β€” status:… trigger:… interaction:… after:…" + className="font-mono text-xs" + /> + +
+ + + + No matching filters. + + + + + {statusSuggestions.map((suggestion) => { + const active = filters.status.includes(suggestion.value); + const tone = statusTone(suggestion.value as RunStatus); + return ( + toggleStatus(suggestion.value)} + > + + + {suggestion.label} + + {active ? ( + βœ“ + ) : null} + {suggestion.hint} + + ); + })} + + + {triggerSuggestions.length > 0 ? ( + + {triggerSuggestions.map((suggestion) => { + const active = filters.trigger.includes(suggestion.value); + const tone = triggerTone(suggestion.value); + return ( + toggleTrigger(suggestion.value)} + > + + + {suggestion.label} + + {active ? ( + βœ“ + ) : null} + {suggestion.hint} + + ); + })} + + ) : null} + + + + + {FILTER_KEYS.map((entry) => ( + insertKey(entry.key)} + > + {entry.key}: + {entry.hint} + + ))} + + + + + + + Apply filters + Enter + + + + +
+ + Use + ↑↓ + to navigate + + Β· + + + Enter + + to apply + + Β· + + + Esc + + to close + + Β· + + Union:{" "} + + status:failed,completed + + + Β· + + Time:{" "} + after:1h + +
+
+ +
+ ); + }, +); diff --git a/packages/plugins/execution-history/src/react/filter-rail.tsx b/packages/plugins/execution-history/src/react/filter-rail.tsx new file mode 100644 index 0000000000..be592dd459 --- /dev/null +++ b/packages/plugins/execution-history/src/react/filter-rail.tsx @@ -0,0 +1,366 @@ +import * as React from "react"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@executor-js/react/components/accordion"; +import { Button } from "@executor-js/react/components/button"; +import { cn } from "@executor-js/react/lib/utils"; + +import type { ExecutionListMeta } from "../sdk/store"; +import { STATUS_ORDER, STATUS_LABELS, TRIGGER_ORDER, statusTone, triggerTone } from "./status"; +import type { RunsFilters } from "./use-runs-list"; + +// --------------------------------------------------------------------------- +// Time range presets +// --------------------------------------------------------------------------- + +export interface TimeRangePreset { + readonly key: string; + readonly label: string; + readonly ms: number | null; +} + +export const TIME_RANGE_PRESETS: readonly TimeRangePreset[] = [ + { key: "1h", label: "Last 1h", ms: 60 * 60 * 1000 }, + { key: "24h", label: "Last 24h", ms: 24 * 60 * 60 * 1000 }, + { key: "7d", label: "Last 7d", ms: 7 * 24 * 60 * 60 * 1000 }, + { key: "30d", label: "Last 30d", ms: 30 * 24 * 60 * 60 * 1000 }, + { key: "all", label: "All time", ms: null }, +]; + +export const resolveTimeRange = (key: string): { from: number | null; to: number | null } => { + const preset = TIME_RANGE_PRESETS.find((p) => p.key === key); + if (!preset || preset.ms === null) return { from: null, to: null }; + const now = Date.now(); + return { from: now - preset.ms, to: now }; +}; + +/** Best-effort match of the current from/to values against a known preset key. */ +const activePresetKey = (from: number | null, to: number | null): string | null => { + if (from === null && to === null) return "all"; + if (from === null || to === null) return null; + const delta = to - from; + for (const preset of TIME_RANGE_PRESETS) { + if (preset.ms !== null && Math.abs(delta - preset.ms) < 60_000) return preset.key; + } + return null; +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +const toggle = (arr: readonly T[], item: T): readonly T[] => + arr.includes(item) ? arr.filter((v) => v !== item) : [...arr, item]; + +const isFiltersActive = (filters: RunsFilters): boolean => + filters.status.length > 0 || + filters.trigger.length > 0 || + filters.interaction !== null || + filters.from !== null || + filters.to !== null; + +// --------------------------------------------------------------------------- +// Sub-components +// --------------------------------------------------------------------------- + +function FacetSection({ + value, + label, + children, +}: { + readonly value: string; + readonly label: string; + readonly children: React.ReactNode; +}) { + return ( + + + {label} + + {children} + + ); +} + +function FacetRow({ + checked, + onToggle, + onOnly, + dotClass, + pulse, + label, + count, + monoLabel, +}: { + readonly checked: boolean; + readonly onToggle: () => void; + readonly onOnly?: () => void; + readonly dotClass: string; + readonly pulse?: boolean; + readonly label: string; + readonly count: number | undefined; + readonly monoLabel?: boolean; +}) { + return ( +
+ + + {onOnly ? ( + + ) : null} +
+ ); +} + +// --------------------------------------------------------------------------- +// Main export +// --------------------------------------------------------------------------- + +export interface RunsFilterRailProps { + readonly filters: RunsFilters; + readonly meta: ExecutionListMeta | null; + readonly onChange: (filters: RunsFilters) => void; + readonly onReset: () => void; +} + +export function RunsFilterRail({ filters, meta, onChange, onReset }: RunsFilterRailProps) { + const filtersActive = isFiltersActive(filters); + + const triggerKeys = React.useMemo(() => { + const set = new Set(TRIGGER_ORDER); + if (meta?.triggerCounts) { + for (const entry of meta.triggerCounts) { + if (entry.triggerKind != null) set.add(entry.triggerKind); + } + } + return [...set]; + }, [meta?.triggerCounts]); + + const currentPresetKey = activePresetKey(filters.from, filters.to); + + return ( +
+ {/* Header */} +
+

Execution history

+

+ Every execution recorded for this scope, newest first. +

+ {meta ? ( +

+ {meta.filterRowCount.toLocaleString()} of {meta.totalRowCount.toLocaleString()} runs +

+ ) : null} +
+ + {/* Filters label + reset */} +
+

+ Filters +

+ {filtersActive ? ( + + ) : null} +
+ +
+ + {/* Status facet */} + +
    + {STATUS_ORDER.map((status) => { + const tone = statusTone(status); + const checked = filters.status.includes(status); + const count = meta?.statusCounts.find((c) => c.status === status)?.count; + return ( +
  • + + onChange({ ...filters, status: toggle(filters.status, status) }) + } + onOnly={() => onChange({ ...filters, status: [status] })} + dotClass={tone.dot} + pulse={tone.pulse} + label={STATUS_LABELS[status]} + count={count} + /> +
    • + ); + })} +
+ + + {/* Trigger facet */} + +
    + {triggerKeys.map((key) => { + const tone = triggerTone(key); + const checked = filters.trigger.includes(key); + const count = meta?.triggerCounts.find((c) => c.triggerKind === key)?.count; + return ( +
  • + + onChange({ ...filters, trigger: toggle(filters.trigger, key) }) + } + onOnly={() => onChange({ ...filters, trigger: [key] })} + dotClass={tone.dot} + label={tone.label} + count={count} + monoLabel + /> +
    • + ); + })} +
+ + + {/* Interaction facet */} + +
    +
  • + + onChange({ + ...filters, + interaction: filters.interaction === "true" ? null : "true", + }) + } + dotClass="bg-amber-500" + label="With interaction" + count={meta?.interactionCounts.withInteraction} + /> +
    • +
  • + + onChange({ + ...filters, + interaction: filters.interaction === "false" ? null : "false", + }) + } + dotClass="bg-muted-foreground/40" + label="Without interaction" + count={meta?.interactionCounts.withoutInteraction} + /> +
    • +
+ + + {/* Time range facet */} + +
    + {TIME_RANGE_PRESETS.map((preset) => { + const active = currentPresetKey === preset.key; + return ( +
  • + +
    • + ); + })} +
+ + +
+
+ ); +} diff --git a/packages/plugins/execution-history/src/react/format.ts b/packages/plugins/execution-history/src/react/format.ts new file mode 100644 index 0000000000..b597b8a7a3 --- /dev/null +++ b/packages/plugins/execution-history/src/react/format.ts @@ -0,0 +1,57 @@ +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import type { RunStatus, ToolCallStatus } from "../sdk/collections"; +import { STATUS_LABELS } from "./status"; + +// --------------------------------------------------------------------------- +// Formatting helpers shared by the runs table, drawer, and chart. Uses native +// Date + Intl (no date-fns dep). The stored `*Json` columns are already +// compact JSON strings; decode through Effect Schema (only JSON.parse is +// lint-banned β€” JSON.stringify is fine and won't throw on a decoded value). +// --------------------------------------------------------------------------- + +const decodeJson = Schema.decodeUnknownOption(Schema.UnknownFromJsonString); +const decodeLogLines = Schema.decodeUnknownOption( + Schema.fromJsonString(Schema.Array(Schema.String)), +); + +export const prettyJson = (raw: string | null): string | null => { + if (!raw) return null; + return Option.match(decodeJson(raw), { + onNone: () => raw, + onSome: (value) => JSON.stringify(value, null, 2), + }); +}; + +export const logLines = (raw: string | null): readonly string[] => + !raw + ? [] + : Option.match(decodeLogLines(raw), { + onNone: () => [raw], + onSome: (value) => value, + }); + +export const statusLabel = (status: RunStatus | ToolCallStatus): string => + status in STATUS_LABELS ? STATUS_LABELS[status as RunStatus] : status; + +export const formatDateTime = (timestamp: number | null): string => + timestamp == null ? "Pending" : new Date(timestamp).toLocaleString(); + +const relativeFormatter = new Intl.RelativeTimeFormat(undefined, { numeric: "auto" }); + +export const formatRelative = (timestamp: number): string => { + const delta = timestamp - Date.now(); + const abs = Math.abs(delta); + if (abs < 60_000) return relativeFormatter.format(Math.round(delta / 1000), "second"); + if (abs < 3_600_000) return relativeFormatter.format(Math.round(delta / 60_000), "minute"); + if (abs < 86_400_000) return relativeFormatter.format(Math.round(delta / 3_600_000), "hour"); + return relativeFormatter.format(Math.round(delta / 86_400_000), "day"); +}; + +export const formatDuration = (value: number | null): string => { + if (value == null) return "running"; + if (value < 1000) return `${value}ms`; + if (value < 60_000) return `${(value / 1000).toFixed(1)}s`; + return `${(value / 60_000).toFixed(1)}m`; +}; diff --git a/packages/plugins/execution-history/src/react/keyboard-help.tsx b/packages/plugins/execution-history/src/react/keyboard-help.tsx new file mode 100644 index 0000000000..ccf8190c2a --- /dev/null +++ b/packages/plugins/execution-history/src/react/keyboard-help.tsx @@ -0,0 +1,56 @@ +import { Keyboard } from "lucide-react"; +import { Button } from "@executor-js/react/components/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuShortcut, + DropdownMenuTrigger, +} from "@executor-js/react/components/dropdown-menu"; + +const SHORTCUTS: readonly { readonly label: string; readonly keys: readonly string[] }[] = [ + { label: "Focus filter", keys: ["/"] }, + { label: "Toggle live", keys: ["j"] }, + { label: "Refresh", keys: ["r"] }, + { label: "Keyboard shortcuts", keys: ["?"] }, +]; + +const NAV_SHORTCUTS: readonly { readonly label: string; readonly keys: readonly string[] }[] = [ + { label: "Toggle filter rail", keys: ["b"] }, + { label: "Navigate rows", keys: ["↑", "↓"] }, + { label: "Close drawer", keys: ["Esc"] }, +]; + +export function KeyboardHelp(props: { + readonly open: boolean; + readonly onOpenChange: (open: boolean) => void; +}) { + return ( + + + + + + Keyboard shortcuts + + {SHORTCUTS.map((shortcut) => ( + + {shortcut.label} + {shortcut.keys.join(" ")} + + ))} + + {NAV_SHORTCUTS.map((shortcut) => ( + + {shortcut.label} + {shortcut.keys.join(" ")} + + ))} + + + ); +} diff --git a/packages/plugins/execution-history/src/react/live-button.tsx b/packages/plugins/execution-history/src/react/live-button.tsx new file mode 100644 index 0000000000..6c82a68496 --- /dev/null +++ b/packages/plugins/execution-history/src/react/live-button.tsx @@ -0,0 +1,36 @@ +import { CirclePause, CirclePlay } from "lucide-react"; +import { Button } from "@executor-js/react/components/button"; +import { cn } from "@executor-js/react/lib/utils"; + +export function LiveButton(props: { readonly active: boolean; readonly onToggle: () => void }) { + return ( + + ); +} diff --git a/packages/plugins/execution-history/src/react/live-row.tsx b/packages/plugins/execution-history/src/react/live-row.tsx new file mode 100644 index 0000000000..c79d4c0b86 --- /dev/null +++ b/packages/plugins/execution-history/src/react/live-row.tsx @@ -0,0 +1,15 @@ +export function LiveDivider() { + return ( +
+
+ + + + + + Live + +
+
+ ); +} diff --git a/packages/plugins/execution-history/src/react/refresh-button.tsx b/packages/plugins/execution-history/src/react/refresh-button.tsx new file mode 100644 index 0000000000..ef3d0d2710 --- /dev/null +++ b/packages/plugins/execution-history/src/react/refresh-button.tsx @@ -0,0 +1,24 @@ +import { LoaderCircle, RefreshCcw } from "lucide-react"; +import { Button } from "@executor-js/react/components/button"; + +export function RefreshButton(props: { + readonly onClick: () => void; + readonly isLoading?: boolean; +}) { + return ( + + ); +} diff --git a/packages/plugins/execution-history/src/react/run-row.tsx b/packages/plugins/execution-history/src/react/run-row.tsx new file mode 100644 index 0000000000..841c9bd0b6 --- /dev/null +++ b/packages/plugins/execution-history/src/react/run-row.tsx @@ -0,0 +1,125 @@ +import { Button } from "@executor-js/react/components/button"; +import { cn } from "@executor-js/react/lib/utils"; +import { Badge } from "@executor-js/react/components/badge"; + +import type { RunRow } from "../sdk/collections"; +import { formatDateTime, formatDuration, formatRelative, statusLabel } from "./format"; +import { STATUS_TONES, triggerTone } from "./status"; +import type { RunColumns } from "./view"; + +// Column slot classes β€” RunListRow and RunsColumnHeader both apply these +// fixed-width, same-breakpoint slots (after the dot/started/status base +// columns) so the header and rows stay aligned across screen sizes. +export const COL_TRIGGER = "hidden 2xl:block w-[120px] shrink-0"; +export const COL_DURATION = "hidden md:block w-[100px] shrink-0"; +export const COL_TOOLS = "hidden xl:block w-[80px] shrink-0"; +export const COL_INTERACTION = "hidden xl:block w-[100px] shrink-0"; + +export interface RunListRowProps { + readonly run: RunRow; + readonly selected: boolean; + readonly isPast: boolean; + readonly columns: RunColumns; + readonly onSelect: () => void; +} + +export function RunListRow({ run, selected, isPast, columns, onSelect }: RunListRowProps) { + const tone = STATUS_TONES[run.status]; + const trigger = triggerTone(run.triggerKind); + const isLive = run.status === "running" || run.status === "waiting_for_interaction"; + + return ( + + ); +} diff --git a/packages/plugins/execution-history/src/react/runs-list-state.test.ts b/packages/plugins/execution-history/src/react/runs-list-state.test.ts new file mode 100644 index 0000000000..5943e87c37 --- /dev/null +++ b/packages/plugins/execution-history/src/react/runs-list-state.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "@effect/vitest"; + +import type { RunRow } from "../sdk/collections"; +import { + initialRunsListState, + runsListReducer, + runsListRows, + type RunsListAction, + type RunsListState, +} from "./runs-list-state"; + +const row = (id: string, startedAt: number): RunRow => ({ + executionId: id, + status: "completed", + code: "noop", + resultJson: null, + errorText: null, + logsJson: null, + triggerKind: "cli", + triggerMetaJson: null, + startedAt, + completedAt: startedAt + 100, + durationMs: 100, + toolCallCount: 0, + hadInteraction: false, +}); + +const run = (actions: readonly RunsListAction[], from: RunsListState = initialRunsListState) => + actions.reduce(runsListReducer, from); + +describe("runsListReducer", () => { + it("appends cursor pages in order and merges into one list", () => { + const state = run([ + { + type: "appendPage", + cursor: undefined, + runs: [row("r3", 3000), row("r2", 2000)], + nextCursor: "c2", + }, + { type: "loadMore" }, + { type: "appendPage", cursor: "c2", runs: [row("r1", 1000)], nextCursor: null }, + ]); + expect(runsListRows(state).map((r) => r.executionId)).toEqual(["r3", "r2", "r1"]); + expect(state.done).toBe(true); + expect(state.cursor).toBe("c2"); + }); + + it("ignores a duplicate appendPage for the same cursor (idempotent)", () => { + const page = { + type: "appendPage", + cursor: undefined, + runs: [row("r1", 1000)], + nextCursor: "c2", + } as const; + const state = run([page, page]); + expect(state.pages).toHaveLength(1); + expect(runsListRows(state).map((r) => r.executionId)).toEqual(["r1"]); + }); + + it("loadMore is a no-op when there is no next cursor", () => { + const state = run([ + { type: "appendPage", cursor: undefined, runs: [row("r1", 1000)], nextCursor: null }, + { type: "loadMore" }, + ]); + expect(state.cursor).toBeUndefined(); + }); + + it("prependLive adds only unseen rows, newest-first, above existing pages", () => { + const base = run([ + { + type: "appendPage", + cursor: undefined, + runs: [row("r2", 2000), row("r1", 1000)], + nextCursor: null, + }, + { type: "enableLive" }, + ]); + expect(base.liveCutoffId).toBe("r2"); + + // A live refresh returns the new top row r3 plus the already-seen r2/r1. + const live = runsListReducer(base, { + type: "prependLive", + runs: [row("r3", 3000), row("r2", 2000), row("r1", 1000)], + }); + expect(runsListRows(live).map((r) => r.executionId)).toEqual(["r3", "r2", "r1"]); + + // A second identical refresh adds nothing. + const again = runsListReducer(live, { + type: "prependLive", + runs: [row("r3", 3000), row("r2", 2000)], + }); + expect(again).toBe(live); + }); + + it("dedupes a run that appears in both a live refresh and a later page", () => { + const state = run([ + { type: "appendPage", cursor: undefined, runs: [row("r2", 2000)], nextCursor: "c2" }, + { type: "prependLive", runs: [row("r3", 3000)] }, + { type: "loadMore" }, + // page 2 happens to re-include r3 (race); it must not double up. + { + type: "appendPage", + cursor: "c2", + runs: [row("r3", 3000), row("r1", 1000)], + nextCursor: null, + }, + ]); + expect(runsListRows(state).map((r) => r.executionId)).toEqual(["r3", "r2", "r1"]); + }); + + it("runsListRows(state, false) places live rows after pages.flat()", () => { + const state = run([ + { + type: "appendPage", + cursor: undefined, + runs: [row("r2", 2000), row("r1", 1000)], + nextCursor: null, + }, + { type: "prependLive", runs: [row("r3", 3000)] }, + ]); + // Default (liveFirst) keeps the live row on top. + expect(runsListRows(state).map((r) => r.executionId)).toEqual(["r3", "r2", "r1"]); + // Ascending sort appends live rows after the paginated rows. + expect(runsListRows(state, false).map((r) => r.executionId)).toEqual(["r2", "r1", "r3"]); + }); + + it("reset clears all accumulated state", () => { + const state = run([ + { type: "appendPage", cursor: undefined, runs: [row("r1", 1000)], nextCursor: "c2" }, + { type: "enableLive" }, + { type: "reset" }, + ]); + expect(state).toEqual(initialRunsListState); + }); +}); diff --git a/packages/plugins/execution-history/src/react/runs-list-state.ts b/packages/plugins/execution-history/src/react/runs-list-state.ts new file mode 100644 index 0000000000..afec739131 --- /dev/null +++ b/packages/plugins/execution-history/src/react/runs-list-state.ts @@ -0,0 +1,113 @@ +import type { RunRow } from "../sdk/collections"; + +// --------------------------------------------------------------------------- +// Pure state machine for the runs list. Kept free of React/atoms so the +// fiddly accumulation + live-prepend logic is unit-testable. +// +// Two row sources merge into one display list (newest first): +// - `live` β€” rows that arrived via live polling after live mode was enabled +// - `pages` β€” the cursor-paginated scroll history (page 1, page 2, ...) +// A `seen` set dedupes across both so the same run never appears twice and a +// live refresh only ever prepends genuinely new runs. Append/prepend are +// idempotent (guarded by `seen`/`appended`), so the React effects that drive +// them can fire repeatedly without corrupting state. +// --------------------------------------------------------------------------- + +export interface RunsListState { + /** Cursor-paginated scroll history; each page is newest-first. */ + readonly pages: readonly (readonly RunRow[])[]; + /** Rows prepended by live polling, newest-first. */ + readonly live: readonly RunRow[]; + /** executionIds present in `pages` or `live`. */ + readonly seen: ReadonlySet; + /** Cursor of the page currently subscribed (undefined = first page). */ + readonly cursor: string | undefined; + /** Page keys already folded in (cursor ?? ""), for idempotent appends. */ + readonly appended: ReadonlySet; + /** nextCursor of the most recently appended page. */ + readonly nextCursor: string | null; + /** True once a page returns no nextCursor. */ + readonly done: boolean; + /** Divider anchor: the top row id when live mode was enabled. */ + readonly liveCutoffId: string | null; +} + +export type RunsListAction = + | { + readonly type: "appendPage"; + readonly cursor: string | undefined; + readonly runs: readonly RunRow[]; + readonly nextCursor: string | null; + } + | { readonly type: "loadMore" } + | { readonly type: "enableLive" } + | { readonly type: "disableLive" } + | { readonly type: "prependLive"; readonly runs: readonly RunRow[] } + | { readonly type: "reset" }; + +export const initialRunsListState: RunsListState = { + pages: [], + live: [], + seen: new Set(), + cursor: undefined, + appended: new Set(), + nextCursor: null, + done: false, + liveCutoffId: null, +}; + +const pageKey = (cursor: string | undefined): string => cursor ?? ""; + +const topRowId = (state: RunsListState): string | null => + state.live[0]?.executionId ?? state.pages[0]?.[0]?.executionId ?? null; + +export const runsListReducer = (state: RunsListState, action: RunsListAction): RunsListState => { + if (action.type === "appendPage") { + const key = pageKey(action.cursor); + if (state.appended.has(key)) return state; + const fresh = action.runs.filter((run) => !state.seen.has(run.executionId)); + const seen = new Set(state.seen); + for (const run of fresh) seen.add(run.executionId); + return { + ...state, + pages: [...state.pages, fresh], + seen, + appended: new Set(state.appended).add(key), + nextCursor: action.nextCursor, + done: action.nextCursor === null, + }; + } + if (action.type === "loadMore") { + if (state.done || state.nextCursor === null || state.nextCursor === state.cursor) { + return state; + } + return { ...state, cursor: state.nextCursor }; + } + if (action.type === "enableLive") { + return { ...state, liveCutoffId: topRowId(state) }; + } + if (action.type === "disableLive") { + return { ...state, liveCutoffId: null }; + } + if (action.type === "prependLive") { + // Append-only: a live refresh only adds genuinely-new runs. A run already + // displayed keeps the data it was first seen with β€” a status transition + // (running -> completed) is not reflected until the list is reset/refiltered + // (the aggregate `meta` does update live). Acceptable for v1. + const fresh = action.runs.filter((run) => !state.seen.has(run.executionId)); + if (fresh.length === 0) return state; + const seen = new Set(state.seen); + for (const run of fresh) seen.add(run.executionId); + return { ...state, live: [...fresh, ...state.live], seen }; + } + // action.type === "reset" + return initialRunsListState; +}; + +/** + * The merged display list. With `liveFirst` (the default, for descending / + * newest-first sort) live rows go on top; otherwise (ascending sort) they are + * appended after the paginated rows so the live tail stays at the bottom. + */ +export const runsListRows = (state: RunsListState, liveFirst = true): readonly RunRow[] => + liveFirst ? [...state.live, ...state.pages.flat()] : [...state.pages.flat(), ...state.live]; diff --git a/packages/plugins/execution-history/src/react/shell.tsx b/packages/plugins/execution-history/src/react/shell.tsx new file mode 100644 index 0000000000..cf4b0b7583 --- /dev/null +++ b/packages/plugins/execution-history/src/react/shell.tsx @@ -0,0 +1,134 @@ +import * as React from "react"; + +import { cn } from "@executor-js/react/lib/utils"; + +// --------------------------------------------------------------------------- +// RunsShell β€” two-column layout (filter rail + main pane) with infinite scroll. +// +// The left
)} + + {props.interactions.length > 0 && ( +
+

+ {interactionsHeading} +

+ {props.interactions.map((interaction) => ( + + ))} +
+ )} - + - - -
); } +/** Reads the detail atom β€” split out so the query only fires for a real id + * (never the closed/sentinel state) and runs solely while the drawer is open. */ +function DetailLoader(props: { + readonly executionId: string; + readonly onPrev?: () => void; + readonly onNext?: () => void; +}) { + const detail = useAtomValue(runDetailAtom(props.executionId)); + return AsyncResult.match(detail, { + onInitial: () =>
Loading run...
, + onFailure: () =>
Unable to load run.
, + onSuccess: ({ value }) => + value ? ( + + ) : ( +
Run not found.
+ ), + }); +} + export function DetailDrawer(props: { readonly executionId: string | null; readonly onClose: () => void; + readonly onPrev?: () => void; + readonly onNext?: () => void; }) { - const detail = useAtomValue(runDetailAtom(props.executionId ?? "__none__")); return ( !open && props.onClose()}> - {props.executionId == null - ? null - : AsyncResult.match(detail, { - onInitial: () => ( -
Loading run...
- ), - onFailure: () => ( -
Unable to load run.
- ), - onSuccess: ({ value }) => - value ? ( - - ) : ( -
Run not found.
- ), - })} + {props.executionId == null ? null : ( + + )} ); From 6cba288c8467d30c63acf83d1eb9e2f95ce90714 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 00:33:06 +0530 Subject: [PATCH 09/57] fix(execution-history): pad the runs toolbar --- packages/plugins/execution-history/src/react/RunsPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/plugins/execution-history/src/react/RunsPage.tsx b/packages/plugins/execution-history/src/react/RunsPage.tsx index 237759d80c..90d5d80d07 100644 --- a/packages/plugins/execution-history/src/react/RunsPage.tsx +++ b/packages/plugins/execution-history/src/react/RunsPage.tsx @@ -149,7 +149,7 @@ export function RunsPage() { } const toolbar = ( -
+

Runs

From 391e431cecd8a725d8b37c13901995f9f0872be1 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 01:46:32 +0530 Subject: [PATCH 10/57] feat(execution-history): attribute runs to a trigger actor + filter by actor (#20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Records **who/what triggered each run** and lets the runs page filter by it. The engine's `ExecutionStarted` already carried an optional `trigger`, but no caller ever populated it β€” so every run rendered with an unknown trigger. This wires it end to end and adds an identity ("actor") dimension on top. ## Changes **Trigger actor plumbing (generic, carrier-agnostic)** - `ExecutionTrigger` gains an `ExecutionActor {kind, id, label}`. - The trigger is populated at both execute call sites: the HTTP execute handler (`kind: "http"`) and the MCP session build (`kind: "mcp"`), each resolving the actor from the request principal. - `Principal`/`AuthContext` gain an `actor`, derived by `executionActorFromPrincipal` (defaults to a `user` actor keyed by `accountId`). The host-mcp `Principal` schema carries it too, so it survives cross-isolate session serialization. The HTTP handler reads `AuthContext` optionally, so it adds no new requirement. A host that authenticates a machine credential acting as a human can set `principal.actor` to keep the credential distinct from the subject it acts as. **execution-history** - `RunRow` persists an indexed `actorId` plus an `actorLabel`/`actorKind` snapshot. - The runs list filters by `actorFilter`; an `actorCounts` facet (keyed on the stable `actorId`, rendering the most-recent label + kind) joins the existing facets. - UI: an **Actor** facet in the filter rail, an **Actor** column, and an Actor field in the run detail drawer. As a side effect this also fixes the long-standing "unknown trigger" β€” `triggerKind` is now set (`http`/`mcp`). ## Verification - `typecheck` 41/41 Β· `lint` + `format` clean on all changed files - tests green: execution-history 17 (incl. a new test covering actor persistence, the facet label/kind, and `actorFilter`), api 45, host-mcp 32, cloud 52 ## Follow-up The seam is fully generic. A host that aliases a machine credential to a human subject populates `principal.actor` with the credential's own id/label so the Actor column/facet distinguish it from the human β€” no further UI change required. --- apps/cloud/src/auth/middleware-live.ts | 20 +++-- apps/cloud/src/engine/execution-usage.ts | 4 +- apps/cloud/src/org/handlers.test.ts | 2 + packages/core/api/src/handlers/executions.ts | 16 +++- packages/core/api/src/server/identity.ts | 28 ++++++ packages/core/api/src/server/mcp-build.ts | 10 ++- packages/core/sdk/src/execution-observer.ts | 19 ++++ packages/core/sdk/src/index.ts | 1 + packages/hosts/mcp/src/seams.ts | 12 +++ packages/hosts/mcp/src/tool-server.ts | 12 ++- .../execution-history/src/api/group.ts | 13 ++- .../src/api/handlers.test.ts | 3 + .../execution-history/src/api/handlers.ts | 2 + .../execution-history/src/react/atoms.ts | 1 + .../src/react/column-header.tsx | 5 +- .../src/react/detail-drawer.tsx | 12 +++ .../src/react/filter-rail.tsx | 41 ++++++++- .../execution-history/src/react/run-row.tsx | 25 +++++- .../src/react/runs-list-state.test.ts | 3 + .../execution-history/src/react/status.ts | 25 ++++++ .../src/react/use-runs-list.ts | 4 + .../src/react/view-options.tsx | 8 +- .../execution-history/src/react/view.ts | 3 + .../execution-history/src/sdk/collections.ts | 10 ++- .../src/sdk/store-list.test.ts | 75 ++++++++++++++++ .../execution-history/src/sdk/store.ts | 88 ++++++++++++++++++- 26 files changed, 420 insertions(+), 22 deletions(-) diff --git a/apps/cloud/src/auth/middleware-live.ts b/apps/cloud/src/auth/middleware-live.ts index 0d35218a6b..a0809737d4 100644 --- a/apps/cloud/src/auth/middleware-live.ts +++ b/apps/cloud/src/auth/middleware-live.ts @@ -6,7 +6,12 @@ import { Effect, Layer, Redacted } from "effect"; import { HttpServerResponse } from "effect/unstable/http"; -import { AuthContext, NoOrganization, Unauthorized } from "@executor-js/api/server"; +import { + AuthContext, + authContextFromPrincipal, + NoOrganization, + Unauthorized, +} from "@executor-js/api/server"; import { OrgAuth, @@ -93,17 +98,20 @@ export const OrgAuthLive = Layer.effect( // (org-settings β†’ domain verification) is niche; URL-scoping it would // mean converting this to an HttpRouter middleware β€” deferred. const session = sessionFromSealed(result, Redacted.value(credential)); - const auth = { + // Wrap the WorkOS session into a neutral `Principal` and let + // `authContextFromPrincipal` derive everything (incl. the run actor), + // so this path never re-implements the actor/label logic. `roles` is + // empty (the WorkOS control plane doesn't resolve them here; no cloud + // handler reads them) and `organizationName` is unused by AuthContext. + const auth = authContextFromPrincipal({ accountId: session.accountId, organizationId: result.organizationId, + organizationName: "", email: session.email, name: session.name, avatarUrl: session.avatarUrl, - // The unified `AuthContext` carries roles; cloud's WorkOS control - // plane does not resolve them here, so pass an empty list (no cloud - // handler reads roles today). roles: [], - }; + }); return yield* Effect.provideService(httpEffect, AuthContext, auth); }), diff --git a/apps/cloud/src/engine/execution-usage.ts b/apps/cloud/src/engine/execution-usage.ts index cd4489984f..c2dafc33d6 100644 --- a/apps/cloud/src/engine/execution-usage.ts +++ b/apps/cloud/src/engine/execution-usage.ts @@ -12,9 +12,9 @@ export const withExecutionUsageTracking = ( engine .execute(code, options) .pipe(Effect.tap(() => Effect.sync(() => trackUsage(organizationId)))), - executeWithPause: (code) => + executeWithPause: (code, options) => engine - .executeWithPause(code) + .executeWithPause(code, options) .pipe(Effect.tap(() => Effect.sync(() => trackUsage(organizationId)))), // resume doesn't count as usage resume: (executionId, response) => engine.resume(executionId, response), diff --git a/apps/cloud/src/org/handlers.test.ts b/apps/cloud/src/org/handlers.test.ts index da60e91834..fc2b08dccc 100644 --- a/apps/cloud/src/org/handlers.test.ts +++ b/apps/cloud/src/org/handlers.test.ts @@ -53,6 +53,7 @@ const adminAuth = { name: "Admin", avatarUrl: null, roles: [], + actor: { kind: "user", id: "user_admin", label: "Admin" }, }; const memberAuth = { @@ -62,6 +63,7 @@ const memberAuth = { name: "Member", avatarUrl: null, roles: [], + actor: { kind: "user", id: "user_member", label: "Member" }, }; const provide = (auth: typeof adminAuth, workosOverrides: StubOverrides = {}) => diff --git a/packages/core/api/src/handlers/executions.ts b/packages/core/api/src/handlers/executions.ts index 86e1c17ac0..ab24313434 100644 --- a/packages/core/api/src/handlers/executions.ts +++ b/packages/core/api/src/handlers/executions.ts @@ -1,10 +1,11 @@ import { HttpApiBuilder } from "effect/unstable/httpapi"; -import { Effect } from "effect"; +import { Effect, Option } from "effect"; import { Schema } from "effect"; import { ExecutorApi } from "../api"; import { formatExecuteResult, formatPausedExecution } from "@executor-js/execution"; import { ExecutionEngineService } from "../services"; +import { AuthContext } from "../server/identity"; import { capture, captureEngineError } from "@executor-js/api"; class ExecutionNotFoundError extends Schema.TaggedErrorClass()( @@ -34,7 +35,18 @@ export const ExecutionsHandlers = HttpApiBuilder.group(ExecutorApi, "executions" capture( Effect.gen(function* () { const engine = yield* ExecutionEngineService; - const outcome = yield* captureEngineError(engine.executeWithPause(payload.code)); + // Read identity OPTIONALLY (like `RequestWebOrigin`) so the handler + // never adds `AuthContext` to its requirements β€” every host provides it + // via middleware, but harnesses that mount handlers directly need not. + // The actor attributes the run; the kind tags the trigger "http" + // regardless of whether an actor was resolved. + const actor = Option.match(yield* Effect.serviceOption(AuthContext), { + onNone: () => undefined, + onSome: (auth) => auth.actor, + }); + const outcome = yield* captureEngineError( + engine.executeWithPause(payload.code, { trigger: { kind: "http", actor } }), + ); if (outcome.status === "completed") { const formatted = formatExecuteResult(outcome.result); diff --git a/packages/core/api/src/server/identity.ts b/packages/core/api/src/server/identity.ts index 15da8ac12a..5fc8d08d8d 100644 --- a/packages/core/api/src/server/identity.ts +++ b/packages/core/api/src/server/identity.ts @@ -22,6 +22,8 @@ import { Context, Effect, Schema } from "effect"; +import type { ExecutionActor } from "@executor-js/sdk/core"; + /** * The provider-neutral resolved identity. Both self-host's AuthProvider impls * (single-admin, Better Auth) and cloud's WorkOS path produce this. Self-host's @@ -36,6 +38,15 @@ export interface Principal { readonly name: string | null; readonly avatarUrl: string | null; readonly roles: readonly string[]; + /** + * The credential identity this principal acts AS, for run attribution. Most + * providers leave it unset β€” the run is attributed to the user + * (`executionActorFromPrincipal` derives a `"user"` actor from `accountId`). + * A host that authenticates a non-human credential which then acts as a human + * (a Cloudflare Access service token aliased to a subject) sets this so the + * machine credential stays distinguishable from the human it impersonates. + */ + readonly actor?: ExecutionActor; } /** @@ -52,9 +63,25 @@ export class AuthContext extends Context.Service< readonly name: string | null; readonly avatarUrl: string | null; readonly roles: readonly string[]; + /** The resolved actor for run attribution (always present β€” derived when the + * principal did not supply one). See `executionActorFromPrincipal`. */ + readonly actor: ExecutionActor; } >()("@executor-js/api/AuthContext") {} +/** + * Resolve the `ExecutionActor` a run started by this principal acts as. When the + * principal supplies its own `actor` (a host that knows the credential is, e.g., + * a service token) that wins; otherwise the run is attributed to the user β€” a + * `"user"` actor keyed by the stable `accountId`, labelled by name or email. + */ +export const executionActorFromPrincipal = (principal: Principal): ExecutionActor => + principal.actor ?? { + kind: "user", + id: principal.accountId, + label: principal.name ?? (principal.email.length > 0 ? principal.email : null), + }; + /** Build the shared `AuthContext` value from a resolved `Principal`. */ export const authContextFromPrincipal = (principal: Principal): AuthContext["Service"] => ({ accountId: principal.accountId, @@ -63,6 +90,7 @@ export const authContextFromPrincipal = (principal: Principal): AuthContext["Ser name: principal.name, avatarUrl: principal.avatarUrl, roles: principal.roles, + actor: executionActorFromPrincipal(principal), }); // Optional per-failure render hints. Self-host produces the bare error (these diff --git a/packages/core/api/src/server/mcp-build.ts b/packages/core/api/src/server/mcp-build.ts index 216413102b..011bc5e83a 100644 --- a/packages/core/api/src/server/mcp-build.ts +++ b/packages/core/api/src/server/mcp-build.ts @@ -10,6 +10,7 @@ import { createExecutorMcpServer } from "@executor-js/host-mcp/tool-server"; import { ErrorCapture } from "../observability"; import { CodeExecutorProvider, EngineDecorator, makeExecutionStack } from "./execution-stack"; import { DbProvider } from "./executor-fuma-db"; +import { executionActorFromPrincipal } from "./identity"; import { HostConfig, PluginsProvider } from "./scoped-executor"; // --------------------------------------------------------------------------- @@ -46,7 +47,14 @@ export const makeMcpBuildServer = Effect.map(({ engine }) => engine), Effect.provide(executionStack), Effect.mapError((cause) => new McpEngineBuildError({ cause })), - Effect.flatMap((engine) => createExecutorMcpServer({ engine })), + Effect.flatMap((engine) => + // Every run from this session is an MCP-triggered run acting as this + // session's principal (a user, or a host-supplied service-token actor). + createExecutorMcpServer({ + engine, + trigger: { kind: "mcp", actor: executionActorFromPrincipal(principal) }, + }), + ), ); /** diff --git a/packages/core/sdk/src/execution-observer.ts b/packages/core/sdk/src/execution-observer.ts index 2c19bcb0f5..e0ec8898ac 100644 --- a/packages/core/sdk/src/execution-observer.ts +++ b/packages/core/sdk/src/execution-observer.ts @@ -18,8 +18,27 @@ export type ExecutionToolCallId = typeof ExecutionToolCallId.Type; export const ExecutionInteractionId = Schema.String.pipe(Schema.brand("ExecutionInteractionId")); export type ExecutionInteractionId = typeof ExecutionInteractionId.Type; +/** + * The credential identity a run acts as β€” the "who" behind a trigger. + * + * `kind` is the credential class (e.g. `"user"`, `"service-token"`); `id` is the + * STABLE identity used for filtering/grouping (a user subject, a token client + * id) β€” it must not change when a display name is edited; `label` is a + * human-facing string (machine name, email, display name) or null. Hosts that + * know more than the neutral `Principal` (e.g. a Cloudflare service token's + * client id) supply this so the actor stays distinguishable from the human + * subject it may be aliased to act as. + */ +export type ExecutionActor = { + readonly kind: string; + readonly id: string; + readonly label: string | null; +}; + export type ExecutionTrigger = { readonly kind: string; + /** Who/what this run acts as; recorded on `ExecutionStarted` for attribution. */ + readonly actor?: ExecutionActor; readonly metadata?: Record; }; diff --git a/packages/core/sdk/src/index.ts b/packages/core/sdk/src/index.ts index 4b822b4310..cacce1e772 100644 --- a/packages/core/sdk/src/index.ts +++ b/packages/core/sdk/src/index.ts @@ -182,6 +182,7 @@ export { ignoreExecutionObserverErrors, composeExecutionObservers, type ExecutionTrigger, + type ExecutionActor, type ToolCallStatus, type InteractionStatus, type ExecutionStatus, diff --git a/packages/hosts/mcp/src/seams.ts b/packages/hosts/mcp/src/seams.ts index 5c82295da2..5760a74d35 100644 --- a/packages/hosts/mcp/src/seams.ts +++ b/packages/hosts/mcp/src/seams.ts @@ -44,6 +44,18 @@ export const Principal = Schema.Struct({ name: Schema.NullOr(Schema.String), avatarUrl: Schema.NullOr(Schema.String), roles: Schema.Array(Schema.String), + // The credential identity this principal acts AS, for run attribution + // (structurally the SDK's `ExecutionActor`). Schema'd here β€” not just passed + // through β€” so it survives the cross-isolate session-store serialization the + // cloud DO does. Unset means "attribute to the user" (derived downstream); a + // host sets it when a machine credential acts as a human (a service token). + actor: Schema.optional( + Schema.Struct({ + kind: Schema.String, + id: Schema.String, + label: Schema.NullOr(Schema.String), + }), + ), }); export type Principal = Schema.Schema.Type; diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index 121c36be0a..6f61b8684e 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -15,6 +15,7 @@ import type { ElicitationContext, ElicitationRequest, } from "@executor-js/sdk"; +import type { ExecutionTrigger } from "@executor-js/sdk/core"; import type * as Tracer from "effect/Tracer"; import { createExecutionEngine, @@ -88,6 +89,13 @@ type SharedMcpServerConfig = { readonly mode: "native"; }; readonly browserApprovalStore?: BrowserApprovalStore; + /** + * What kicked off runs from this MCP connection, recorded on every + * `ExecutionStarted` for attribution. The principal-aware build site (which + * knows who the session belongs to) supplies `{ kind: "mcp", actor }`; absent + * it, runs carry no trigger (the historical behaviour). + */ + readonly trigger?: ExecutionTrigger; }; export type ExecutorMcpServerConfig = @@ -358,6 +366,7 @@ export const createExecutorMcpServer = ( ): Effect.Effect => Effect.gen(function* () { const engine = "engine" in config ? config.engine : createExecutionEngine(config); + const trigger = config.trigger; const description = config.description ?? (yield* engine.getDescription.pipe(Effect.withSpan("mcp.host.get_description"))); @@ -419,10 +428,11 @@ export const createExecutorMcpServer = ( if (elicitationMode.mode === "native") { const result = yield* engine.execute(code, { onElicitation: makeMcpElicitationHandler(server, debugLog), + trigger, }); return toMcpResult(formatExecuteResult(result)); } - const outcome = yield* engine.executeWithPause(code); + const outcome = yield* engine.executeWithPause(code, { trigger }); debugLog("execute.paused_flow_result", { status: outcome.status, executionId: outcome.status === "paused" ? outcome.execution.id : undefined, diff --git a/packages/plugins/execution-history/src/api/group.ts b/packages/plugins/execution-history/src/api/group.ts index 446388d07f..8acd143d09 100644 --- a/packages/plugins/execution-history/src/api/group.ts +++ b/packages/plugins/execution-history/src/api/group.ts @@ -34,6 +34,12 @@ const RunTriggerCount = Schema.Struct({ triggerKind: Schema.NullOr(Schema.String), count: Schema.Number, }); +const RunActorCount = Schema.Struct({ + actorId: Schema.NullOr(Schema.String), + actorLabel: Schema.NullOr(Schema.String), + actorKind: Schema.NullOr(Schema.String), + count: Schema.Number, +}); const RunInteractionCounts = Schema.Struct({ withInteraction: Schema.Number, withoutInteraction: Schema.Number, @@ -58,6 +64,7 @@ export const ExecutionListMeta = Schema.Struct({ filterRowCount: Schema.Number, statusCounts: Schema.Array(RunStatusCount), triggerCounts: Schema.Array(RunTriggerCount), + actorCounts: Schema.Array(RunActorCount), interactionCounts: RunInteractionCounts, chartBucketMs: Schema.Number, chartData: Schema.Array(RunChartBucket), @@ -93,8 +100,8 @@ export const ListToolCallsResponse = Schema.Struct({ // Query / path params // // Numeric filters arrive as strings and decode via `NumberFromString`. The -// `status`/`trigger` CSV lists stay raw strings the handler splits, mirroring -// how the store's `list` takes `statusFilter`/`triggerFilter` arrays. The +// `status`/`trigger`/`actor` CSV lists stay raw strings the handler splits, +// mirroring how the store's `list` takes the matching filter arrays. The // boolean `interaction` is carried as a string ("true"/"false") and // interpreted in the handler β€” the same convention core uses for // `includeAnnotations`/`includeBlocked`. @@ -103,6 +110,8 @@ export const ListToolCallsResponse = Schema.Struct({ const ListRunsQuery = Schema.Struct({ status: Schema.optional(Schema.String), trigger: Schema.optional(Schema.String), + // CSV of actor ids (token client ids / user subjects) to filter runs by. + actor: Schema.optional(Schema.String), from: Schema.optional(Schema.NumberFromString), to: Schema.optional(Schema.NumberFromString), interaction: Schema.optional(Schema.String), diff --git a/packages/plugins/execution-history/src/api/handlers.test.ts b/packages/plugins/execution-history/src/api/handlers.test.ts index 99d4c36543..48c0ae1fcf 100644 --- a/packages/plugins/execution-history/src/api/handlers.test.ts +++ b/packages/plugins/execution-history/src/api/handlers.test.ts @@ -45,6 +45,9 @@ const runRow = (overrides: Partial): RunRow => ({ logsJson: null, triggerKind: "manual", triggerMetaJson: null, + actorId: null, + actorLabel: null, + actorKind: null, startedAt: 1000, completedAt: 2000, durationMs: 1000, diff --git a/packages/plugins/execution-history/src/api/handlers.ts b/packages/plugins/execution-history/src/api/handlers.ts index 519f5d84e2..714db7e185 100644 --- a/packages/plugins/execution-history/src/api/handlers.ts +++ b/packages/plugins/execution-history/src/api/handlers.ts @@ -84,6 +84,7 @@ export const ExecutionHistoryHandlers = HttpApiBuilder.group( // must not reach the storage `where` clause. const statusFilter = splitCsv(query.status).filter(isRunStatus); const triggerFilter = splitCsv(query.trigger); + const actorFilter = splitCsv(query.actor); const timeRange = query.from !== undefined || query.to !== undefined ? { from: query.from, to: query.to } @@ -96,6 +97,7 @@ export const ExecutionHistoryHandlers = HttpApiBuilder.group( const options: ExecutionHistoryListOptions = { statusFilter: statusFilter.length > 0 ? statusFilter : undefined, triggerFilter: triggerFilter.length > 0 ? triggerFilter : undefined, + actorFilter: actorFilter.length > 0 ? actorFilter : undefined, timeRange, hadInteraction: parseBooleanFlag(query.interaction), after: query.after, diff --git a/packages/plugins/execution-history/src/react/atoms.ts b/packages/plugins/execution-history/src/react/atoms.ts index 3589c1f710..8ce7d9b84e 100644 --- a/packages/plugins/execution-history/src/react/atoms.ts +++ b/packages/plugins/execution-history/src/react/atoms.ts @@ -16,6 +16,7 @@ import { ExecutionHistoryClient } from "./client"; export interface RunsQuery { readonly status?: string; readonly trigger?: string; + readonly actor?: string; readonly from?: number; readonly to?: number; readonly interaction?: string; diff --git a/packages/plugins/execution-history/src/react/column-header.tsx b/packages/plugins/execution-history/src/react/column-header.tsx index 64df6ff758..b69bdeae16 100644 --- a/packages/plugins/execution-history/src/react/column-header.tsx +++ b/packages/plugins/execution-history/src/react/column-header.tsx @@ -4,7 +4,7 @@ import { cn } from "@executor-js/react/lib/utils"; import type { RunsSortField } from "./use-runs-list"; import type { RunColumns } from "./view"; -import { COL_DURATION, COL_INTERACTION, COL_TOOLS, COL_TRIGGER } from "./run-row"; +import { COL_ACTOR, COL_DURATION, COL_INTERACTION, COL_TOOLS, COL_TRIGGER } from "./run-row"; // --------------------------------------------------------------------------- // RunsColumnHeader β€” sticky header row aligned to RunListRow's layout. @@ -86,6 +86,9 @@ export function RunsColumnHeader({ {/* Trigger */} {columns.trigger ? trigger : null} + {/* Actor */} + {columns.actor ? actor : null} + {/* Duration (sortable) */} {columns.duration ? ( {run.durationMs != null ? formatDuration(run.durationMs) : "β€”"} + + {run.actorId !== null ? ( + + {run.actorLabel ?? run.actorId} + {run.actorKind !== null ? ( + Β· {run.actorKind} + ) : null} + + ) : ( + β€” + )} + {formatDateTime(run.startedAt)} {formatDateTime(run.completedAt)}
diff --git a/packages/plugins/execution-history/src/react/filter-rail.tsx b/packages/plugins/execution-history/src/react/filter-rail.tsx index be592dd459..9a15b07448 100644 --- a/packages/plugins/execution-history/src/react/filter-rail.tsx +++ b/packages/plugins/execution-history/src/react/filter-rail.tsx @@ -9,7 +9,14 @@ import { Button } from "@executor-js/react/components/button"; import { cn } from "@executor-js/react/lib/utils"; import type { ExecutionListMeta } from "../sdk/store"; -import { STATUS_ORDER, STATUS_LABELS, TRIGGER_ORDER, statusTone, triggerTone } from "./status"; +import { + STATUS_ORDER, + STATUS_LABELS, + TRIGGER_ORDER, + actorTone, + statusTone, + triggerTone, +} from "./status"; import type { RunsFilters } from "./use-runs-list"; // --------------------------------------------------------------------------- @@ -58,6 +65,7 @@ const toggle = (arr: readonly T[], item: T): readonly T[] => const isFiltersActive = (filters: RunsFilters): boolean => filters.status.length > 0 || filters.trigger.length > 0 || + filters.actor.length > 0 || filters.interaction !== null || filters.from !== null || filters.to !== null; @@ -234,7 +242,7 @@ export function RunsFilterRail({ filters, meta, onChange, onReset }: RunsFilterR
{/* Status facet */} @@ -289,6 +297,35 @@ export function RunsFilterRail({ filters, meta, onChange, onReset }: RunsFilterR + {/* Actor facet β€” dynamic keys (token client ids / user subjects) from + the run set; only filterable (non-null) actors are shown. */} + {meta?.actorCounts && meta.actorCounts.some((entry) => entry.actorId !== null) ? ( + +
    + {meta.actorCounts.map((entry) => { + const id = entry.actorId; + // null actor (unattributed runs) isn't filterable β€” skip it. + if (id === null) return null; + const tone = actorTone(entry.actorKind); + const checked = filters.actor.includes(id); + return ( +
  • + onChange({ ...filters, actor: toggle(filters.actor, id) })} + onOnly={() => onChange({ ...filters, actor: [id] })} + dotClass={tone.dot} + label={entry.actorLabel ?? id} + count={entry.count} + monoLabel + /> +
    • + ); + })} +
+ + ) : null} + {/* Interaction facet */}
    diff --git a/packages/plugins/execution-history/src/react/run-row.tsx b/packages/plugins/execution-history/src/react/run-row.tsx index 11d3fd6df5..290ef27ffd 100644 --- a/packages/plugins/execution-history/src/react/run-row.tsx +++ b/packages/plugins/execution-history/src/react/run-row.tsx @@ -4,7 +4,7 @@ import { Badge } from "@executor-js/react/components/badge"; import type { RunRow } from "../sdk/collections"; import { formatDateTime, formatDuration, formatRelative, statusLabel } from "./format"; -import { STATUS_TONES, triggerTone } from "./status"; +import { actorTone, STATUS_TONES, triggerTone } from "./status"; import type { RunColumns } from "./view"; // Column slot classes β€” RunListRow and RunsColumnHeader both apply these @@ -18,6 +18,9 @@ import type { RunColumns } from "./view"; // desyncs header and row (the trigger column) or wraps the sort label onto a // second line (duration). A single flex display keeps both in lockstep. export const COL_TRIGGER = "hidden w-[120px] shrink-0 xl:flex"; +// Actor carries a dot + label (machine name / email / user), so it is a flex +// slot like trigger. Wider, since labels are longer than trigger kinds. +export const COL_ACTOR = "hidden w-[170px] shrink-0 lg:flex"; export const COL_DURATION = "hidden w-[100px] shrink-0 md:flex"; export const COL_TOOLS = "hidden xl:block w-[80px] shrink-0"; export const COL_INTERACTION = "hidden xl:block w-[100px] shrink-0"; @@ -33,6 +36,7 @@ export interface RunListRowProps { export function RunListRow({ run, selected, isPast, columns, onSelect }: RunListRowProps) { const tone = STATUS_TONES[run.status]; const trigger = triggerTone(run.triggerKind); + const actor = actorTone(run.actorKind); const isLive = run.status === "running" || run.status === "waiting_for_interaction"; return ( @@ -81,6 +85,25 @@ export function RunListRow({ run, selected, isPast, columns, onSelect }: RunList ) : null} + {/* Actor (optional) */} + {columns.actor ? ( + + {run.actorId !== null ? ( + <> + + + {run.actorLabel ?? run.actorId} + + + ) : ( + β€” + )} + + ) : null} + {/* Duration (optional) */} {columns.duration ? ( diff --git a/packages/plugins/execution-history/src/react/runs-list-state.test.ts b/packages/plugins/execution-history/src/react/runs-list-state.test.ts index 5943e87c37..3fd9dea1ab 100644 --- a/packages/plugins/execution-history/src/react/runs-list-state.test.ts +++ b/packages/plugins/execution-history/src/react/runs-list-state.test.ts @@ -18,6 +18,9 @@ const row = (id: string, startedAt: number): RunRow => ({ logsJson: null, triggerKind: "cli", triggerMetaJson: null, + actorId: null, + actorLabel: null, + actorKind: null, startedAt, completedAt: startedAt + 100, durationMs: 100, diff --git a/packages/plugins/execution-history/src/react/status.ts b/packages/plugins/execution-history/src/react/status.ts index 379abc51c7..498f20c2c4 100644 --- a/packages/plugins/execution-history/src/react/status.ts +++ b/packages/plugins/execution-history/src/react/status.ts @@ -93,3 +93,28 @@ export const triggerTone = (kind: string | null | undefined): TriggerTone => { } return { dot: "bg-muted-foreground/40", text: "text-muted-foreground", label: "unknown" }; }; + +// --------------------------------------------------------------------------- +// Actor tones β€” colour the dot in the Actor facet/column by credential class +// (`actorKind`: "user", "service-token", …). The label itself is the actor's +// own display string (machine name, email), not a fixed vocabulary, so there is +// no ACTOR_ORDER β€” facet keys come from `meta.actorCounts`. +// --------------------------------------------------------------------------- + +export interface ActorTone { + readonly dot: string; + readonly text: string; +} + +export const ACTOR_TONES: Record = { + user: { dot: "bg-sky-500", text: "text-foreground/80" }, + "service-token": { dot: "bg-violet-500", text: "text-foreground/80" }, +}; + +export const actorTone = (kind: string | null | undefined): ActorTone => { + if (kind != null) { + const known = ACTOR_TONES[kind]; + if (known) return known; + } + return { dot: "bg-muted-foreground/40", text: "text-muted-foreground" }; +}; diff --git a/packages/plugins/execution-history/src/react/use-runs-list.ts b/packages/plugins/execution-history/src/react/use-runs-list.ts index a2cca72a12..93477c2874 100644 --- a/packages/plugins/execution-history/src/react/use-runs-list.ts +++ b/packages/plugins/execution-history/src/react/use-runs-list.ts @@ -21,6 +21,7 @@ export type RunsSortField = "startedAt" | "durationMs"; export interface RunsFilters { readonly status: readonly string[]; readonly trigger: readonly string[]; + readonly actor: readonly string[]; readonly interaction: "true" | "false" | null; readonly from: number | null; readonly to: number | null; @@ -34,6 +35,7 @@ const LIVE_INTERVAL_MS = 5000; export const emptyRunsFilters: RunsFilters = { status: [], trigger: [], + actor: [], interaction: null, from: null, to: null, @@ -47,6 +49,7 @@ export const buildRunsQuery = (filters: RunsFilters, cursor: string | undefined) dir: filters.sortDirection, ...(filters.status.length > 0 ? { status: filters.status.join(",") } : {}), ...(filters.trigger.length > 0 ? { trigger: filters.trigger.join(",") } : {}), + ...(filters.actor.length > 0 ? { actor: filters.actor.join(",") } : {}), ...(filters.interaction != null ? { interaction: filters.interaction } : {}), ...(filters.from != null ? { from: filters.from } : {}), ...(filters.to != null ? { to: filters.to } : {}), @@ -57,6 +60,7 @@ const filtersKey = (filters: RunsFilters): string => JSON.stringify([ filters.status, filters.trigger, + filters.actor, filters.interaction, filters.from, filters.to, diff --git a/packages/plugins/execution-history/src/react/view-options.tsx b/packages/plugins/execution-history/src/react/view-options.tsx index 40e822c100..37456a907c 100644 --- a/packages/plugins/execution-history/src/react/view-options.tsx +++ b/packages/plugins/execution-history/src/react/view-options.tsx @@ -5,7 +5,13 @@ import { cn } from "@executor-js/react/lib/utils"; import { RUN_COLUMN_LABELS, type RunColumnKey, type RunColumns } from "./view"; -const COLUMN_KEYS: readonly RunColumnKey[] = ["trigger", "duration", "tools", "interaction"]; +const COLUMN_KEYS: readonly RunColumnKey[] = [ + "trigger", + "actor", + "duration", + "tools", + "interaction", +]; export function ViewOptions(props: { readonly columns: RunColumns; diff --git a/packages/plugins/execution-history/src/react/view.ts b/packages/plugins/execution-history/src/react/view.ts index 66c94eef35..b8bf27019d 100644 --- a/packages/plugins/execution-history/src/react/view.ts +++ b/packages/plugins/execution-history/src/react/view.ts @@ -6,6 +6,7 @@ export interface RunColumns { readonly trigger: boolean; + readonly actor: boolean; readonly duration: boolean; readonly tools: boolean; readonly interaction: boolean; @@ -15,6 +16,7 @@ export type RunColumnKey = keyof RunColumns; export const DEFAULT_COLUMNS: RunColumns = { trigger: true, + actor: true, duration: true, tools: true, interaction: true, @@ -22,6 +24,7 @@ export const DEFAULT_COLUMNS: RunColumns = { export const RUN_COLUMN_LABELS: Record = { trigger: "Trigger", + actor: "Actor", duration: "Duration", tools: "Tools", interaction: "Interaction", diff --git a/packages/plugins/execution-history/src/sdk/collections.ts b/packages/plugins/execution-history/src/sdk/collections.ts index 01cdb6277c..7214403cb6 100644 --- a/packages/plugins/execution-history/src/sdk/collections.ts +++ b/packages/plugins/execution-history/src/sdk/collections.ts @@ -47,6 +47,14 @@ export const RunRow = Schema.Struct({ logsJson: Schema.NullOr(Schema.String), triggerKind: Schema.NullOr(Schema.String), triggerMetaJson: Schema.NullOr(Schema.String), + // Who/what the run acted as (from the trigger's `ExecutionActor`). `actorId` + // is the STABLE filter/facet key (a token client id, a user subject); + // `actorLabel` is the display snapshot at run time (machine name, email); + // `actorKind` is the credential class ("user", "service-token"). Null on runs + // recorded before a trigger actor was supplied. + actorId: Schema.NullOr(Schema.String), + actorLabel: Schema.NullOr(Schema.String), + actorKind: Schema.NullOr(Schema.String), startedAt: Schema.Number, completedAt: Schema.NullOr(Schema.Number), durationMs: Schema.NullOr(Schema.Number), @@ -56,7 +64,7 @@ export const RunRow = Schema.Struct({ export type RunRow = typeof RunRow.Type; export const runs = definePluginStorageCollection("runs", RunRow, { - indexes: ["status", "triggerKind", "startedAt", "durationMs", "hadInteraction"], + indexes: ["status", "triggerKind", "actorId", "startedAt", "durationMs", "hadInteraction"], }); export const ToolCallRow = Schema.Struct({ diff --git a/packages/plugins/execution-history/src/sdk/store-list.test.ts b/packages/plugins/execution-history/src/sdk/store-list.test.ts index faf6a2c41b..e0856f8a9a 100644 --- a/packages/plugins/execution-history/src/sdk/store-list.test.ts +++ b/packages/plugins/execution-history/src/sdk/store-list.test.ts @@ -154,4 +154,79 @@ describe("execution-history list β€” meta, facets, keyset", () => { expect(page.meta).toBeNull(); }), ); + + it.effect("attributes runs to their actor: facet (label+kind) + filter", () => + Effect.gen(function* () { + const executor = yield* makeExecutor(); + const seedActor = ( + id: string, + startedAt: number, + actor: { kind: string; id: string; label: string | null }, + ) => + Effect.gen(function* () { + const executionId = ExecutionId.make(id); + yield* executor.executionHistory.handleEvent( + new ExecutionStarted({ + executionId, + owner, + code: "noop", + trigger: { kind: "mcp", actor }, + startedAt: new Date(startedAt), + }), + ); + yield* executor.executionHistory.handleEvent( + new ExecutionFinished({ + executionId, + owner, + status: "completed", + result: { ok: true }, + logs: [], + completedAt: new Date(startedAt + 100), + }), + ); + }); + + const token = { kind: "service-token", id: "tok_abc.access", label: "executor-phoenix" }; + const user = { kind: "user", id: "sub_123", label: "saatvik@example.com" }; + yield* seedActor("a1", 1000, token); + yield* seedActor("a2", 2000, token); + yield* seedActor("a3", 3000, user); + + // The actor facet keys on the stable `actorId` and renders the most-recent + // run's label + kind snapshot. + const page = yield* executor.executionHistory.list({ limit: 10 }); + expect( + Object.fromEntries((page.meta?.actorCounts ?? []).map((entry) => [entry.actorId, entry])), + ).toEqual({ + "tok_abc.access": { + actorId: "tok_abc.access", + actorLabel: "executor-phoenix", + actorKind: "service-token", + count: 2, + }, + sub_123: { + actorId: "sub_123", + actorLabel: "saatvik@example.com", + actorKind: "user", + count: 1, + }, + }); + + // Filtering by the token returns only its runs; the facet omits its own + // filter so every actor stays visible in the rail. + const filtered = yield* executor.executionHistory.list({ + limit: 10, + actorFilter: ["tok_abc.access"], + }); + expect(filtered.runs.map((r) => r.executionId)).toEqual(["a2", "a1"]); + expect(filtered.meta?.filterRowCount).toBe(2); + expect(filtered.meta?.actorCounts).toHaveLength(2); + + // The persisted run carries the actor snapshot for the column/detail. + const run = filtered.runs[0]; + expect(run?.actorId).toBe("tok_abc.access"); + expect(run?.actorLabel).toBe("executor-phoenix"); + expect(run?.actorKind).toBe("service-token"); + }), + ); }); diff --git a/packages/plugins/execution-history/src/sdk/store.ts b/packages/plugins/execution-history/src/sdk/store.ts index 5bda6d307a..f8526c6990 100644 --- a/packages/plugins/execution-history/src/sdk/store.ts +++ b/packages/plugins/execution-history/src/sdk/store.ts @@ -89,10 +89,13 @@ interface RunBuffer { owner: Owner; startedAt: number; // Retained from ExecutionStarted so every re-write of the run row (waiting, - // terminal) keeps the code + trigger β€” later events don't carry them. + // terminal) keeps the code + trigger + actor β€” later events don't carry them. code: string; triggerKind: string | null; triggerMetaJson: string | null; + actorId: string | null; + actorLabel: string | null; + actorKind: string | null; hadInteraction: boolean; toolCalls: Map; interactions: Map; @@ -121,6 +124,7 @@ export interface RunsCursor { export interface ExecutionHistoryListOptions { readonly statusFilter?: readonly RunStatus[]; readonly triggerFilter?: readonly string[]; + readonly actorFilter?: readonly string[]; readonly timeRange?: { readonly from?: number; readonly to?: number }; readonly hadInteraction?: boolean; /** Live-tail floor: only runs whose `startedAt` is strictly greater. */ @@ -141,6 +145,14 @@ export interface RunTriggerCount { readonly count: number; } +export interface RunActorCount { + readonly actorId: string | null; + /** A display label for the actor (the most-recent run's snapshot), or null. */ + readonly actorLabel: string | null; + readonly actorKind: string | null; + readonly count: number; +} + export interface RunInteractionCounts { readonly withInteraction: number; readonly withoutInteraction: number; @@ -168,6 +180,7 @@ export interface ExecutionListMeta { readonly filterRowCount: number; readonly statusCounts: readonly RunStatusCount[]; readonly triggerCounts: readonly RunTriggerCount[]; + readonly actorCounts: readonly RunActorCount[]; readonly interactionCounts: RunInteractionCounts; readonly chartBucketMs: number; readonly chartData: readonly RunChartBucket[]; @@ -206,6 +219,7 @@ export interface ExecutionHistoryStore { type RunsWhere = { status?: { in: readonly RunStatus[] }; triggerKind?: { in: readonly string[] }; + actorId?: { in: readonly string[] }; startedAt?: { gte?: number; lte?: number; gt?: number }; hadInteraction?: { eq: boolean }; }; @@ -215,7 +229,7 @@ type RunsWhere = { * shows every option for the field you're filtering on). */ const buildRunsWhere = ( options: ExecutionHistoryListOptions, - omit?: "status" | "triggerKind" | "hadInteraction", + omit?: "status" | "triggerKind" | "actorId" | "hadInteraction", ): RunsWhere => { const where: RunsWhere = {}; if (omit !== "status" && options.statusFilter && options.statusFilter.length > 0) { @@ -224,6 +238,9 @@ const buildRunsWhere = ( if (omit !== "triggerKind" && options.triggerFilter && options.triggerFilter.length > 0) { where.triggerKind = { in: options.triggerFilter }; } + if (omit !== "actorId" && options.actorFilter && options.actorFilter.length > 0) { + where.actorId = { in: options.actorFilter }; + } const startedAt: { gte?: number; lte?: number; gt?: number } = {}; if (options.timeRange?.from != null) startedAt.gte = options.timeRange.from; if (options.timeRange?.to != null) startedAt.lte = options.timeRange.to; @@ -273,12 +290,19 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt const startedAt = event.startedAt.getTime(); const triggerKind = event.trigger?.kind ?? null; const triggerMetaJson = toJson(event.trigger?.metadata); + const actor = event.trigger?.actor; + const actorId = actor?.id ?? null; + const actorLabel = actor?.label ?? null; + const actorKind = actor?.kind ?? null; buffers.set(event.executionId, { owner, startedAt, code: event.code, triggerKind, triggerMetaJson, + actorId, + actorLabel, + actorKind, hadInteraction: false, toolCalls: new Map(), interactions: new Map(), @@ -292,6 +316,9 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt logsJson: null, triggerKind, triggerMetaJson, + actorId, + actorLabel, + actorKind, startedAt, completedAt: null, durationMs: null, @@ -369,6 +396,9 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt logsJson: null, triggerKind: buffer.triggerKind, triggerMetaJson: buffer.triggerMetaJson, + actorId: buffer.actorId, + actorLabel: buffer.actorLabel, + actorKind: buffer.actorKind, startedAt: buffer.startedAt, completedAt: null, durationMs: null, @@ -412,6 +442,9 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt const code = buffer?.code ?? existing?.data.code ?? ""; const triggerKind = buffer?.triggerKind ?? existing?.data.triggerKind ?? null; const triggerMetaJson = buffer?.triggerMetaJson ?? existing?.data.triggerMetaJson ?? null; + const actorId = buffer?.actorId ?? existing?.data.actorId ?? null; + const actorLabel = buffer?.actorLabel ?? existing?.data.actorLabel ?? null; + const actorKind = buffer?.actorKind ?? existing?.data.actorKind ?? null; const startedAt = buffer?.startedAt ?? existing?.data.startedAt ?? completedAt; const hadInteraction = buffer?.hadInteraction ?? (existing?.data.hadInteraction || interactionEntries.length > 0); @@ -425,6 +458,9 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt logsJson: toJson(event.logs), triggerKind, triggerMetaJson, + actorId, + actorLabel, + actorKind, startedAt, completedAt, durationMs: completedAt - startedAt, @@ -507,6 +543,7 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt filterRowCount, statusGroups, triggerGroups, + actorGroups, interactionGroups, stats, startedAtStats, @@ -522,6 +559,10 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt field: "triggerKind", where: buildRunsWhere(options, "triggerKind"), }), + runsC.aggregate.groupCount({ + field: "actorId", + where: buildRunsWhere(options, "actorId"), + }), runsC.aggregate.groupCount({ field: "hadInteraction", where: buildRunsWhere(options, "hadInteraction"), @@ -593,6 +634,48 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt triggerKind: typeof group.value === "string" ? group.value : null, count: group.count, })); + + // Resolve a display label + kind per actor. The facet keys on the STABLE + // `actorId`, but renders the human snapshot (`actorLabel`/`actorKind`) from + // that actor's most-recent run WITHIN the current filter window β€” so the + // label stays consistent with the (filtered) count shown beside it, rather + // than a globally-newest snapshot that could predate the window. One query + // per distinct actor; the null-actor group needs no lookup. + const actorFacetWhere = buildRunsWhere(options, "actorId"); + const actorIds = actorGroups + .map((group) => (typeof group.value === "string" ? group.value : null)) + .filter(Predicate.isNotNull); + const actorMeta = yield* Effect.forEach( + actorIds, + (actorId) => + runsC + .query({ + where: { ...actorFacetWhere, actorId: { in: [actorId] } }, + orderBy: [{ field: "startedAt", direction: "desc" }], + limit: 1, + }) + .pipe( + Effect.map((rows) => ({ + actorId, + label: rows[0]?.data.actorLabel ?? null, + kind: rows[0]?.data.actorKind ?? null, + })), + ), + // Cap the burst: distinct-actor cardinality is normally small, but a + // workspace with many service tokens shouldn't fan out unboundedly. + { concurrency: 10 }, + ); + const actorMetaById = new Map(actorMeta.map((entry) => [entry.actorId, entry])); + const actorCounts: RunActorCount[] = actorGroups.map((group) => { + const actorId = typeof group.value === "string" ? group.value : null; + const resolved = actorId !== null ? actorMetaById.get(actorId) : undefined; + return { + actorId, + actorLabel: resolved?.label ?? null, + actorKind: resolved?.kind ?? null, + count: group.count, + }; + }); const withInteraction = interactionGroups.find((group) => group.value === true)?.count ?? 0; const withoutInteraction = interactionGroups.find((group) => group.value === false)?.count ?? 0; @@ -604,6 +687,7 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt filterRowCount, statusCounts, triggerCounts, + actorCounts, interactionCounts: { withInteraction, withoutInteraction }, chartBucketMs: bucketMs, chartData, From 1d6ef75828171b3b749623c9e970c1157443c492 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 02:41:01 +0530 Subject: [PATCH 11/57] feat(execution-history): restore parity gaps from the old runs UI (#21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Restores the user-facing affordances the execution-history plugin dropped when it replaced the old observability runs UI (`a952d08ad`). Scoped from a full old-vs-new audit; **Groups A + B** (all client-side β€” no API/store changes). ### Timestamps - **`HoverCardTimestamp`** (new) β€” hovering a timestamp reveals **epoch / UTC / local-tz / relative** renderings, each **click-to-copy**. On the list-row started cell (keeps its relative-time display) and the drawer's Started/Completed. Native `Date`/`Intl`, reusing the shared Radix `HoverCard` (no `date-fns`). ### List - **Log column** β€” optional column counting `[error]`/`[warn]` lines from the row's `logsJson` (off by default; toggle in view options). Case-insensitive. - **Slow-run highlight** β€” durations > 5s render in the destructive color. - Code preview widened **80 β†’ 160** chars. - Toolbar **row count** ("N of M"); **end-of-history** footer; empty-state hint line. ### Filters / controls - **`Last 15m`** time-range preset; **title hints** on Live/Refresh buttons. ### Chart - **Seconds-resolution** axis + tooltip for sub-minute buckets; tooltip escapes the view box and ignores pointer events. ### Persistence - **Column visibility** persists across reloads (localStorage, comma-joined keys β€” no `JSON.parse`). ## Deferred (called out, not done here) - **Filter-state persistence** β€” needs a `RunsFilters` schema (the repo lints against `JSON.parse`) and is better paired with URL params for shareability. - **`tool:` / `code:` / `duration_ms:` filters + Tools facet** β€” these need API + store changes (cross-collection tool faceting especially), so they belong in a separate PR next to the data work. ## Verification `oxfmt --check`, `oxlint`, `turbo typecheck` (9 packages), plugin tests (17) all pass. UI-only; no API, store, or event-contract changes β€” independent of the in-flight data/actor work. --- .../execution-history/src/react/RunsPage.tsx | 39 ++++++- .../src/react/column-header.tsx | 12 +- .../src/react/detail-drawer.tsx | 13 ++- .../src/react/filter-rail.tsx | 1 + .../src/react/hover-card-timestamp.tsx | 104 ++++++++++++++++++ .../src/react/live-button.tsx | 1 + .../src/react/refresh-button.tsx | 1 + .../execution-history/src/react/run-row.tsx | 53 +++++++-- .../execution-history/src/react/shell.tsx | 5 + .../src/react/timeline-chart.tsx | 21 ++-- .../src/react/view-options.tsx | 1 + .../execution-history/src/react/view.ts | 3 + 12 files changed, 234 insertions(+), 20 deletions(-) create mode 100644 packages/plugins/execution-history/src/react/hover-card-timestamp.tsx diff --git a/packages/plugins/execution-history/src/react/RunsPage.tsx b/packages/plugins/execution-history/src/react/RunsPage.tsx index 90d5d80d07..931a278718 100644 --- a/packages/plugins/execution-history/src/react/RunsPage.tsx +++ b/packages/plugins/execution-history/src/react/RunsPage.tsx @@ -36,11 +36,35 @@ interface ShortcutContext { readonly selected: string | null; } +const COLUMNS_STORAGE_KEY = "executionHistory.columns"; + +// Column visibility persists across reloads as a comma-joined list of the +// visible keys (avoids JSON.parse, which the repo lints against). Keys absent +// from the stored list β€” including columns added in a later release β€” read as +// hidden, falling back to DEFAULT_COLUMNS only when nothing is stored. +const readStoredColumns = (): RunColumns => { + if (typeof window === "undefined") return DEFAULT_COLUMNS; + const raw = window.localStorage.getItem(COLUMNS_STORAGE_KEY); + if (raw == null) return DEFAULT_COLUMNS; + // Start from the current defaults and override only the keys actually stored, + // so a column added in a later release keeps its default visibility instead + // of being read as hidden for existing users. + const next = { ...DEFAULT_COLUMNS }; + for (const pair of raw.split(",")) { + const [key, value] = pair.split("="); + if (key != null && key in next) next[key as RunColumnKey] = value === "1"; + } + return next; +}; + +const serializeColumns = (columns: RunColumns): string => + (Object.keys(columns) as RunColumnKey[]).map((key) => `${key}=${columns[key] ? 1 : 0}`).join(","); + export function RunsPage() { const [filters, setFilters] = useState(emptyRunsFilters); const [live, setLive] = useState(false); const [selected, setSelected] = useState(null); - const [columns, setColumns] = useState(DEFAULT_COLUMNS); + const [columns, setColumns] = useState(readStoredColumns); const [commandOpen, setCommandOpen] = useState(false); const [helpOpen, setHelpOpen] = useState(false); const [railCollapsed, setRailCollapsed] = useState(false); @@ -60,6 +84,11 @@ export function RunsPage() { setColumns((current) => ({ ...current, [key]: !current[key] })); }, []); + useEffect(() => { + if (typeof window === "undefined") return; + window.localStorage.setItem(COLUMNS_STORAGE_KEY, serializeColumns(columns)); + }, [columns]); + // Keep the latest view/selected reachable from a stable keydown listener. const shortcutRef = useRef({ view, selected }); shortcutRef.current = { view, selected }; @@ -153,6 +182,11 @@ export function RunsPage() {

    Runs

    + {view.meta != null && ( + + {view.rows.length} of {view.meta.filterRowCount} + + )}
    } railCollapsed={railCollapsed} diff --git a/packages/plugins/execution-history/src/react/column-header.tsx b/packages/plugins/execution-history/src/react/column-header.tsx index b69bdeae16..644e73cb49 100644 --- a/packages/plugins/execution-history/src/react/column-header.tsx +++ b/packages/plugins/execution-history/src/react/column-header.tsx @@ -4,7 +4,14 @@ import { cn } from "@executor-js/react/lib/utils"; import type { RunsSortField } from "./use-runs-list"; import type { RunColumns } from "./view"; -import { COL_ACTOR, COL_DURATION, COL_INTERACTION, COL_TOOLS, COL_TRIGGER } from "./run-row"; +import { + COL_ACTOR, + COL_DURATION, + COL_INTERACTION, + COL_LOG, + COL_TOOLS, + COL_TRIGGER, +} from "./run-row"; // --------------------------------------------------------------------------- // RunsColumnHeader β€” sticky header row aligned to RunListRow's layout. @@ -107,6 +114,9 @@ export function RunsColumnHeader({ {/* Interaction */} {columns.interaction ? interaction : null} + {/* Log */} + {columns.log ? log : null} + {/* Code */} code
    diff --git a/packages/plugins/execution-history/src/react/detail-drawer.tsx b/packages/plugins/execution-history/src/react/detail-drawer.tsx index 48e37f1d35..73d403427d 100644 --- a/packages/plugins/execution-history/src/react/detail-drawer.tsx +++ b/packages/plugins/execution-history/src/react/detail-drawer.tsx @@ -20,6 +20,7 @@ import { cn } from "@executor-js/react/lib/utils"; import type { InteractionRow, InteractionStatus, RunRow, ToolCallRow } from "../sdk/collections"; import { runDetailAtom, runToolCallsAtom } from "./atoms"; import { formatDateTime, formatDuration, logLines, prettyJson, statusLabel } from "./format"; +import { HoverCardTimestamp } from "./hover-card-timestamp"; import { STATUS_TONES, triggerTone } from "./status"; // --------------------------------------------------------------------------- @@ -340,8 +341,16 @@ function DetailContent(props: { β€” )} - {formatDateTime(run.startedAt)} - {formatDateTime(run.completedAt)} + + + + + {run.completedAt != null ? ( + + ) : ( + formatDateTime(run.completedAt) + )} +
diff --git a/packages/plugins/execution-history/src/react/filter-rail.tsx b/packages/plugins/execution-history/src/react/filter-rail.tsx index 9a15b07448..41e46d6736 100644 --- a/packages/plugins/execution-history/src/react/filter-rail.tsx +++ b/packages/plugins/execution-history/src/react/filter-rail.tsx @@ -30,6 +30,7 @@ export interface TimeRangePreset { } export const TIME_RANGE_PRESETS: readonly TimeRangePreset[] = [ + { key: "15m", label: "Last 15m", ms: 15 * 60 * 1000 }, { key: "1h", label: "Last 1h", ms: 60 * 60 * 1000 }, { key: "24h", label: "Last 24h", ms: 24 * 60 * 60 * 1000 }, { key: "7d", label: "Last 7d", ms: 7 * 24 * 60 * 60 * 1000 }, diff --git a/packages/plugins/execution-history/src/react/hover-card-timestamp.tsx b/packages/plugins/execution-history/src/react/hover-card-timestamp.tsx new file mode 100644 index 0000000000..1d18d771a9 --- /dev/null +++ b/packages/plugins/execution-history/src/react/hover-card-timestamp.tsx @@ -0,0 +1,104 @@ +import { useCallback, useState } from "react"; +import { Check, Copy } from "lucide-react"; +import { + HoverCard, + HoverCardContent, + HoverCardTrigger, +} from "@executor-js/react/components/hover-card"; +import { cn } from "@executor-js/react/lib/utils"; + +import { formatRelative } from "./format"; + +// --------------------------------------------------------------------------- +// Hover popup over a timestamp: the trigger shows whatever `display` is given +// (the list row passes a relative "1 hour ago"; the drawer falls back to the +// absolute format), and hovering reveals the epoch ms, UTC, local-tz, and +// relative renderings β€” each a click-to-copy row. Native Date/Intl only (the +// plugin deliberately carries no date-fns dep); reuses the shared Radix +// HoverCard from @executor-js/react. +// --------------------------------------------------------------------------- + +const absolute = (timestamp: number, timeZone?: string): string => + new Intl.DateTimeFormat(undefined, { + year: "numeric", + month: "short", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hour12: false, + ...(timeZone ? { timeZone } : {}), + }).format(timestamp); + +const LOCAL_TZ = Intl.DateTimeFormat().resolvedOptions().timeZone; + +export interface HoverCardTimestampProps { + readonly timestamp: number; + /** Trigger text; defaults to the absolute local format. */ + readonly display?: React.ReactNode; + readonly side?: "top" | "right" | "bottom" | "left"; + readonly align?: "start" | "center" | "end"; + readonly className?: string; +} + +export function HoverCardTimestamp({ + timestamp, + display, + side = "right", + align = "start", + className, +}: HoverCardTimestampProps) { + return ( + + + + {display ?? absolute(timestamp)} + + + +
+ + + + +
+ + + ); +} + +function CopyRow({ label, value }: { readonly label: string; readonly value: string }) { + const [copied, setCopied] = useState(false); + const copy = useCallback( + (event: React.MouseEvent | React.KeyboardEvent) => { + event.stopPropagation(); + void navigator.clipboard.writeText(value).then(() => { + setCopied(true); + window.setTimeout(() => setCopied(false), 1500); + }); + }, + [value], + ); + return ( +
{ + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + copy(event); + } + }} + className="group flex items-center justify-between gap-4 text-xs" + > +
{label}
+
+ + {copied ? : } + + {value} +
+
+ ); +} diff --git a/packages/plugins/execution-history/src/react/live-button.tsx b/packages/plugins/execution-history/src/react/live-button.tsx index 6c82a68496..d9f802204c 100644 --- a/packages/plugins/execution-history/src/react/live-button.tsx +++ b/packages/plugins/execution-history/src/react/live-button.tsx @@ -9,6 +9,7 @@ export function LiveButton(props: { readonly active: boolean; readonly onToggle: variant="outline" size="sm" aria-pressed={props.active} + title={props.active ? "Pause live refresh (j)" : "Start live refresh (j)"} onClick={props.onToggle} className={cn( "gap-1.5", diff --git a/packages/plugins/execution-history/src/react/refresh-button.tsx b/packages/plugins/execution-history/src/react/refresh-button.tsx index ef3d0d2710..d7e778e2cf 100644 --- a/packages/plugins/execution-history/src/react/refresh-button.tsx +++ b/packages/plugins/execution-history/src/react/refresh-button.tsx @@ -11,6 +11,7 @@ export function RefreshButton(props: { variant="ghost" size="icon-sm" aria-label="Refresh" + title="Refresh (r)" onClick={props.onClick} disabled={props.isLoading} > diff --git a/packages/plugins/execution-history/src/react/run-row.tsx b/packages/plugins/execution-history/src/react/run-row.tsx index 290ef27ffd..dbb753402e 100644 --- a/packages/plugins/execution-history/src/react/run-row.tsx +++ b/packages/plugins/execution-history/src/react/run-row.tsx @@ -3,8 +3,9 @@ import { cn } from "@executor-js/react/lib/utils"; import { Badge } from "@executor-js/react/components/badge"; import type { RunRow } from "../sdk/collections"; -import { formatDateTime, formatDuration, formatRelative, statusLabel } from "./format"; +import { formatDuration, formatRelative, logLines, statusLabel } from "./format"; import { actorTone, STATUS_TONES, triggerTone } from "./status"; +import { HoverCardTimestamp } from "./hover-card-timestamp"; import type { RunColumns } from "./view"; // Column slot classes β€” RunListRow and RunsColumnHeader both apply these @@ -24,6 +25,8 @@ export const COL_ACTOR = "hidden w-[170px] shrink-0 lg:flex"; export const COL_DURATION = "hidden w-[100px] shrink-0 md:flex"; export const COL_TOOLS = "hidden xl:block w-[80px] shrink-0"; export const COL_INTERACTION = "hidden xl:block w-[100px] shrink-0"; +// Log error/warn counts β€” least critical, so only on the widest screens. +export const COL_LOG = "hidden 2xl:block w-[80px] shrink-0"; export interface RunListRowProps { readonly run: RunRow; @@ -38,6 +41,9 @@ export function RunListRow({ run, selected, isPast, columns, onSelect }: RunList const trigger = triggerTone(run.triggerKind); const actor = actorTone(run.actorKind); const isLive = run.status === "running" || run.status === "waiting_for_interaction"; + const logs = columns.log ? logLines(run.logsJson) : []; + const logErrors = logs.filter((line) => line.toLowerCase().includes("[error]")).length; + const logWarns = logs.filter((line) => line.toLowerCase().includes("[warn]")).length; return ( diff --git a/packages/plugins/execution-history/src/react/shell.tsx b/packages/plugins/execution-history/src/react/shell.tsx index cf4b0b7583..b4eb9c0ee4 100644 --- a/packages/plugins/execution-history/src/react/shell.tsx +++ b/packages/plugins/execution-history/src/react/shell.tsx @@ -125,6 +125,11 @@ export function RunsShell(props: RunsShellProps) { Loading more…
)} + {!hasMore && !isLoadingMore && ( +
+ End of history +
+ )} )}
diff --git a/packages/plugins/execution-history/src/react/timeline-chart.tsx b/packages/plugins/execution-history/src/react/timeline-chart.tsx index 7b89837079..d221cc049b 100644 --- a/packages/plugins/execution-history/src/react/timeline-chart.tsx +++ b/packages/plugins/execution-history/src/react/timeline-chart.tsx @@ -51,15 +51,17 @@ const buildAxisFormatter = const date = new Date(value); if (Number.isNaN(date.getTime())) return "β€”"; const options: Intl.DateTimeFormatOptions = - bucketMs < DAY_MS - ? { hour: "2-digit", minute: "2-digit", hour12: false } - : bucketMs >= 7 * DAY_MS - ? { month: "short", day: "numeric" } - : { month: "2-digit", day: "2-digit" }; + bucketMs <= 60_000 + ? { hour: "2-digit", minute: "2-digit", second: "2-digit", hour12: false } + : bucketMs < DAY_MS + ? { hour: "2-digit", minute: "2-digit", hour12: false } + : bucketMs >= 7 * DAY_MS + ? { month: "short", day: "numeric" } + : { month: "2-digit", day: "2-digit" }; return new Intl.DateTimeFormat(undefined, options).format(date); }; -const formatTooltipLabel = (value: number): string => { +const formatTooltipLabel = (value: number, bucketMs: number): string => { const date = new Date(value); if (Number.isNaN(date.getTime())) return "β€”"; return new Intl.DateTimeFormat(undefined, { @@ -67,6 +69,7 @@ const formatTooltipLabel = (value: number): string => { day: "numeric", hour: "2-digit", minute: "2-digit", + ...(bucketMs <= 60_000 ? { second: "2-digit" as const } : {}), hour12: false, }).format(date); }; @@ -163,11 +166,15 @@ export function RunsTimelineChart(props: { /> { const first = payload?.[0]?.payload as { bucketStart?: number } | undefined; - return first?.bucketStart == null ? "β€”" : formatTooltipLabel(first.bucketStart); + return first?.bucketStart == null + ? "β€”" + : formatTooltipLabel(first.bucketStart, props.bucketMs); }} /> } diff --git a/packages/plugins/execution-history/src/react/view-options.tsx b/packages/plugins/execution-history/src/react/view-options.tsx index 37456a907c..05d6e75375 100644 --- a/packages/plugins/execution-history/src/react/view-options.tsx +++ b/packages/plugins/execution-history/src/react/view-options.tsx @@ -11,6 +11,7 @@ const COLUMN_KEYS: readonly RunColumnKey[] = [ "duration", "tools", "interaction", + "log", ]; export function ViewOptions(props: { diff --git a/packages/plugins/execution-history/src/react/view.ts b/packages/plugins/execution-history/src/react/view.ts index b8bf27019d..c19aae9c47 100644 --- a/packages/plugins/execution-history/src/react/view.ts +++ b/packages/plugins/execution-history/src/react/view.ts @@ -10,6 +10,7 @@ export interface RunColumns { readonly duration: boolean; readonly tools: boolean; readonly interaction: boolean; + readonly log: boolean; } export type RunColumnKey = keyof RunColumns; @@ -20,6 +21,7 @@ export const DEFAULT_COLUMNS: RunColumns = { duration: true, tools: true, interaction: true, + log: false, }; export const RUN_COLUMN_LABELS: Record = { @@ -28,4 +30,5 @@ export const RUN_COLUMN_LABELS: Record = { duration: "Duration", tools: "Tools", interaction: "Interaction", + log: "Log", }; From 7c77922da8af44fd37a08f192afd8e21345de254 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 02:46:25 +0530 Subject: [PATCH 12/57] feat(host-cloudflare): attribute MCP-session runs to a trigger actor (#22) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Threads the run **actor** through the Cloudflare MCP session Durable Object so MCP-triggered runs carry a `{kind:"mcp", actor}` trigger β€” the MCP half of run actor attribution (HTTP runs already got one). ## Changes - `McpSessionInit` gains an optional `actor`; the worker dispatcher stamps it from the gate's resolved principal at session create. - The shared DO base carries `actor` onto the persisted `SessionMeta` (alongside `webOrigin`), so a cold isolate rebuilds with the same attribution. - Each host's `buildMcpServer` (`apps/host-cloudflare`) stamps the trigger, falling back to a `user` actor keyed by the session user when the principal supplied none. Generic plumbing only β€” it threads whatever actor the principal carries (no service-token specifics here). Pairs with the actor contract from #20. ## Verification - `typecheck` 41/41 Β· lint + format clean on all changed files (Build/Deploy-preview checks fail on the fork's missing R2 secrets, as on prior PRs β€” unrelated.) --- .../src/mcp/session-durable-object.ts | 9 ++++++++- packages/hosts/cloudflare/src/mcp/seams.ts | 7 +++++++ .../src/mcp/session-durable-object.ts | 20 +++++++++++++------ .../hosts/cloudflare/src/mcp/session-store.ts | 6 +++++- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/apps/host-cloudflare/src/mcp/session-durable-object.ts b/apps/host-cloudflare/src/mcp/session-durable-object.ts index 258e70c411..2640aba23c 100644 --- a/apps/host-cloudflare/src/mcp/session-durable-object.ts +++ b/apps/host-cloudflare/src/mcp/session-durable-object.ts @@ -77,7 +77,14 @@ export class McpSessionDO extends McpSessionDOBase { sessionMeta.organizationId, sessionMeta.organizationName, ).pipe(Effect.provide(makeCloudflareExecutionStackLayer(config, dbHandle))); - const mcpServer = yield* createExecutorMcpServer({ engine }); + // Attribute every run from this MCP session to its actor β€” carried from the + // gate's principal (a host that sets a non-user actor, e.g. a service token) + // when present, else the session user. Makes MCP runs filterable by actor. + const actor = sessionMeta.actor ?? { kind: "user", id: sessionMeta.userId, label: null }; + const mcpServer = yield* createExecutorMcpServer({ + engine, + trigger: { kind: "mcp", actor }, + }); return { mcpServer, engine } satisfies BuiltMcpServer; }).pipe( Effect.withSpan("McpSessionDO.buildMcpServer"), diff --git a/packages/hosts/cloudflare/src/mcp/seams.ts b/packages/hosts/cloudflare/src/mcp/seams.ts index 330f782005..4a8b10e810 100644 --- a/packages/hosts/cloudflare/src/mcp/seams.ts +++ b/packages/hosts/cloudflare/src/mcp/seams.ts @@ -1,3 +1,5 @@ +import type { ExecutionActor } from "@executor-js/sdk/core"; + import type { IncomingPropagationHeaders, McpElicitationMode } from "./do-headers"; // --------------------------------------------------------------------------- @@ -14,6 +16,11 @@ export interface McpSessionInit { /** Public origin of the create request (`https://host`), so the DO derives a * web base URL zero-config when the host configures no static one. */ readonly webOrigin?: string; + /** The credential identity this session's runs act as β€” set from the gate's + * resolved principal so MCP runs attribute to the actual actor (a service + * token stays distinct from the human subject it shares a partition with). + * Session-stable, so it rides the create init rather than per-request headers. */ + readonly actor?: ExecutionActor; } /** diff --git a/packages/hosts/cloudflare/src/mcp/session-durable-object.ts b/packages/hosts/cloudflare/src/mcp/session-durable-object.ts index 250911f315..6404a09441 100644 --- a/packages/hosts/cloudflare/src/mcp/session-durable-object.ts +++ b/packages/hosts/cloudflare/src/mcp/session-durable-object.ts @@ -18,6 +18,7 @@ import type { TransportState } from "agents/mcp"; import { jsonRpcErrorBody } from "@executor-js/host-mcp"; import { RequestWebOrigin } from "@executor-js/api/server"; +import type { ExecutionActor } from "@executor-js/sdk/core"; import { formatPausedExecution, type ExecutionEngine, @@ -140,6 +141,10 @@ export interface SessionMeta { /** Public origin captured at session create β€” used to derive the runtime's * web base URL when the host configures no static one. */ readonly webOrigin?: string; + /** The session's run actor (carried from the create init), so the host's + * `buildMcpServer` can stamp it as the MCP trigger actor. Persisted with the + * meta so a cold isolate rebuilds the runtime with the same attribution. */ + readonly actor?: ExecutionActor; } /** What a host's `buildMcpServer` seam returns: the connected MCP server plus @@ -559,12 +564,15 @@ export abstract class McpSessionDOBase< const self = this; return Effect.gen(function* () { const resolved = yield* self.resolveSessionMeta(token); - // Carry the create request's origin onto the persisted meta (the host's - // resolveSessionMeta is identity-only and doesn't see it), so a cold - // isolate rebuilds the runtime with the same web base URL. - const sessionMeta: SessionMeta = token.webOrigin - ? { ...resolved, webOrigin: token.webOrigin } - : resolved; + // Carry the create request's origin + run actor onto the persisted meta + // (the host's resolveSessionMeta is identity-only and sees neither), so a + // cold isolate rebuilds the runtime with the same web base URL + the same + // run attribution. + const sessionMeta: SessionMeta = { + ...resolved, + ...(token.webOrigin ? { webOrigin: token.webOrigin } : {}), + ...(token.actor ? { actor: token.actor } : {}), + }; yield* Effect.promise(() => self.saveSessionMeta(sessionMeta)).pipe( Effect.withSpan("mcp.session.save_meta"), ); diff --git a/packages/hosts/cloudflare/src/mcp/session-store.ts b/packages/hosts/cloudflare/src/mcp/session-store.ts index c2f29f8ac9..9717b33b29 100644 --- a/packages/hosts/cloudflare/src/mcp/session-store.ts +++ b/packages/hosts/cloudflare/src/mcp/session-store.ts @@ -26,6 +26,7 @@ import { type McpDispatchInput, type McpDispatchResult, } from "@executor-js/host-mcp"; +import type { ExecutionActor } from "@executor-js/sdk/core"; import { currentPropagationHeaders, @@ -87,6 +88,7 @@ const createSession = ( config: DurableObjectStoreConfig, request: Request, token: VerifiedTokenHeaders, + actor: ExecutionActor | undefined, ): Effect.Effect => Effect.gen(function* () { const stub = config.newStub(); @@ -101,6 +103,8 @@ const createSession = ( // base URL with no static config (we read the real URL, not a spoofable // forwarded host). webOrigin: new URL(request.url).origin, + // Stamp the session's run actor once at create (session-stable). + actor, }, propagation, ), @@ -167,7 +171,7 @@ export const makeDurableObjectMcpSessionStore = ( }; return sessionId ? forwardToExistingSession(config, request, sessionId, request.method !== "GET", token) - : createSession(config, request, token); + : createSession(config, request, token, principal.actor); }, dispose: (sessionId, request) => clearExistingSession(config, sessionId, request), }); From 75a2273148288b75d6e3921f9aa2bc4d95afd756 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 03:15:27 +0530 Subject: [PATCH 13/57] fix(execution-history): tolerate runs stored before the actor fields (#23) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## The bug The runs list returns an **empty-body 400** for everyone. Root cause is a schema/data-evolution bug from #20, not anything resource-related: `RunRow.actorId/actorLabel/actorKind` were `Schema.NullOr(...)` β€” present-but-nullable, which means **the key must exist**. Runs are stored as JSON documents, and every run written *before* #20 has a `data` doc with no such key. The store passes stored docs straight to the HTTP response encoder, which rejects the newest row with `Missing key at ["runs"][0]["actorId"]`. A response-encode failure emits a bare 400 (no body, no content-type) β€” the empty-body signature. Since no executions have happened since the actor store deployed, 100% of stored runs are legacy and every request fails. ## The fix Make the three actor keys optional with a **decoding default of null**: ```ts actorId: Schema.optional(Schema.NullOr(Schema.String)).pipe( Schema.withDecodingDefaultType(Effect.succeed(null)), ) ``` - **Decoded type stays `string | null`** (always present) β€” every reader still treats the field as required; it just defaults to `null` for pre-actor runs. - **Wire/storage tolerates an absent key** β€” legacy docs decode *and* encode unchanged. - Immunizes the collection against the next field added the same way. No data migration, no prod writes β€” deploy-forward and the page is back with all run history intact. ## Verification - New regression test round-trips a legacy-shaped doc (decode β†’ null; **encode with the keys absent succeeds** β€” the exact failure). - execution-history suite 20/20, typecheck 41/41, lint + format clean. (Build/Deploy-preview checks fail on the fork's missing R2 secrets, as on prior PRs β€” unrelated.) --- .../src/sdk/collections.test.ts | 80 +++++++++++++++++++ .../execution-history/src/sdk/collections.ts | 26 ++++-- 2 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 packages/plugins/execution-history/src/sdk/collections.test.ts diff --git a/packages/plugins/execution-history/src/sdk/collections.test.ts b/packages/plugins/execution-history/src/sdk/collections.test.ts new file mode 100644 index 0000000000..98aa2346c3 --- /dev/null +++ b/packages/plugins/execution-history/src/sdk/collections.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Schema } from "effect"; + +import { RunRow } from "./collections"; + +// --------------------------------------------------------------------------- +// Schema-evolution tolerance for the JSON-document `runs` collection. +// +// `#20` added actorId/actorLabel/actorKind. Runs stored BEFORE that have no such +// key at all. The store passes the raw stored doc straight to the HTTP response +// encoder, so a legacy doc must both decode AND encode through `RunRow` without a +// "Missing key" error β€” otherwise the entire runs list 400s on the newest page. +// These fields use an optional key + a decoding default of null precisely so +// that holds. +// --------------------------------------------------------------------------- + +// Hoisted: `Schema.*Sync` compiles a function, so it must not be rebuilt per call. +const decodeRunRow = Schema.decodeUnknownSync(RunRow); +const encodeRunRow = Schema.encodeUnknownSync(RunRow); + +// A run document as written before the actor fields existed: the three actor +// keys are physically absent. +const legacyRunDoc = { + executionId: "exec_legacy", + status: "completed", + code: "noop", + resultJson: null, + errorText: null, + logsJson: null, + triggerKind: null, + triggerMetaJson: null, + startedAt: 1000, + completedAt: 2000, + durationMs: 1000, + toolCallCount: 0, + hadInteraction: false, +}; + +describe("RunRow legacy-document tolerance", () => { + it("decodes a legacy doc, defaulting the absent actor fields to null", () => { + const decoded = decodeRunRow(legacyRunDoc); + expect(decoded.actorId).toBeNull(); + expect(decoded.actorLabel).toBeNull(); + expect(decoded.actorKind).toBeNull(); + // The decoded type is `string | null` (always present) β€” readers treat it as + // required; it just defaults to null for pre-actor runs. + expect(decoded.executionId).toBe("exec_legacy"); + }); + + it("encodes a legacy doc with the actor keys absent (the runs-response regression)", () => { + // The store hands the raw stored doc to the response encoder, so the encode + // input is the legacy shape itself. With `NullOr` this threw + // "Missing key at [actorId]"; with the optional key it must succeed, and the + // absent actor keys stay absent (the encoder must not fabricate them). + const encoded = encodeRunRow(legacyRunDoc); + expect(encoded.executionId).toBe("exec_legacy"); + expect(encoded.actorId ?? null).toBeNull(); + expect(encoded.actorLabel ?? null).toBeNull(); + expect(encoded.actorKind ?? null).toBeNull(); + }); + + it("round-trips a run that DOES carry an actor through decode AND encode", () => { + const withActor = { + ...legacyRunDoc, + actorId: "tok.access", + actorLabel: "phoenix", + actorKind: "service-token", + }; + const decoded = decodeRunRow(withActor); + expect(decoded.actorId).toBe("tok.access"); + expect(decoded.actorLabel).toBe("phoenix"); + expect(decoded.actorKind).toBe("service-token"); + // Encode must carry the present keys through unchanged (the path the runs + // response actually takes for an attributed run). + const encoded = encodeRunRow(decoded); + expect(encoded.actorId).toBe("tok.access"); + expect(encoded.actorLabel).toBe("phoenix"); + expect(encoded.actorKind).toBe("service-token"); + }); +}); diff --git a/packages/plugins/execution-history/src/sdk/collections.ts b/packages/plugins/execution-history/src/sdk/collections.ts index 7214403cb6..737c7db6fc 100644 --- a/packages/plugins/execution-history/src/sdk/collections.ts +++ b/packages/plugins/execution-history/src/sdk/collections.ts @@ -1,4 +1,4 @@ -import { Schema } from "effect"; +import { Effect, Schema } from "effect"; import { definePluginStorageCollection } from "@executor-js/sdk/core"; @@ -50,11 +50,25 @@ export const RunRow = Schema.Struct({ // Who/what the run acted as (from the trigger's `ExecutionActor`). `actorId` // is the STABLE filter/facet key (a token client id, a user subject); // `actorLabel` is the display snapshot at run time (machine name, email); - // `actorKind` is the credential class ("user", "service-token"). Null on runs - // recorded before a trigger actor was supplied. - actorId: Schema.NullOr(Schema.String), - actorLabel: Schema.NullOr(Schema.String), - actorKind: Schema.NullOr(Schema.String), + // `actorKind` is the credential class ("user", "service-token"). + // + // Optional-key + decoding default, NOT `NullOr`: runs are stored as JSON + // documents, and rows written BEFORE these fields existed have no such key at + // all. `NullOr` requires the key to be present, so a legacy doc fails the + // response encoder ("Missing key at runs[0].actorId"). Making the key optional + // (tolerant on the wire) while defaulting an absent key to null on decode lets + // older docs decode/encode unchanged β€” and immunizes the collection against + // the next field added the same way. The decoded type stays `string | null` + // (always present), so every reader treats it as required. + actorId: Schema.optional(Schema.NullOr(Schema.String)).pipe( + Schema.withDecodingDefaultType(Effect.succeed(null)), + ), + actorLabel: Schema.optional(Schema.NullOr(Schema.String)).pipe( + Schema.withDecodingDefaultType(Effect.succeed(null)), + ), + actorKind: Schema.optional(Schema.NullOr(Schema.String)).pipe( + Schema.withDecodingDefaultType(Effect.succeed(null)), + ), startedAt: Schema.Number, completedAt: Schema.NullOr(Schema.Number), durationMs: Schema.NullOr(Schema.Number), From 3e3bb25ce0f1446148958716e4123f14ba68b8cf Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 03:39:17 +0530 Subject: [PATCH 14/57] feat(execution-history): horizontal-scroll runs table instead of hiding columns Merge the runs table header and body into a single overflow-auto scroller with a min-width grid so the column header and rows share one horizontal scroll. Every column is now always rendered; when the viewport is narrower than the table's natural width the body scrolls horizontally instead of hiding columns at breakpoints. The code column flexes to fill on wider screens. Removes the ResizeObserver/stickyTop machinery that positioned the header, which the single scroller makes unnecessary. --- .../src/react/column-header.tsx | 2 +- .../execution-history/src/react/run-row.tsx | 34 +++--- .../execution-history/src/react/shell.tsx | 102 +++++++----------- 3 files changed, 58 insertions(+), 80 deletions(-) diff --git a/packages/plugins/execution-history/src/react/column-header.tsx b/packages/plugins/execution-history/src/react/column-header.tsx index 644e73cb49..100631110d 100644 --- a/packages/plugins/execution-history/src/react/column-header.tsx +++ b/packages/plugins/execution-history/src/react/column-header.tsx @@ -84,7 +84,7 @@ export function RunsColumnHeader({ sortField={sortField} sortDirection={sortDirection} onSort={onSort} - className="w-[150px] shrink-0 md:w-[190px]" + className="w-[180px] shrink-0" /> {/* Status */} diff --git a/packages/plugins/execution-history/src/react/run-row.tsx b/packages/plugins/execution-history/src/react/run-row.tsx index dbb753402e..4e13d1077a 100644 --- a/packages/plugins/execution-history/src/react/run-row.tsx +++ b/packages/plugins/execution-history/src/react/run-row.tsx @@ -9,24 +9,22 @@ import { HoverCardTimestamp } from "./hover-card-timestamp"; import type { RunColumns } from "./view"; // Column slot classes β€” RunListRow and RunsColumnHeader both apply these -// fixed-width, same-breakpoint slots (after the dot/started/status base -// columns) so the header and rows stay aligned across screen sizes. +// fixed-width slots (after the dot/started/status base columns) so the header +// and rows stay pixel-aligned. Every column is always rendered; when the +// viewport is narrower than the table's natural width the body scrolls +// horizontally (see RunsShell's min-width wrapper) instead of hiding columns. // -// Trigger/duration carry inline content (a dot+label, a sort button), so their -// visible state must be a flex display β€” `xl:flex`/`md:flex` rather than -// `xl:block`/`md:block`. A `block` slot composed with an element's own -// `inline-flex` makes tailwind-merge pick one display value, which silently -// desyncs header and row (the trigger column) or wraps the sort label onto a -// second line (duration). A single flex display keeps both in lockstep. -export const COL_TRIGGER = "hidden w-[120px] shrink-0 xl:flex"; -// Actor carries a dot + label (machine name / email / user), so it is a flex -// slot like trigger. Wider, since labels are longer than trigger kinds. -export const COL_ACTOR = "hidden w-[170px] shrink-0 lg:flex"; -export const COL_DURATION = "hidden w-[100px] shrink-0 md:flex"; -export const COL_TOOLS = "hidden xl:block w-[80px] shrink-0"; -export const COL_INTERACTION = "hidden xl:block w-[100px] shrink-0"; -// Log error/warn counts β€” least critical, so only on the widest screens. -export const COL_LOG = "hidden 2xl:block w-[80px] shrink-0"; +// Trigger/duration/actor carry inline content (a dot+label, a sort button), so +// their display must be `flex` β€” composing `block` with an element's own +// `inline-flex` makes tailwind-merge collapse to one display value, which +// silently desyncs the header slot from the row slot (trigger) or wraps the +// sort label onto a second line (duration). A `flex` slot keeps both in lockstep. +export const COL_TRIGGER = "flex w-[120px] shrink-0"; +export const COL_ACTOR = "flex w-[170px] shrink-0"; +export const COL_DURATION = "flex w-[100px] shrink-0"; +export const COL_TOOLS = "block w-[80px] shrink-0"; +export const COL_INTERACTION = "block w-[100px] shrink-0"; +export const COL_LOG = "block w-[80px] shrink-0"; export interface RunListRowProps { readonly run: RunRow; @@ -69,7 +67,7 @@ export function RunListRow({ run, selected, isPast, columns, onSelect }: RunList {/* Status label */} diff --git a/packages/plugins/execution-history/src/react/shell.tsx b/packages/plugins/execution-history/src/react/shell.tsx index b4eb9c0ee4..9118e55060 100644 --- a/packages/plugins/execution-history/src/react/shell.tsx +++ b/packages/plugins/execution-history/src/react/shell.tsx @@ -6,10 +6,11 @@ import { cn } from "@executor-js/react/lib/utils"; // RunsShell β€” two-column layout (filter rail + main pane) with infinite scroll. // // The left
- - - {run.errorText && ( + + + {props.errorText && (

Error

-                  {run.errorText}
+                  {props.errorText}
)} @@ -384,10 +381,10 @@ function DetailContent(props: { )} - + - + @@ -410,6 +407,11 @@ function DetailLoader(props: { value ? ( line.toLowerCase().includes("[error]")).length; - const logWarns = logs.filter((line) => line.toLowerCase().includes("[warn]")).length; + // Log error/warn counts are denormalized onto the slim row at write time, so + // the list never fetches/parses the full logs (now in the R2 detail object). + const logErrors = run.logErrorCount; + const logWarns = run.logWarnCount; return ( ); diff --git a/packages/plugins/execution-history/src/react/runs-list-state.test.ts b/packages/plugins/execution-history/src/react/runs-list-state.test.ts index 3fd9dea1ab..449530be86 100644 --- a/packages/plugins/execution-history/src/react/runs-list-state.test.ts +++ b/packages/plugins/execution-history/src/react/runs-list-state.test.ts @@ -12,12 +12,10 @@ import { const row = (id: string, startedAt: number): RunRow => ({ executionId: id, status: "completed", - code: "noop", - resultJson: null, - errorText: null, - logsJson: null, + codePreview: "noop", triggerKind: "cli", - triggerMetaJson: null, + logErrorCount: 0, + logWarnCount: 0, actorId: null, actorLabel: null, actorKind: null, diff --git a/packages/plugins/execution-history/src/sdk/collections.test.ts b/packages/plugins/execution-history/src/sdk/collections.test.ts index 98aa2346c3..dd0f63ee4b 100644 --- a/packages/plugins/execution-history/src/sdk/collections.test.ts +++ b/packages/plugins/execution-history/src/sdk/collections.test.ts @@ -23,12 +23,10 @@ const encodeRunRow = Schema.encodeUnknownSync(RunRow); const legacyRunDoc = { executionId: "exec_legacy", status: "completed", - code: "noop", - resultJson: null, - errorText: null, - logsJson: null, + codePreview: "noop", triggerKind: null, - triggerMetaJson: null, + logErrorCount: 0, + logWarnCount: 0, startedAt: 1000, completedAt: 2000, durationMs: 1000, diff --git a/packages/plugins/execution-history/src/sdk/collections.ts b/packages/plugins/execution-history/src/sdk/collections.ts index 737c7db6fc..bffe84a468 100644 --- a/packages/plugins/execution-history/src/sdk/collections.ts +++ b/packages/plugins/execution-history/src/sdk/collections.ts @@ -5,14 +5,21 @@ import { definePluginStorageCollection } from "@executor-js/sdk/core"; // --------------------------------------------------------------------------- // Execution-history storage collections. // -// Three owner-scoped plugin-storage collections back the run history: one row -// per execution (`runs`), per tool call (`toolCalls`), and per interaction -// (`interactions`). Every payload that the engine hands us as `unknown` (tool -// args/results, interaction payloads/responses, execution results/logs) is -// stored as an already-serialized JSON string in a `*Json` column so the -// indexed columns stay primitive and query-friendly. Indexes are declared so -// the read surface can filter/sort on them (the facade type-enforces that only -// declared fields appear in `where`/`orderBy`). +// ONE owner-scoped plugin-storage collection backs the run history: a slim +// `runs` row per execution carrying only the fields the list + aggregate +// surface needs (status, trigger, actor, timing, counts) plus two bounded +// denormalized fields β€” `codePreview` (the list's snippet) and +// `logErrorCount`/`logWarnCount` (the list's optional log column) β€” so the list +// renders entirely from D1 with no per-row blob fetch. +// +// The bulky, drawer-only detail (full `code`, `resultJson`, `errorText`, +// `logsJson`, `triggerMetaJson`, and the per-tool-call / per-interaction rows) +// lives in an append-only R2 object per run (see `detail-types.ts` + the store's +// `deps.blobs` writes), keeping the D1 row tiny and uncapped. `ToolCallRow` / +// `InteractionRow` remain here as the shared detail row shapes (they back the R2 +// detail object and the read response), but no longer have their own collection. +// Indexes are declared so the read surface can filter/sort on them (the facade +// type-enforces that only declared fields appear in `where`/`orderBy`). // --------------------------------------------------------------------------- /** Terminal + transient lifecycle state of a single execution. */ @@ -41,12 +48,16 @@ export type InteractionStatus = typeof InteractionStatus.Type; export const RunRow = Schema.Struct({ executionId: Schema.String, status: RunStatus, - code: Schema.String, - resultJson: Schema.NullOr(Schema.String), - errorText: Schema.NullOr(Schema.String), - logsJson: Schema.NullOr(Schema.String), + // Bounded list-snippet of the code (full source lives in the R2 detail blob). + // Normalized + truncated at write time so the list renders it directly. + // Required: this is a breaking change with no backfill β€” the cutover clears + // pre-migration rows, so every row the store writes carries these. + codePreview: Schema.String, triggerKind: Schema.NullOr(Schema.String), - triggerMetaJson: Schema.NullOr(Schema.String), + // Denormalized log-line counts for the list's optional log column, so the + // list never has to fetch + parse the full `logsJson` (now in R2). + logErrorCount: Schema.Number, + logWarnCount: Schema.Number, // Who/what the run acted as (from the trigger's `ExecutionActor`). `actorId` // is the STABLE filter/facet key (a token client id, a user subject); // `actorLabel` is the display snapshot at run time (machine name, email); @@ -81,6 +92,9 @@ export const runs = definePluginStorageCollection("runs", RunRow, { indexes: ["status", "triggerKind", "actorId", "startedAt", "durationMs", "hadInteraction"], }); +// Per-tool-call and per-interaction detail rows. No longer their own D1 +// collections β€” they are serialized into the run's R2 detail object (see +// `detail-types.ts`) and returned in the `get` detail response. export const ToolCallRow = Schema.Struct({ executionId: Schema.String, toolCallId: Schema.String, @@ -96,10 +110,6 @@ export const ToolCallRow = Schema.Struct({ }); export type ToolCallRow = typeof ToolCallRow.Type; -export const toolCalls = definePluginStorageCollection("toolCalls", ToolCallRow, { - indexes: ["executionId", "startedAt"], -}); - export const InteractionRow = Schema.Struct({ executionId: Schema.String, interactionId: Schema.String, @@ -113,7 +123,3 @@ export const InteractionRow = Schema.Struct({ completedAt: Schema.NullOr(Schema.Number), }); export type InteractionRow = typeof InteractionRow.Type; - -export const interactions = definePluginStorageCollection("interactions", InteractionRow, { - indexes: ["executionId", "startedAt"], -}); diff --git a/packages/plugins/execution-history/src/sdk/detail-types.ts b/packages/plugins/execution-history/src/sdk/detail-types.ts new file mode 100644 index 0000000000..988dd42647 --- /dev/null +++ b/packages/plugins/execution-history/src/sdk/detail-types.ts @@ -0,0 +1,30 @@ +import { Schema } from "effect"; + +import { InteractionRow, ToolCallRow } from "./collections"; + +// --------------------------------------------------------------------------- +// Run detail β€” the append-only R2 object written once per finished execution +// (and as a code-only stub on start). Holds the bulky, drawer-only payload that +// used to live in the fat `runs` row + the `toolCalls`/`interactions` +// collections: full code, the serialized result/error/logs/trigger-metadata, +// and the per-tool-call / per-interaction rows. +// +// Serialized to/from a JSON string at the blob boundary via Effect Schema +// (`Schema.fromJsonString`) β€” never raw `JSON.parse`/`JSON.stringify`. The +// `*Json` fields are already JSON strings (the store pre-serializes each +// `unknown` payload), so they round-trip as escaped strings the drawer parses. +// --------------------------------------------------------------------------- + +export const RunDetail = Schema.Struct({ + code: Schema.String, + resultJson: Schema.NullOr(Schema.String), + errorText: Schema.NullOr(Schema.String), + logsJson: Schema.NullOr(Schema.String), + triggerMetaJson: Schema.NullOr(Schema.String), + toolCalls: Schema.Array(ToolCallRow), + interactions: Schema.Array(InteractionRow), +}); +export type RunDetail = typeof RunDetail.Type; + +/** Codec between the R2 blob's JSON string and a typed {@link RunDetail}. */ +export const RunDetailFromJsonString = Schema.fromJsonString(RunDetail); diff --git a/packages/plugins/execution-history/src/sdk/index.ts b/packages/plugins/execution-history/src/sdk/index.ts index f562df4b9b..66d00f6161 100644 --- a/packages/plugins/execution-history/src/sdk/index.ts +++ b/packages/plugins/execution-history/src/sdk/index.ts @@ -1,9 +1,7 @@ export { executionHistoryPlugin } from "./plugin"; export { - interactions, runs, - toolCalls, InteractionRow, InteractionStatus, RunRow, @@ -12,6 +10,8 @@ export { ToolCallStatus, } from "./collections"; +export { RunDetail, RunDetailFromJsonString } from "./detail-types"; + export { makeExecutionHistoryObserver, makeExecutionHistoryStore, diff --git a/packages/plugins/execution-history/src/sdk/plugin.ts b/packages/plugins/execution-history/src/sdk/plugin.ts index b34857adc3..a16663040d 100644 --- a/packages/plugins/execution-history/src/sdk/plugin.ts +++ b/packages/plugins/execution-history/src/sdk/plugin.ts @@ -1,13 +1,14 @@ import { definePlugin } from "@executor-js/sdk/core"; -import { interactions, runs, toolCalls } from "./collections"; +import { runs } from "./collections"; import { makeExecutionHistoryObserver, makeExecutionHistoryStore } from "./store"; // --------------------------------------------------------------------------- // Execution-history plugin (SDK surface). A pure sink: it contributes no tools -// or integrations, only the three storage collections, a read surface -// (`list`/`get`/`listToolCalls`) on `executor.executionHistory`, and a runtime -// ExecutionObserver that records the engine's event stream. +// or integrations, only the slim `runs` storage collection (the bulky per-run +// detail lives in an R2 object via `deps.blobs`), a read surface (`list`/`get`) +// on `executor.executionHistory`, and a runtime ExecutionObserver that records +// the engine's event stream. // // One store instance is shared: `storage(deps)` builds it (read methods + // buffered `handleEvent` writer), `extension(ctx)` surfaces the read methods @@ -22,12 +23,11 @@ import { makeExecutionHistoryObserver, makeExecutionHistoryStore } from "./store export const executionHistoryPlugin = definePlugin(() => ({ id: "executionHistory" as const, packageName: "@executor-js/plugin-execution-history", - pluginStorage: { runs, toolCalls, interactions }, + pluginStorage: { runs }, storage: (deps) => makeExecutionHistoryStore(deps), extension: (ctx) => ({ list: ctx.storage.list, get: ctx.storage.get, - listToolCalls: ctx.storage.listToolCalls, handleEvent: ctx.storage.handleEvent, }), runtime: { diff --git a/packages/plugins/execution-history/src/sdk/store.test.ts b/packages/plugins/execution-history/src/sdk/store.test.ts index e2ddc820c2..95e28a397b 100644 --- a/packages/plugins/execution-history/src/sdk/store.test.ts +++ b/packages/plugins/execution-history/src/sdk/store.test.ts @@ -83,21 +83,20 @@ describe("execution-history store", () => { expect(run?.toolCallCount).toBe(1); expect(run?.durationMs).toBe(2000); expect(run?.hadInteraction).toBe(false); - // code + trigger from ExecutionStarted survive the terminal re-write. - expect(run?.code).toBe("await tools.shell({ command: 'ls' })"); + // codePreview + trigger from ExecutionStarted survive the terminal re-write. + expect(run?.codePreview).toBe("await tools.shell({ command: 'ls' })"); expect(run?.triggerKind).toBe("manual"); const detail = yield* executor.executionHistory.get("exec_1"); expect(detail?.run.status).toBe("completed"); + // Full code + the tool call now come from the R2 detail object. + expect(detail?.code).toBe("await tools.shell({ command: 'ls' })"); expect(detail?.toolCalls).toHaveLength(1); expect(detail?.toolCalls[0]?.toolCallId).toBe("call_1"); expect(detail?.toolCalls[0]?.status).toBe("completed"); expect(detail?.toolCalls[0]?.durationMs).toBe(1000); + expect(detail?.toolCalls[0]?.path).toBe("tools.shell.org.default.run"); expect(detail?.interactions).toHaveLength(0); - - const toolCallRows = yield* executor.executionHistory.listToolCalls("exec_1"); - expect(toolCallRows).toHaveLength(1); - expect(toolCallRows[0]?.path).toBe("tools.shell.org.default.run"); }), ); }); diff --git a/packages/plugins/execution-history/src/sdk/store.ts b/packages/plugins/execution-history/src/sdk/store.ts index f8526c6990..a00513f686 100644 --- a/packages/plugins/execution-history/src/sdk/store.ts +++ b/packages/plugins/execution-history/src/sdk/store.ts @@ -11,6 +11,7 @@ import { type PluginStorageFacade, type StorageDeps, type StorageFailure, + StorageError, } from "@executor-js/sdk/core"; import { @@ -20,26 +21,33 @@ import { type ToolCallRow, type ToolCallStatus, RunStatus, - interactions, runs, - toolCalls, } from "./collections"; +import { type RunDetail, RunDetailFromJsonString } from "./detail-types"; // --------------------------------------------------------------------------- -// Execution-history store. Translates the engine's ExecutionEvent stream into -// durable run/tool-call/interaction rows and exposes the read surface. +// Execution-history store. Translates the engine's ExecutionEvent stream into a +// slim D1 `runs` index row + an append-only R2 detail object, and exposes the +// read surface. +// +// Storage split: +// - D1 `runs` row (via pluginStorage): slim, indexed, list-renderable β€” status, +// trigger, actor, timing, counts, plus `codePreview` and `logErrorCount`/ +// `logWarnCount` so the list never fetches a blob. +// - R2 detail object (via `deps.blobs`, keyed `run-detail/`): the +// bulky drawer-only payload β€” full code, result/error/logs/trigger-metadata, +// and the per-tool-call / per-interaction rows. Written once. // // Write model β€” buffered batch: tool-call and interaction detail is held in an -// in-memory buffer keyed by executionId and only flushed when the execution -// finishes, so a completed run lands as one batch of writes rather than a -// write-per-event. Two points are written eagerly for durability even before -// the buffer flushes: the `runs` row on ExecutionStarted (status "running") and -// again on InteractionStarted (status "waiting_for_interaction"), so a paused -// run survives a restart while it waits on the user. +// in-memory buffer keyed by executionId and flushed into the R2 detail object +// when the execution finishes. Eager writes for durability: on ExecutionStarted +// the slim "running" row AND a code-only stub detail blob (so a live/paused run +// shows its code immediately and survives a restart); on InteractionStarted the +// "waiting_for_interaction" row. // -// Every `unknown` payload (tool args/results, interaction payload/response, -// execution result/logs) is serialized to a JSON string via Effect Schema -// (`Schema.UnknownFromJsonString`) β€” no raw `JSON.stringify` in domain code. +// Every `unknown` payload is serialized to a JSON string via Effect Schema +// (`Schema.UnknownFromJsonString`); the detail object round-trips via +// `Schema.fromJsonString` β€” no raw `JSON.stringify`/`JSON.parse` in domain code. // --------------------------------------------------------------------------- /** Serialize an arbitrary value to a JSON string, or null when absent or when @@ -60,6 +68,35 @@ const namespaceOf = (path: string): string | null => { return index > 0 ? path.slice(0, index) : null; }; +/** Bounded, whitespace-normalized snippet of the code for the list column. The + * full code lives in the R2 detail object. */ +const codePreviewOf = (code: string): string => code.trim().replace(/\s+/g, " ").slice(0, 256); + +/** Count `[error]` / `[warn]` log lines (case-insensitive substring, counted + * independently β€” matches the list's prior logsJson parsing) so the list reads + * precomputed scalars instead of fetching + parsing the full logs from R2. */ +const logCountsOf = (logs: unknown): { errors: number; warns: number } => { + if (!Array.isArray(logs)) return { errors: 0, warns: 0 }; + let errors = 0; + let warns = 0; + for (const line of logs) { + if (typeof line !== "string") continue; + const lower = line.toLowerCase(); + if (lower.includes("[error]")) errors += 1; + if (lower.includes("[warn]")) warns += 1; + } + return { errors, warns }; +}; + +/** R2 blob key for a run's detail object. */ +const detailBlobKey = (executionId: string): string => `run-detail/${executionId}`; + +/** Detail codec helpers (Schema-based; never raw JSON). Encode runs in the + * Effect channel so a (never-in-practice) failure maps to a typed StorageError + * rather than a `die`; decode reads gracefully as an `Option`. */ +const encodeDetail = Schema.encodeUnknownEffect(RunDetailFromJsonString); +const decodeDetail = Schema.decodeUnknownOption(RunDetailFromJsonString); + interface BufferedToolCall { toolCallId: ExecutionToolCallId; status: ToolCallStatus; @@ -195,6 +232,14 @@ export interface ExecutionHistoryListResult { export interface ExecutionHistoryDetail { readonly run: RunRow; + /** Bulky drawer payload, sourced from the R2 detail object (flat-merged onto + * the slim run row). For in-flight or detail-less runs these are the + * stub/empty values. */ + readonly code: string; + readonly resultJson: string | null; + readonly errorText: string | null; + readonly logsJson: string | null; + readonly triggerMetaJson: string | null; readonly toolCalls: readonly ToolCallRow[]; readonly interactions: readonly InteractionRow[]; } @@ -207,9 +252,6 @@ export interface ExecutionHistoryStore { readonly get: ( executionId: string, ) => Effect.Effect; - readonly listToolCalls: ( - executionId: string, - ) => Effect.Effect; } // --------------------------------------------------------------------------- @@ -275,16 +317,36 @@ const chooseBucketMs = (timeRange: { from?: number; to?: number } | undefined): export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistoryStore => { const pluginStorage: PluginStorageFacade = deps.pluginStorage; const runsC: PluginStorageCollectionFacade = pluginStorage.collection(runs); - const toolCallsC: PluginStorageCollectionFacade = - pluginStorage.collection(toolCalls); - const interactionsC: PluginStorageCollectionFacade = - pluginStorage.collection(interactions); + const blobs = deps.blobs; const buffers = new Map(); const putRun = (owner: Owner, row: RunRow): Effect.Effect => runsC.put({ owner, key: row.executionId, data: row }).pipe(Effect.asVoid); + /** Write the run's R2 detail object. A (never-in-practice) encode failure maps + * to a typed StorageError so it stays in the storage error channel. */ + const putDetail = ( + owner: Owner, + executionId: string, + detail: RunDetail, + ): Effect.Effect => + encodeDetail(detail).pipe( + Effect.mapError( + (cause) => + new StorageError({ message: "execution-history: failed to encode run detail", cause }), + ), + Effect.flatMap((json) => blobs.put(detailBlobKey(executionId), json, { owner })), + ); + + /** Read + decode the run's R2 detail object; `None` when absent or malformed. */ + const readDetail = ( + executionId: string, + ): Effect.Effect, StorageFailure> => + blobs + .get(detailBlobKey(executionId)) + .pipe(Effect.map((raw) => (raw === null ? Option.none() : decodeDetail(raw)))); + const onExecutionStarted = (event: Extract) => { const owner = ownerOf(event.owner); const startedAt = event.startedAt.getTime(); @@ -307,24 +369,38 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt toolCalls: new Map(), interactions: new Map(), }); - return putRun(owner, { - executionId: event.executionId, - status: "running", - code: event.code, - resultJson: null, - errorText: null, - logsJson: null, - triggerKind, - triggerMetaJson, - actorId, - actorLabel, - actorKind, - startedAt, - completedAt: null, - durationMs: null, - toolCallCount: 0, - hadInteraction: false, - }); + return Effect.all( + [ + putRun(owner, { + executionId: event.executionId, + status: "running", + codePreview: codePreviewOf(event.code), + triggerKind, + logErrorCount: 0, + logWarnCount: 0, + actorId, + actorLabel, + actorKind, + startedAt, + completedAt: null, + durationMs: null, + toolCallCount: 0, + hadInteraction: false, + }), + // Eager code-only stub: the drawer shows code while the run is live, and + // the code survives a restart that loses the in-memory buffer. + putDetail(owner, event.executionId, { + code: event.code, + resultJson: null, + errorText: null, + logsJson: null, + triggerMetaJson, + toolCalls: [], + interactions: [], + }), + ], + { concurrency: "unbounded", discard: true }, + ); }; const onToolCallStarted = (event: Extract) => { @@ -390,12 +466,10 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt return putRun(buffer.owner, { executionId: event.executionId, status: "waiting_for_interaction", - code: buffer.code, - resultJson: null, - errorText: null, - logsJson: null, + codePreview: codePreviewOf(buffer.code), triggerKind: buffer.triggerKind, - triggerMetaJson: buffer.triggerMetaJson, + logErrorCount: 0, + logWarnCount: 0, actorId: buffer.actorId, actorLabel: buffer.actorLabel, actorKind: buffer.actorKind, @@ -434,30 +508,29 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt const completedAt = event.completedAt.getTime(); const toolCallEntries = buffer ? Array.from(buffer.toolCalls.values()) : []; const interactionEntries = buffer ? Array.from(buffer.interactions.values()) : []; + const { errors: logErrorCount, warns: logWarnCount } = logCountsOf(event.logs); return Effect.gen(function* () { - // Preserve code/trigger/startedAt from the buffer, or from the persisted - // "running" row if the buffer was lost (e.g. a restart mid-run). + // Recover slim fields from the persisted "running" row if the buffer was + // lost (restart mid-run). Full code/trigger-metadata live only in the + // buffer + the start-stub blob. const existing = buffer ? null : yield* runsC.get({ key: event.executionId }); - const code = buffer?.code ?? existing?.data.code ?? ""; const triggerKind = buffer?.triggerKind ?? existing?.data.triggerKind ?? null; - const triggerMetaJson = buffer?.triggerMetaJson ?? existing?.data.triggerMetaJson ?? null; const actorId = buffer?.actorId ?? existing?.data.actorId ?? null; const actorLabel = buffer?.actorLabel ?? existing?.data.actorLabel ?? null; const actorKind = buffer?.actorKind ?? existing?.data.actorKind ?? null; const startedAt = buffer?.startedAt ?? existing?.data.startedAt ?? completedAt; + const codePreview = buffer ? codePreviewOf(buffer.code) : (existing?.data.codePreview ?? ""); const hadInteraction = buffer?.hadInteraction ?? (existing?.data.hadInteraction || interactionEntries.length > 0); yield* putRun(owner, { executionId: event.executionId, status: event.status, - code, - resultJson: toJson(event.result), - errorText: event.error ?? null, - logsJson: toJson(event.logs), + codePreview, triggerKind, - triggerMetaJson, + logErrorCount, + logWarnCount, actorId, actorLabel, actorKind, @@ -468,13 +541,18 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt hadInteraction, }); - yield* Effect.forEach( - toolCallEntries, - (entry) => - toolCallsC.put({ - owner, - key: entry.toolCallId, - data: { + // Write the full detail object. Normal path: assembled from the buffer. + // Buffer-lost path: preserve the start-stub's code + trigger-metadata and + // attach the terminal result/error/logs (per-call children are + // unrecoverable after a restart). + const detail: RunDetail = buffer + ? { + code: buffer.code, + resultJson: toJson(event.result), + errorText: event.error ?? null, + logsJson: toJson(event.logs), + triggerMetaJson: buffer.triggerMetaJson, + toolCalls: toolCallEntries.map((entry) => ({ executionId: event.executionId, toolCallId: entry.toolCallId, status: entry.status, @@ -486,18 +564,8 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt startedAt: entry.startedAt, completedAt: entry.completedAt, durationMs: entry.durationMs, - }, - }), - { discard: true }, - ); - - yield* Effect.forEach( - interactionEntries, - (entry) => - interactionsC.put({ - owner, - key: entry.interactionId, - data: { + })), + interactions: interactionEntries.map((entry) => ({ executionId: event.executionId, interactionId: entry.interactionId, status: entry.status, @@ -508,10 +576,24 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt errorText: entry.errorText, startedAt: entry.startedAt, completedAt: entry.completedAt, - }, - }), - { discard: true }, - ); + })), + } + : yield* readDetail(event.executionId).pipe( + Effect.map((stubOpt) => { + const stub = Option.getOrNull(stubOpt); + return { + code: stub?.code ?? "", + resultJson: toJson(event.result), + errorText: event.error ?? null, + logsJson: toJson(event.logs), + triggerMetaJson: stub?.triggerMetaJson ?? null, + toolCalls: [], + interactions: [], + }; + }), + ); + + yield* putDetail(owner, event.executionId, detail); }).pipe( // Always release the buffer, even if a write fails or is interrupted β€” // otherwise a StorageFailure during the flush leaks the RunBuffer forever. @@ -738,29 +820,22 @@ export const makeExecutionHistoryStore = (deps: StorageDeps): ExecutionHistorySt Effect.gen(function* () { const run = yield* runsC.get({ key: executionId }); if (run === null) return null; - const toolCallRows = yield* toolCallsC.query({ - where: { executionId }, - orderBy: [{ field: "startedAt" }], - }); - const interactionRows = yield* interactionsC.query({ - where: { executionId }, - orderBy: [{ field: "startedAt" }], - }); + // Detail comes from the R2 object. Absent/malformed (or an in-flight run + // whose stub is gone) β†’ empty detail so the drawer still renders the row. + const detail = Option.getOrNull(yield* readDetail(executionId)); return { run: run.data, - toolCalls: toolCallRows.map((entry) => entry.data), - interactions: interactionRows.map((entry) => entry.data), + code: detail?.code ?? "", + resultJson: detail?.resultJson ?? null, + errorText: detail?.errorText ?? null, + logsJson: detail?.logsJson ?? null, + triggerMetaJson: detail?.triggerMetaJson ?? null, + toolCalls: detail?.toolCalls ?? [], + interactions: detail?.interactions ?? [], }; }); - const listToolCalls = ( - executionId: string, - ): Effect.Effect => - toolCallsC - .query({ where: { executionId }, orderBy: [{ field: "startedAt" }] }) - .pipe(Effect.map((rows) => rows.map((entry) => entry.data))); - - return { handleEvent, list, get, listToolCalls }; + return { handleEvent, list, get }; }; /** Build an ExecutionObserver over a store instance β€” every engine event is diff --git a/packages/plugins/service-tokens/CHANGELOG.md b/packages/plugins/service-tokens/CHANGELOG.md new file mode 100644 index 0000000000..84aa5ad4d8 --- /dev/null +++ b/packages/plugins/service-tokens/CHANGELOG.md @@ -0,0 +1 @@ +# @executor-js/plugin-service-tokens From 1304189426cb6a8107acb1945c57ba46d6a02be4 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 21:06:40 +0530 Subject: [PATCH 25/57] feat(plugin-service-tokens): edit a machine name on an existing alias Each alias card gets an inline Edit (or "Add name" when unset) action that swaps the Name row for an input with Save/Cancel. Save reuses the existing `alias` upsert, so an empty value clears the name. Enter saves, Escape cancels. --- .../plugins/service-tokens/src/client.tsx | 141 ++++++++++++++---- 1 file changed, 113 insertions(+), 28 deletions(-) diff --git a/packages/plugins/service-tokens/src/client.tsx b/packages/plugins/service-tokens/src/client.tsx index f7bec8cbba..9f1ec8a2ce 100644 --- a/packages/plugins/service-tokens/src/client.tsx +++ b/packages/plugins/service-tokens/src/client.tsx @@ -51,6 +51,13 @@ function ServiceTokensPage() { const [machineName, setMachineName] = useState(""); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); + // Inline edit of an existing alias's machine name. `editing` holds the + // commonName of the card in edit mode (null = none). Saving reuses the `alias` + // upsert, so it also re-captures the editor's acts-as identity β€” fine here + // because you only ever edit your own aliases. + const [editing, setEditing] = useState(null); + const [editValue, setEditValue] = useState(""); + const [editBusy, setEditBusy] = useState(false); const aliasesResult = useAtomValue(aliasesAtom); const refresh = useAtomRefresh(aliasesAtom); @@ -105,6 +112,32 @@ function ServiceTokensPage() { if (!Exit.isFailure(exit)) refresh(); }; + const startEdit = (alias: ServiceTokenAlias) => { + setEditing(alias.commonName); + setEditValue(alias.machineName ?? ""); + setError(null); + }; + + const handleSaveEdit = async (commonNameToEdit: string) => { + const machine = editValue.trim(); + setEditBusy(true); + setError(null); + const exit = await doAlias({ + payload: { + commonName: commonNameToEdit, + ...(machine ? { machineName: machine } : {}), + }, + reactivityKeys: [], + }); + setEditBusy(false); + if (Exit.isFailure(exit)) { + setError("Failed to update machine name"); + return; + } + setEditing(null); + refresh(); + }; + return (
@@ -176,36 +209,88 @@ function ServiceTokensPage() { ) : ( - aliases.map((alias) => ( - - -
- {alias.machineName ? ( + aliases.map((alias) => { + const isEditing = editing === alias.commonName; + return ( + + +
+ {isEditing ? ( + <> + Name + setEditValue((e.target as HTMLInputElement).value)} + className="h-7 text-[12px]" + onKeyDown={(e) => { + if (e.key === "Enter") void handleSaveEdit(alias.commonName); + if (e.key === "Escape") setEditing(null); + }} + /> + + ) : alias.machineName ? ( + <> + Name + {alias.machineName} + + ) : null} + Token + + {alias.commonName} + + Acts as + + {alias.email ?? alias.name ?? alias.subject} + +
+ + + {isEditing ? ( + <> + + + + ) : ( <> - Name - {alias.machineName} + + - ) : null} - Token - {alias.commonName} - Acts as - - {alias.email ?? alias.name ?? alias.subject} - -
- - - - - - )) + )} + + + ); + }) )}
From aa3c278e08b08b762085ba1e31624c3acba7d81d Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 21:52:37 +0530 Subject: [PATCH 26/57] feat(execution-history): content-fit runs table columns + centered text (#25) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Rebuilds the runs list as a **real semantic `` with content-fit columns** (`table-layout: auto`), replacing the bespoke fixed-width flexbox layout. - **Content-fit**: each column sizes to the maximum width its rows need β€” the sticky `` and `` rows share one column model, so columns size *and* stay aligned automatically (no more shared `COL_*` fixed-width constants, no `min-w` scroll floor). - **Alignment**: short / numeric / badge columns (status, duration, tools, interaction, log) are **centered**; text columns (started, trigger, actor, code) are left-aligned. - **No blowout**: the two columns that can run long β€” `actor` (token id / email) and `code` β€” are width-capped + truncated (hover shows the full value). - The row `` (click + keyboard, accessible). The live divider / loading / empty / end-of-history states become `colSpan` rows. ## Why this shape (vs openstatus) Studied `openstatus-data-table`: it renders a real shadcn `
` but sets **fixed per-column `size`** values (CSS vars β†’ inline widths) β€” *not* content-fit. True "snap each column to its content" comes from a real table's default `table-layout: auto`, which is what this uses. Sticky-header border rendering uses `border-separate border-spacing-0` (the same browser-bug workaround openstatus uses). ## Notes - `
` carries a narrow `oxlint-disable react/forbid-elements` (with a boundary reason) β€” the runs table needs its own infinite-scroll container + sticky thead, which the shared `
`'s built-in overflow wrapper would conflict with. Same suppression the shared `Table` component itself uses. - Trade-off: content-fit means the table is its natural content width β€” narrower than the pane β†’ left-aligned with slack on the right; wider β†’ horizontal scroll. ## Verification typecheck 0 Β· lint 0/0 Β· format clean Β· tests 19/19. --- .../execution-history-content-fit-table.md | 13 ++ .../execution-history/src/react/RunsPage.tsx | 33 ++-- .../src/react/column-header.tsx | 133 ++++++++------- .../execution-history/src/react/live-row.tsx | 30 ++-- .../execution-history/src/react/run-row.tsx | 156 +++++++++--------- .../execution-history/src/react/shell.tsx | 101 +++++++----- 6 files changed, 266 insertions(+), 200 deletions(-) create mode 100644 .changeset/execution-history-content-fit-table.md diff --git a/.changeset/execution-history-content-fit-table.md b/.changeset/execution-history-content-fit-table.md new file mode 100644 index 0000000000..7ff22521ae --- /dev/null +++ b/.changeset/execution-history-content-fit-table.md @@ -0,0 +1,13 @@ +--- +"@executor-js/plugin-execution-history": patch +--- + +Rebuild the runs list as a real semantic `
` with content-fit columns. +Each column now sizes to the maximum width its rows need (browser +`table-layout: auto`), instead of fixed pixel widths β€” the sticky `` and +the rows share one column model so they stay aligned for free. Short / numeric / +badge columns (status, duration, tools, interaction, log) are centered; text +columns (started, trigger, actor, code) are left-aligned. The two columns that +can run long β€” `actor` and `code` β€” are width-capped and truncated (hover shows +the full value). Removes the fixed-width `COL_*` slot constants and the +`min-w` horizontal-scroll floor. diff --git a/packages/plugins/execution-history/src/react/RunsPage.tsx b/packages/plugins/execution-history/src/react/RunsPage.tsx index 931a278718..6c4173e0f7 100644 --- a/packages/plugins/execution-history/src/react/RunsPage.tsx +++ b/packages/plugins/execution-history/src/react/RunsPage.tsx @@ -10,7 +10,7 @@ import { LiveButton } from "./live-button"; import { LiveDivider } from "./live-row"; import { RefreshButton } from "./refresh-button"; import { RunListRow } from "./run-row"; -import { RunsShell } from "./shell"; +import { RunsShell, RUNS_TABLE_COLSPAN } from "./shell"; import { RunsTimelineChart } from "./timeline-chart"; import { emptyRunsFilters, @@ -165,15 +165,16 @@ export function RunsPage() { if (view.isLoadMoreError) { rowItems.push( -
-

Failed to load more

- -
, + + + , ); } @@ -226,9 +227,17 @@ export function RunsPage() { ) : undefined; const body = view.isLoading ? ( -
Loading runs…
+ + + ) : view.isError ? ( -
Unable to load runs.
+ + + ) : ( rowItems ); diff --git a/packages/plugins/execution-history/src/react/column-header.tsx b/packages/plugins/execution-history/src/react/column-header.tsx index 100631110d..1a2e9f47e9 100644 --- a/packages/plugins/execution-history/src/react/column-header.tsx +++ b/packages/plugins/execution-history/src/react/column-header.tsx @@ -4,38 +4,26 @@ import { cn } from "@executor-js/react/lib/utils"; import type { RunsSortField } from "./use-runs-list"; import type { RunColumns } from "./view"; -import { - COL_ACTOR, - COL_DURATION, - COL_INTERACTION, - COL_LOG, - COL_TOOLS, - COL_TRIGGER, -} from "./run-row"; // --------------------------------------------------------------------------- -// RunsColumnHeader β€” sticky header row aligned to RunListRow's layout. -// Imports the shared column slot classes from run-row so that header and rows -// use the same widths and breakpoints. +// RunsColumnHeader β€” the row. Cells are + {/* dot column (no label) */} + + + {/* Status (centered) */} + + + {/* Trigger (left) */} + {columns.trigger ? ( + + ) : null} + + {/* Actor (left) */} + {columns.actor ? ( + ) : null} - {/* Tools */} - {columns.tools ? tools : null} + {/* Duration (sortable, centered) */} + {columns.duration ? ( + + ) : null} - {/* Interaction */} - {columns.interaction ? interaction : null} + {/* Tools (centered) */} + {columns.tools ? ( + + ) : null} - {/* Log */} - {columns.log ? log : null} + {/* Interaction (centered) */} + {columns.interaction ? ( + + ) : null} + + {/* Log (centered) */} + {columns.log ? ( + + ) : null} - {/* Code */} - code - + {/* Code (left) */} + + ); } diff --git a/packages/plugins/execution-history/src/react/live-row.tsx b/packages/plugins/execution-history/src/react/live-row.tsx index c79d4c0b86..3bbd93da70 100644 --- a/packages/plugins/execution-history/src/react/live-row.tsx +++ b/packages/plugins/execution-history/src/react/live-row.tsx @@ -1,15 +1,23 @@ +import { RUNS_TABLE_COLSPAN } from "./shell"; + +// A full-width "Live" divider row between past and newly-arrived runs. Rendered +// as a . export function LiveDivider() { return ( -
-
- - - - - - Live - -
-
+
+ + ); } diff --git a/packages/plugins/execution-history/src/react/run-row.tsx b/packages/plugins/execution-history/src/react/run-row.tsx index 83df093a10..704398cf0a 100644 --- a/packages/plugins/execution-history/src/react/run-row.tsx +++ b/packages/plugins/execution-history/src/react/run-row.tsx @@ -1,4 +1,3 @@ -import { Button } from "@executor-js/react/components/button"; import { cn } from "@executor-js/react/lib/utils"; import { Badge } from "@executor-js/react/components/badge"; @@ -8,23 +7,13 @@ import { actorTone, STATUS_TONES, triggerTone } from "./status"; import { HoverCardTimestamp } from "./hover-card-timestamp"; import type { RunColumns } from "./view"; -// Column slot classes β€” RunListRow and RunsColumnHeader both apply these -// fixed-width slots (after the dot/started/status base columns) so the header -// and rows stay pixel-aligned. Every column is always rendered; when the -// viewport is narrower than the table's natural width the body scrolls -// horizontally (see RunsShell's min-width wrapper) instead of hiding columns. -// -// Trigger/duration/actor carry inline content (a dot+label, a sort button), so -// their display must be `flex` β€” composing `block` with an element's own -// `inline-flex` makes tailwind-merge collapse to one display value, which -// silently desyncs the header slot from the row slot (trigger) or wraps the -// sort label onto a second line (duration). A `flex` slot keeps both in lockstep. -export const COL_TRIGGER = "flex w-[120px] shrink-0"; -export const COL_ACTOR = "flex w-[170px] shrink-0"; -export const COL_DURATION = "flex w-[100px] shrink-0"; -export const COL_TOOLS = "block w-[80px] shrink-0"; -export const COL_INTERACTION = "block w-[100px] shrink-0"; -export const COL_LOG = "block w-[80px] shrink-0"; +// One semantic
+ + Failed to load more + + +
+ Loading runs… +
+ Unable to load runs. +
with no fixed widths; the +// table's content-fit layout sizes each column to its widest cell across the +// header and all rows. Alignment matches RunListRow (short/numeric center, text +// left). `bg-background` keeps rows from bleeding through the sticky header. // --------------------------------------------------------------------------- +const HEAD = + "border-b border-border/60 bg-background px-3 py-1.5 text-[10px] font-medium uppercase tracking-wider text-muted-foreground/60 whitespace-nowrap"; + interface SortButtonProps { readonly label: string; readonly field: RunsSortField; readonly sortField: RunsSortField; readonly sortDirection: "asc" | "desc"; readonly onSort: (field: RunsSortField) => void; - readonly className?: string; } -function SortButton({ - label, - field, - sortField, - sortDirection, - onSort, - className, -}: SortButtonProps) { +function SortButton({ label, field, sortField, sortDirection, onSort }: SortButtonProps) { const isActive = sortField === field; const Icon = isActive ? (sortDirection === "desc" ? ArrowDown : ArrowUp) : ArrowUpDown; @@ -46,11 +34,10 @@ function SortButton({ size="sm" onClick={() => onSort(field)} className={cn( - "h-auto p-0 inline-flex items-center gap-1 text-left", + "inline-flex h-auto items-center gap-1 p-0 text-left", "text-[10px] font-medium uppercase tracking-wider", - "text-muted-foreground/60 hover:text-foreground hover:bg-transparent", + "text-muted-foreground/60 hover:bg-transparent hover:text-foreground", isActive && "text-foreground", - className, )} > {label} @@ -73,52 +60,78 @@ export function RunsColumnHeader({ columns, }: RunsColumnHeaderProps) { return ( -
- {/* dot spacer */} - - - {/* Started (sortable) */} - - - {/* Status */} - status - - {/* Trigger */} - {columns.trigger ? trigger : null} - - {/* Actor */} - {columns.actor ? actor : null} - - {/* Duration (sortable) */} - {columns.duration ? ( +
+ + {/* Started (sortable, left) */} + + + status + + trigger + + actor + + + + tools + + interaction + + log + + code +
so it is valid inside the runs
+
+
+ + + + + + Live + +
+
+
row per run. Columns are content-fit β€” widths come from +// the browser's `table-layout: auto` over all rows + the header (see RunsShell), +// so there are no fixed column widths. Short / numeric / badge columns center; +// text columns left-align. The two columns that can run long β€” `actor` (a token +// id / email) and `code` β€” are width-capped and truncated (hover shows the full +// value) so content-fit can't blow the table out. +const CELL = "border-b border-border/40 px-3 py-2 align-middle whitespace-nowrap"; export interface RunListRowProps { readonly run: RunRow; @@ -39,106 +28,112 @@ export function RunListRow({ run, selected, isPast, columns, onSelect }: RunList const trigger = triggerTone(run.triggerKind); const actor = actorTone(run.actorKind); const isLive = run.status === "running" || run.status === "waiting_for_interaction"; - // Log error/warn counts are denormalized onto the slim row at write time, so - // the list never fetches/parses the full logs (now in the R2 detail object). const logErrors = run.logErrorCount; const logWarns = run.logWarnCount; return ( - - {/* Started */} - + {/* Started (left) */} + - {/* Status label */} - + {/* Status (centered) */} + - {/* Trigger (optional) */} + {/* Trigger (left, optional) */} {columns.trigger ? ( - - - {trigger.label} - + ) : null} - {/* Actor (optional) */} + {/* Actor (left, capped + truncated, optional) */} {columns.actor ? ( - + ) : null} - {/* Duration (optional) β€” slow runs (>5s) flagged. */} + {/* Duration (centered, optional) β€” slow runs (>5s) flagged */} {columns.duration ? ( - 5000 ? "text-destructive" : "text-muted-foreground", )} > {formatDuration(run.durationMs)} - + ) : null} - {/* Tool call count (optional) */} + {/* Tool call count (centered, optional) */} {columns.tools ? ( - 0 ? "text-foreground/80" : "text-muted-foreground/60", )} > {run.toolCallCount} - + ) : null} - {/* Interaction (optional) */} + {/* Interaction (centered, optional) */} {columns.interaction ? ( - + ) : null} - {/* Log error/warn counts (optional) */} + {/* Log error/warn counts (centered, optional) */} {columns.log ? ( - + ) : null} - {/* Code preview (always visible, fills remaining space). Pre-normalized at - write time; sliced here only to bound the rendered width. */} - + {/* Code preview (left, capped + truncated; hover reveals the full + write-bounded preview) */} + + ); } diff --git a/packages/plugins/execution-history/src/react/shell.tsx b/packages/plugins/execution-history/src/react/shell.tsx index 9118e55060..31300e7f47 100644 --- a/packages/plugins/execution-history/src/react/shell.tsx +++ b/packages/plugins/execution-history/src/react/shell.tsx @@ -7,12 +7,19 @@ import { cn } from "@executor-js/react/lib/utils"; // // The left
+ + + + {statusLabel(run.status)} - {isLive && ( - - {run.status === "waiting_for_interaction" ? " Β·" : ""} - - )} - + {isLive && run.status === "waiting_for_interaction" ? ( + Β· + ) : null} + + + + {trigger.label} + + {run.actorId !== null ? ( - <> + {run.actorLabel ?? run.actorId} - + ) : ( β€” )} - + {run.hadInteraction ? ( β€” )} - + {logErrors === 0 && logWarns === 0 ? ( β€” ) : ( @@ -171,15 +166,18 @@ export function RunListRow({ run, selected, isPast, columns, onSelect }: RunList )} - + code: - "{run.codePreview.slice(0, 160)}" - - + "{run.codePreview}" +
with content-fit columns (table-layout: auto): the sticky +// and the rows share one column model, so columns size to their +// widest cell and stay aligned for free β€” no fixed widths. The scroller fires +// onLoadMore when the user scrolls within 320px of the bottom, guarded against +// re-firing while loading. // --------------------------------------------------------------------------- +// Upper bound on the visible column count (dot + started + status + up to six +// optional columns + code = 10). Full-width message rows span the whole table +// with this; an over-count colSpan is harmless (the browser clamps it). +export const RUNS_TABLE_COLSPAN = 12; + export interface RunsShellProps { readonly filterRail: React.ReactNode; readonly toolbar: React.ReactNode; @@ -77,41 +84,59 @@ export function RunsShell(props: RunsShellProps) { {chart != null &&
{chart}
} - {/* Table scroller β€” owns both axes. The inner min-width wrapper pins the - column grid: above it the code column flexes to fill, below it the - body scrolls horizontally instead of collapsing columns. The header - sticks to the top of this scroller and rides the same horizontal - scroll as the rows (top-only sticky), so the two stay aligned. */} + {/* Table scroller β€” owns both axes. The sticky sticks to the top + of this scroller (below the toolbar/chart) and rides the same + horizontal scroll as the rows. `border-separate border-spacing-0` + keeps cell borders rendering on the sticky header (a known + border-collapse + sticky browser bug). No table width is forced, so + columns content-fit: narrower than the pane β†’ left-aligned with slack + on the right; wider β†’ horizontal scroll. */}
-
-
- {columnHeader} -
- - {isEmpty ? ( -
- {emptyState ?? ( -

- No runs match the current filters. -

- )} -
- ) : ( - <> - {children} - {isLoadingMore && ( -
- Loading more… -
- )} - {!hasMore && !isLoadingMore && ( -
- End of history -
- )} - - )} -
+ {/* oxlint-disable-next-line react/forbid-elements -- the runs table needs + this custom scroll container (the onScroll infinite-scroll handler) + + a sticky thead inside it; the shared
wraps its own + overflow-x-auto div, which would conflict. Same suppression the + shared Table component itself uses on its raw
. */} +
+ {columnHeader} + + {isEmpty ? ( + + + + ) : ( + <> + {children} + {isLoadingMore && ( + + + + )} + {!hasMore && !isLoadingMore && ( + + + + )} + + )} + +
+ {emptyState ?? ( + + No runs match the current filters. + + )} +
+ Loading more… +
+ End of history +
From 7efcd1725090f77d288090eecab3d73e31bd9a31 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 21:16:26 +0530 Subject: [PATCH 27/57] refactor(plugin-service-tokens): table layout for the alias list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the card-stack list with a minimal 3-column table (Name, Token, Acts as) using the shared Table primitives β€” openstatus-flavored: muted headers, font-mono token/identity, truncation, and row actions (Edit/Add name/Remove) revealed on hover. Inline machine-name edit unchanged. --- .../plugins/service-tokens/src/client.tsx | 223 +++++++++--------- 1 file changed, 117 insertions(+), 106 deletions(-) diff --git a/packages/plugins/service-tokens/src/client.tsx b/packages/plugins/service-tokens/src/client.tsx index 9f1ec8a2ce..326ad5bcfe 100644 --- a/packages/plugins/service-tokens/src/client.tsx +++ b/packages/plugins/service-tokens/src/client.tsx @@ -25,11 +25,13 @@ import { import { Button } from "@executor-js/react/components/button"; import { Input } from "@executor-js/react/components/input"; import { - CardStackEntry, - CardStackEntryActions, - CardStackEntryContent, - CardStackEntryDescription, -} from "@executor-js/react/components/card-stack"; + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@executor-js/react/components/table"; import type { ServiceTokenAlias } from "./shared"; import { ServiceTokensApi } from "./shared"; @@ -140,8 +142,8 @@ function ServiceTokensPage() { return (
-
-
+
+

Service Tokens

@@ -153,7 +155,7 @@ function ServiceTokensPage() {

-
+
{error && ( -
+

{error}

)} - {isLoading ? ( - - - Loading… - - - ) : isError ? ( - - - - Failed to load aliases - - - - ) : aliases.length === 0 ? ( - - - - No service-token aliases yet. Create a token in Cloudflare Access, then paste its - Client ID above. - - - - ) : ( - aliases.map((alias) => { - const isEditing = editing === alias.commonName; - return ( - - -
- {isEditing ? ( - <> - Name + + + + + Name + + Token + + Acts as + + + + + + {isLoading ? ( + + + Loading… + + + ) : isError ? ( + + + Failed to load aliases + + + ) : aliases.length === 0 ? ( + + + No service-token aliases yet. Create a token in Cloudflare Access, then paste its + Client ID above. + + + ) : ( + aliases.map((alias) => { + const isEditing = editing === alias.commonName; + const actsAs = alias.email ?? alias.name ?? alias.subject; + return ( + + + {isEditing ? ( setEditValue((e.target as HTMLInputElement).value)} - className="h-7 text-[12px]" + className="h-7 max-w-[180px] text-xs" onKeyDown={(e) => { if (e.key === "Enter") void handleSaveEdit(alias.commonName); if (e.key === "Escape") setEditing(null); }} /> - - ) : alias.machineName ? ( - <> - Name - {alias.machineName} - - ) : null} - Token - - {alias.commonName} - - Acts as - - {alias.email ?? alias.name ?? alias.subject} - - - - - {isEditing ? ( - <> - - - - ) : ( - <> - - - - )} - - - ); - }) - )} + ) : alias.machineName ? ( + {alias.machineName} + ) : ( + β€” + )} + + + + {alias.commonName} + + + + + {actsAs} + + + + {isEditing ? ( +
+ + +
+ ) : ( +
+ + +
+ )} + + + ); + }) + )} + +
); From 50d6e2a6c4fb1643d78bca58372c27923045a870 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 15 Jun 2026 21:45:07 +0530 Subject: [PATCH 28/57] feat(execution-history): render live machine names for run actors Add a neutral actor-label registry in @executor-js/react: plugins publish a (kind+id)->label source atom, and the runs UI resolves through the merged map. The service-tokens plugin publishes commonName->machineName; run-row, detail-drawer, and the actor facet prefer the resolved label over the run-time snapshot, falling back to the snapshot then the id. Resolution is live, so renaming a machine name updates past and future runs alike, with no cross-plugin import (both sides depend only on @executor-js/react). --- .../src/react/detail-drawer.tsx | 7 ++++- .../src/react/filter-rail.tsx | 4 ++- .../execution-history/src/react/run-row.tsx | 12 ++++---- .../plugins/service-tokens/src/client.tsx | 28 ++++++++++++++++++ packages/react/src/lib/actor-labels.ts | Bin 0 -> 2819 bytes 5 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 packages/react/src/lib/actor-labels.ts diff --git a/packages/plugins/execution-history/src/react/detail-drawer.tsx b/packages/plugins/execution-history/src/react/detail-drawer.tsx index 596660962e..ff321ed56e 100644 --- a/packages/plugins/execution-history/src/react/detail-drawer.tsx +++ b/packages/plugins/execution-history/src/react/detail-drawer.tsx @@ -4,6 +4,7 @@ import { useState } from "react"; import { ChevronDown, ChevronUp } from "lucide-react"; import { Badge } from "@executor-js/react/components/badge"; import { Button } from "@executor-js/react/components/button"; +import { useResolveActorLabel } from "@executor-js/react/lib/actor-labels"; import { CodeBlock } from "@executor-js/react/components/code-block"; import { CopyButton } from "@executor-js/react/components/copy-button"; import { ScrollArea } from "@executor-js/react/components/scroll-area"; @@ -247,6 +248,10 @@ function DetailContent(props: { const { run } = props; const tone = STATUS_TONES[run.status]; const trigger = triggerTone(run.triggerKind); + // Prefer the live friendly label (e.g. a service-token machine name) over the + // run-time snapshot, falling back to the snapshot then the id. + const resolveActorLabel = useResolveActorLabel(); + const actorLabel = resolveActorLabel(run.actorKind, run.actorId) ?? run.actorLabel ?? run.actorId; const pendingCount = props.interactions.filter( (interaction) => interaction.status === "pending", ).length; @@ -329,7 +334,7 @@ function DetailContent(props: { {run.actorId !== null ? ( - {run.actorLabel ?? run.actorId} + {actorLabel} {run.actorKind !== null ? ( Β· {run.actorKind} ) : null} diff --git a/packages/plugins/execution-history/src/react/filter-rail.tsx b/packages/plugins/execution-history/src/react/filter-rail.tsx index 41e46d6736..93ab55ed33 100644 --- a/packages/plugins/execution-history/src/react/filter-rail.tsx +++ b/packages/plugins/execution-history/src/react/filter-rail.tsx @@ -7,6 +7,7 @@ import { } from "@executor-js/react/components/accordion"; import { Button } from "@executor-js/react/components/button"; import { cn } from "@executor-js/react/lib/utils"; +import { useResolveActorLabel } from "@executor-js/react/lib/actor-labels"; import type { ExecutionListMeta } from "../sdk/store"; import { @@ -194,6 +195,7 @@ export interface RunsFilterRailProps { export function RunsFilterRail({ filters, meta, onChange, onReset }: RunsFilterRailProps) { const filtersActive = isFiltersActive(filters); + const resolveActorLabel = useResolveActorLabel(); const triggerKeys = React.useMemo(() => { const set = new Set(TRIGGER_ORDER); @@ -316,7 +318,7 @@ export function RunsFilterRail({ filters, meta, onChange, onReset }: RunsFilterR onToggle={() => onChange({ ...filters, actor: toggle(filters.actor, id) })} onOnly={() => onChange({ ...filters, actor: [id] })} dotClass={tone.dot} - label={entry.actorLabel ?? id} + label={resolveActorLabel(entry.actorKind, id) ?? entry.actorLabel ?? id} count={entry.count} monoLabel /> diff --git a/packages/plugins/execution-history/src/react/run-row.tsx b/packages/plugins/execution-history/src/react/run-row.tsx index 704398cf0a..ccf7381651 100644 --- a/packages/plugins/execution-history/src/react/run-row.tsx +++ b/packages/plugins/execution-history/src/react/run-row.tsx @@ -1,5 +1,6 @@ import { cn } from "@executor-js/react/lib/utils"; import { Badge } from "@executor-js/react/components/badge"; +import { useResolveActorLabel } from "@executor-js/react/lib/actor-labels"; import type { RunRow } from "../sdk/collections"; import { formatDuration, formatRelative, statusLabel } from "./format"; @@ -27,6 +28,10 @@ export function RunListRow({ run, selected, isPast, columns, onSelect }: RunList const tone = STATUS_TONES[run.status]; const trigger = triggerTone(run.triggerKind); const actor = actorTone(run.actorKind); + // Prefer the live friendly label (e.g. a service-token machine name) over the + // snapshot captured at run time, falling back to the snapshot then the id. + const resolveActorLabel = useResolveActorLabel(); + const actorLabel = resolveActorLabel(run.actorKind, run.actorId) ?? run.actorLabel ?? run.actorId; const isLive = run.status === "running" || run.status === "waiting_for_interaction"; const logErrors = run.logErrorCount; const logWarns = run.logWarnCount; @@ -90,11 +95,8 @@ export function RunListRow({ run, selected, isPast, columns, onSelect }: RunList {run.actorId !== null ? ( - - {run.actorLabel ?? run.actorId} + + {actorLabel} ) : ( diff --git a/packages/plugins/service-tokens/src/client.tsx b/packages/plugins/service-tokens/src/client.tsx index 326ad5bcfe..7bc80d66d2 100644 --- a/packages/plugins/service-tokens/src/client.tsx +++ b/packages/plugins/service-tokens/src/client.tsx @@ -17,6 +17,7 @@ import { useState } from "react"; import { useAtomRefresh, useAtomSet, useAtomValue } from "@effect/atom-react"; import * as Exit from "effect/Exit"; import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; +import * as Atom from "effect/unstable/reactivity/Atom"; import { createPluginAtomClient, defineClientPlugin } from "@executor-js/sdk/client"; import { getExecutorApiBaseUrl, @@ -32,6 +33,7 @@ import { TableHeader, TableRow, } from "@executor-js/react/components/table"; +import { actorLabelKey, registerActorLabelSource } from "@executor-js/react/lib/actor-labels"; import type { ServiceTokenAlias } from "./shared"; import { ServiceTokensApi } from "./shared"; @@ -48,6 +50,32 @@ const aliasesAtom = Client.query("serviceTokens", "list", { const aliasMutation = Client.mutation("serviceTokens", "alias"); const unaliasMutation = Client.mutation("serviceTokens", "unalias"); +// Publish `commonName β†’ machineName` into the shared actor-label registry so the +// execution-history runs UI renders the machine name for service-token actors +// (run actorKind "service-token", actorId = the token's commonName). Only +// aliases WITH a machine name are published β€” runs without one fall back to the +// client id. Registered at module load (this module is imported eagerly as a +// plugin), and resolved live, so a rename updates past and future runs alike. +const serviceTokenActorLabelsAtom: Atom.Atom> = Atom.make((get) => + AsyncResult.match( + get(aliasesAtom) as AsyncResult.AsyncResult, + { + onInitial: () => new Map(), + onFailure: () => new Map(), + onSuccess: ({ value }) => { + const map = new Map(); + for (const alias of value) { + if (alias.machineName) { + map.set(actorLabelKey("service-token", alias.commonName), alias.machineName); + } + } + return map; + }, + }, + ), +); +registerActorLabelSource(serviceTokenActorLabelsAtom); + function ServiceTokensPage() { const [commonName, setCommonName] = useState(""); const [machineName, setMachineName] = useState(""); diff --git a/packages/react/src/lib/actor-labels.ts b/packages/react/src/lib/actor-labels.ts new file mode 100644 index 0000000000000000000000000000000000000000..d8f4f279799ebae1eba90b19655d4361202cd0ef GIT binary patch literal 2819 zcmb_d&2Aev5bjw|F)#vj-y>lYN>oYVTeP7KbDLhJ6rtF?V%av|)CGB+0HtM+V5k4UjPg7mUZ7=Q zzta`Zqua0z%l<7HJCKroj|!+Vz&PR%Dg=wn3!Ypfgk4{~7BCVY!qrw)Yi*eXW_({u zzaCz|uH%gEbX>bGKC>V3bYjU zHf{1vT|3D*(>-;8uaq&js@l*)7T6JFUB#o9kBTngiQkp!_)zsDHw|E0h+F1QUO6Te zNM7kX_NA~*vhX66>m5|PkSd%tSsWkJFT593W$*yyX#dgs3BZJyXslJ*XFIkK1?*YU z@nXUsMMYFOoR_=(D(NKc&!~_LAh>A^;*36C$~&5rw7{?(Gd@A`%YQLOw-q`{^^HLW-wb zZpWj|MNp7PQDCR9MIK7V(0eWuCyBH}W*&BX!eyFLe2syG^mJnsV=g4&B&2(kzqo#6 xPRi*l5u7>~B5x=|BKl&CXgbN%hEjMP+auspx~KQ2Vr%@o&qIBXCman&`~!-&uA%?{ literal 0 HcmV?d00001 From b0fa3a0bc9f300b79c53de7ae1e43b89ae36f725 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Tue, 16 Jun 2026 01:05:55 +0530 Subject: [PATCH 29/57] feat: semantic tools.search via a Cloudflare Vectorize plugin (#26) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Makes `tools.search` pluggable and adds a plugin that answers it with semantic search backed by Cloudflare Vectorize + Gemini embeddings. The plugin is opt-in: until its binding and embedding key are present the engine keeps its built-in lexical search, so nothing changes for hosts that don't enable it. ## Changes **Core β€” a `runtime.toolDiscoveryProvider` plugin hook** - Adds `runtime.toolDiscoveryProvider` to `PluginSpec`, mirroring `runtime.executionObserver`. `composeToolDiscoveryProviders` picks the first plugin that supplies one and threads it into the engine in `makeExecutionStack`; the built-in lexical scorer stays the fallback when none do. - Relocates the tool-discovery contract (`ToolDiscoveryProvider`/`Input`/`Result`, `PagedResult`) and `ExecutionToolError` from `@executor-js/execution` into `@executor-js/sdk`, so the plugin spec (in sdk) can name the contract without an `sdk -> execution` cycle. Both are re-exported from `@executor-js/execution/core`, so existing importers are unaffected. **New plugin `@executor-js/plugin-vectorize-search`** - `runtime.toolDiscoveryProvider`: embeds the query (Gemini via the Vercel AI SDK), nearest-neighbour queries a Vectorize index, and maps matches to `ToolDiscoveryResult` straight from stored metadata (no per-tool describe). - Indexer: embeds the tool catalog (`path`/`name`/`description`/`integration`) and upserts namespaced vectors. - `/api` subpath adds `POST /vectorize-search/reindex` (the handler yields the scoped executor, since the catalog isn't on the plugin ctx). - Binding-optional: inert until both a Vectorize binding and an embedding API key are supplied. **Cloudflare host wiring** - Adds the Vectorize binding + embedding-key plumbing to the host config and threads them into the plugin tuple. The plugin is always present (stable API shape) but inert until bound β€” the same opt-in-by-binding shape as the metrics plugin. ## Tests - `vitest run` β€” new plugin 5/5 (provider mapping/pagination/blank-query; indexer projection + empty-catalog no-op). - Full suites green for the touched core packages: sdk 341, execution 34, api 45, host-mcp 32. - `typecheck` 20/20 across the host chain; lint 0/0; format clean. ## Known tradeoffs / deferred - **Not yet exercised live** β€” bringing it up requires provisioning a Vectorize index and an embedding API key; there is no end-to-end run in this PR. - Single-namespace scoping (per-request multi-tenant scoping is a follow-up). - Explicit reindex only (no incremental/eventful catalog sync yet). - Pagination is sliced within Vectorize's top-K cap; deep pagination is out of scope. --- apps/host-cloudflare/package.json | 1 + apps/host-cloudflare/src/app.ts | 8 +- apps/host-cloudflare/src/config.ts | 14 + apps/host-cloudflare/src/plugins.ts | 17 +- bun.lock | 44 ++- .../core/api/src/server/execution-stack.ts | 19 +- packages/core/execution/src/errors.ts | 12 +- packages/core/execution/src/tool-invoker.ts | 52 +--- packages/core/sdk/src/index.ts | 12 + packages/core/sdk/src/plugin.ts | 12 +- packages/core/sdk/src/tool-discovery.ts | 87 ++++++ .../plugins/vectorize-search/CHANGELOG.md | 1 + .../plugins/vectorize-search/package.json | 39 +++ .../plugins/vectorize-search/src/api/group.ts | 30 ++ .../vectorize-search/src/api/handlers.ts | 41 +++ .../plugins/vectorize-search/src/api/index.ts | 20 ++ .../vectorize-search/src/sdk/documents.ts | 46 +++ .../vectorize-search/src/sdk/embedder.ts | 92 ++++++ .../vectorize-search/src/sdk/errors.ts | 10 + .../plugins/vectorize-search/src/sdk/index.ts | 24 ++ .../vectorize-search/src/sdk/indexer.ts | 60 ++++ .../vectorize-search/src/sdk/plugin.ts | 96 +++++++ .../vectorize-search/src/sdk/provider.ts | 91 ++++++ .../vectorize-search/src/sdk/search.test.ts | 268 ++++++++++++++++++ .../vectorize-search/src/sdk/vectorize.ts | 106 +++++++ .../plugins/vectorize-search/tsconfig.json | 24 ++ .../plugins/vectorize-search/tsup.config.ts | 13 + .../plugins/vectorize-search/vitest.config.ts | 8 + 28 files changed, 1183 insertions(+), 64 deletions(-) create mode 100644 packages/core/sdk/src/tool-discovery.ts create mode 100644 packages/plugins/vectorize-search/CHANGELOG.md create mode 100644 packages/plugins/vectorize-search/package.json create mode 100644 packages/plugins/vectorize-search/src/api/group.ts create mode 100644 packages/plugins/vectorize-search/src/api/handlers.ts create mode 100644 packages/plugins/vectorize-search/src/api/index.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/documents.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/embedder.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/errors.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/index.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/indexer.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/plugin.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/provider.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/search.test.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/vectorize.ts create mode 100644 packages/plugins/vectorize-search/tsconfig.json create mode 100644 packages/plugins/vectorize-search/tsup.config.ts create mode 100644 packages/plugins/vectorize-search/vitest.config.ts diff --git a/apps/host-cloudflare/package.json b/apps/host-cloudflare/package.json index 167ca03189..983f3d96bf 100644 --- a/apps/host-cloudflare/package.json +++ b/apps/host-cloudflare/package.json @@ -29,6 +29,7 @@ "@executor-js/plugin-mcp": "workspace:*", "@executor-js/plugin-openapi": "workspace:*", "@executor-js/plugin-service-tokens": "workspace:*", + "@executor-js/plugin-vectorize-search": "workspace:*", "@executor-js/react": "workspace:*", "@executor-js/runtime-quickjs": "workspace:*", "@executor-js/sdk": "workspace:*", diff --git a/apps/host-cloudflare/src/app.ts b/apps/host-cloudflare/src/app.ts index 81186c5845..9efb71facf 100644 --- a/apps/host-cloudflare/src/app.ts +++ b/apps/host-cloudflare/src/app.ts @@ -34,7 +34,13 @@ import { preloadQuickJs } from "./quickjs"; export const makeCloudflareApp = async (env: CloudflareEnv) => { const config = loadConfig(env); - const plugins = makeCloudflarePlugins(config.secretKey, env.ANALYTICS); + const plugins = makeCloudflarePlugins( + config.secretKey, + env.ANALYTICS, + env.VECTORIZE, + config.geminiApiKey, + config.organizationId, + ); // Load the Workers-compatible (WASM-inlined) QuickJS variant before any // executor is built β€” the default variant can't fetch its .wasm on Workers. diff --git a/apps/host-cloudflare/src/config.ts b/apps/host-cloudflare/src/config.ts index f01968cd84..e718525df3 100644 --- a/apps/host-cloudflare/src/config.ts +++ b/apps/host-cloudflare/src/config.ts @@ -1,5 +1,6 @@ import type { D1Database, DurableObjectNamespace, R2Bucket } from "@cloudflare/workers-types"; import type { AnalyticsEngineDataset } from "@executor-js/plugin-execution-metrics/cloudflare"; +import type { VectorizeIndex } from "@executor-js/plugin-vectorize-search"; import { isValidOrgSlug } from "@executor-js/api"; @@ -23,6 +24,15 @@ export interface CloudflareEnv { * bound (uncomment `analytics_engine_datasets` in wrangler.jsonc), each * finished execution/tool call writes a data point; absent, metrics are off. */ readonly ANALYTICS?: AnalyticsEngineDataset; + /** Vectorize index binding β€” opt-in semantic `tools.search`. When bound (add a + * `vectorize` binding in wrangler.jsonc) the vectorize-search plugin embeds + * the tool catalog and answers `tools.search` from it; absent, the engine + * keeps its built-in lexical search. */ + readonly VECTORIZE?: VectorizeIndex; + /** Gemini API key (a `wrangler secret`) powering the embeddings for the + * Vectorize search. Absent β†’ vectorize search stays inert even if the index + * is bound. */ + readonly GEMINI_API_KEY?: string; /** MCP session Durable Object namespace β€” one addressable isolate per MCP * session (the DO id IS the session id), so a session survives across the * Worker's stateless isolates. */ @@ -66,6 +76,9 @@ export interface CloudflareConfig { /** URL slug for org-prefixed console paths (`//policies`). */ readonly organizationSlug: string; readonly secretKey: string; + /** Gemini API key for the Vectorize search embeddings (a `wrangler secret`). + * Unset β†’ vectorize search is inert. */ + readonly geminiApiKey?: string; readonly allowLocalNetwork: boolean; /** Explicit web base URL (`VITE_PUBLIC_SITE_URL`). Unset on a Worker with no * static URL β€” the per-request origin is used instead (see RequestWebOrigin). */ @@ -112,6 +125,7 @@ export const loadConfig = (env: CloudflareEnv): CloudflareConfig => { organizationName: env.SELF_HOSTED_ORG_NAME ?? "Default", organizationSlug: resolveOrgSlug(env.SELF_HOSTED_ORG_SLUG), secretKey, + geminiApiKey: env.GEMINI_API_KEY?.trim() || undefined, allowLocalNetwork: env.ALLOW_LOCAL_NETWORK === "true", // No static URL on a Worker β€” leave unset when VITE_PUBLIC_SITE_URL is absent // and let the request origin drive it (RequestWebOrigin). Explicit still wins. diff --git a/apps/host-cloudflare/src/plugins.ts b/apps/host-cloudflare/src/plugins.ts index df3a06e3df..d1055d8f7e 100644 --- a/apps/host-cloudflare/src/plugins.ts +++ b/apps/host-cloudflare/src/plugins.ts @@ -9,6 +9,8 @@ import { } from "@executor-js/plugin-execution-metrics/cloudflare"; import { noopExecutionObserver } from "@executor-js/sdk"; import { serviceTokensPlugin } from "@executor-js/plugin-service-tokens/server"; +import { vectorizeSearchHttpPlugin } from "@executor-js/plugin-vectorize-search/api"; +import type { VectorizeIndex } from "@executor-js/plugin-vectorize-search"; // --------------------------------------------------------------------------- // The Cloudflare host's plugin list β€” the same protocol/provider plugins as @@ -27,9 +29,21 @@ import { serviceTokensPlugin } from "@executor-js/plugin-service-tokens/server"; // fleet), so the local Prometheus scrape is deliberately NOT mounted here; WAE // is the durable sink. To enable: uncomment `analytics_engine_datasets` in // wrangler.jsonc. +// +// Vectorize search follows the same opt-in-by-binding shape: the plugin is +// always in the tuple (its reindex route keeps the API shape stable), but it is +// inert β€” the engine keeps its lexical `tools.search` β€” until BOTH a `vectorize` +// binding and the `GEMINI_API_KEY` secret are present. To enable: create a +// Vectorize index + add the binding in wrangler.jsonc and set the secret. // --------------------------------------------------------------------------- -export const makeCloudflarePlugins = (secretKey: string, analytics?: AnalyticsEngineDataset) => +export const makeCloudflarePlugins = ( + secretKey: string, + analytics?: AnalyticsEngineDataset, + vectorize?: VectorizeIndex, + geminiApiKey?: string, + searchNamespace?: string, +) => [ openApiHttpPlugin(), mcpHttpPlugin({ dangerouslyAllowStdioMCP: false }), @@ -39,6 +53,7 @@ export const makeCloudflarePlugins = (secretKey: string, analytics?: AnalyticsEn observer: () => (analytics ? createWaeMetricsObserver(analytics) : noopExecutionObserver), }), serviceTokensPlugin(), + vectorizeSearchHttpPlugin({ vectorize, geminiApiKey, namespace: searchNamespace }), ] as const; export type CloudflarePlugins = ReturnType; diff --git a/bun.lock b/bun.lock index 1692dfd65d..9f62e5f307 100644 --- a/bun.lock +++ b/bun.lock @@ -172,6 +172,7 @@ "@executor-js/plugin-mcp": "workspace:*", "@executor-js/plugin-openapi": "workspace:*", "@executor-js/plugin-service-tokens": "workspace:*", + "@executor-js/plugin-vectorize-search": "workspace:*", "@executor-js/react": "workspace:*", "@executor-js/runtime-quickjs": "workspace:*", "@executor-js/sdk": "workspace:*", @@ -1113,6 +1114,31 @@ "react", ], }, + "packages/plugins/vectorize-search": { + "name": "@executor-js/plugin-vectorize-search", + "version": "0.0.0", + "dependencies": { + "@ai-sdk/google": "^3.0.80", + "@executor-js/sdk": "workspace:*", + "ai": "^6.0.191", + "effect": "catalog:", + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20250620.0", + "@effect/vitest": "catalog:", + "@executor-js/api": "workspace:*", + "@types/node": "catalog:", + "bun-types": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:", + }, + "peerDependencies": { + "@executor-js/api": "workspace:*", + }, + "optionalPeers": [ + "@executor-js/api", + ], + }, "packages/plugins/workos-vault": { "name": "@executor-js/plugin-workos-vault", "version": "0.0.2", @@ -1242,11 +1268,13 @@ "@1password/sdk-core": ["@1password/sdk-core@0.4.1-beta.1", "", {}, "sha512-/otbg1JVhsEn6oUIeReoT9TmFr8J7KBwr9UuRVfJFwwGG3bHPF8ewT+LhRimQeJtypqQ69ZVuOYkxknD4iQHxw=="], - "@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.99", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23", "@vercel/oidc": "3.1.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8/UuzFY8p+T8j4XP/9m841pUb5bhnFt8cecSnJpd2zhBttNZ6GbfjZTmsqnvM/RwJOvzIsdFULZrU+E9QFREsQ=="], + "@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.131", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.29", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-CnjOZdywQaUnCyZ0N5wVNm7Sm63+NeHDVZQJKFX2IDq+t03SLwiiuoi3ILTLPlM+YSOhkQ/pvIDoR4qa98Zp5A=="], + + "@ai-sdk/google": ["@ai-sdk/google@3.0.82", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.29" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-md+M92ZJuPIMU2p4v1rGLpJJWTmTh/vpJPkMnQbEdcLaPTZxRaroIKSnmL/9UGJV0BORJlHNDJegkcnhVpTmDA=="], - "@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="], + "@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="], - "@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.23", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-z8GlDaCmRSDlqkMF2f4/RFgWxdarvIbyuk+m6WXT1LYgsnGiXRJGTD2Z1+SDl3LqtFuRtGX1aghYvQLoHL/9pg=="], + "@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.29", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-uhukHaCBvqkwBHkT8C2PrnqKTCoLn3pdHXqtcR9I8ErH+flbzgW4o7VHSNIup9LRu+WBvZIZDQLsx6rwl2tiOA=="], "@alcalzone/ansi-tokenize": ["@alcalzone/ansi-tokenize@0.2.5", "", { "dependencies": { "ansi-styles": "^6.2.1", "is-fullwidth-code-point": "^5.0.0" } }, "sha512-3NX/MpTdroi0aKz134A6RC2Gb2iXVECN4QaAXnvCIxxIm3C3AVB1mkUe8NaaiyvOpDfsrqWhYtj+Q6a62RrTsw=="], @@ -1714,6 +1742,8 @@ "@executor-js/plugin-service-tokens": ["@executor-js/plugin-service-tokens@workspace:packages/plugins/service-tokens"], + "@executor-js/plugin-vectorize-search": ["@executor-js/plugin-vectorize-search@workspace:packages/plugins/vectorize-search"], + "@executor-js/plugin-workos-vault": ["@executor-js/plugin-workos-vault@workspace:packages/plugins/workos-vault"], "@executor-js/react": ["@executor-js/react@workspace:packages/react"], @@ -3042,7 +3072,7 @@ "@use-gesture/react": ["@use-gesture/react@10.3.1", "", { "dependencies": { "@use-gesture/core": "10.3.1" }, "peerDependencies": { "react": ">= 16.8.0" } }, "sha512-Yy19y6O2GJq8f7CHf7L0nxL8bf4PZCPaVOCgJrusOeFHY1LvHgYXnmnXg6N5iwAnbgbZCDjo60SiM6IPJi9C5g=="], - "@vercel/oidc": ["@vercel/oidc@3.1.0", "", {}, "sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w=="], + "@vercel/oidc": ["@vercel/oidc@3.2.0", "", {}, "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug=="], "@vitejs/plugin-react": ["@vitejs/plugin-react@6.0.1", "", { "dependencies": { "@rolldown/pluginutils": "1.0.0-rc.7" }, "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", "vite": "^8.0.0" }, "optionalPeers": ["@rolldown/plugin-babel", "babel-plugin-react-compiler"] }, "sha512-l9X/E3cDb+xY3SWzlG1MOGt2usfEHGMNIaegaUGFsLkb3RCn/k8/TOXBcab+OndDI4TBtktT8/9BwwW8Vi9KUQ=="], @@ -3098,7 +3128,7 @@ "ahooks": ["ahooks@3.9.7", "", { "dependencies": { "@babel/runtime": "^7.21.0", "@types/js-cookie": "^3.0.6", "dayjs": "^1.9.1", "intersection-observer": "^0.12.0", "js-cookie": "^3.0.5", "lodash": "^4.17.21", "react-fast-compare": "^3.2.2", "resize-observer-polyfill": "^1.5.1", "screenfull": "^5.0.0", "tslib": "^2.4.1" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-S0lvzhbdlhK36RFBkGv+RbOM/dbbweym+BIHM/bwwuWVSVN5TuVErHPMWo4w0t1NDYg5KPp2iEf7Y7E5LASYiw=="], - "ai": ["ai@6.0.162", "", { "dependencies": { "@ai-sdk/gateway": "3.0.99", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23", "@opentelemetry/api": "1.9.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-1PSvNEK1PEbpUXahnFrcey6l7DJXMVWmg0ibQ8h8oMSe9V1Vx5d+R3xNu0hzBtwqfxYj21ddZo+EUYVs6GOEyA=="], + "ai": ["ai@6.0.205", "", { "dependencies": { "@ai-sdk/gateway": "3.0.131", "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.29", "@opentelemetry/api": "^1.9.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-F4akEGF41UdgJO3L4v+D5noVD1/czhJy6x0k9R/i1EXfxqrkBh/PdYSgRSLPiGFvrw76dzI8h4w3NYmLrTb8dw=="], "ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="], @@ -5494,6 +5524,8 @@ "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + "@ai-sdk/provider-utils/eventsource-parser": ["eventsource-parser@3.1.0", "", {}, "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg=="], + "@antfu/install-pkg/package-manager-detector": ["package-manager-detector@1.6.0", "", {}, "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA=="], "@astrojs/cloudflare/vite": ["vite@7.3.2", "", { "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg=="], @@ -5980,8 +6012,6 @@ "agents/yargs": ["yargs@18.0.0", "", { "dependencies": { "cliui": "^9.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "string-width": "^7.2.0", "y18n": "^5.0.5", "yargs-parser": "^22.0.0" } }, "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg=="], - "ai/@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="], - "ajv-keywords/ajv": ["ajv@6.15.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw=="], "anymatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], diff --git a/packages/core/api/src/server/execution-stack.ts b/packages/core/api/src/server/execution-stack.ts index ebfe63bf63..bbcc07d3d1 100644 --- a/packages/core/api/src/server/execution-stack.ts +++ b/packages/core/api/src/server/execution-stack.ts @@ -26,7 +26,7 @@ import { Context, Effect, Layer } from "effect"; import type * as Cause from "effect/Cause"; -import { composeExecutionObservers } from "@executor-js/sdk"; +import { composeExecutionObservers, composeToolDiscoveryProviders } from "@executor-js/sdk"; import type { AnyPlugin, Executor, StorageFailure } from "@executor-js/sdk"; import { createExecutionEngine, @@ -118,11 +118,18 @@ export const makeExecutionStack = < // PluginsProvider erases the tuple to AnyPlugin[]; recover the caller's // TPlugins phantom so the extensions arg (the executor) lines up. const observer = composeExecutionObservers(plugins() as TPlugins, executor); + // Pick a plugin-supplied `tools.search` backend (e.g. semantic Vectorize + // search). `undefined` when no plugin provides one, so the engine falls + // back to its built-in lexical scorer. + const toolDiscoveryProvider = composeToolDiscoveryProviders(plugins() as TPlugins, executor); - const engine = decorate(createExecutionEngine({ executor, codeExecutor, observer }), { - accountId, - organizationId, - organizationName, - }); + const engine = decorate( + createExecutionEngine({ executor, codeExecutor, observer, toolDiscoveryProvider }), + { + accountId, + organizationId, + organizationName, + }, + ); return { executor, engine }; }); diff --git a/packages/core/execution/src/errors.ts b/packages/core/execution/src/errors.ts index 8470b89c76..f29d5b67cc 100644 --- a/packages/core/execution/src/errors.ts +++ b/packages/core/execution/src/errors.ts @@ -1,9 +1,9 @@ -import * as Data from "effect/Data"; - -export class ExecutionToolError extends Data.TaggedError("ExecutionToolError")<{ - readonly message: string; - readonly cause?: unknown; -}> {} +// `ExecutionToolError` is the tool-discovery error channel. It now lives in +// `@executor-js/sdk` beside the tool-discovery contract (`ToolDiscoveryProvider`) +// so the plugin spec can name that contract without a `sdk -> execution` cycle. +// Re-exported here so in-package imports (`./errors`) and the public +// `@executor-js/execution/core` surface keep resolving the same class. +export { ExecutionToolError } from "@executor-js/sdk/core"; // `CodeExecutionError` lives in `@executor-js/codemode-core` β€” the `CodeExecutor` // interface uses it as the default error channel, so the runtime packages diff --git a/packages/core/execution/src/tool-invoker.ts b/packages/core/execution/src/tool-invoker.ts index 619170b619..2458281443 100644 --- a/packages/core/execution/src/tool-invoker.ts +++ b/packages/core/execution/src/tool-invoker.ts @@ -7,7 +7,17 @@ import type { ToolError, Tool, ToolSchemaView, + PagedResult, + ToolDiscoveryInput, + ToolDiscoveryProvider, + ToolDiscoveryResult, } from "@executor-js/sdk/core"; + +// The tool-discovery contract moved to `@executor-js/sdk` (so the plugin spec +// can name it without a cycle). Re-export from this module's prior home so +// in-package importers (`engine.ts`, tests) and `@executor-js/execution/core` +// keep their import paths. +export type { PagedResult, ToolDiscoveryInput, ToolDiscoveryProvider, ToolDiscoveryResult }; import { authToolFailure, isToolResult, @@ -304,14 +314,6 @@ const isElicitationDeclinedError = ( "action" in value && (value.action === "cancel" || value.action === "decline"); -export type ToolDiscoveryResult = { - readonly path: string; - readonly name: string; - readonly description?: string; - readonly integration: string; - readonly score: number; -}; - export type ExecutorSourceListItem = { readonly id: string; readonly name: string; @@ -321,40 +323,6 @@ export type ExecutorSourceListItem = { readonly toolCount: number; }; -export type ToolDiscoveryInput = { - readonly executor: Executor; - readonly query: string; - readonly namespace?: string; - readonly limit: number; - readonly offset: number; -}; - -export interface ToolDiscoveryProvider { - readonly searchTools: ( - input: ToolDiscoveryInput, - ) => Effect.Effect, ExecutionToolError>; -} - -/** - * Page of results from a list-style discovery tool. Shared by - * `tools.search` and `tools.executor.sources.list` so the model sees one - * consistent shape: - * - * - `items` β€” the page (slice). - * - `total` β€” count after filtering, before pagination. The model - * can use this to detect truncation. - * - `hasMore` β€” convenience flag for `(offset + items.length) < total`. - * - `nextOffset` β€” concrete offset for the next page when `hasMore`, - * `null` otherwise. Pre-computing it removes a class of - * off-by-one mistakes when the model paginates. - */ -export type PagedResult = { - readonly items: readonly T[]; - readonly total: number; - readonly hasMore: boolean; - readonly nextOffset: number | null; -}; - const paginate = (all: readonly T[], offset: number, limit: number): PagedResult => { const total = all.length; const start = Math.min(Math.max(offset, 0), total); diff --git a/packages/core/sdk/src/index.ts b/packages/core/sdk/src/index.ts index cacce1e772..3a1c5f4478 100644 --- a/packages/core/sdk/src/index.ts +++ b/packages/core/sdk/src/index.ts @@ -190,6 +190,18 @@ export { type ExecutionObserver, } from "./execution-observer"; +// Tool discovery β€” the backend behind the sandbox `tools.search` call. Plugins +// supply a custom provider (e.g. semantic Vectorize search) via +// `runtime.toolDiscoveryProvider`; `composeToolDiscoveryProviders` picks it. +export { + ExecutionToolError, + composeToolDiscoveryProviders, + type ToolDiscoveryProvider, + type ToolDiscoveryInput, + type ToolDiscoveryResult, + type PagedResult, +} from "./tool-discovery"; + // Blob store β€” the plugin-facing contract (`BlobStore`/`PluginBlobStore`) // plus the platform-neutral backends (`makeFumaBlobStore` default, // `makeInMemoryBlobStore` for tests). Platform-specific backends live with diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 6ab97f45ba..111f5080de 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -39,6 +39,7 @@ import type { InvalidConnectionInputError, } from "./errors"; import type { ExecutionObserver } from "./execution-observer"; +import type { ToolDiscoveryProvider } from "./tool-discovery"; import type { OAuthService } from "./oauth-client"; import type { CredentialProvider, ProviderEntry } from "./provider"; import type { PluginStorageConfig, PluginStorageFacade } from "./plugin-storage"; @@ -502,9 +503,18 @@ export interface PluginSpec< /** Runtime hooks invoked while the engine executes code. `executionObserver` * receives this plugin's extension and returns an observer that is fanned - * every {@link ExecutionEvent} β€” the seam history/metrics sinks build on. */ + * every {@link ExecutionEvent} β€” the seam history/metrics sinks build on. + * `toolDiscoveryProvider` supplies the backend for the sandbox `tools.search` + * call (e.g. a semantic search): the first plugin to provide one wins, else + * the engine keeps its built-in lexical scorer. */ readonly runtime?: { readonly executionObserver?: (self: NoInfer) => ExecutionObserver; + /** Returns this plugin's `tools.search` backend, or `undefined` to opt out + * (e.g. the binding it needs is unbound) β€” letting a later provider or the + * engine's lexical default answer instead. */ + readonly toolDiscoveryProvider?: ( + self: NoInfer, + ) => ToolDiscoveryProvider | undefined; }; readonly close?: () => Effect.Effect; diff --git a/packages/core/sdk/src/tool-discovery.ts b/packages/core/sdk/src/tool-discovery.ts new file mode 100644 index 0000000000..d623ebe1bc --- /dev/null +++ b/packages/core/sdk/src/tool-discovery.ts @@ -0,0 +1,87 @@ +import { Data, type Effect } from "effect"; + +import type { Executor } from "./executor"; +import type { AnyPlugin, PluginExtensions } from "./plugin"; + +/* The tool-discovery contract: the backend behind the sandbox `tools.search` + * call. The engine holds a single provider. Plugins opt in via + * `plugin.runtime.toolDiscoveryProvider` (mirroring `executionObserver`) to + * supply a custom one β€” e.g. a semantic, Vectorize-backed search. When no + * plugin provides one the engine keeps its built-in lexical scorer. + * + * This contract lives in `sdk` (not `execution`) so the plugin spec can name it + * without a `sdk -> execution` cycle; `@executor-js/execution` re-exports it. */ + +/** Raised when a discovery provider fails to search. */ +export class ExecutionToolError extends Data.TaggedError("ExecutionToolError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +/** One match returned by `tools.search`. */ +export type ToolDiscoveryResult = { + readonly path: string; + readonly name: string; + readonly description?: string; + readonly integration: string; + readonly score: number; +}; + +export type ToolDiscoveryInput = { + readonly executor: Executor; + readonly query: string; + readonly namespace?: string; + readonly limit: number; + readonly offset: number; +}; + +/** + * Page of results from a list-style discovery tool. Shared by `tools.search` + * and `tools.executor.sources.list` so the model sees one consistent shape: + * + * - `items` β€” the page (slice). + * - `total` β€” count after filtering, before pagination. The model can + * use this to detect truncation. + * - `hasMore` β€” convenience flag for `(offset + items.length) < total`. + * - `nextOffset` β€” concrete offset for the next page when `hasMore`, `null` + * otherwise. Pre-computing it removes a class of off-by-one + * mistakes when the model paginates. + */ +export type PagedResult = { + readonly items: readonly T[]; + readonly total: number; + readonly hasMore: boolean; + readonly nextOffset: number | null; +}; + +export interface ToolDiscoveryProvider { + readonly searchTools: ( + input: ToolDiscoveryInput, + ) => Effect.Effect, ExecutionToolError>; +} + +/** + * Pick the tool-discovery provider a plugin contributes via + * `runtime.toolDiscoveryProvider`, bound to that plugin's extension. Returns + * the FIRST plugin (in registration order) that supplies one, or `undefined` + * when none do β€” in which case the engine keeps its built-in lexical scorer. + * + * Unlike observers, search is singular: one provider answers `tools.search`, so + * providers do not compose/fan out. Registration order is the tiebreak; a host + * that wants the semantic provider to win simply registers it before any other + * search-providing plugin. + */ +export const composeToolDiscoveryProviders = ( + plugins: TPlugins, + extensions: PluginExtensions, +): ToolDiscoveryProvider | undefined => { + for (const plugin of plugins) { + const provider = plugin.runtime?.toolDiscoveryProvider?.( + extensions[plugin.id as keyof PluginExtensions] as never, + ); + if (provider) { + return provider; + } + } + return undefined; +}; diff --git a/packages/plugins/vectorize-search/CHANGELOG.md b/packages/plugins/vectorize-search/CHANGELOG.md new file mode 100644 index 0000000000..6deaf9ada0 --- /dev/null +++ b/packages/plugins/vectorize-search/CHANGELOG.md @@ -0,0 +1 @@ +# @executor-js/plugin-vectorize-search diff --git a/packages/plugins/vectorize-search/package.json b/packages/plugins/vectorize-search/package.json new file mode 100644 index 0000000000..a9cae12b28 --- /dev/null +++ b/packages/plugins/vectorize-search/package.json @@ -0,0 +1,39 @@ +{ + "name": "@executor-js/plugin-vectorize-search", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/sdk/index.ts", + "./api": "./src/api/index.ts" + }, + "scripts": { + "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src || true)", + "typecheck": "tsgo --noEmit", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@ai-sdk/google": "^3.0.80", + "@executor-js/sdk": "workspace:*", + "ai": "^6.0.191", + "effect": "catalog:" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20250620.0", + "@effect/vitest": "catalog:", + "@executor-js/api": "workspace:*", + "@types/node": "catalog:", + "bun-types": "catalog:", + "tsup": "catalog:", + "vitest": "catalog:" + }, + "peerDependencies": { + "@executor-js/api": "workspace:*" + }, + "peerDependenciesMeta": { + "@executor-js/api": { + "optional": true + } + } +} diff --git a/packages/plugins/vectorize-search/src/api/group.ts b/packages/plugins/vectorize-search/src/api/group.ts new file mode 100644 index 0000000000..d2ff0228b9 --- /dev/null +++ b/packages/plugins/vectorize-search/src/api/group.ts @@ -0,0 +1,30 @@ +import { Schema } from "effect"; +import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; + +import { InternalError } from "@executor-js/api"; + +// --------------------------------------------------------------------------- +// HTTP surface for the Vectorize search plugin: a single explicit reindex. +// +// The route is flat and plugin-id-prefixed (`/vectorize-search/reindex`), +// matching the execution-history/graphql convention β€” the per-request executor +// is already owner-scoped at the host edge, so there is no `:scopeId` segment. +// +// Reindex takes no body: it embeds the whole tool catalog for the scoped tenant +// and upserts it into Vectorize. `VectorizeSearchError` on the extension flows +// through the typed channel and `capture` downgrades it to `InternalError`. +// --------------------------------------------------------------------------- + +/** Result of a reindex: the namespace touched + the number of tools embedded + * and upserted into Vectorize. */ +export const ReindexResponse = Schema.Struct({ + namespace: Schema.String, + indexedToolCount: Schema.Number, +}); + +export const VectorizeSearchGroup = HttpApiGroup.make("vectorizeSearch").add( + HttpApiEndpoint.post("reindex", "/vectorize-search/reindex", { + success: ReindexResponse, + error: InternalError, + }), +); diff --git a/packages/plugins/vectorize-search/src/api/handlers.ts b/packages/plugins/vectorize-search/src/api/handlers.ts new file mode 100644 index 0000000000..1675809f2b --- /dev/null +++ b/packages/plugins/vectorize-search/src/api/handlers.ts @@ -0,0 +1,41 @@ +import { Context, Effect } from "effect"; +import { HttpApiBuilder } from "effect/unstable/httpapi"; + +import { addGroup, captureEngineError } from "@executor-js/api"; +import { ExecutorService } from "@executor-js/api/server"; + +import type { VectorizeSearchExtension } from "../sdk/plugin"; +import { VectorizeSearchGroup } from "./group"; + +// --------------------------------------------------------------------------- +// Service tag +// +// Holds the `executor.vectorizeSearch` surface. The host provides an +// already-bound extension via +// `Layer.succeed(VectorizeSearchExtensionService, executor.vectorizeSearch)`. +// The handler also yields the per-request `ExecutorService` (the scoped +// executor) and hands it to `reindex`, since the catalog lives on the executor, +// not the plugin ctx. +// --------------------------------------------------------------------------- + +export class VectorizeSearchExtensionService extends Context.Service< + VectorizeSearchExtensionService, + VectorizeSearchExtension +>()("VectorizeSearchExtensionService") {} + +const ExecutorApiWithVectorizeSearch = addGroup(VectorizeSearchGroup); + +export const VectorizeSearchHandlers = HttpApiBuilder.group( + ExecutorApiWithVectorizeSearch, + "vectorizeSearch", + (handlers) => + handlers.handle("reindex", () => + captureEngineError( + Effect.gen(function* () { + const search = yield* VectorizeSearchExtensionService; + const executor = yield* ExecutorService; + return yield* search.reindex(executor); + }), + ), + ), +); diff --git a/packages/plugins/vectorize-search/src/api/index.ts b/packages/plugins/vectorize-search/src/api/index.ts new file mode 100644 index 0000000000..2d0d6474eb --- /dev/null +++ b/packages/plugins/vectorize-search/src/api/index.ts @@ -0,0 +1,20 @@ +import { definePlugin } from "@executor-js/sdk/core"; + +import { vectorizeSearchPlugin, type VectorizeSearchPluginOptions } from "../sdk/plugin"; +import { VectorizeSearchGroup } from "./group"; +import { VectorizeSearchExtensionService, VectorizeSearchHandlers } from "./handlers"; + +export { VectorizeSearchGroup, ReindexResponse } from "./group"; +export { VectorizeSearchHandlers, VectorizeSearchExtensionService } from "./handlers"; + +// HTTP-augmented variant of `vectorizeSearchPlugin`. The returned plugin carries +// the reindex `routes`/`handlers`/`extensionService` so a host can mount the +// reindex endpoint. Hosts that compose an `HttpApi` import this; SDK-only +// consumers stay on `@executor-js/plugin-vectorize-search` and never load +// `@executor-js/api`. +export const vectorizeSearchHttpPlugin = definePlugin((options?: VectorizeSearchPluginOptions) => ({ + ...vectorizeSearchPlugin(options), + routes: () => VectorizeSearchGroup, + handlers: () => VectorizeSearchHandlers, + extensionService: VectorizeSearchExtensionService, +})); diff --git a/packages/plugins/vectorize-search/src/sdk/documents.ts b/packages/plugins/vectorize-search/src/sdk/documents.ts new file mode 100644 index 0000000000..d041d7b73e --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/documents.ts @@ -0,0 +1,46 @@ +import type { Tool } from "@executor-js/sdk/core"; + +const ADDRESS_PREFIX = "tools."; + +/** Strip the proxy-root `tools.` prefix so the address becomes the + * sandbox-callable path the model writes after `tools.` β€” mirrors the engine's + * own `addressToPath` so the `path` we index matches what `tools.search` + * callers expect (and can pass back to `describe`/invoke). */ +const addressToPath = (address: string): string => + address.startsWith(ADDRESS_PREFIX) ? address.slice(ADDRESS_PREFIX.length) : address; + +/** The indexed view of a tool: the `ToolDiscoveryResult` fields (returned at + * query time straight from Vectorize metadata) plus the text we embed. */ +export interface ToolSearchDocument { + readonly id: string; + readonly path: string; + readonly name: string; + readonly description: string; + readonly integration: string; + readonly embeddingText: string; +} + +const joinText = (parts: readonly (string | undefined)[]): string => + parts + .map((part) => part?.trim()) + .filter((part): part is string => part !== undefined && part.length > 0) + .join("\n"); + +/** Vector id for a tool within a namespace. Vectorize ids are global, so the + * namespace is folded into the id to keep tenants' vectors distinct. */ +export const toolVectorId = (namespace: string, path: string): string => `${namespace}#${path}`; + +export const projectToolDocument = (namespace: string, tool: Tool): ToolSearchDocument => { + const path = addressToPath(String(tool.address)); + const name = String(tool.name); + const description = tool.description; + const integration = String(tool.integration); + return { + id: toolVectorId(namespace, path), + path, + name, + description, + integration, + embeddingText: joinText([`${integration} ${path}`, name, description]), + }; +}; diff --git a/packages/plugins/vectorize-search/src/sdk/embedder.ts b/packages/plugins/vectorize-search/src/sdk/embedder.ts new file mode 100644 index 0000000000..e044ca2bd8 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/embedder.ts @@ -0,0 +1,92 @@ +import { createGoogleGenerativeAI } from "@ai-sdk/google"; +import { embed, embedMany } from "ai"; +import { Effect } from "effect"; + +import { VectorizeSearchError } from "./errors"; + +/** Turns tool documents and queries into embedding vectors. Document and query + * embeddings use different task types (Gemini distinguishes + * `RETRIEVAL_DOCUMENT` from `RETRIEVAL_QUERY`), so the two methods are + * separate. The plugin treats this as a seam: tests inject a deterministic + * embedder, production uses {@link makeGeminiEmbedder}. */ +export interface ToolEmbedder { + readonly model: string; + readonly dimensions: number; + readonly embedDocuments: ( + texts: readonly string[], + ) => Effect.Effect; + readonly embedQuery: (text: string) => Effect.Effect; +} + +export interface GeminiEmbedderOptions { + readonly apiKey: string; + /** Gemini embedding model id. Defaults to the v2 model. */ + readonly model?: string; + /** Output dimensionality (MRL truncation). MUST match the Vectorize index. */ + readonly dimensions?: number; + readonly batchSize?: number; +} + +/** Gemini Embedding 2: natively multimodal, default 3072d, MRL-truncatable to + * 1536/768. We default to 1536d as a quality/size balance for a tool catalog. */ +export const DEFAULT_GEMINI_MODEL = "gemini-embedding-2"; +export const DEFAULT_EMBEDDING_DIMENSIONS = 1536; +const DEFAULT_BATCH_SIZE = 32; +const DOCUMENT_TASK_TYPE = "RETRIEVAL_DOCUMENT"; +const QUERY_TASK_TYPE = "RETRIEVAL_QUERY"; + +const chunk = (items: readonly A[], size: number): readonly (readonly A[])[] => { + const safe = Math.max(1, Math.floor(size)); + const out: A[][] = []; + for (let i = 0; i < items.length; i += safe) { + out.push(items.slice(i, i + safe)); + } + return out; +}; + +export const makeGeminiEmbedder = (options: GeminiEmbedderOptions): ToolEmbedder => { + const model = options.model ?? DEFAULT_GEMINI_MODEL; + const dimensions = options.dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS; + const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE; + const google = createGoogleGenerativeAI({ apiKey: options.apiKey }); + + const embedTexts = ( + texts: readonly string[], + taskType: typeof DOCUMENT_TASK_TYPE | typeof QUERY_TASK_TYPE, + ): Effect.Effect => + Effect.tryPromise({ + try: async () => { + if (texts.length === 0) return []; + const textModel = google.textEmbedding(model); + const providerOptions = { google: { outputDimensionality: dimensions, taskType } }; + if (texts.length === 1) { + const { embedding } = await embed({ + model: textModel, + value: texts[0]!, + providerOptions, + }); + return [embedding]; + } + const { embeddings } = await embedMany({ + model: textModel, + values: [...texts], + maxParallelCalls: 5, + providerOptions, + }); + return embeddings; + }, + catch: (cause) => + new VectorizeSearchError({ message: `Gemini embedding failed for ${model}.`, cause }), + }); + + return { + model, + dimensions, + embedDocuments: (texts) => + Effect.forEach(chunk(texts, batchSize), (group) => embedTexts(group, DOCUMENT_TASK_TYPE), { + concurrency: 1, + }).pipe(Effect.map((groups) => groups.flat())), + embedQuery: (text) => + embedTexts([text], QUERY_TASK_TYPE).pipe(Effect.map((vectors) => vectors[0] ?? [])), + }; +}; diff --git a/packages/plugins/vectorize-search/src/sdk/errors.ts b/packages/plugins/vectorize-search/src/sdk/errors.ts new file mode 100644 index 0000000000..f1a3de63db --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/errors.ts @@ -0,0 +1,10 @@ +import { Data } from "effect"; + +/** A failure in the Vectorize search plugin's index or query path β€” an + * embedding call, the Vectorize binding, or a catalog read. The query path + * maps this into the engine's `ExecutionToolError` contract; the reindex path + * surfaces it directly. */ +export class VectorizeSearchError extends Data.TaggedError("VectorizeSearchError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} diff --git a/packages/plugins/vectorize-search/src/sdk/index.ts b/packages/plugins/vectorize-search/src/sdk/index.ts new file mode 100644 index 0000000000..7800327e04 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/index.ts @@ -0,0 +1,24 @@ +export { + vectorizeSearchPlugin, + type VectorizeSearchPluginOptions, + type VectorizeSearchExtension, +} from "./plugin"; +export { makeVectorizeToolDiscoveryProvider } from "./provider"; +export { + makeGeminiEmbedder, + DEFAULT_GEMINI_MODEL, + DEFAULT_EMBEDDING_DIMENSIONS, + type ToolEmbedder, + type GeminiEmbedderOptions, +} from "./embedder"; +export { + makeVectorizeStore, + type VectorizeIndex, + type VectorizeStore, + type VectorizeMatch, + type VectorizeMatches, + type VectorizeVectorInput, +} from "./vectorize"; +export { reindexToolCatalog, collectToolDocuments, type ReindexResult } from "./indexer"; +export { projectToolDocument, toolVectorId, type ToolSearchDocument } from "./documents"; +export { VectorizeSearchError } from "./errors"; diff --git a/packages/plugins/vectorize-search/src/sdk/indexer.ts b/packages/plugins/vectorize-search/src/sdk/indexer.ts new file mode 100644 index 0000000000..f9c062f7d6 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/indexer.ts @@ -0,0 +1,60 @@ +import type { Executor } from "@executor-js/sdk/core"; +import { Effect } from "effect"; + +import { projectToolDocument, type ToolSearchDocument } from "./documents"; +import type { ToolEmbedder } from "./embedder"; +import { VectorizeSearchError } from "./errors"; +import type { VectorizeStore, VectorizeVectorInput } from "./vectorize"; + +export interface ReindexResult { + readonly namespace: string; + readonly indexedToolCount: number; +} + +/** Project the whole tool catalog into indexable documents. Lightweight: it + * uses only `tools.list` fields (path/name/description/integration) and does + * not `describe` each tool, so a reindex is one catalog read + one batched + * embedding call rather than N per-tool round-trips. */ +export const collectToolDocuments = ( + namespace: string, + executor: Executor, +): Effect.Effect => + executor.tools.list({ includeAnnotations: false }).pipe( + Effect.map((tools) => tools.map((tool) => projectToolDocument(namespace, tool))), + Effect.mapError( + (cause) => new VectorizeSearchError({ message: "Failed to list tools for indexing.", cause }), + ), + ); + +/** Embed the current tool catalog and upsert it into Vectorize under + * `namespace`. Explicit (v1): callers invoke it on demand via the `/api` + * reindex route (or a cron). Returns the indexed count. */ +export const reindexToolCatalog = (input: { + readonly namespace: string; + readonly executor: Executor; + readonly embedder: ToolEmbedder; + readonly store: VectorizeStore; +}): Effect.Effect => + Effect.gen(function* () { + const { namespace, executor, embedder, store } = input; + const documents = yield* collectToolDocuments(namespace, executor); + if (documents.length === 0) { + return { namespace, indexedToolCount: 0 }; + } + const vectors = yield* embedder.embedDocuments( + documents.map((document) => document.embeddingText), + ); + const records: readonly VectorizeVectorInput[] = documents.map((document, index) => ({ + id: document.id, + values: [...(vectors[index] ?? [])], + namespace, + metadata: { + path: document.path, + name: document.name, + description: document.description, + integration: document.integration, + }, + })); + yield* store.upsert(records); + return { namespace, indexedToolCount: records.length }; + }); diff --git a/packages/plugins/vectorize-search/src/sdk/plugin.ts b/packages/plugins/vectorize-search/src/sdk/plugin.ts new file mode 100644 index 0000000000..b30de0da3e --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/plugin.ts @@ -0,0 +1,96 @@ +import { definePlugin, type Executor } from "@executor-js/sdk/core"; +import { Effect } from "effect"; + +import { makeGeminiEmbedder, type ToolEmbedder } from "./embedder"; +import { VectorizeSearchError } from "./errors"; +import { reindexToolCatalog, type ReindexResult } from "./indexer"; +import { makeVectorizeToolDiscoveryProvider } from "./provider"; +import { makeVectorizeStore, type VectorizeIndex, type VectorizeStore } from "./vectorize"; + +export interface VectorizeSearchPluginOptions { + /** The Cloudflare Vectorize binding the tool catalog is indexed into. Absent + * (binding unbound) β†’ the plugin is inert and the engine keeps its built-in + * lexical search, mirroring how the metrics plugin no-ops without its binding. */ + readonly vectorize?: VectorizeIndex; + /** Gemini API key (a wrangler secret on the Cloudflare host). Absent β†’ inert, + * unless a custom `embedder` is supplied. */ + readonly geminiApiKey?: string; + /** Namespace isolating this tenant's vectors. The single-org self-host passes + * its org id; multi-tenant per-request scoping is a follow-up. Defaults to + * `"default"`. */ + readonly namespace?: string; + /** Gemini embedding model id. Defaults to the v2 model (see embedder). */ + readonly model?: string; + /** Embedding dimensionality β€” MUST equal the Vectorize index's dimensions. */ + readonly dimensions?: number; + /** Inject a custom embedder (tests). Overrides `model`/`geminiApiKey`/`dimensions`. */ + readonly embedder?: ToolEmbedder; +} + +const notConfigured = (): Effect.Effect => + Effect.fail( + new VectorizeSearchError({ + message: + "Vectorize search is not configured (missing the Vectorize binding or Gemini API key).", + }), + ); + +/** Build the `executor.vectorizeSearch` surface. `reindex` embeds the current + * tool catalog and upserts it into Vectorize; it takes the scoped executor as + * an argument because only the request/API layer holds it (the plugin ctx does + * not expose the catalog). Inert β€” `reindex` fails clearly β€” until both a + * Vectorize binding and an embedder are present. */ +const makeVectorizeSearchExtension = (deps: { + readonly namespace: string; + readonly embedder: ToolEmbedder | undefined; + readonly store: VectorizeStore | undefined; +}) => ({ + reindex: (executor: Executor): Effect.Effect => + deps.embedder && deps.store + ? reindexToolCatalog({ + namespace: deps.namespace, + executor, + embedder: deps.embedder, + store: deps.store, + }) + : notConfigured(), +}); + +/** The `executor.vectorizeSearch` surface, derived from its factory. */ +export type VectorizeSearchExtension = ReturnType; + +/** + * Semantic `tools.search` backed by Cloudflare Vectorize + Gemini embeddings. + * Supplies a `runtime.toolDiscoveryProvider`, so it replaces the engine's + * built-in lexical scorer wherever it is registered. The tool catalog is + * indexed explicitly via the `reindex` extension method (see the `/api` subpath + * for the HTTP route). + */ +export const vectorizeSearchPlugin = definePlugin((options?: VectorizeSearchPluginOptions) => { + const namespace = options?.namespace ?? "default"; + const embedder = + options?.embedder ?? + (options?.geminiApiKey + ? makeGeminiEmbedder({ + apiKey: options.geminiApiKey, + model: options.model, + dimensions: options.dimensions, + }) + : undefined); + // Inert without both a Vectorize binding and an embedder β€” the engine then + // keeps its built-in lexical search (mirrors the metrics plugin's no-op). + const store = options?.vectorize ? makeVectorizeStore(options.vectorize) : undefined; + + return { + id: "vectorizeSearch" as const, + packageName: "@executor-js/plugin-vectorize-search", + storage: () => ({}), + extension: () => makeVectorizeSearchExtension({ namespace, embedder, store }), + runtime: { + toolDiscoveryProvider: () => + embedder && store + ? makeVectorizeToolDiscoveryProvider({ embedder, store, namespace }) + : undefined, + }, + }; +}); diff --git a/packages/plugins/vectorize-search/src/sdk/provider.ts b/packages/plugins/vectorize-search/src/sdk/provider.ts new file mode 100644 index 0000000000..10b9a9a245 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/provider.ts @@ -0,0 +1,91 @@ +import { + ExecutionToolError, + type ToolDiscoveryProvider, + type ToolDiscoveryResult, +} from "@executor-js/sdk/core"; +import { Effect } from "effect"; + +import type { ToolEmbedder } from "./embedder"; +import { MAX_TOP_K } from "./vectorize"; +import type { VectorizeMatch, VectorizeStore } from "./vectorize"; + +const asString = (value: unknown): string => (typeof value === "string" ? value : ""); + +const toResult = (match: VectorizeMatch): ToolDiscoveryResult => { + const description = asString(match.metadata?.description); + return { + path: asString(match.metadata?.path), + name: asString(match.metadata?.name), + description: description.length > 0 ? description : undefined, + integration: asString(match.metadata?.integration), + score: match.score, + }; +}; + +/** Narrow results to a search `namespace` (an integration/path prefix), + * mirroring the lexical provider's `matchesNamespace`. Applied to the fetched + * page, so it best-effort narrows within the topK window rather than across the + * whole index. */ +const matchesNamespace = (result: ToolDiscoveryResult, namespace: string | undefined): boolean => { + if (namespace === undefined) return true; + const ns = namespace.trim().toLowerCase(); + if (ns.length === 0) return true; + return ( + result.integration.toLowerCase().startsWith(ns) || result.path.toLowerCase().startsWith(ns) + ); +}; + +/** + * A semantic `tools.search` backend: embed the query (Gemini), nearest-neighbour + * query Vectorize within the tenant `namespace`, and map matches back to + * `ToolDiscoveryResult` straight from the stored metadata (no per-tool describe + * round-trip). + * + * Pagination fetches the full topK window (Vectorize never reports a true total) + * and slices it: `hasMore` means "the filtered window holds more than this page". + * Fetching the whole window β€” rather than probing one item past the page β€” keeps + * `hasMore` correct even when the `input.namespace` filter discards the item that + * would otherwise have been the probe. Deep pagination past the topK cap is out + * of scope for v1. + * + * `input.executor` is unused β€” results come from the index, not a live catalog + * scan. `input.namespace` narrows the page to an integration/path prefix. + */ +export const makeVectorizeToolDiscoveryProvider = (deps: { + readonly embedder: ToolEmbedder; + readonly store: VectorizeStore; + readonly namespace: string; +}): ToolDiscoveryProvider => ({ + searchTools: ({ query, namespace, limit, offset }) => + Effect.gen(function* () { + if (query.trim().length === 0) { + return { items: [], total: 0, hasMore: false, nextOffset: null }; + } + const safeOffset = Math.max(offset, 0); + const vector = yield* deps.embedder.embedQuery(query); + // Fetch the whole topK window (not a tight probe) so the post-fetch + // namespace filter can't drop the one item that signals hasMore. + const matches = yield* deps.store.query({ + vector, + namespace: deps.namespace, + topK: MAX_TOP_K, + }); + const ranked = matches + .filter((match) => asString(match.metadata?.path).length > 0) + .map(toResult) + .filter((result) => matchesNamespace(result, namespace)); + const start = Math.min(safeOffset, ranked.length); + const items = ranked.slice(start, start + limit); + const hasMore = ranked.length > start + items.length; + return { + items, + total: ranked.length, + hasMore, + nextOffset: hasMore ? start + items.length : null, + }; + }).pipe( + Effect.mapError( + (cause) => new ExecutionToolError({ message: "Vectorize tool search failed.", cause }), + ), + ), +}); diff --git a/packages/plugins/vectorize-search/src/sdk/search.test.ts b/packages/plugins/vectorize-search/src/sdk/search.test.ts new file mode 100644 index 0000000000..a688e3f7e4 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/search.test.ts @@ -0,0 +1,268 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import type { Executor, Tool } from "@executor-js/sdk/core"; + +import type { ToolEmbedder } from "./embedder"; +import { reindexToolCatalog } from "./indexer"; +import { makeVectorizeToolDiscoveryProvider } from "./provider"; +import { makeVectorizeStore } from "./vectorize"; +import type { + VectorizeIndex, + VectorizeMatch, + VectorizeStore, + VectorizeVectorInput, +} from "./vectorize"; + +const fakeEmbedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => Effect.succeed(texts.map(() => [0.1, 0.2, 0.3])), + embedQuery: () => Effect.succeed([1, 0, 0]), +}; + +// The fake store honours topK (slicing to it) so pagination is exercised against +// a realistic window β€” a store that ignored topK would mask hasMore bugs. +const makeQueryStore = (matches: readonly VectorizeMatch[]): VectorizeStore => ({ + query: ({ topK }) => Effect.succeed(matches.slice(0, topK)), + upsert: () => Effect.void, + deleteByIds: () => Effect.void, +}); + +const matchIn = (path: string, integration: string, score: number): VectorizeMatch => ({ + id: `org#${path}`, + score, + metadata: { path, name: path, description: `desc ${path}`, integration }, +}); + +const match = (path: string, score: number): VectorizeMatch => matchIn(path, "github", score); + +// Minimal stubs β€” the indexer only calls `tools.list()` and reads a few Tool +// fields, so a full branded Tool/Executor is unnecessary for these unit tests. +const stubTool = (address: string, name: string, integration: string, description: string): Tool => + // oxlint-disable-next-line executor/no-double-cast -- test stub: the indexer reads only these Tool fields + ({ address, name, integration, description }) as unknown as Tool; +const stubExecutor = (tools: readonly Tool[]): Executor => + // oxlint-disable-next-line executor/no-double-cast -- test stub: only tools.list() is exercised + ({ tools: { list: () => Effect.succeed(tools) } }) as unknown as Executor; + +describe("makeVectorizeToolDiscoveryProvider", () => { + it.effect("maps Vectorize matches to ToolDiscoveryResult", () => + Effect.gen(function* () { + const provider = makeVectorizeToolDiscoveryProvider({ + embedder: fakeEmbedder, + store: makeQueryStore([match("repos.get", 0.9), match("repos.list", 0.8)]), + namespace: "org", + }); + const page = yield* provider.searchTools({ + executor: undefined as never, + query: "get a repo", + limit: 10, + offset: 0, + }); + expect(page.items).toHaveLength(2); + expect(page.items[0]).toEqual({ + path: "repos.get", + name: "repos.get", + description: "desc repos.get", + integration: "github", + score: 0.9, + }); + expect(page.total).toBe(2); + expect(page.hasMore).toBe(false); + expect(page.nextOffset).toBeNull(); + }), + ); + + it.effect("paginates within the returned matches", () => + Effect.gen(function* () { + const provider = makeVectorizeToolDiscoveryProvider({ + embedder: fakeEmbedder, + store: makeQueryStore([match("a", 0.9), match("b", 0.8), match("c", 0.7)]), + namespace: "org", + }); + const page = yield* provider.searchTools({ + executor: undefined as never, + query: "x", + limit: 1, + offset: 1, + }); + expect(page.items.map((item) => item.path)).toEqual(["b"]); + expect(page.hasMore).toBe(true); + expect(page.nextOffset).toBe(2); + }), + ); + + it.effect("reports hasMore on a full first page", () => + Effect.gen(function* () { + const provider = makeVectorizeToolDiscoveryProvider({ + embedder: fakeEmbedder, + store: makeQueryStore([match("a", 0.9), match("b", 0.8), match("c", 0.7)]), + namespace: "org", + }); + const page = yield* provider.searchTools({ + executor: undefined as never, + query: "x", + limit: 2, + offset: 0, + }); + expect(page.items.map((item) => item.path)).toEqual(["a", "b"]); + expect(page.hasMore).toBe(true); + expect(page.nextOffset).toBe(2); + }), + ); + + it.effect("narrows results to input.namespace (integration prefix)", () => + Effect.gen(function* () { + const provider = makeVectorizeToolDiscoveryProvider({ + embedder: fakeEmbedder, + store: makeQueryStore([ + matchIn("github.repos.get", "github", 0.9), + matchIn("slack.chat.post", "slack", 0.8), + matchIn("github.issues.list", "github", 0.7), + ]), + namespace: "org", + }); + const page = yield* provider.searchTools({ + executor: undefined as never, + query: "x", + namespace: "github", + limit: 10, + offset: 0, + }); + expect(page.items.every((item) => item.integration === "github")).toBe(true); + expect(page.items.map((item) => item.path)).toEqual([ + "github.repos.get", + "github.issues.list", + ]); + }), + ); + + it.effect("keeps hasMore correct when the namespace filter drops interspersed items", () => + Effect.gen(function* () { + // Greptile scenario: a non-matching item sits where a tight probe would be. + const provider = makeVectorizeToolDiscoveryProvider({ + embedder: fakeEmbedder, + store: makeQueryStore([ + matchIn("github.a", "github", 0.9), + matchIn("slack.b", "slack", 0.8), + matchIn("github.c", "github", 0.7), + matchIn("github.d", "github", 0.6), + ]), + namespace: "org", + }); + const page = yield* provider.searchTools({ + executor: undefined as never, + query: "x", + namespace: "github", + limit: 2, + offset: 0, + }); + expect(page.items.map((item) => item.path)).toEqual(["github.a", "github.c"]); + expect(page.hasMore).toBe(true); + expect(page.nextOffset).toBe(2); + }), + ); + + it.effect("returns empty for a blank query (no embedding call)", () => + Effect.gen(function* () { + const provider = makeVectorizeToolDiscoveryProvider({ + embedder: { + ...fakeEmbedder, + embedQuery: () => Effect.die("embedQuery must not be called for a blank query") as never, + }, + store: makeQueryStore([match("a", 0.9)]), + namespace: "org", + }); + const page = yield* provider.searchTools({ + executor: undefined as never, + query: " ", + limit: 10, + offset: 0, + }); + expect(page.items).toHaveLength(0); + expect(page.total).toBe(0); + }), + ); +}); + +describe("reindexToolCatalog", () => { + it.effect("projects the catalog and upserts namespaced records", () => + Effect.gen(function* () { + const upserts: VectorizeVectorInput[][] = []; + const store: VectorizeStore = { + query: () => Effect.succeed([]), + upsert: (vectors) => + Effect.sync(() => { + upserts.push([...vectors]); + }), + deleteByIds: () => Effect.void, + }; + const tool = stubTool("tools.github.repos.get", "repos.get", "github", "Get a repository"); + const executor = stubExecutor([tool]); + + const result = yield* reindexToolCatalog({ + namespace: "org", + executor, + embedder: fakeEmbedder, + store, + }); + + expect(result.indexedToolCount).toBe(1); + expect(upserts).toHaveLength(1); + const record = upserts[0]![0]!; + expect(record.id).toBe("org#github.repos.get"); + expect(record.namespace).toBe("org"); + expect(record.values).toEqual([0.1, 0.2, 0.3]); + expect(record.metadata).toMatchObject({ + path: "github.repos.get", + integration: "github", + name: "repos.get", + }); + }), + ); + + it.effect("no-ops on an empty catalog", () => + Effect.gen(function* () { + const store: VectorizeStore = { + query: () => Effect.succeed([]), + upsert: () => Effect.die("upsert must not be called for an empty catalog") as never, + deleteByIds: () => Effect.void, + }; + const executor = stubExecutor([]); + const result = yield* reindexToolCatalog({ + namespace: "org", + executor, + embedder: fakeEmbedder, + store, + }); + expect(result.indexedToolCount).toBe(0); + }), + ); +}); + +describe("makeVectorizeStore", () => { + it.effect("chunks upserts under the Vectorize per-call cap", () => + Effect.gen(function* () { + const batches: number[] = []; + const index: VectorizeIndex = { + query: () => Promise.resolve({ matches: [] }), + upsert: (vectors) => { + batches.push(vectors.length); + return Promise.resolve({}); + }, + deleteByIds: () => Promise.resolve({}), + }; + const store = makeVectorizeStore(index); + const vectors: readonly VectorizeVectorInput[] = Array.from({ length: 120 }, (_, i) => ({ + id: `v${i}`, + values: [0, 0, 0], + })); + + yield* store.upsert(vectors); + + // 120 records at batch size 50 -> [50, 50, 20]; never a single 120 call. + expect(batches).toEqual([50, 50, 20]); + }), + ); +}); diff --git a/packages/plugins/vectorize-search/src/sdk/vectorize.ts b/packages/plugins/vectorize-search/src/sdk/vectorize.ts new file mode 100644 index 0000000000..72159928a7 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/vectorize.ts @@ -0,0 +1,106 @@ +import { Effect } from "effect"; + +import { VectorizeSearchError } from "./errors"; + +/** Minimal structural view of a Cloudflare Vectorize binding (the subset this + * plugin uses). A real `Vectorize` / `VectorizeIndex` binding from + * `@cloudflare/workers-types` satisfies it structurally, so the plugin stays + * free of the Workers global type lib β€” the same approach the metrics plugin + * takes for the Analytics Engine binding. */ +export interface VectorizeIndex { + query( + vector: readonly number[], + options?: { + readonly topK?: number; + readonly namespace?: string; + readonly returnMetadata?: boolean | "all" | "indexed" | "none"; + readonly returnValues?: boolean; + }, + ): Promise; + upsert(vectors: readonly VectorizeVectorInput[]): Promise; + deleteByIds(ids: readonly string[]): Promise; +} + +export interface VectorizeMatches { + readonly matches: readonly VectorizeMatch[]; + readonly count?: number; +} + +export interface VectorizeMatch { + readonly id: string; + readonly score: number; + readonly namespace?: string; + readonly metadata?: Record; +} + +export interface VectorizeVectorInput { + readonly id: string; + readonly values: readonly number[]; + readonly namespace?: string; + readonly metadata?: Record; +} + +/** Vectorize caps `topK` at 100 (with metadata). The provider fetches this full + * window and paginates within it, so the store clamps any request to it. */ +export const MAX_TOP_K = 100; + +/** Vectorize caps a single upsert at ~1000 vectors / ~2 MB. At 1536-d a few + * dozen vectors already approach the size cap, so chunk conservatively well + * under both limits and upsert the batches sequentially. */ +const UPSERT_BATCH_SIZE = 50; + +const chunk = (items: readonly A[], size: number): readonly (readonly A[])[] => { + const safe = Math.max(1, Math.floor(size)); + const out: A[][] = []; + for (let i = 0; i < items.length; i += safe) { + out.push(items.slice(i, i + safe)); + } + return out; +}; + +/** Effect-wrapped, plugin-facing view over a Vectorize binding. */ +export interface VectorizeStore { + readonly query: (input: { + readonly vector: readonly number[]; + readonly namespace: string; + readonly topK: number; + }) => Effect.Effect; + readonly upsert: ( + vectors: readonly VectorizeVectorInput[], + ) => Effect.Effect; + readonly deleteByIds: (ids: readonly string[]) => Effect.Effect; +} + +export const makeVectorizeStore = (index: VectorizeIndex): VectorizeStore => ({ + query: ({ vector, namespace, topK }) => + Effect.tryPromise({ + try: () => + index.query([...vector], { + topK: Math.min(Math.max(1, topK), MAX_TOP_K), + namespace, + returnMetadata: "all", + }), + catch: (cause) => new VectorizeSearchError({ message: "Vectorize query failed.", cause }), + }).pipe(Effect.map((result) => result.matches)), + upsert: (vectors) => + vectors.length === 0 + ? Effect.void + : Effect.forEach( + chunk([...vectors], UPSERT_BATCH_SIZE), + (batch) => + Effect.tryPromise({ + try: () => index.upsert(batch), + catch: (cause) => + new VectorizeSearchError({ message: "Vectorize upsert failed.", cause }), + }), + { concurrency: 1, discard: true }, + ), + deleteByIds: (ids) => + ids.length === 0 + ? Effect.void + : Effect.tryPromise({ + try: () => index.deleteByIds([...ids]), + catch: (cause) => + new VectorizeSearchError({ message: "Vectorize delete failed.", cause }), + }).pipe(Effect.asVoid), +}); diff --git a/packages/plugins/vectorize-search/tsconfig.json b/packages/plugins/vectorize-search/tsconfig.json new file mode 100644 index 0000000000..eebc1e6f14 --- /dev/null +++ b/packages/plugins/vectorize-search/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "skipLibCheck": true, + "lib": ["ES2022"], + "types": ["bun-types", "node"], + "noUnusedLocals": true, + "noImplicitOverride": true, + "plugins": [ + { + "name": "@effect/language-service", + "ignoreEffectSuggestionsInTscExitCode": true, + "ignoreEffectWarningsInTscExitCode": true, + "diagnosticSeverity": { + "preferSchemaOverJson": "off" + } + } + ] + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/plugins/vectorize-search/tsup.config.ts b/packages/plugins/vectorize-search/tsup.config.ts new file mode 100644 index 0000000000..a5ba012963 --- /dev/null +++ b/packages/plugins/vectorize-search/tsup.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: { + index: "src/sdk/index.ts", + api: "src/api/index.ts", + }, + format: ["esm"], + dts: false, + sourcemap: true, + clean: true, + external: [/^@executor-js\//, /^effect/, /^@effect\//, "ai", /^@ai-sdk\//], +}); diff --git a/packages/plugins/vectorize-search/vitest.config.ts b/packages/plugins/vectorize-search/vitest.config.ts new file mode 100644 index 0000000000..5bfa2d586e --- /dev/null +++ b/packages/plugins/vectorize-search/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + passWithNoTests: true, + }, +}); From 7747efcc61c39bf13d2026db4955eda8792169c8 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Tue, 16 Jun 2026 13:05:07 +0530 Subject: [PATCH 30/57] feat(plugin-vectorize-search): facet chunking, hybrid RRF, and incremental reindex (#27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Extends the Cloudflare Vectorize `tools.search` plugin (#26) with three retrieval improvements, each behind a pluggable seam. - **Structure-aware facet chunker** β€” a tool is chunked by its structure (an always-present identity facet, plus optional description / input-schema / output-schema facets derived from `tools.schema`) instead of a single blind vector. Long descriptions split recursively on paragraph β†’ sentence boundaries β€” no fixed windows, no truncation; schema-less tools collapse to one identity chunk. `ChunkerService` makes the chunker swappable, and `makeWholeChunker` is retained as a single-vector baseline. - **Hybrid lexical + vector search (RRF)** β€” because a plugin discovery provider fully supersedes the engine's built-in lexical scorer (first-provider-wins), a vector-only provider would drop lexical matching entirely. `makeHybridToolDiscoveryProvider` fuses an injected lexical provider with the vector provider via Reciprocal Rank Fusion (weights 0.7 / 0.3, k = 60). The lexical provider is a plugin option, so the plugin never has to depend on the engine package β€” the host passes the built-in `defaultToolDiscoveryProvider` to opt into hybrid. - **Incremental fingerprint reindex** β€” `reconcileToolCatalog` fingerprints each tool and diffs against a per-tool side-index in plugin storage; only new and changed tools are re-chunked and re-embedded, and a changed tool's old chunk vectors are deleted before the new ones upsert. Tools absent from the live catalog are counted and logged, not deleted β€” per-isolate catalog variance makes deletion unsafe in v1. Also adds `withCloudflareLimits`, a store decorator that fails loudly on Cloudflare's hard limits (64-byte ids, topK ≀ 20 with full metadata) rather than surfacing opaque Vectorize 4xx errors mid-reindex; the plugin wraps its bound store with it, and the provider requests topK 20 on the full-metadata path. ## Testing `typecheck`, `lint`, `format:check`, and the package test suite (46 tests) pass on this branch in isolation. The downstream Cloudflare host typechecks against the changed plugin API. ## Stack Base of a two-part stack. The local eval harness that tunes this offline (pluggable embedder/store, zvec, golden-set eval) is stacked on top. ### [Stack](https://github.com/kitlangton/stack) 1. **#27** πŸ‘ˆ current 2. #28 --- .../plugins/vectorize-search/src/api/group.ts | 15 +- .../src/sdk/chunker-service.ts | 24 ++ .../vectorize-search/src/sdk/chunker.test.ts | 205 ++++++++++ .../vectorize-search/src/sdk/chunker.ts | 320 +++++++++++++++ .../vectorize-search/src/sdk/collections.ts | 29 ++ .../vectorize-search/src/sdk/documents.ts | 97 +++-- .../src/sdk/fingerprint.test.ts | 90 +++++ .../vectorize-search/src/sdk/fingerprint.ts | 48 +++ .../vectorize-search/src/sdk/hybrid.test.ts | 283 ++++++++++++++ .../vectorize-search/src/sdk/hybrid.ts | 159 ++++++++ .../plugins/vectorize-search/src/sdk/index.ts | 30 +- .../vectorize-search/src/sdk/indexer.test.ts | 368 ++++++++++++++++++ .../vectorize-search/src/sdk/indexer.ts | 256 ++++++++++-- .../vectorize-search/src/sdk/plugin.ts | 88 ++++- .../vectorize-search/src/sdk/provider.ts | 26 +- .../vectorize-search/src/sdk/search.test.ts | 78 +--- .../src/sdk/store-cloudflare-limits.test.ts | 160 ++++++++ .../src/sdk/store-cloudflare-limits.ts | 61 +++ 18 files changed, 2170 insertions(+), 167 deletions(-) create mode 100644 packages/plugins/vectorize-search/src/sdk/chunker-service.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/chunker.test.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/chunker.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/collections.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/fingerprint.test.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/fingerprint.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/hybrid.test.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/hybrid.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/indexer.test.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.test.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.ts diff --git a/packages/plugins/vectorize-search/src/api/group.ts b/packages/plugins/vectorize-search/src/api/group.ts index d2ff0228b9..97dd70cdd5 100644 --- a/packages/plugins/vectorize-search/src/api/group.ts +++ b/packages/plugins/vectorize-search/src/api/group.ts @@ -10,16 +10,19 @@ import { InternalError } from "@executor-js/api"; // matching the execution-history/graphql convention β€” the per-request executor // is already owner-scoped at the host edge, so there is no `:scopeId` segment. // -// Reindex takes no body: it embeds the whole tool catalog for the scoped tenant -// and upserts it into Vectorize. `VectorizeSearchError` on the extension flows -// through the typed channel and `capture` downgrades it to `InternalError`. +// Reindex takes no body: it reconciles the whole tool catalog for the scoped +// tenant against Vectorize (incremental fingerprint diff). `VectorizeSearchError` +// on the extension flows through the typed channel and `capture` downgrades it +// to `InternalError`. // --------------------------------------------------------------------------- -/** Result of a reindex: the namespace touched + the number of tools embedded - * and upserted into Vectorize. */ +/** Result of a reindex reconcile: counts for each category of tool processed. */ export const ReindexResponse = Schema.Struct({ namespace: Schema.String, - indexedToolCount: Schema.Number, + total: Schema.Number, + reembedded: Schema.Number, + unchanged: Schema.Number, + removedSkipped: Schema.Number, }); export const VectorizeSearchGroup = HttpApiGroup.make("vectorizeSearch").add( diff --git a/packages/plugins/vectorize-search/src/sdk/chunker-service.ts b/packages/plugins/vectorize-search/src/sdk/chunker-service.ts new file mode 100644 index 0000000000..7854b3d39b --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/chunker-service.ts @@ -0,0 +1,24 @@ +import { Context, Layer } from "effect"; + +import { + type Chunker, + type FacetChunkerOptions, + makeFacetChunker, + makeWholeChunker, +} from "./chunker"; + +// --------------------------------------------------------------------------- +// Pluggable chunker seam. The indexer depends on `ChunkerService`; the host +// (or eval harness) supplies ONE of the layers β€” the facet chunker (default, +// production) or the whole chunker (benchmark baseline). +// --------------------------------------------------------------------------- + +export class ChunkerService extends Context.Service()( + "@executor-js/plugin-vectorize-search/ChunkerService", +) {} + +export const facetChunkerLayer = (options?: FacetChunkerOptions): Layer.Layer => + Layer.succeed(ChunkerService)(makeFacetChunker(options)); + +export const wholeChunkerLayer = (): Layer.Layer => + Layer.succeed(ChunkerService)(makeWholeChunker()); diff --git a/packages/plugins/vectorize-search/src/sdk/chunker.test.ts b/packages/plugins/vectorize-search/src/sdk/chunker.test.ts new file mode 100644 index 0000000000..09627b6c04 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/chunker.test.ts @@ -0,0 +1,205 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { type ToolDocumentInput, makeFacetChunker, makeWholeChunker } from "./chunker"; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +/** A minimal, schema-less tool β€” should produce exactly one identity chunk. */ +const schemalessDoc: ToolDocumentInput = { + path: "stripe.customers.list", + name: "customers.list", + integration: "stripe", + description: "List all customers.", +}; + +/** A realistic Stripe subscriptions tool with a 5-paragraph description and + * representative TypeScript input/output schemas. */ +const stripeSubscriptionsDoc: ToolDocumentInput = { + path: "stripe.subscriptions.create", + name: "subscriptions.create", + integration: "stripe", + description: `Create a new subscription for a customer to one or more products. + +A subscription is the core billing object in Stripe. It schedules recurring charges by anchoring to a billing cycle (monthly, yearly, or a custom interval) and a collection of price items. The customer is billed automatically at the start of each period unless the subscription is in trial or paused. + +When creating a subscription, you must specify at least one item referencing a recurring Price object. If the customer has a default payment method on file, Stripe will attempt the first charge immediately; otherwise you can pass payment_behavior: 'default_incomplete' to create the subscription in an incomplete state and collect payment separately. + +Subscriptions support a rich set of lifecycle features: free trial periods via trial_period_days or trial_end, proration when upgrading or downgrading mid-cycle, metered billing via usage records, and multiple tax rates. You can also attach metadata, set a custom billing anchor, or pass coupon and promotion codes at creation time. + +After creation the subscription moves through statuses: incomplete, trialing, active, past_due, unpaid, and canceled. Webhooks such as customer.subscription.created and invoice.payment_succeeded are the recommended integration path for reacting to these transitions reliably.`, + inputTypeScript: `{ + customer: string; + items: Array<{ + price: string; + quantity?: number; + }>; + payment_behavior?: "default_incomplete" | "error_if_incomplete" | "allow_incomplete"; + trial_period_days?: number; + trial_end?: number | "now"; + coupon?: string; + promotion_code?: string; + metadata?: Record; +}`, + outputTypeScript: `{ + id: string; + object: "subscription"; + customer: string; + status: "incomplete" | "trialing" | "active" | "past_due" | "unpaid" | "canceled"; + current_period_start: number; + current_period_end: number; + items: { + data: Array<{ + id: string; + price: { id: string; currency: string; unit_amount: number | null }; + quantity: number; + }>; + }; + latest_invoice: string | null; + trial_start: number | null; + trial_end: number | null; +}`, +}; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("makeFacetChunker", () => { + it("schema-less short tool β†’ exactly one identity chunk", () => { + const chunker = makeFacetChunker(); + const chunks = chunker.chunk("ns", schemalessDoc); + + expect(chunks).toHaveLength(1); + expect(chunks[0]!.facet).toBe("identity"); + expect(chunks[0]!.chunkIndex).toBe(0); + expect(chunks[0]!.path).toBe("stripe.customers.list"); + expect(chunks[0]!.integration).toBe("stripe"); + expect(chunks[0]!.name).toBe("customers.list"); + expect(chunks[0]!.embeddingText).toContain("stripe"); + expect(chunks[0]!.embeddingText).toContain("customers.list"); + }); + + it("5-paragraph description β†’ identity + multiple description chunks on paragraph boundaries", () => { + const budget = 400; // small budget so paragraphs definitely fan out + const chunker = makeFacetChunker({ facetCharBudget: budget }); + const chunks = chunker.chunk("ns", stripeSubscriptionsDoc); + + // Must have the identity chunk. + const identityChunks = chunks.filter((c) => c.facet === "identity"); + expect(identityChunks).toHaveLength(1); + + // Must have description continuation chunks (5-paragraph description with + // remainder beyond the lead). + const descChunks = chunks.filter((c) => c.facet === "description"); + expect(descChunks.length).toBeGreaterThan(0); + + // No chunk embedding text exceeds the budget. + chunks + .filter((c) => c.facet === "identity" || c.facet === "description") + .forEach((c) => expect(c.embeddingText.length).toBeLessThanOrEqual(budget)); + + // Splits land on paragraph boundaries: no description chunk should contain + // the exact paragraph separator (\n\n) as an internal segment that spans + // two different source paragraphs in the same budget group. We verify this + // by checking that each description chunk is a substring of the original + // description (possibly trimmed) β€” i.e., no chunk was built by stitching + // mid-word across a boundary. + const fullDesc = stripeSubscriptionsDoc.description.trim(); + for (const chunk of descChunks) { + const text = chunk.embeddingText.trim(); + expect(fullDesc).toContain(text.split("\n\n")[0]!.trim()); + } + }); + + it("tool with inputTypeScript and outputTypeScript β†’ has input and output facets", () => { + const chunker = makeFacetChunker(); + const chunks = chunker.chunk("ns", stripeSubscriptionsDoc); + + const inputChunks = chunks.filter((c) => c.facet === "input"); + const outputChunks = chunks.filter((c) => c.facet === "output"); + + expect(inputChunks).toHaveLength(1); + expect(outputChunks).toHaveLength(1); + + // The input chunk's embeddingText contains the schema header and schema body. + expect(inputChunks[0]!.embeddingText).toContain("stripe.subscriptions.create input"); + expect(inputChunks[0]!.embeddingText).toContain("customer: string"); + + // The output chunk's embeddingText contains the output schema. + expect(outputChunks[0]!.embeddingText).toContain("stripe.subscriptions.create output"); + expect(outputChunks[0]!.embeddingText).toContain("status:"); + }); + + it("trivial outputTypeScript β†’ no output facet", () => { + const chunker = makeFacetChunker(); + const doc: ToolDocumentInput = { + ...stripeSubscriptionsDoc, + outputTypeScript: "{}", + }; + const chunks = chunker.chunk("ns", doc); + expect(chunks.filter((c) => c.facet === "output")).toHaveLength(0); + }); + + it("all ids are ≀ 64 bytes", () => { + const chunker = makeFacetChunker({ facetCharBudget: 400 }); + const chunks = chunker.chunk( + "very-long-namespace-that-might-cause-id-bloat", + stripeSubscriptionsDoc, + ); + for (const chunk of chunks) { + expect(chunk.id.length).toBeLessThanOrEqual(64); + } + }); + + it("chunkIndex increments monotonically across facets", () => { + const chunker = makeFacetChunker({ facetCharBudget: 400 }); + const chunks = chunker.chunk("ns", stripeSubscriptionsDoc); + for (let i = 0; i < chunks.length; i++) { + expect(chunks[i]!.chunkIndex).toBe(i); + } + }); + + it("ids are stable and deterministic across calls", () => { + const chunker = makeFacetChunker(); + const a = chunker.chunk("ns", schemalessDoc); + const b = chunker.chunk("ns", schemalessDoc); + expect(a[0]!.id).toBe(b[0]!.id); + }); + + it("ids differ across namespaces for the same tool", () => { + const chunker = makeFacetChunker(); + const a = chunker.chunk("ns1", schemalessDoc); + const b = chunker.chunk("ns2", schemalessDoc); + expect(a[0]!.id).not.toBe(b[0]!.id); + }); +}); + +describe("makeWholeChunker", () => { + it("always produces exactly one identity chunk", () => { + const chunker = makeWholeChunker(); + + const single = chunker.chunk("ns", schemalessDoc); + expect(single).toHaveLength(1); + expect(single[0]!.facet).toBe("identity"); + + // Even a rich schema tool produces one chunk. + const rich = chunker.chunk("ns", stripeSubscriptionsDoc); + expect(rich).toHaveLength(1); + expect(rich[0]!.facet).toBe("identity"); + }); + + it("embeddingText contains the full description", () => { + const chunker = makeWholeChunker(); + const chunks = chunker.chunk("ns", stripeSubscriptionsDoc); + // Spot-check a phrase from the 4th paragraph. + expect(chunks[0]!.embeddingText).toContain("trial_period_days"); + }); + + it("id ≀ 64 bytes", () => { + const chunker = makeWholeChunker(); + const chunks = chunker.chunk("long-namespace-string-for-safety", stripeSubscriptionsDoc); + expect(chunks[0]!.id.length).toBeLessThanOrEqual(64); + }); +}); diff --git a/packages/plugins/vectorize-search/src/sdk/chunker.ts b/packages/plugins/vectorize-search/src/sdk/chunker.ts new file mode 100644 index 0000000000..54978240ce --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/chunker.ts @@ -0,0 +1,320 @@ +// --------------------------------------------------------------------------- +// Facet-aware chunker (P1). A tool becomes one OR MORE chunks, each targeting +// a distinct semantic facet: identity (header + description lead), description +// continuation paragraphs, input schema, and output schema. +// +// Design goals: +// - Structure-aware splitting: paragraph β†’ sentence β†’ hard-cap (almost never). +// No fixed char windows; chunks always end on coherent text boundaries. +// - Schema-less tools degrade cleanly to a single identity chunk. +// - Hashed ids keep every id ≀ 64 bytes (Vectorize hard cap). +// - makeWholeChunker() mirrors the pre-facet documents.ts behavior for +// benchmarking / regression comparisons. +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// cyrb53 β€” deterministic 53-bit hash, dependency-free. Copied locally so +// chunker.ts has no import dependency on the old chunking.ts. +// --------------------------------------------------------------------------- +const cyrb53 = (str: string): string => { + let h1 = 0xdeadbeef; + let h2 = 0x41c6ce57; + for (let i = 0; i < str.length; i++) { + const ch = str.charCodeAt(i); + h1 = Math.imul(h1 ^ ch, 2654435761); + h2 = Math.imul(h2 ^ ch, 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909); + return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(36); +}; + +// --------------------------------------------------------------------------- +// Public types +// --------------------------------------------------------------------------- + +/** The input representation of a tool used by the chunker β€” richer than the + * core Tool contract, because it includes the schema strings produced by + * tools.schema(). */ +export interface ToolDocumentInput { + readonly path: string; + readonly name: string; + readonly integration: string; + readonly description: string; + readonly inputTypeScript?: string; + readonly outputTypeScript?: string; + readonly typeScriptDefinitions?: Record; +} + +/** Which semantic facet a chunk represents. */ +export type ChunkFacet = "identity" | "description" | "input" | "output"; + +/** A single embeddable unit of a tool, carrying the facet tag and a stable + * hashed id that is guaranteed to be ≀ 64 bytes. */ +export interface ToolChunk { + readonly id: string; + readonly path: string; + readonly name: string; + readonly integration: string; + readonly facet: ChunkFacet; + readonly chunkIndex: number; + readonly embeddingText: string; +} + +/** The chunker contract. The namespace is the logical partition key (matches the + * Vectorize namespace) and is baked into each chunk id so ids are globally + * unique across namespaces. */ +export interface Chunker { + readonly chunk: (namespace: string, doc: ToolDocumentInput) => readonly ToolChunk[]; +} + +/** Options for the facet chunker. */ +export interface FacetChunkerOptions { + /** Per-facet character budget (~4 chars/token). Default β‰ˆ 1600 (400 tokens). */ + readonly facetCharBudget?: number; +} + +// --------------------------------------------------------------------------- +// Internal constants +// --------------------------------------------------------------------------- + +const DEFAULT_FACET_CHAR_BUDGET = 1600; + +// Sentence-boundary regex: split after . ! ? followed by whitespace or end. +const SENTENCE_BOUNDARY = /(?<=[.!?])\s+/; + +// Paragraph boundary: one or more blank lines. +const PARAGRAPH_BOUNDARY = /\n\s*\n/; + +// A "trivial" schema string contributes nothing useful to the embedding. +const isTrivialSchema = (ts: string | undefined): ts is undefined => { + if (ts === undefined) return true; + const trimmed = ts.trim(); + return ( + trimmed.length === 0 || + trimmed === "{}" || + trimmed === "void" || + trimmed === "undefined" || + trimmed === "null" || + trimmed === "unknown" + ); +}; + +// --------------------------------------------------------------------------- +// Chunk-id factory +// --------------------------------------------------------------------------- + +/** `t_${cyrb53(namespace#path#facet#index)}` β€” always ≀ 64 bytes. */ +const makeId = (namespace: string, path: string, facet: ChunkFacet, chunkIndex: number): string => + `t_${cyrb53(`${namespace}#${path}#${facet}#${chunkIndex}`)}`; + +// --------------------------------------------------------------------------- +// Structure-aware splitting +// --------------------------------------------------------------------------- + +/** Extract the first paragraph ("lead") from text. Returns `[lead, remainder]`. + * The lead never includes the blank-line separator. */ +const splitLead = (text: string): [string, string] => { + const match = text.search(PARAGRAPH_BOUNDARY); + if (match === -1) return [text.trim(), ""]; + return [text.slice(0, match).trim(), text.slice(match).trim()]; +}; + +/** Split `text` into pieces each ≀ `budget` characters, respecting structure: + * paragraphs first, then sentences, then hard-cap as a last resort. + * Adjacent paragraphs are greedily grouped up to `budget`. */ +const recursiveSplit = (text: string, budget: number): readonly string[] => { + if (text.length === 0) return []; + if (text.length <= budget) return [text]; + + // --- paragraph split --- + const rawParagraphs = text + .split(PARAGRAPH_BOUNDARY) + .map((p) => p.trim()) + .filter(Boolean); + if (rawParagraphs.length > 1) { + // Greedily group adjacent paragraphs up to budget. + const groups: string[] = []; + let current = ""; + for (const para of rawParagraphs) { + const candidate = current.length === 0 ? para : `${current}\n\n${para}`; + if (candidate.length <= budget) { + current = candidate; + } else { + if (current.length > 0) groups.push(current); + // The paragraph itself may exceed the budget β€” recurse into it. + if (para.length > budget) { + groups.push(...recursiveSplit(para, budget)); + current = ""; + } else { + current = para; + } + } + } + if (current.length > 0) groups.push(current); + return groups; + } + + // --- sentence split --- + const sentences = text + .split(SENTENCE_BOUNDARY) + .map((s) => s.trim()) + .filter(Boolean); + if (sentences.length > 1) { + const groups: string[] = []; + let current = ""; + for (const sentence of sentences) { + const candidate = current.length === 0 ? sentence : `${current} ${sentence}`; + if (candidate.length <= budget) { + current = candidate; + } else { + if (current.length > 0) groups.push(current); + if (sentence.length > budget) { + // Hard-cap last resort: slice at budget. + groups.push(...hardSlice(sentence, budget)); + current = ""; + } else { + current = sentence; + } + } + } + if (current.length > 0) groups.push(current); + return groups; + } + + // --- hard-cap last resort --- + return hardSlice(text, budget); +}; + +const hardSlice = (text: string, budget: number): readonly string[] => { + const out: string[] = []; + for (let i = 0; i < text.length; i += budget) { + out.push(text.slice(i, i + budget)); + } + return out; +}; + +// --------------------------------------------------------------------------- +// makeFacetChunker +// --------------------------------------------------------------------------- + +export const makeFacetChunker = (options?: FacetChunkerOptions): Chunker => { + const budget = Math.max(1, options?.facetCharBudget ?? DEFAULT_FACET_CHAR_BUDGET); + + return { + chunk(namespace, doc) { + const { path, name, integration, description } = doc; + const chunks: ToolChunk[] = []; + + // ----------------------------------------------------------------------- + // Identity facet (chunkIndex 0, always present) + // ----------------------------------------------------------------------- + const header = `${integration} ${path}\n${name}`; + const [lead, remainder] = splitLead(description.trim()); + + // Budget remaining after the header for the lead. + const headerLen = header.length + (lead.length > 0 ? 1 : 0); // +1 for '\n' + const leadBudget = Math.max(0, budget - headerLen); + const clippedLead = lead.slice(0, leadBudget); // Trim lead to budget; it rarely exceeds it. + const identityText = clippedLead.length > 0 ? `${header}\n${clippedLead}` : header; + + let globalIndex = 0; + chunks.push({ + id: makeId(namespace, path, "identity", globalIndex), + path, + name, + integration, + facet: "identity", + chunkIndex: globalIndex, + embeddingText: identityText, + }); + globalIndex++; + + // ----------------------------------------------------------------------- + // Description facets (remainder split by structure) + // ----------------------------------------------------------------------- + if (remainder.length > 0) { + const pieces = recursiveSplit(remainder, budget); + for (const piece of pieces) { + if (piece.trim().length === 0) continue; + chunks.push({ + id: makeId(namespace, path, "description", globalIndex), + path, + name, + integration, + facet: "description", + chunkIndex: globalIndex, + embeddingText: piece.trim(), + }); + globalIndex++; + } + } + + // ----------------------------------------------------------------------- + // Input facet + // ----------------------------------------------------------------------- + if (!isTrivialSchema(doc.inputTypeScript)) { + const text = `${integration} ${path} input\n${doc.inputTypeScript}`; + // If the schema exceeds the budget, keep the structural head (entire + // text is preferable; hard-slicing is safer than silently losing it). + const embeddingText = text.length <= budget ? text : text.slice(0, budget); + chunks.push({ + id: makeId(namespace, path, "input", globalIndex), + path, + name, + integration, + facet: "input", + chunkIndex: globalIndex, + embeddingText, + }); + globalIndex++; + } + + // ----------------------------------------------------------------------- + // Output facet + // ----------------------------------------------------------------------- + if (!isTrivialSchema(doc.outputTypeScript)) { + const text = `${integration} ${path} output\n${doc.outputTypeScript}`; + const embeddingText = text.length <= budget ? text : text.slice(0, budget); + chunks.push({ + id: makeId(namespace, path, "output", globalIndex), + path, + name, + integration, + facet: "output", + chunkIndex: globalIndex, + embeddingText, + }); + globalIndex++; + } + + return chunks; + }, + }; +}; + +// --------------------------------------------------------------------------- +// makeWholeChunker β€” benchmark baseline (mirrors pre-P1 documents.ts behavior) +// --------------------------------------------------------------------------- + +/** Produces a single identity chunk per tool with the full description joined. + * Used as a search-quality baseline to compare against the facet chunker. */ +export const makeWholeChunker = (): Chunker => ({ + chunk(namespace, doc) { + const { path, name, integration, description } = doc; + const header = `${integration} ${path}\n${name}`; + const embeddingText = + description.trim().length > 0 ? `${header}\n${description.trim()}` : header; + return [ + { + id: makeId(namespace, path, "identity", 0), + path, + name, + integration, + facet: "identity", + chunkIndex: 0, + embeddingText, + }, + ]; + }, +}); diff --git a/packages/plugins/vectorize-search/src/sdk/collections.ts b/packages/plugins/vectorize-search/src/sdk/collections.ts new file mode 100644 index 0000000000..3ed562c185 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/collections.ts @@ -0,0 +1,29 @@ +import { Schema } from "effect"; + +import { definePluginStorageCollection } from "@executor-js/sdk/core"; + +// --------------------------------------------------------------------------- +// Vectorize-search plugin storage collections. +// +// `toolFingerprints` stores one row per indexed tool, keyed by `path`, to +// support incremental reindex: on each reindex pass the stored fingerprint is +// compared against the freshly computed one; only changed or new tools are +// re-embedded and upserted. Removed tools are skipped in v1 (logged only). +// --------------------------------------------------------------------------- + +export const FingerprintRow = Schema.Struct({ + /** Canonical tool path (e.g. `github.repos.get`). Row key for the diff. */ + path: Schema.String, + /** Integration slug (e.g. `github`). Carried for namespace-filter queries. */ + integration: Schema.String, + /** cyrb53-based content hash over path+name+description+inputTS+outputTS. */ + fingerprint: Schema.String, + /** Vectorize vector ids produced by the chunker for this tool's last embed. + * Stored so the reindex can issue a targeted deleteByIds before re-upserting. */ + chunkIds: Schema.Array(Schema.String), +}); +export type FingerprintRow = typeof FingerprintRow.Type; + +export const toolFingerprints = definePluginStorageCollection("toolFingerprints", FingerprintRow, { + indexes: ["path"], +}); diff --git a/packages/plugins/vectorize-search/src/sdk/documents.ts b/packages/plugins/vectorize-search/src/sdk/documents.ts index d041d7b73e..fcd9026388 100644 --- a/packages/plugins/vectorize-search/src/sdk/documents.ts +++ b/packages/plugins/vectorize-search/src/sdk/documents.ts @@ -1,4 +1,8 @@ -import type { Tool } from "@executor-js/sdk/core"; +import type { Executor } from "@executor-js/sdk/core"; +import { Effect } from "effect"; + +import type { ToolDocumentInput } from "./chunker"; +import { VectorizeSearchError } from "./errors"; const ADDRESS_PREFIX = "tools."; @@ -6,41 +10,60 @@ const ADDRESS_PREFIX = "tools."; * sandbox-callable path the model writes after `tools.` β€” mirrors the engine's * own `addressToPath` so the `path` we index matches what `tools.search` * callers expect (and can pass back to `describe`/invoke). */ -const addressToPath = (address: string): string => +export const addressToPath = (address: string): string => address.startsWith(ADDRESS_PREFIX) ? address.slice(ADDRESS_PREFIX.length) : address; -/** The indexed view of a tool: the `ToolDiscoveryResult` fields (returned at - * query time straight from Vectorize metadata) plus the text we embed. */ -export interface ToolSearchDocument { - readonly id: string; - readonly path: string; - readonly name: string; - readonly description: string; - readonly integration: string; - readonly embeddingText: string; -} - -const joinText = (parts: readonly (string | undefined)[]): string => - parts - .map((part) => part?.trim()) - .filter((part): part is string => part !== undefined && part.length > 0) - .join("\n"); - -/** Vector id for a tool within a namespace. Vectorize ids are global, so the - * namespace is folded into the id to keep tenants' vectors distinct. */ -export const toolVectorId = (namespace: string, path: string): string => `${namespace}#${path}`; - -export const projectToolDocument = (namespace: string, tool: Tool): ToolSearchDocument => { - const path = addressToPath(String(tool.address)); - const name = String(tool.name); - const description = tool.description; - const integration = String(tool.integration); - return { - id: toolVectorId(namespace, path), - path, - name, - description, - integration, - embeddingText: joinText([`${integration} ${path}`, name, description]), - }; -}; +/** Collect the full `ToolDocumentInput` set from the live catalog. + * + * For each tool returned by `tools.list`, we attempt to fetch its schema via + * `tools.schema(address)` to populate the TypeScript facets (inputTypeScript, + * outputTypeScript, typeScriptDefinitions). If the schema fetch fails for any + * individual tool we degrade gracefully to an identity-only document (no TS + * fields) β€” never failing the whole collection for one tool. + * + * Bounded concurrency (16) keeps the catalog walk fast while avoiding + * unbounded fan-out. */ +export const collectToolDocumentInputs = ( + namespace: string, + executor: Executor, +): Effect.Effect => + executor.tools.list({ includeAnnotations: false }).pipe( + Effect.mapError( + (cause) => new VectorizeSearchError({ message: "Failed to list tools for indexing.", cause }), + ), + Effect.flatMap((tools) => + Effect.forEach( + tools, + (tool) => { + const path = addressToPath(String(tool.address)); + const base: ToolDocumentInput = { + path, + name: String(tool.name), + integration: String(tool.integration), + description: tool.description, + }; + // Attempt schema fetch; on any failure degrade to identity-only. + return executor.tools.schema(tool.address).pipe( + Effect.map((view): ToolDocumentInput => { + if (view === null) return base; + return { + ...base, + ...(view.inputTypeScript !== undefined + ? { inputTypeScript: view.inputTypeScript } + : {}), + ...(view.outputTypeScript !== undefined + ? { outputTypeScript: view.outputTypeScript } + : {}), + ...(view.typeScriptDefinitions !== undefined + ? { typeScriptDefinitions: view.typeScriptDefinitions } + : {}), + }; + }), + // Degrade: schema fetch failed β€” use identity-only document. + Effect.catch(() => Effect.succeed(base)), + ); + }, + { concurrency: 16 }, + ), + ), + ); diff --git a/packages/plugins/vectorize-search/src/sdk/fingerprint.test.ts b/packages/plugins/vectorize-search/src/sdk/fingerprint.test.ts new file mode 100644 index 0000000000..85bb03166c --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/fingerprint.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { fingerprintTool } from "./fingerprint"; + +const base = { + path: "github.repos.get", + name: "repos.get", + description: "Get a GitHub repository by owner and name.", + inputTypeScript: "{ owner: string; repo: string }", + outputTypeScript: "{ id: number; full_name: string }", +} as const; + +describe("fingerprintTool", () => { + it("is deterministic β€” same input yields the same hash", () => { + const a = fingerprintTool(base); + const b = fingerprintTool({ ...base }); + expect(a).toBe(b); + }); + + it("changes when description changes", () => { + const original = fingerprintTool(base); + const changed = fingerprintTool({ ...base, description: "Updated description." }); + expect(changed).not.toBe(original); + }); + + it("changes when inputTypeScript changes", () => { + const original = fingerprintTool(base); + const changed = fingerprintTool({ + ...base, + inputTypeScript: "{ owner: string; repo: string; ref?: string }", + }); + expect(changed).not.toBe(original); + }); + + it("changes when outputTypeScript changes", () => { + const original = fingerprintTool(base); + const changed = fingerprintTool({ + ...base, + outputTypeScript: "{ id: number; full_name: string; private: boolean }", + }); + expect(changed).not.toBe(original); + }); + + it("changes when path changes", () => { + const original = fingerprintTool(base); + const changed = fingerprintTool({ ...base, path: "github.repos.list" }); + expect(changed).not.toBe(original); + }); + + it("changes when name changes", () => { + const original = fingerprintTool(base); + const changed = fingerprintTool({ ...base, name: "repos.list" }); + expect(changed).not.toBe(original); + }); + + it("two different tools produce different hashes", () => { + const tool1 = fingerprintTool(base); + const tool2 = fingerprintTool({ + path: "github.issues.create", + name: "issues.create", + description: "Create an issue in a repository.", + inputTypeScript: "{ title: string; body?: string }", + outputTypeScript: "{ id: number; number: number }", + }); + expect(tool1).not.toBe(tool2); + }); + + it("does not blur field boundaries β€” shifting content across fields changes the hash", () => { + // Under a space separator these collide ("a" + " " + "b c" === "a b" + " " + "c"); + // the NUL separator keeps the boundary distinct. + const a = fingerprintTool({ path: "a", name: "b c" }); + const b = fingerprintTool({ path: "a b", name: "c" }); + expect(a).not.toBe(b); + }); + + it("treats absent optional fields the same as empty strings", () => { + const withEmpty = fingerprintTool({ + path: "x.y", + name: "y", + description: "", + inputTypeScript: "", + outputTypeScript: "", + }); + const withAbsent = fingerprintTool({ + path: "x.y", + name: "y", + }); + expect(withEmpty).toBe(withAbsent); + }); +}); diff --git a/packages/plugins/vectorize-search/src/sdk/fingerprint.ts b/packages/plugins/vectorize-search/src/sdk/fingerprint.ts new file mode 100644 index 0000000000..9ffb59e4b2 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/fingerprint.ts @@ -0,0 +1,48 @@ +// --------------------------------------------------------------------------- +// Deterministic content hash over the five fields that define a tool's +// semantic identity. Used by the incremental reindex to detect changes and +// skip unchanged tools. +// --------------------------------------------------------------------------- + +export interface FingerprintInput { + readonly path: string; + readonly name: string; + readonly description?: string; + readonly inputTypeScript?: string; + readonly outputTypeScript?: string; +} + +// cyrb53 β€” a fast, stable 53-bit hash with good avalanche properties. +// Produces a consistent unsigned integer across all JS environments. +// Source: https://github.com/bryc/code/blob/master/jshash/experimental/cyrb53.js +function cyrb53(str: string, seed = 0): number { + let h1 = 0xdeadbeef ^ seed; + let h2 = 0x41c6ce57 ^ seed; + for (let i = 0; i < str.length; i++) { + const ch = str.charCodeAt(i); + h1 = Math.imul(h1 ^ ch, 2654435761); + h2 = Math.imul(h2 ^ ch, 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909); + return 4294967296 * (2097151 & h2) + (h1 >>> 0); +} + +// Field separator: the NUL byte cannot appear in TypeScript source or +// natural-language text, so adjacent field values can never blur across the +// boundary. A space separator would let e.g. {name:"a b"} and +// {name:"a", description:"b"} hash to the same value. +const FIELD_SEPARATOR = String.fromCharCode(0); + +/** Stable, deterministic fingerprint over all five semantic fields of a tool. + * Changes when ANY field changes and is identical for identical input. */ +export const fingerprintTool = (input: FingerprintInput): string => { + const content = [ + input.path, + input.name, + input.description ?? "", + input.inputTypeScript ?? "", + input.outputTypeScript ?? "", + ].join(FIELD_SEPARATOR); + return cyrb53(content).toString(36); +}; diff --git a/packages/plugins/vectorize-search/src/sdk/hybrid.test.ts b/packages/plugins/vectorize-search/src/sdk/hybrid.test.ts new file mode 100644 index 0000000000..905c9ab6e8 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/hybrid.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + ExecutionToolError, + type ToolDiscoveryProvider, + type ToolDiscoveryResult, +} from "@executor-js/sdk/core"; +import { Effect, Exit } from "effect"; + +import { makeHybridToolDiscoveryProvider } from "./hybrid"; + +// --------------------------------------------------------------------------- +// Test helpers +// --------------------------------------------------------------------------- + +const makeResult = (path: string, score: number, description?: string): ToolDiscoveryResult => ({ + path, + name: path, + description, + integration: "test", + score, +}); + +/** Build an inline fake provider that returns a fixed ordered list of results. */ +const makeFakeProvider = (results: readonly ToolDiscoveryResult[]): ToolDiscoveryProvider => ({ + searchTools: () => + Effect.succeed({ items: results, total: results.length, hasMore: false, nextOffset: null }), +}); + +/** A provider that always fails with ExecutionToolError. */ +const failingProvider: ToolDiscoveryProvider = { + searchTools: () => Effect.fail(new ExecutionToolError({ message: "provider failure" })), +}; + +const defaultInput = { + executor: undefined as never, + query: "test query", + limit: 10, + offset: 0, +}; + +// --------------------------------------------------------------------------- +// RRF score helpers used in assertions. +// weight_lex=0.7 weight_vec=0.3 k=60 +// rank is 0-based. +// --------------------------------------------------------------------------- +const rrfScore = ( + lexRank: number | null, + vecRank: number | null, + opts?: { k?: number; wLex?: number; wVec?: number }, +): number => { + const k = opts?.k ?? 60; + const wLex = opts?.wLex ?? 0.7; + const wVec = opts?.wVec ?? 0.3; + let score = 0; + if (lexRank !== null) score += wLex / (k + lexRank); + if (vecRank !== null) score += wVec / (k + vecRank); + return score; +}; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("makeHybridToolDiscoveryProvider β€” RRF fusion", () => { + it.effect("an item ranked #1 in one source and absent in the other still surfaces high", () => + Effect.gen(function* () { + // lexical: exact-name match "repos.create" first, "repos.list" second + const lexical = makeFakeProvider([ + makeResult("repos.create", 0.95), + makeResult("repos.list", 0.8), + ]); + // vector: intent match "issues.open" first, "repos.create" second + const vector = makeFakeProvider([ + makeResult("issues.open", 0.92), + makeResult("repos.create", 0.85), + ]); + + const provider = makeHybridToolDiscoveryProvider({ lexical, vector }); + const page = yield* provider.searchTools(defaultInput); + + // "repos.create" is rank-0 in lex and rank-1 in vec β†’ highest fused score. + expect(page.items[0]!.path).toBe("repos.create"); + // "issues.open" is rank-0 in vec only β†’ still surfaces above "repos.list" + // (which is lex-only rank-1). + const paths = page.items.map((r) => r.path); + const issuesIdx = paths.indexOf("issues.open"); + const reposListIdx = paths.indexOf("repos.list"); + expect(issuesIdx).toBeGreaterThan(-1); + expect(reposListIdx).toBeGreaterThan(-1); + // issues.open (vec rank 0): 0.3/60 = 0.005 + // repos.list (lex rank 1): 0.7/61 β‰ˆ 0.01147 + // repos.list beats issues.open because wLex is larger. + expect(reposListIdx).toBeLessThan(issuesIdx); + }), + ); + + it.effect("a path in both sources gets a higher fused score than one in only one source", () => + Effect.gen(function* () { + // "shared" appears in both at rank 1. + // "lex-only" appears only in lexical at rank 0. + const lexical = makeFakeProvider([makeResult("lex-only", 0.99), makeResult("shared", 0.8)]); + const vector = makeFakeProvider([makeResult("vec-only", 0.99), makeResult("shared", 0.9)]); + + const provider = makeHybridToolDiscoveryProvider({ lexical, vector }); + const page = yield* provider.searchTools(defaultInput); + + const scoreMap = new Map(page.items.map((r) => [r.path, r.score])); + + // "shared": lex rank-1 + vec rank-1 + const sharedScore = rrfScore(1, 1); + // "lex-only": lex rank-0 only + const lexOnlyScore = rrfScore(0, null); + // "vec-only": vec rank-0 only + const vecOnlyScore = rrfScore(null, 0); + + expect(scoreMap.get("shared")).toBeCloseTo(sharedScore, 10); + expect(scoreMap.get("lex-only")).toBeCloseTo(lexOnlyScore, 10); + expect(scoreMap.get("vec-only")).toBeCloseTo(vecOnlyScore, 10); + + // The path in both sources outscores anything only-one-source at the same rank. + // shared (lex-1+vec-1): 0.7/61 + 0.3/61 = 1/61 β‰ˆ 0.01639 + // lex-only (lex-0): 0.7/60 β‰ˆ 0.01167 β€” less than shared + // vec-only (vec-0): 0.3/60 = 0.005 β€” less than shared + expect(sharedScore).toBeGreaterThan(lexOnlyScore); + expect(sharedScore).toBeGreaterThan(vecOnlyScore); + + const paths = page.items.map((r) => r.path); + expect(paths.indexOf("shared")).toBeLessThan(paths.indexOf("lex-only")); + expect(paths.indexOf("shared")).toBeLessThan(paths.indexOf("vec-only")); + }), + ); + + it.effect("pagination (offset/limit) slices the FUSED list", () => + Effect.gen(function* () { + // Build a scenario with 4 known paths, all-in-lexical so ordering is predictable. + const lexical = makeFakeProvider([ + makeResult("a", 0.99), + makeResult("b", 0.9), + makeResult("c", 0.8), + makeResult("d", 0.7), + ]); + const vector = makeFakeProvider([]); // no vector results + + const provider = makeHybridToolDiscoveryProvider({ lexical, vector }); + + // First page: offset=0, limit=2 + const page1 = yield* provider.searchTools({ ...defaultInput, limit: 2, offset: 0 }); + expect(page1.items.map((r) => r.path)).toEqual(["a", "b"]); + expect(page1.total).toBe(4); + expect(page1.hasMore).toBe(true); + expect(page1.nextOffset).toBe(2); + + // Second page: offset=2, limit=2 + const page2 = yield* provider.searchTools({ ...defaultInput, limit: 2, offset: 2 }); + expect(page2.items.map((r) => r.path)).toEqual(["c", "d"]); + expect(page2.total).toBe(4); + expect(page2.hasMore).toBe(false); + expect(page2.nextOffset).toBeNull(); + }), + ); + + it.effect("tie-break by path asc for determinism", () => + Effect.gen(function* () { + // Two paths both appear only in lexical at the same rank β€” they cannot + // both be rank-0, so put them at ranks 0 and 1, then have vector put two + // OTHER paths at the same vector ranks, leaving "tie-a" and "tie-b" as + // lex-only at the same relative scoring vis-Γ -vis each other. + // Simpler: two items only from vector at the same rank via a custom k. + // Use k=0 so that all lex-only items tied by rank differ only by path. + // We can achieve an exact tie by having one item in each source at rank 0. + const lexical = makeFakeProvider([makeResult("z-path", 0.9)]); + const vector = makeFakeProvider([makeResult("a-path", 0.9)]); + + // With default weights: z-path gets 0.7/60; a-path gets 0.3/60. + // They are NOT tied, so use equal weights to get a real tie. + const provider = makeHybridToolDiscoveryProvider({ + lexical, + vector, + options: { weights: { lexical: 0.5, vector: 0.5 } }, + }); + + const page = yield* provider.searchTools(defaultInput); + // Both have score 0.5/60; tie-break alphabetically β†’ "a-path" first. + expect(page.items.map((r) => r.path)).toEqual(["a-path", "z-path"]); + }), + ); + + it.effect("prefers the richer description when a path appears in both sources", () => + Effect.gen(function* () { + const lexical = makeFakeProvider([makeResult("tool.x", 0.9, undefined)]); + const vector = makeFakeProvider([makeResult("tool.x", 0.8, "A helpful description")]); + + const provider = makeHybridToolDiscoveryProvider({ lexical, vector }); + const page = yield* provider.searchTools(defaultInput); + + expect(page.items).toHaveLength(1); + expect(page.items[0]!.description).toBe("A helpful description"); + }), + ); + + it.effect("description from lexical wins when vector has none", () => + Effect.gen(function* () { + const lexical = makeFakeProvider([makeResult("tool.y", 0.9, "Lexical description")]); + const vector = makeFakeProvider([makeResult("tool.y", 0.8, undefined)]); + + const provider = makeHybridToolDiscoveryProvider({ lexical, vector }); + const page = yield* provider.searchTools(defaultInput); + + expect(page.items[0]!.description).toBe("Lexical description"); + }), + ); + + it.effect("propagates ExecutionToolError from a failing provider", () => + Effect.gen(function* () { + const provider = makeHybridToolDiscoveryProvider({ + lexical: failingProvider, + vector: makeFakeProvider([makeResult("x", 0.9)]), + }); + + const result = yield* provider.searchTools(defaultInput).pipe(Effect.exit); + // The error should be mapped to ExecutionToolError (cause wraps the original). + expect(Exit.isFailure(result)).toBe(true); + }), + ); + + it.effect("custom weights, k, and fuseDepth are respected", () => + Effect.gen(function* () { + const k = 10; + const wLex = 0.4; + const wVec = 0.6; + + const lexical = makeFakeProvider([makeResult("lex-item", 0.9)]); + const vector = makeFakeProvider([makeResult("vec-item", 0.9)]); + + const provider = makeHybridToolDiscoveryProvider({ + lexical, + vector, + options: { weights: { lexical: wLex, vector: wVec }, k, fuseDepth: 50 }, + }); + + const page = yield* provider.searchTools(defaultInput); + const scoreMap = new Map(page.items.map((r) => [r.path, r.score])); + + expect(scoreMap.get("lex-item")).toBeCloseTo(rrfScore(0, null, { k, wLex, wVec }), 10); + expect(scoreMap.get("vec-item")).toBeCloseTo(rrfScore(null, 0, { k, wLex, wVec }), 10); + + // With wVec > wLex, vec-item (rank-0 in vector) > lex-item (rank-0 in lexical). + // wVec/10 > wLex/10 + expect(scoreMap.get("vec-item")!).toBeGreaterThan(scoreMap.get("lex-item")!); + expect(page.items[0]!.path).toBe("vec-item"); + }), + ); + + it.effect("empty results from both providers yields an empty page", () => + Effect.gen(function* () { + const provider = makeHybridToolDiscoveryProvider({ + lexical: makeFakeProvider([]), + vector: makeFakeProvider([]), + }); + + const page = yield* provider.searchTools(defaultInput); + expect(page.items).toHaveLength(0); + expect(page.total).toBe(0); + expect(page.hasMore).toBe(false); + expect(page.nextOffset).toBeNull(); + }), + ); + + it.effect("offset beyond total returns an empty page with correct total", () => + Effect.gen(function* () { + const lexical = makeFakeProvider([makeResult("a", 0.9), makeResult("b", 0.8)]); + const vector = makeFakeProvider([]); + + const provider = makeHybridToolDiscoveryProvider({ lexical, vector }); + const page = yield* provider.searchTools({ ...defaultInput, limit: 5, offset: 100 }); + + expect(page.items).toHaveLength(0); + expect(page.total).toBe(2); + expect(page.hasMore).toBe(false); + expect(page.nextOffset).toBeNull(); + }), + ); +}); diff --git a/packages/plugins/vectorize-search/src/sdk/hybrid.ts b/packages/plugins/vectorize-search/src/sdk/hybrid.ts new file mode 100644 index 0000000000..056809fc37 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/hybrid.ts @@ -0,0 +1,159 @@ +import { + ExecutionToolError, + type PagedResult, + type ToolDiscoveryProvider, + type ToolDiscoveryResult, +} from "@executor-js/sdk/core"; +import { Effect } from "effect"; + +// --------------------------------------------------------------------------- +// Hybrid Reciprocal Rank Fusion (RRF) provider. +// +// Calls both a lexical and a vector provider concurrently, fuses their +// per-path scores with RRF, and paginates the fused list. The module is +// intentionally PURE β€” it takes both providers as arguments and never imports +// the engine's default provider. The plugin wires that in. +// --------------------------------------------------------------------------- + +/** Tuning knobs for Reciprocal Rank Fusion. */ +export interface HybridOptions { + /** + * Per-source weight applied before summing contributions. + * Default: `{ lexical: 0.7, vector: 0.3 }`. + */ + readonly weights?: { + readonly lexical: number; + readonly vector: number; + }; + /** + * The RRF rank-offset constant (`k` in `w / (k + rank)`). + * Default: 60 (Pi's value; large k dampens early-rank dominance). + */ + readonly k?: number; + /** + * How many items to request from EACH provider before fusing. + * Default: 50. + */ + readonly fuseDepth?: number; +} + +// Resolved defaults used throughout. +const DEFAULT_WEIGHTS = { lexical: 0.7, vector: 0.3 } as const; +const DEFAULT_K = 60; +const DEFAULT_FUSE_DEPTH = 50; + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +/** Per-path accumulator keyed by `path`. */ +type FusedEntry = { + readonly path: string; + score: number; + // We keep the richest ToolDiscoveryResult seen for this path. + result: ToolDiscoveryResult; +}; + +/** + * Accumulate RRF contributions from one provider's ranked list into `acc`. + * `weight` is the source-specific weight; `k` is the RRF constant. + */ +const accumulateSource = ( + acc: Map, + items: readonly ToolDiscoveryResult[], + weight: number, + k: number, +): void => { + for (let rank = 0; rank < items.length; rank++) { + const item = items[rank]!; + const contribution = weight / (k + rank); + const existing = acc.get(item.path); + if (existing === undefined) { + acc.set(item.path, { + path: item.path, + score: contribution, + result: item, + }); + } else { + existing.score += contribution; + // Prefer the richer result: the one with a non-empty description wins. + if ( + (existing.result.description === undefined || existing.result.description.length === 0) && + item.description !== undefined && + item.description.length > 0 + ) { + existing.result = item; + } + } + } +}; + +// --------------------------------------------------------------------------- +// Public factory +// --------------------------------------------------------------------------- + +/** + * Build a `ToolDiscoveryProvider` that fuses a lexical and a vector provider + * using Reciprocal Rank Fusion. + * + * Both providers are called concurrently with `{ ...input, limit: fuseDepth, + * offset: 0 }` so the full over-fetch window is available for ranking, + * regardless of the caller's pagination parameters. The caller's `offset` and + * `limit` are applied to the fused, sorted list. + */ +export const makeHybridToolDiscoveryProvider = (deps: { + readonly lexical: ToolDiscoveryProvider; + readonly vector: ToolDiscoveryProvider; + readonly options?: HybridOptions; +}): ToolDiscoveryProvider => { + const weights = deps.options?.weights ?? DEFAULT_WEIGHTS; + const k = deps.options?.k ?? DEFAULT_K; + const fuseDepth = deps.options?.fuseDepth ?? DEFAULT_FUSE_DEPTH; + + return { + searchTools: (input) => + Effect.gen(function* () { + const fetchInput = { ...input, limit: fuseDepth, offset: 0 }; + + // Fetch both sources concurrently; propagate any failure immediately. + const [lexicalPage, vectorPage] = yield* Effect.all( + [deps.lexical.searchTools(fetchInput), deps.vector.searchTools(fetchInput)] as const, + { concurrency: "unbounded" }, + ); + + // Build the fused score map. + const acc = new Map(); + accumulateSource(acc, lexicalPage.items, weights.lexical, k); + accumulateSource(acc, vectorPage.items, weights.vector, k); + + // Sort by fused score desc; tie-break by path asc for determinism. + const fused = [...acc.values()].sort((a, b) => { + const diff = b.score - a.score; + if (diff !== 0) return diff; + return a.path < b.path ? -1 : a.path > b.path ? 1 : 0; + }); + + // Apply the caller's pagination to the fused list. + const safeOffset = Math.max(input.offset, 0); + const start = Math.min(safeOffset, fused.length); + const pageEntries = fused.slice(start, start + input.limit); + const hasMore = fused.length > start + pageEntries.length; + + const items: readonly ToolDiscoveryResult[] = pageEntries.map((entry) => ({ + ...entry.result, + score: entry.score, + })); + + return { + items, + total: fused.length, + hasMore, + nextOffset: hasMore ? start + pageEntries.length : null, + } satisfies PagedResult; + }).pipe( + Effect.mapError( + (cause) => new ExecutionToolError({ message: "Hybrid RRF tool search failed.", cause }), + ), + ), + }; +}; diff --git a/packages/plugins/vectorize-search/src/sdk/index.ts b/packages/plugins/vectorize-search/src/sdk/index.ts index 7800327e04..12b14fd279 100644 --- a/packages/plugins/vectorize-search/src/sdk/index.ts +++ b/packages/plugins/vectorize-search/src/sdk/index.ts @@ -3,7 +3,7 @@ export { type VectorizeSearchPluginOptions, type VectorizeSearchExtension, } from "./plugin"; -export { makeVectorizeToolDiscoveryProvider } from "./provider"; +export { makeVectorizeToolDiscoveryProvider, MAX_METADATA_TOP_K } from "./provider"; export { makeGeminiEmbedder, DEFAULT_GEMINI_MODEL, @@ -18,7 +18,31 @@ export { type VectorizeMatch, type VectorizeMatches, type VectorizeVectorInput, + MAX_TOP_K, } from "./vectorize"; -export { reindexToolCatalog, collectToolDocuments, type ReindexResult } from "./indexer"; -export { projectToolDocument, toolVectorId, type ToolSearchDocument } from "./documents"; +export { reconcileToolCatalog, type ReconcileResult } from "./indexer"; export { VectorizeSearchError } from "./errors"; + +// Chunker +export { + makeFacetChunker, + makeWholeChunker, + type Chunker, + type ToolDocumentInput, + type ToolChunk, + type ChunkFacet, + type FacetChunkerOptions, +} from "./chunker"; +export { ChunkerService, facetChunkerLayer, wholeChunkerLayer } from "./chunker-service"; + +// Hybrid RRF provider +export { makeHybridToolDiscoveryProvider, type HybridOptions } from "./hybrid"; + +// Fingerprint +export { fingerprintTool, type FingerprintInput } from "./fingerprint"; + +// Collections (plugin storage) +export { toolFingerprints, FingerprintRow } from "./collections"; + +// Store limit decorator +export { withCloudflareLimits } from "./store-cloudflare-limits"; diff --git a/packages/plugins/vectorize-search/src/sdk/indexer.test.ts b/packages/plugins/vectorize-search/src/sdk/indexer.test.ts new file mode 100644 index 0000000000..acfacd8323 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/indexer.test.ts @@ -0,0 +1,368 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import type { + Executor, + Owner, + PluginStorageCollectionFacade, + PluginStorageEntry, +} from "@executor-js/sdk/core"; + +import { makeFacetChunker } from "./chunker"; +import type { FingerprintRow } from "./collections"; +import { toolFingerprints } from "./collections"; +import type { ToolEmbedder } from "./embedder"; +import { reconcileToolCatalog } from "./indexer"; +import type { VectorizeStore, VectorizeVectorInput } from "./vectorize"; + +// --------------------------------------------------------------------------- +// Test helpers +// --------------------------------------------------------------------------- + +/** Build a minimal fake Executor that exposes a fixed tool list + schema map. */ +const makeExecutor = ( + tools: ReadonlyArray<{ + address: string; + name: string; + integration: string; + description: string; + inputTypeScript?: string; + outputTypeScript?: string; + }>, + schemas: Map = new Map(), +): Executor => { + const toolObjects = tools.map((t) => ({ + address: t.address as never, + name: t.name as never, + integration: t.integration as never, + description: t.description, + owner: "org" as never, + connection: "default" as never, + pluginId: "test", + })); + // oxlint-disable-next-line executor/no-double-cast -- test stub: only the exercised methods are implemented + return { + tools: { + list: () => Effect.succeed(toolObjects as never), + schema: (address: never) => { + const addrStr = String(address); + const key = addrStr.startsWith("tools.") ? addrStr.slice(6) : addrStr; + const view = schemas.get(key) ?? null; + if (view === null) return Effect.succeed(null); + return Effect.succeed({ + address: address as never, + ...view, + }); + }, + }, + } as unknown as Executor; +}; + +/** Build a fake VectorizeStore that records upserts and deletes. */ +const makeStore = () => { + const upserted: VectorizeVectorInput[] = []; + const deleted: string[] = []; + const store: VectorizeStore = { + query: () => Effect.succeed([]), + upsert: (vectors) => + Effect.sync(() => { + upserted.push(...vectors); + }), + deleteByIds: (ids) => + Effect.sync(() => { + deleted.push(...ids); + }), + }; + return { store, upserted, deleted }; +}; + +/** Build a fake PluginStorageCollectionFacade backed by an in-test Map. */ +const makeFingerprints = ( + initial: Map = new Map(), +): PluginStorageCollectionFacade => { + const storage = new Map(initial); + let idCounter = 0; + + const makeEntry = (key: string, data: FingerprintRow): PluginStorageEntry => ({ + id: String(idCounter++), + owner: "org" as Owner, + pluginId: "vectorize-search", + collection: "toolFingerprints", + key, + data, + createdAt: new Date(0), + updatedAt: new Date(0), + }); + + // oxlint-disable-next-line executor/no-double-cast -- test stub: only the exercised methods are implemented + return { + get: ({ key }: { key: string }) => + Effect.succeed(storage.has(key) ? makeEntry(key, storage.get(key)!) : null), + getForOwner: ({ key }: { key: string }) => + Effect.succeed(storage.has(key) ? makeEntry(key, storage.get(key)!) : null), + list: () => Effect.succeed([...storage.entries()].map(([key, data]) => makeEntry(key, data))), + put: ({ key, data }: { key: string; owner: string; data: unknown }) => { + storage.set(key, data as FingerprintRow); + return Effect.succeed(makeEntry(key, data as FingerprintRow)); + }, + query: (input?: { where?: unknown }) => { + let entries = [...storage.entries()].map(([key, data]) => makeEntry(key, data)); + if (input?.where) { + // Simple path-eq filter for tests. + const where = input.where as Record; + if (typeof where["path"] === "string") { + entries = entries.filter((e) => e.data.path === where["path"]); + } + } + return Effect.succeed(entries); + }, + count: () => Effect.succeed(storage.size), + queryKeyset: () => Effect.succeed({ entries: [], nextCursor: null }), + aggregate: { + count: () => Effect.succeed(storage.size), + groupCount: () => Effect.succeed([]), + timeBuckets: () => Effect.succeed([]), + stats: () => Effect.succeed({ count: 0, min: null, max: null, percentiles: [] }), + }, + remove: ({ key }: { key: string }) => + Effect.sync(() => { + storage.delete(key); + }), + } as unknown as PluginStorageCollectionFacade; +}; + +const owner: Owner = "org" as Owner; +const namespace = "test-ns"; +const chunker = makeFacetChunker(); +const fakeEmbedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => Effect.succeed(texts.map(() => [0.1, 0.2, 0.3])), + embedQuery: () => Effect.succeed([1, 0, 0]), +}; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("reconcileToolCatalog", () => { + it.effect("first run: embeds and upserts all tools, writes fingerprint rows", () => + Effect.gen(function* () { + const { store, upserted, deleted } = makeStore(); + const fingerprints = makeFingerprints(); + + const result = yield* reconcileToolCatalog({ + namespace, + executor: makeExecutor([ + { + address: "tools.github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + }, + { + address: "tools.github.repos.list", + name: "repos.list", + integration: "github", + description: "List repositories", + }, + ]), + embedder: fakeEmbedder, + store, + chunker, + fingerprints, + owner, + }); + + expect(result.total).toBe(2); + expect(result.reembedded).toBe(2); + expect(result.unchanged).toBe(0); + expect(result.removedSkipped).toBe(0); + // Each tool produces at least 1 identity chunk. + expect(upserted.length).toBeGreaterThanOrEqual(2); + expect(deleted).toHaveLength(0); + // All upserted records belong to our namespace. + for (const v of upserted) { + expect(v.namespace).toBe(namespace); + } + }), + ); + + it.effect("second run with no changes: reembeds nothing, unchanged equals total", () => + Effect.gen(function* () { + const executor = makeExecutor([ + { + address: "tools.github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + }, + ]); + + // First run to populate fingerprints. + const fingerprints = makeFingerprints(); + const { store: store1 } = makeStore(); + yield* reconcileToolCatalog({ + namespace, + executor, + embedder: fakeEmbedder, + store: store1, + chunker, + fingerprints, + owner, + }); + + // Second run β€” same executor, same data. + const { store: store2, upserted: upserted2 } = makeStore(); + const result2 = yield* reconcileToolCatalog({ + namespace, + executor, + embedder: fakeEmbedder, + store: store2, + chunker, + fingerprints, + owner, + }); + + expect(result2.total).toBe(1); + expect(result2.reembedded).toBe(0); + expect(result2.unchanged).toBe(1); + expect(upserted2).toHaveLength(0); + }), + ); + + it.effect( + "changing one tool's description re-embeds only that tool AND deletes old chunk ids", + () => + Effect.gen(function* () { + const executor1 = makeExecutor([ + { + address: "tools.github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + }, + { + address: "tools.github.repos.list", + name: "repos.list", + integration: "github", + description: "List repositories", + }, + ]); + + // First run. + const fingerprints = makeFingerprints(); + const { store: store1, upserted: upserted1 } = makeStore(); + yield* reconcileToolCatalog({ + namespace, + executor: executor1, + embedder: fakeEmbedder, + store: store1, + chunker, + fingerprints, + owner, + }); + + // Capture old chunk ids for repos.get. + const oldGetChunkIds = upserted1 + .filter((v) => v.metadata?.["path"] === "github.repos.get") + .map((v) => v.id); + expect(oldGetChunkIds.length).toBeGreaterThan(0); + + // Second run β€” repos.get description changed; repos.list is unchanged. + const executor2 = makeExecutor([ + { + address: "tools.github.repos.get", + name: "repos.get", + integration: "github", + description: "Fetch a repository β€” updated description", + }, + { + address: "tools.github.repos.list", + name: "repos.list", + integration: "github", + description: "List repositories", + }, + ]); + const { store: store2, upserted: upserted2, deleted: deleted2 } = makeStore(); + const result2 = yield* reconcileToolCatalog({ + namespace, + executor: executor2, + embedder: fakeEmbedder, + store: store2, + chunker, + fingerprints, + owner, + }); + + expect(result2.reembedded).toBe(1); + expect(result2.unchanged).toBe(1); + // Old chunk ids for repos.get must have been deleted. + for (const id of oldGetChunkIds) { + expect(deleted2).toContain(id); + } + // Only repos.get chunks were re-upserted. + for (const v of upserted2) { + expect(v.metadata?.["path"]).toBe("github.repos.get"); + } + }), + ); + + it.effect( + "a tool removed from the live list increments removedSkipped and is NOT deleted from the store", + () => + Effect.gen(function* () { + const executor1 = makeExecutor([ + { + address: "tools.github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + }, + { + address: "tools.github.repos.delete", + name: "repos.delete", + integration: "github", + description: "Delete a repository", + }, + ]); + + // First run β€” both tools indexed. + const fingerprints = makeFingerprints(); + const { store: store1 } = makeStore(); + yield* reconcileToolCatalog({ + namespace, + executor: executor1, + embedder: fakeEmbedder, + store: store1, + chunker, + fingerprints, + owner, + }); + + // Second run β€” repos.delete is no longer in the catalog. + const executor2 = makeExecutor([ + { + address: "tools.github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + }, + ]); + const { store: store2, deleted: deleted2 } = makeStore(); + const result2 = yield* reconcileToolCatalog({ + namespace, + executor: executor2, + embedder: fakeEmbedder, + store: store2, + chunker, + fingerprints, + owner, + }); + + expect(result2.total).toBe(1); + expect(result2.removedSkipped).toBe(1); + // repos.delete's chunk ids must NOT appear in the delete list. + expect(deleted2).toHaveLength(0); + }), + ); +}); diff --git a/packages/plugins/vectorize-search/src/sdk/indexer.ts b/packages/plugins/vectorize-search/src/sdk/indexer.ts index f9c062f7d6..ff2270e055 100644 --- a/packages/plugins/vectorize-search/src/sdk/indexer.ts +++ b/packages/plugins/vectorize-search/src/sdk/indexer.ts @@ -1,60 +1,236 @@ import type { Executor } from "@executor-js/sdk/core"; +import type { Owner, PluginStorageCollectionFacade } from "@executor-js/sdk/core"; import { Effect } from "effect"; -import { projectToolDocument, type ToolSearchDocument } from "./documents"; +import type { Chunker } from "./chunker"; +import type { FingerprintRow } from "./collections"; +import { toolFingerprints } from "./collections"; +import { collectToolDocumentInputs } from "./documents"; import type { ToolEmbedder } from "./embedder"; import { VectorizeSearchError } from "./errors"; +import { fingerprintTool } from "./fingerprint"; import type { VectorizeStore, VectorizeVectorInput } from "./vectorize"; -export interface ReindexResult { +// --------------------------------------------------------------------------- +// ReconcileResult +// --------------------------------------------------------------------------- + +export interface ReconcileResult { + /** The namespace this reconcile ran against. */ readonly namespace: string; - readonly indexedToolCount: number; + /** Total tools in the live catalog. */ + readonly total: number; + /** Number of tools whose embeddings were refreshed (new + changed). */ + readonly reembedded: number; + /** Number of tools whose stored fingerprint matched β€” skipped entirely. */ + readonly unchanged: number; + /** Number of tools present in storage but absent from the current catalog + * (not deleted in v1 β€” deletion is unsafe across per-isolate catalog variance; + * logged only). */ + readonly removedSkipped: number; } -/** Project the whole tool catalog into indexable documents. Lightweight: it - * uses only `tools.list` fields (path/name/description/integration) and does - * not `describe` each tool, so a reindex is one catalog read + one batched - * embedding call rather than N per-tool round-trips. */ -export const collectToolDocuments = ( - namespace: string, - executor: Executor, -): Effect.Effect => - executor.tools.list({ includeAnnotations: false }).pipe( - Effect.map((tools) => tools.map((tool) => projectToolDocument(namespace, tool))), - Effect.mapError( - (cause) => new VectorizeSearchError({ message: "Failed to list tools for indexing.", cause }), - ), - ); - -/** Embed the current tool catalog and upsert it into Vectorize under - * `namespace`. Explicit (v1): callers invoke it on demand via the `/api` - * reindex route (or a cron). Returns the indexed count. */ -export const reindexToolCatalog = (input: { +// --------------------------------------------------------------------------- +// reconcileToolCatalog β€” incremental fingerprint-based reindex +// --------------------------------------------------------------------------- + +/** Reconcile the live tool catalog against the Vectorize index. + * + * Algorithm: + * 1. Collect `ToolDocumentInput` for every tool (tools.list + tools.schema). + * 2. Fingerprint each tool. + * 3. Load stored `FingerprintRow`s; diff: new βˆͺ changed β†’ re-embed + upsert. + * For CHANGED tools, delete old chunk ids before upserting new. + * 4. Tools in storage but absent from the live catalog β†’ skip deletion (v1), + * count them into `removedSkipped`. + * 5. Unchanged tools β†’ skip entirely. + * + * The owner parameter is the org-level scope used for all storage writes. */ +export const reconcileToolCatalog = (input: { readonly namespace: string; readonly executor: Executor; readonly embedder: ToolEmbedder; readonly store: VectorizeStore; -}): Effect.Effect => + readonly chunker: Chunker; + readonly fingerprints: PluginStorageCollectionFacade; + readonly owner: Owner; +}): Effect.Effect => Effect.gen(function* () { - const { namespace, executor, embedder, store } = input; - const documents = yield* collectToolDocuments(namespace, executor); - if (documents.length === 0) { - return { namespace, indexedToolCount: 0 }; + const { namespace, executor, embedder, store, chunker, fingerprints, owner } = input; + + // ------------------------------------------------------------------------- + // Step 1 β€” Collect tool documents (list + schema). + // ------------------------------------------------------------------------- + const docs = yield* collectToolDocumentInputs(namespace, executor); + const total = docs.length; + + if (total === 0) { + return { namespace, total: 0, reembedded: 0, unchanged: 0, removedSkipped: 0 }; } - const vectors = yield* embedder.embedDocuments( - documents.map((document) => document.embeddingText), + + // ------------------------------------------------------------------------- + // Step 2 β€” Fingerprint each live tool. + // ------------------------------------------------------------------------- + const liveByPath = new Map( + docs.map((doc) => [doc.path, { doc, fingerprint: fingerprintTool(doc) }]), ); - const records: readonly VectorizeVectorInput[] = documents.map((document, index) => ({ - id: document.id, - values: [...(vectors[index] ?? [])], - namespace, - metadata: { - path: document.path, - name: document.name, - description: document.description, - integration: document.integration, - }, + + // ------------------------------------------------------------------------- + // Step 3 β€” Load all stored fingerprints. + // ------------------------------------------------------------------------- + const storedEntries = yield* fingerprints.list().pipe( + Effect.mapError( + (cause) => + new VectorizeSearchError({ + message: "Failed to load stored fingerprints for reconcile.", + cause, + }), + ), + ); + const storedByPath = new Map(storedEntries.map((entry) => [entry.key, entry.data])); + + // ------------------------------------------------------------------------- + // Step 4 β€” Diff: classify each live tool as new, changed, or unchanged. + // ------------------------------------------------------------------------- + const toEmbed: Array<{ + doc: (typeof docs)[number]; + fingerprint: string; + oldChunkIds: readonly string[] | null; + }> = []; + let unchanged = 0; + + for (const [path, { doc, fingerprint }] of liveByPath) { + const stored = storedByPath.get(path); + if (stored !== undefined && stored.fingerprint === fingerprint) { + unchanged++; + } else { + toEmbed.push({ + doc, + fingerprint, + oldChunkIds: stored?.chunkIds ?? null, + }); + } + } + + // ------------------------------------------------------------------------- + // Step 5 β€” Count removed (stored but no longer live) β€” skip deletion in v1. + // ------------------------------------------------------------------------- + let removedSkipped = 0; + for (const storedPath of storedByPath.keys()) { + if (!liveByPath.has(storedPath)) { + removedSkipped++; + // Intentionally NOT deleting: per-isolate catalog variance makes + // deletion unsafe in v1 β€” log the count so the operator is aware. + } + } + if (removedSkipped > 0) { + yield* Effect.logWarning( + `reconcileToolCatalog [${namespace}]: ${removedSkipped} tool(s) present in ` + + `storage but absent from the live catalog β€” skipping deletion (v1).`, + ); + } + + // ------------------------------------------------------------------------- + // Step 6 β€” Re-embed and upsert changed/new tools. + // ------------------------------------------------------------------------- + if (toEmbed.length === 0) { + return { namespace, total, reembedded: 0, unchanged, removedSkipped }; + } + + // Chunk all tools to re-embed. + const chunkedGroups = toEmbed.map((item) => ({ + ...item, + chunks: chunker.chunk(namespace, item.doc), })); + + // Gather all embedding texts, preserving their group + chunk positions. + const allTexts: string[] = []; + for (const group of chunkedGroups) { + for (const chunk of group.chunks) { + allTexts.push(chunk.embeddingText); + } + } + + // Single batched embedding call over all changed chunks. + const allVectors = yield* embedder.embedDocuments(allTexts); + + // Build VectorizeVectorInput records and collect new chunkIds per tool. + let vectorOffset = 0; + const records: VectorizeVectorInput[] = []; + const fingerprintUpdates: Array<{ path: string; row: FingerprintRow }> = []; + + for (const group of chunkedGroups) { + const { doc, fingerprint, oldChunkIds, chunks } = group; + const newChunkIds: string[] = []; + + for (const chunkItem of chunks) { + const vec = allVectors[vectorOffset++]; + if (vec === undefined) { + return yield* new VectorizeSearchError({ + message: `reconcileToolCatalog: embedding vector missing at offset ${vectorOffset - 1}`, + }); + } + records.push({ + id: chunkItem.id, + values: [...vec], + namespace, + metadata: { + path: doc.path, + name: doc.name, + description: doc.description, + integration: doc.integration, + facet: chunkItem.facet, + chunkIndex: chunkItem.chunkIndex, + }, + }); + newChunkIds.push(chunkItem.id); + } + + fingerprintUpdates.push({ + path: doc.path, + row: { + path: doc.path, + integration: doc.integration, + fingerprint, + chunkIds: newChunkIds, + }, + }); + + // Delete OLD chunk ids for changed tools before upserting new ones. + if (oldChunkIds !== null && oldChunkIds.length > 0) { + yield* store.deleteByIds([...oldChunkIds]); + } + } + + // Upsert all new vectors in one call. `store.upsert` (makeVectorizeStore) + // chunks internally into UPSERT_BATCH_SIZE (50) sequential batches, so the + // underlying Vectorize binding never receives more than 50 vectors per + // call β€” well under Cloudflare's 1,000-vector / 2 MB per-upsert caps β€” + // regardless of how many tools changed. yield* store.upsert(records); - return { namespace, indexedToolCount: records.length }; + + // Persist updated fingerprint rows. + yield* Effect.forEach( + fingerprintUpdates, + ({ path, row }) => + fingerprints.put({ owner, key: path, data: row }).pipe( + Effect.mapError( + (cause) => + new VectorizeSearchError({ + message: `Failed to persist fingerprint row for tool "${path}".`, + cause, + }), + ), + Effect.asVoid, + ), + { concurrency: 8, discard: true }, + ); + + return { + namespace, + total, + reembedded: toEmbed.length, + unchanged, + removedSkipped, + }; }); diff --git a/packages/plugins/vectorize-search/src/sdk/plugin.ts b/packages/plugins/vectorize-search/src/sdk/plugin.ts index b30de0da3e..a29253dfcf 100644 --- a/packages/plugins/vectorize-search/src/sdk/plugin.ts +++ b/packages/plugins/vectorize-search/src/sdk/plugin.ts @@ -1,10 +1,14 @@ -import { definePlugin, type Executor } from "@executor-js/sdk/core"; +import { definePlugin, type Executor, type ToolDiscoveryProvider } from "@executor-js/sdk/core"; import { Effect } from "effect"; +import { type Chunker, makeFacetChunker } from "./chunker"; +import { toolFingerprints } from "./collections"; import { makeGeminiEmbedder, type ToolEmbedder } from "./embedder"; import { VectorizeSearchError } from "./errors"; -import { reindexToolCatalog, type ReindexResult } from "./indexer"; +import { makeHybridToolDiscoveryProvider } from "./hybrid"; +import { reconcileToolCatalog, type ReconcileResult } from "./indexer"; import { makeVectorizeToolDiscoveryProvider } from "./provider"; +import { withCloudflareLimits } from "./store-cloudflare-limits"; import { makeVectorizeStore, type VectorizeIndex, type VectorizeStore } from "./vectorize"; export interface VectorizeSearchPluginOptions { @@ -25,6 +29,16 @@ export interface VectorizeSearchPluginOptions { readonly dimensions?: number; /** Inject a custom embedder (tests). Overrides `model`/`geminiApiKey`/`dimensions`. */ readonly embedder?: ToolEmbedder; + /** Chunker used when indexing the tool catalog. Defaults to `makeFacetChunker()`. + * Override in tests or to benchmark the whole chunker. */ + readonly chunker?: Chunker; + /** Lexical discovery provider to fuse with the vector provider (RRF). The host + * supplies the engine's built-in `defaultToolDiscoveryProvider` here β€” kept as + * an option so the plugin never depends on `@executor-js/execution`. Absent β†’ + * the plugin replaces tool search with the vector provider ALONE (the engine's + * lexical scorer does NOT run, because a plugin provider supersedes it). Supply + * it to get hybrid lexical+vector search. */ + readonly lexical?: ToolDiscoveryProvider; } const notConfigured = (): Effect.Effect => @@ -35,23 +49,30 @@ const notConfigured = (): Effect.Effect => }), ); -/** Build the `executor.vectorizeSearch` surface. `reindex` embeds the current - * tool catalog and upserts it into Vectorize; it takes the scoped executor as - * an argument because only the request/API layer holds it (the plugin ctx does - * not expose the catalog). Inert β€” `reindex` fails clearly β€” until both a - * Vectorize binding and an embedder are present. */ +/** Build the `executor.vectorizeSearch` surface. `reindex` reconciles the + * current tool catalog incrementally (fingerprint diff) and upserts changed + * tools into Vectorize; it takes the scoped executor as an argument because + * only the request/API layer holds it (the plugin ctx does not expose the + * catalog). Inert β€” `reindex` fails clearly β€” until both a Vectorize binding + * and an embedder are present. */ const makeVectorizeSearchExtension = (deps: { readonly namespace: string; readonly embedder: ToolEmbedder | undefined; readonly store: VectorizeStore | undefined; + readonly chunker: Chunker; + readonly fingerprints: Parameters[0]["fingerprints"] | undefined; + readonly owner: Parameters[0]["owner"] | undefined; }) => ({ - reindex: (executor: Executor): Effect.Effect => - deps.embedder && deps.store - ? reindexToolCatalog({ + reindex: (executor: Executor): Effect.Effect => + deps.embedder && deps.store && deps.fingerprints && deps.owner + ? reconcileToolCatalog({ namespace: deps.namespace, executor, embedder: deps.embedder, store: deps.store, + chunker: deps.chunker, + fingerprints: deps.fingerprints, + owner: deps.owner, }) : notConfigured(), }); @@ -61,10 +82,15 @@ export type VectorizeSearchExtension = ReturnType { const namespace = options?.namespace ?? "default"; @@ -79,18 +105,44 @@ export const vectorizeSearchPlugin = definePlugin((options?: VectorizeSearchPlug : undefined); // Inert without both a Vectorize binding and an embedder β€” the engine then // keeps its built-in lexical search (mirrors the metrics plugin's no-op). - const store = options?.vectorize ? makeVectorizeStore(options.vectorize) : undefined; + // The store is wrapped so Cloudflare's hard limits (64-byte ids, topK ≀ 20 + // with full metadata) fail loudly at the boundary rather than as opaque + // Vectorize 4xx errors deep in a reindex. + const store = options?.vectorize + ? withCloudflareLimits(makeVectorizeStore(options.vectorize)) + : undefined; + const chunker = options?.chunker ?? makeFacetChunker(); + const lexical = options?.lexical; return { id: "vectorizeSearch" as const, packageName: "@executor-js/plugin-vectorize-search", - storage: () => ({}), - extension: () => makeVectorizeSearchExtension({ namespace, embedder, store }), + pluginStorage: { toolFingerprints }, + storage: (deps) => ({ + fingerprints: deps.pluginStorage.collection(toolFingerprints), + // The tool catalog is an org-level artifact, so fingerprints are ALWAYS + // org-scoped. Scoping by the triggering principal (user vs cron) would + // split the fingerprint store into disjoint partitions, so each reindex + // would see an empty store and re-embed the whole catalog. + owner: "org" as const, + }), + extension: (ctx) => + makeVectorizeSearchExtension({ + namespace, + embedder, + store, + chunker, + fingerprints: ctx.storage.fingerprints, + owner: ctx.storage.owner, + }), runtime: { - toolDiscoveryProvider: () => - embedder && store - ? makeVectorizeToolDiscoveryProvider({ embedder, store, namespace }) - : undefined, + toolDiscoveryProvider: () => { + if (!embedder || !store) return undefined; + const vector = makeVectorizeToolDiscoveryProvider({ embedder, store, namespace }); + // Fuse with the host-supplied lexical provider (RRF) when present; else + // the vector provider answers alone. + return lexical ? makeHybridToolDiscoveryProvider({ lexical, vector }) : vector; + }, }, }; }); diff --git a/packages/plugins/vectorize-search/src/sdk/provider.ts b/packages/plugins/vectorize-search/src/sdk/provider.ts index 10b9a9a245..8d51012c6b 100644 --- a/packages/plugins/vectorize-search/src/sdk/provider.ts +++ b/packages/plugins/vectorize-search/src/sdk/provider.ts @@ -6,9 +6,13 @@ import { import { Effect } from "effect"; import type { ToolEmbedder } from "./embedder"; -import { MAX_TOP_K } from "./vectorize"; import type { VectorizeMatch, VectorizeStore } from "./vectorize"; +/** Cloudflare caps `topK` at 20 when `returnMetadata:"all"` is set. + * The vector store always queries with full metadata, so the provider must + * never exceed this limit on the Vectorize path. */ +export const MAX_METADATA_TOP_K = 20; + const asString = (value: unknown): string => (typeof value === "string" ? value : ""); const toResult = (match: VectorizeMatch): ToolDiscoveryResult => { @@ -68,12 +72,28 @@ export const makeVectorizeToolDiscoveryProvider = (deps: { const matches = yield* deps.store.query({ vector, namespace: deps.namespace, - topK: MAX_TOP_K, + // Cloudflare caps topK at 20 when returnMetadata:"all" is used (the + // store always sets it). MAX_TOP_K (100) is kept for non-metadata + // paths; the metadata path must respect MAX_METADATA_TOP_K. + topK: MAX_METADATA_TOP_K, }); - const ranked = matches + const mapped = matches .filter((match) => asString(match.metadata?.path).length > 0) .map(toResult) .filter((result) => matchesNamespace(result, namespace)); + // The facet chunker indexes a tool as several chunks (identity, input, + // output, description), so one tool can surface as multiple matches. + // Collapse to the best-scoring chunk per `path` before paginating β€” + // otherwise a tool occupies several result slots and, on the hybrid path, + // accrues inflated RRF weight from its repeated ranks. + const bestByPath = new Map(); + for (const result of mapped) { + const prev = bestByPath.get(result.path); + if (prev === undefined || result.score > prev.score) { + bestByPath.set(result.path, result); + } + } + const ranked = [...bestByPath.values()].sort((a, b) => b.score - a.score); const start = Math.min(safeOffset, ranked.length); const items = ranked.slice(start, start + limit); const hasMore = ranked.length > start + items.length; diff --git a/packages/plugins/vectorize-search/src/sdk/search.test.ts b/packages/plugins/vectorize-search/src/sdk/search.test.ts index a688e3f7e4..d9256ed65c 100644 --- a/packages/plugins/vectorize-search/src/sdk/search.test.ts +++ b/packages/plugins/vectorize-search/src/sdk/search.test.ts @@ -1,10 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; -import type { Executor, Tool } from "@executor-js/sdk/core"; - import type { ToolEmbedder } from "./embedder"; -import { reindexToolCatalog } from "./indexer"; import { makeVectorizeToolDiscoveryProvider } from "./provider"; import { makeVectorizeStore } from "./vectorize"; import type { @@ -37,15 +34,6 @@ const matchIn = (path: string, integration: string, score: number): VectorizeMat const match = (path: string, score: number): VectorizeMatch => matchIn(path, "github", score); -// Minimal stubs β€” the indexer only calls `tools.list()` and reads a few Tool -// fields, so a full branded Tool/Executor is unnecessary for these unit tests. -const stubTool = (address: string, name: string, integration: string, description: string): Tool => - // oxlint-disable-next-line executor/no-double-cast -- test stub: the indexer reads only these Tool fields - ({ address, name, integration, description }) as unknown as Tool; -const stubExecutor = (tools: readonly Tool[]): Executor => - // oxlint-disable-next-line executor/no-double-cast -- test stub: only tools.list() is exercised - ({ tools: { list: () => Effect.succeed(tools) } }) as unknown as Executor; - describe("makeVectorizeToolDiscoveryProvider", () => { it.effect("maps Vectorize matches to ToolDiscoveryResult", () => Effect.gen(function* () { @@ -184,59 +172,29 @@ describe("makeVectorizeToolDiscoveryProvider", () => { expect(page.total).toBe(0); }), ); -}); -describe("reindexToolCatalog", () => { - it.effect("projects the catalog and upserts namespaced records", () => + it.effect("dedupes multi-facet matches by path, keeping the best score", () => Effect.gen(function* () { - const upserts: VectorizeVectorInput[][] = []; - const store: VectorizeStore = { - query: () => Effect.succeed([]), - upsert: (vectors) => - Effect.sync(() => { - upserts.push([...vectors]); - }), - deleteByIds: () => Effect.void, - }; - const tool = stubTool("tools.github.repos.get", "repos.get", "github", "Get a repository"); - const executor = stubExecutor([tool]); - - const result = yield* reindexToolCatalog({ - namespace: "org", - executor, + // The facet chunker indexes a tool as several chunks, so the same path can + // come back more than once β€” the provider must collapse to the best score. + const provider = makeVectorizeToolDiscoveryProvider({ embedder: fakeEmbedder, - store, - }); - - expect(result.indexedToolCount).toBe(1); - expect(upserts).toHaveLength(1); - const record = upserts[0]![0]!; - expect(record.id).toBe("org#github.repos.get"); - expect(record.namespace).toBe("org"); - expect(record.values).toEqual([0.1, 0.2, 0.3]); - expect(record.metadata).toMatchObject({ - path: "github.repos.get", - integration: "github", - name: "repos.get", - }); - }), - ); - - it.effect("no-ops on an empty catalog", () => - Effect.gen(function* () { - const store: VectorizeStore = { - query: () => Effect.succeed([]), - upsert: () => Effect.die("upsert must not be called for an empty catalog") as never, - deleteByIds: () => Effect.void, - }; - const executor = stubExecutor([]); - const result = yield* reindexToolCatalog({ + store: makeQueryStore([ + match("repos.get", 0.6), // identity-facet chunk + match("repos.list", 0.7), + match("repos.get", 0.9), // a higher-scoring facet chunk of the same tool + ]), namespace: "org", - executor, - embedder: fakeEmbedder, - store, }); - expect(result.indexedToolCount).toBe(0); + const page = yield* provider.searchTools({ + executor: undefined as never, + query: "x", + limit: 10, + offset: 0, + }); + expect(page.items.map((item) => item.path)).toEqual(["repos.get", "repos.list"]); + expect(page.items[0]!.score).toBe(0.9); + expect(page.total).toBe(2); }), ); }); diff --git a/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.test.ts b/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.test.ts new file mode 100644 index 0000000000..1ae1d4647e --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + +import type { VectorizeMatch, VectorizeStore, VectorizeVectorInput } from "./vectorize"; +import { withCloudflareLimits } from "./store-cloudflare-limits"; + +// --------------------------------------------------------------------------- +// Inline fake VectorizeStore β€” records every call it receives so assertions +// can inspect exactly what the decorator forwarded (or withheld). +// --------------------------------------------------------------------------- + +interface FakeStoreState { + upsertCalls: ReadonlyArray; + queryCalls: ReadonlyArray<{ vector: readonly number[]; namespace: string; topK: number }>; + deleteByIdsCalls: ReadonlyArray; +} + +const makeFakeStore = (): { store: VectorizeStore; state: FakeStoreState } => { + const state: FakeStoreState = { + upsertCalls: [], + queryCalls: [], + deleteByIdsCalls: [], + }; + const store: VectorizeStore = { + upsert: (vectors) => + Effect.sync(() => { + // @ts-expect-error -- mutating a readonly array tracked for assertions + state.upsertCalls.push(vectors); + }), + query: (input) => + Effect.sync(() => { + // @ts-expect-error -- mutating a readonly array tracked for assertions + state.queryCalls.push(input); + return [] as readonly VectorizeMatch[]; + }), + deleteByIds: (ids) => + Effect.sync(() => { + // @ts-expect-error -- mutating a readonly array tracked for assertions + state.deleteByIdsCalls.push(ids); + }), + }; + return { store, state }; +}; + +/** Build a string whose UTF-8 byte length is exactly `n` bytes (ASCII). */ +const asciiId = (n: number): string => "x".repeat(n); + +describe("withCloudflareLimits", () => { + describe("upsert β€” id byte-length guard", () => { + it.effect("rejects a vector whose id is 65 bytes (one over the limit)", () => + Effect.gen(function* () { + const { store, state } = makeFakeStore(); + const wrapped = withCloudflareLimits(store); + const id = asciiId(65); // 65 ASCII chars = 65 UTF-8 bytes + expect(Buffer.byteLength(id, "utf8")).toBe(65); + + const exit = yield* wrapped.upsert([{ id, values: [1, 0, 0] }]).pipe(Effect.exit); + + expect(Exit.isFailure(exit)).toBe(true); + // Inner store must NOT have been called. + expect(state.upsertCalls).toHaveLength(0); + }), + ); + + it.effect("passes through a vector whose id is exactly 64 bytes", () => + Effect.gen(function* () { + const { store, state } = makeFakeStore(); + const wrapped = withCloudflareLimits(store); + const id = asciiId(64); // exactly at the limit β€” must pass + expect(Buffer.byteLength(id, "utf8")).toBe(64); + + const exit = yield* wrapped.upsert([{ id, values: [1, 0, 0] }]).pipe(Effect.exit); + + expect(Exit.isSuccess(exit)).toBe(true); + // Inner store must have received the call. + expect(state.upsertCalls).toHaveLength(1); + expect(state.upsertCalls[0]![0]!.id).toBe(id); + }), + ); + + it.effect("fails on ANY offending id, even when earlier ids are valid", () => + Effect.gen(function* () { + const { store, state } = makeFakeStore(); + const wrapped = withCloudflareLimits(store); + + const exit = yield* wrapped + .upsert([ + { id: asciiId(10), values: [1, 0, 0] }, // valid + { id: asciiId(65), values: [0, 1, 0] }, // over limit + ]) + .pipe(Effect.exit); + + expect(Exit.isFailure(exit)).toBe(true); + expect(state.upsertCalls).toHaveLength(0); + }), + ); + + it.effect("counts UTF-8 bytes, not characters (multibyte id over the limit)", () => + Effect.gen(function* () { + const { store, state } = makeFakeStore(); + const wrapped = withCloudflareLimits(store); + // "Γ©" is 2 UTF-8 bytes: 33 characters = 66 bytes, so it is over the + // 64-byte limit despite being well under 64 *characters*. + const id = "Γ©".repeat(33); + expect(id.length).toBe(33); + expect(Buffer.byteLength(id, "utf8")).toBe(66); + + const exit = yield* wrapped.upsert([{ id, values: [1, 0, 0] }]).pipe(Effect.exit); + + expect(Exit.isFailure(exit)).toBe(true); + expect(state.upsertCalls).toHaveLength(0); + }), + ); + }); + + describe("query β€” topK cap guard", () => { + it.effect("rejects topK = 21 (one over the metadata-all cap)", () => + Effect.gen(function* () { + const { store, state } = makeFakeStore(); + const wrapped = withCloudflareLimits(store); + + const exit = yield* wrapped + .query({ vector: [1, 0, 0], namespace: "default", topK: 21 }) + .pipe(Effect.exit); + + expect(Exit.isFailure(exit)).toBe(true); + expect(state.queryCalls).toHaveLength(0); + }), + ); + + it.effect("delegates topK = 20 (exactly at the cap) to the inner store", () => + Effect.gen(function* () { + const { store, state } = makeFakeStore(); + const wrapped = withCloudflareLimits(store); + + const exit = yield* wrapped + .query({ vector: [1, 0, 0], namespace: "default", topK: 20 }) + .pipe(Effect.exit); + + expect(Exit.isSuccess(exit)).toBe(true); + expect(state.queryCalls).toHaveLength(1); + expect(state.queryCalls[0]!.topK).toBe(20); + }), + ); + }); + + describe("deleteByIds β€” passes through unchanged", () => { + it.effect("forwards deleteByIds directly to the inner store", () => + Effect.gen(function* () { + const { store, state } = makeFakeStore(); + const wrapped = withCloudflareLimits(store); + + yield* wrapped.deleteByIds(["id-a", "id-b"]); + + expect(state.deleteByIdsCalls).toHaveLength(1); + expect(state.deleteByIdsCalls[0]).toEqual(["id-a", "id-b"]); + }), + ); + }); +}); diff --git a/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.ts b/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.ts new file mode 100644 index 0000000000..7aea860619 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.ts @@ -0,0 +1,61 @@ +import { Effect } from "effect"; + +import { VectorizeSearchError } from "./errors"; +import type { VectorizeStore, VectorizeVectorInput } from "./vectorize"; + +// --------------------------------------------------------------------------- +// Cloudflare Vectorize runtime limits β€” enforced as a decorator so the same +// validation applies equally to the real Vectorize binding and to zvec-based +// local / test setups. +// +// Limits (as of 2025): +// β€’ id: max 64 bytes (UTF-8 encoded). The raw `${namespace}#${path}` +// format used before the facet chunker could exceed this for long +// OpenAPI paths β€” a failure found by deploying, not testing. +// β€’ topK: max 20 when `returnMetadata` is "all". `makeVectorizeStore` always +// passes `returnMetadata: "all"`, so the query path is capped at 20. +// We reject rather than clamp so the caller is forced to stay within +// the budget instead of silently receiving fewer results than asked. +// --------------------------------------------------------------------------- + +const MAX_ID_BYTES = 64; +const MAX_QUERY_TOP_K = 20; + +// UTF-8 byte length via the Web-standard TextEncoder β€” available in the +// Cloudflare Workers runtime (and Node) without the `nodejs_compat` flag that +// `Buffer` would require. +const utf8ByteLength = (value: string): number => new TextEncoder().encode(value).length; + +/** Wraps `inner` with Cloudflare Vectorize runtime-limit validation. Any call + * that would violate a limit fails immediately with `VectorizeSearchError` + * before the inner store is touched. All other calls pass through unchanged. */ +export const withCloudflareLimits = (inner: VectorizeStore): VectorizeStore => ({ + upsert: (vectors: readonly VectorizeVectorInput[]) => { + const offending = vectors.find((v) => utf8ByteLength(v.id) > MAX_ID_BYTES); + if (offending !== undefined) { + return Effect.fail( + new VectorizeSearchError({ + message: `Vector id exceeds the Cloudflare Vectorize 64-byte limit (${utf8ByteLength(offending.id)} bytes): "${offending.id}".`, + }), + ); + } + return inner.upsert(vectors); + }, + + query: (input: { + readonly vector: readonly number[]; + readonly namespace: string; + readonly topK: number; + }) => { + if (input.topK > MAX_QUERY_TOP_K) { + return Effect.fail( + new VectorizeSearchError({ + message: `topK ${input.topK} exceeds the Cloudflare Vectorize limit of ${MAX_QUERY_TOP_K} when returnMetadata is "all". Lower topK or switch to a metadata strategy that supports a higher cap.`, + }), + ); + } + return inner.query(input); + }, + + deleteByIds: inner.deleteByIds.bind(inner), +}); From 6bd7a235ff91f2b26c1816e93110c51c9248801a Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Tue, 16 Jun 2026 13:07:34 +0530 Subject: [PATCH 31/57] feat(plugin-vectorize-search): local eval harness with pluggable embedder/store (#28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What A dev-only harness for tuning chunking and retrieval relevance offline, before anything touches Cloudflare. - `EmbedderService` and `VectorStoreService` make the embedder and vector store swappable via Effect layers. - `makeOpenAiCompatibleEmbedder` targets any OpenAI-compatible `/embeddings` endpoint (a local model server, vLLM, TEI, llama.cpp, etc.); `makeHashEmbedder` is a deterministic, key-free embedder for fully offline runs. - `makeZVecStore` is a file-backed local ANN index (`@zvec/zvec`, a devDependency) standing in for Vectorize. - `eval.ts` runs the full chunk β†’ embed β†’ query pipeline against a golden set, reporting precision@k and comparing the facet chunker against the whole-document baseline. None of this ships to the worker: zvec is a devDependency, and these modules are not part of the package's public exports. ## Testing Full package gate green on this branch (49 tests, lint 0/0, typecheck, format). The eval runs end-to-end offline: ``` EMBEDDER=hash STORE=zvec CHUNKER=facet bun run src/sdk/eval.ts ``` β†’ facet P@1 0.833 / P@5 1.000 vs whole 0.750 / 1.000. The hash embedder is intentionally weak β€” what's being validated is that the facet pipeline runs end-to-end through zvec and that facets improve ranking, not absolute accuracy. ## Stack Stacked on the facet/hybrid/incremental PR; review and merge that first. ### [Stack](https://github.com/kitlangton/stack) 1. #27 2. **#28** πŸ‘ˆ current --- bun.lock | 11 + .../plugins/vectorize-search/package.json | 4 +- .../vectorize-search/src/sdk/embedder-hash.ts | 42 ++ .../src/sdk/embedder-openai.ts | 108 ++++ .../src/sdk/embedding-service.ts | 30 + .../vectorize-search/src/sdk/eval-golden.ts | 102 ++++ .../plugins/vectorize-search/src/sdk/eval.ts | 520 ++++++++++++++++++ .../vectorize-search/src/sdk/harness.test.ts | 80 +++ .../vectorize-search/src/sdk/store-service.ts | 21 + .../vectorize-search/src/sdk/store-zvec.ts | 174 ++++++ 10 files changed, 1091 insertions(+), 1 deletion(-) create mode 100644 packages/plugins/vectorize-search/src/sdk/embedder-hash.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/embedder-openai.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/embedding-service.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/eval-golden.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/eval.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/harness.test.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/store-service.ts create mode 100644 packages/plugins/vectorize-search/src/sdk/store-zvec.ts diff --git a/bun.lock b/bun.lock index 9f62e5f307..3d9309b2bc 100644 --- a/bun.lock +++ b/bun.lock @@ -1128,6 +1128,7 @@ "@effect/vitest": "catalog:", "@executor-js/api": "workspace:*", "@types/node": "catalog:", + "@zvec/zvec": "^0.4.1", "bun-types": "catalog:", "tsup": "catalog:", "vitest": "catalog:", @@ -3110,6 +3111,16 @@ "@zip.js/zip.js": ["@zip.js/zip.js@2.8.26", "", {}, "sha512-RQ4h9F6DOiHxpdocUDrOl6xBM+yOtz+LkUol47AVWcfebGBDpZ7w7Xvz9PS24JgXvLGiXXzSAfdCdVy1tPlaFA=="], + "@zvec/bindings-darwin-arm64": ["@zvec/bindings-darwin-arm64@0.4.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-UZAUGEigHVAxeoqWuuS2QjiH7K8tQ6JV0AlAdbJiLdDvS3RTGtW2kxlxi9dmdwbF3cZOYOmzQYpwBQlvrjM70w=="], + + "@zvec/bindings-linux-arm64": ["@zvec/bindings-linux-arm64@0.4.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-3ZbvpSH2SdwLmMyK4KjpfBZkSVCgMf3XhbP2dEObVOHYjlC/BHIzg5ASuy3hk3Ei8HMYsCzSdZjUjF2KcmWSUA=="], + + "@zvec/bindings-linux-x64": ["@zvec/bindings-linux-x64@0.4.1", "", { "os": "linux", "cpu": "x64" }, "sha512-bNakqv5q1l1E2s8IKFJ0j/tz+5IUOka/tWG7DQsWngBab0XwnOvCPuIXPSOpdEYM9C2B26cliK3DvGfMgRLh8Q=="], + + "@zvec/bindings-win32-x64": ["@zvec/bindings-win32-x64@0.4.1", "", { "os": "win32", "cpu": "x64" }, "sha512-Pz/llUgPd2hvhA1UbrixWy4/x6pntsuA3aX42zLQKz/UOgJo2F2+h0TLa1Oye4yVsllXqdiGnvBY1oKz9zB3BA=="], + + "@zvec/zvec": ["@zvec/zvec@0.4.1", "", { "dependencies": { "bindings": "^1.5.0" }, "optionalDependencies": { "@zvec/bindings-darwin-arm64": "0.4.1", "@zvec/bindings-linux-arm64": "0.4.1", "@zvec/bindings-linux-x64": "0.4.1", "@zvec/bindings-win32-x64": "0.4.1" } }, "sha512-Rh2XgVOuZy3fxjyO4aVxFfqxyZQZEdxMcr6nV3yyYjToaJSfdixKemSMCoe2mpHCq4cA06B9qCpJFb8A7jzpDQ=="], + "abbrev": ["abbrev@4.0.0", "", {}, "sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA=="], "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], diff --git a/packages/plugins/vectorize-search/package.json b/packages/plugins/vectorize-search/package.json index a9cae12b28..d73c34a9f4 100644 --- a/packages/plugins/vectorize-search/package.json +++ b/packages/plugins/vectorize-search/package.json @@ -11,7 +11,8 @@ "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src || true)", "typecheck": "tsgo --noEmit", "test": "vitest run", - "test:watch": "vitest" + "test:watch": "vitest", + "eval": "bun run src/sdk/eval.ts" }, "dependencies": { "@ai-sdk/google": "^3.0.80", @@ -24,6 +25,7 @@ "@effect/vitest": "catalog:", "@executor-js/api": "workspace:*", "@types/node": "catalog:", + "@zvec/zvec": "^0.4.1", "bun-types": "catalog:", "tsup": "catalog:", "vitest": "catalog:" diff --git a/packages/plugins/vectorize-search/src/sdk/embedder-hash.ts b/packages/plugins/vectorize-search/src/sdk/embedder-hash.ts new file mode 100644 index 0000000000..3456cd8d8b --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/embedder-hash.ts @@ -0,0 +1,42 @@ +import { Effect } from "effect"; + +import type { ToolEmbedder } from "./embedder"; + +// --------------------------------------------------------------------------- +// Deterministic, dependency-free embedder for the harness + unit tests: a hashed +// bag-of-tokens vector, so cosine similarity reflects token overlap. It is NOT +// semantic β€” use the real Gemini embedder for relevance β€” but it is stable and +// meaningful enough to exercise the indexer β†’ store β†’ provider pipeline and to +// assert rankings in CI without an API key. +// --------------------------------------------------------------------------- + +const tokenize = (text: string): readonly string[] => + text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((token) => token.length > 1); + +const hashToken = (token: string, dimensions: number): number => { + let h = 2166136261; + for (let i = 0; i < token.length; i++) { + h ^= token.charCodeAt(i); + h = Math.imul(h, 16777619); + } + return Math.abs(h) % dimensions; +}; + +const embed = (text: string, dimensions: number): readonly number[] => { + const vector = new Array(dimensions).fill(0); + for (const token of tokenize(text)) { + vector[hashToken(token, dimensions)]! += 1; + } + const norm = Math.sqrt(vector.reduce((sum, x) => sum + x * x, 0)) || 1; + return vector.map((x) => x / norm); +}; + +export const makeHashEmbedder = (dimensions = 256): ToolEmbedder => ({ + model: "hash-test", + dimensions, + embedDocuments: (texts) => Effect.succeed(texts.map((text) => embed(text, dimensions))), + embedQuery: (text) => Effect.succeed(embed(text, dimensions)), +}); diff --git a/packages/plugins/vectorize-search/src/sdk/embedder-openai.ts b/packages/plugins/vectorize-search/src/sdk/embedder-openai.ts new file mode 100644 index 0000000000..f4f99aa71c --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/embedder-openai.ts @@ -0,0 +1,108 @@ +import { Effect } from "effect"; + +import type { ToolEmbedder } from "./embedder"; +import { VectorizeSearchError } from "./errors"; + +// --------------------------------------------------------------------------- +// OpenAI-compatible embedder β€” POSTs to `${baseUrl}/embeddings` in the standard +// OpenAI shape, so any endpoint that speaks that protocol works (a local model +// server for cheap experimentation, OpenAI, vLLM, TEI, llama.cpp, Ollama's +// `/v1`, etc.). Provided as a layer behind `EmbedderService`, it's a drop-in +// swap for the Gemini embedder. +// --------------------------------------------------------------------------- + +export interface OpenAiCompatibleEmbedderOptions { + /** Base URL up to (not including) `/embeddings`, e.g. `http://localhost:1234/v1`. */ + readonly baseUrl: string; + readonly model: string; + readonly dimensions: number; + /** Optional bearer token; local servers usually need none. */ + readonly apiKey?: string; + /** Inputs per request. */ + readonly batchSize?: number; +} + +const DEFAULT_BATCH_SIZE = 64; + +interface OpenAiEmbeddingsResponse { + readonly data?: readonly { readonly embedding?: readonly number[] }[]; +} + +const chunk = (items: readonly A[], size: number): readonly (readonly A[])[] => { + const safe = Math.max(1, Math.floor(size)); + const out: A[][] = []; + for (let i = 0; i < items.length; i += safe) out.push(items.slice(i, i + safe)); + return out; +}; + +export const makeOpenAiCompatibleEmbedder = ( + options: OpenAiCompatibleEmbedderOptions, +): ToolEmbedder => { + const endpoint = `${options.baseUrl.replace(/\/+$/, "")}/embeddings`; + const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE; + + const embedBatch = ( + texts: readonly string[], + ): Effect.Effect => { + if (texts.length === 0) return Effect.succeed([]); + return Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: () => + fetch(endpoint, { + method: "POST", + headers: { + "content-type": "application/json", + ...(options.apiKey ? { authorization: `Bearer ${options.apiKey}` } : {}), + }, + // `dimensions` is honored by OpenAI + most compatible servers; ignored otherwise. + body: JSON.stringify({ + model: options.model, + input: [...texts], + dimensions: options.dimensions, + }), + }), + catch: (cause) => + new VectorizeSearchError({ + message: `OpenAI-compatible embedding failed for ${options.model} at ${endpoint}.`, + cause, + }), + }); + if (!response.ok) { + // Read the error body for a better message; tolerate a read failure by + // falling back to "". `Effect.catch(fn)` is the catch-all in this Effect + // version (4.0.0-beta) β€” `Effect.catchAll` does not exist on `Effect`. + const body = yield* Effect.tryPromise(() => response.text()).pipe( + Effect.catch(() => Effect.succeed("")), + ); + return yield* new VectorizeSearchError({ + message: `embeddings HTTP ${response.status}: ${body.slice(0, 300)}`, + }); + } + const json = yield* Effect.tryPromise({ + try: () => response.json() as Promise, + catch: (cause) => + new VectorizeSearchError({ + message: `OpenAI-compatible embedding failed for ${options.model} at ${endpoint}.`, + cause, + }), + }); + const embeddings = json.data?.map((item) => item.embedding ?? []); + if (!embeddings || embeddings.length !== texts.length) { + return yield* new VectorizeSearchError({ + message: `embeddings response returned ${embeddings?.length ?? 0} vectors for ${texts.length} inputs`, + }); + } + return embeddings; + }); + }; + + return { + model: options.model, + dimensions: options.dimensions, + embedDocuments: (texts) => + Effect.forEach(chunk(texts, batchSize), (group) => embedBatch(group), { + concurrency: 1, + }).pipe(Effect.map((groups) => groups.flat())), + embedQuery: (text) => embedBatch([text]).pipe(Effect.map((vectors) => vectors[0] ?? [])), + }; +}; diff --git a/packages/plugins/vectorize-search/src/sdk/embedding-service.ts b/packages/plugins/vectorize-search/src/sdk/embedding-service.ts new file mode 100644 index 0000000000..a9a7ef6bfd --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/embedding-service.ts @@ -0,0 +1,30 @@ +import { Context, Layer } from "effect"; + +import { makeHashEmbedder } from "./embedder-hash"; +import { + makeOpenAiCompatibleEmbedder, + type OpenAiCompatibleEmbedderOptions, +} from "./embedder-openai"; +import { makeGeminiEmbedder, type GeminiEmbedderOptions, type ToolEmbedder } from "./embedder"; + +// --------------------------------------------------------------------------- +// Pluggable embedder seam. The indexer + query provider depend on +// `EmbedderService`; the host (or the eval) supplies ONE of the layers below β€” +// a local OpenAI-compatible endpoint for cheap experimentation, Gemini for +// production, or the deterministic hash embedder for key-free CI. +// --------------------------------------------------------------------------- + +export class EmbedderService extends Context.Service()( + "@executor-js/plugin-vectorize-search/EmbedderService", +) {} + +export const geminiEmbedderLayer = (options: GeminiEmbedderOptions): Layer.Layer => + Layer.succeed(EmbedderService)(makeGeminiEmbedder(options)); + +export const openAiCompatibleEmbedderLayer = ( + options: OpenAiCompatibleEmbedderOptions, +): Layer.Layer => + Layer.succeed(EmbedderService)(makeOpenAiCompatibleEmbedder(options)); + +export const hashEmbedderLayer = (dimensions?: number): Layer.Layer => + Layer.succeed(EmbedderService)(makeHashEmbedder(dimensions)); diff --git a/packages/plugins/vectorize-search/src/sdk/eval-golden.ts b/packages/plugins/vectorize-search/src/sdk/eval-golden.ts new file mode 100644 index 0000000000..8782a41689 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/eval-golden.ts @@ -0,0 +1,102 @@ +// --------------------------------------------------------------------------- +// Eval golden set β€” expected top tool path per query. +// +// Used by eval.ts to compute precision@1 and precision@5 against the embedded +// + ranked results. Keep it small and representative: a few clear-intent +// queries and a few that require deeper semantic matching (e.g. sub-features +// hidden in long descriptions or input schemas). +// --------------------------------------------------------------------------- + +export interface GoldenQuery { + /** The natural-language query used for retrieval. */ + readonly query: string; + /** The canonical tool path that should rank in the top result. */ + readonly expectedPath: string; + /** The top-5 set that ALL contain relevant results (for precision@5). */ + readonly expectedTop5: readonly string[]; +} + +export const GOLDEN: readonly GoldenQuery[] = [ + // --- Google Calendar --- + { + query: "create a calendar event", + expectedPath: "tools.google.org.x.calendar.events.insert", + expectedTop5: [ + "tools.google.org.x.calendar.events.insert", + "tools.google.org.x.calendar.events.quickAdd", + ], + }, + { + query: "delete a calendar event", + expectedPath: "tools.google.org.x.calendar.events.delete", + expectedTop5: ["tools.google.org.x.calendar.events.delete"], + }, + + // --- Gmail --- + { + query: "send an email to recipients", + expectedPath: "tools.google.org.x.gmail.users.messages.send", + expectedTop5: ["tools.google.org.x.gmail.users.messages.send"], + }, + + // --- GitHub --- + { + query: "open a bug report on a repository", + expectedPath: "tools.github.org.x.repos.issues.create", + expectedTop5: [ + "tools.github.org.x.repos.issues.create", + "tools.github.org.x.repos.pulls.create", + ], + }, + { + query: "make a pull request", + expectedPath: "tools.github.org.x.repos.pulls.create", + expectedTop5: [ + "tools.github.org.x.repos.pulls.create", + "tools.github.org.x.repos.issues.create", + ], + }, + + // --- Stripe charges + refunds --- + { + query: "charge a credit card", + expectedPath: "tools.stripe.org.x.charges.create", + expectedTop5: ["tools.stripe.org.x.charges.create", "tools.stripe.org.x.customers.create"], + }, + { + query: "give a refund for a payment", + expectedPath: "tools.stripe.org.x.refunds.create", + expectedTop5: ["tools.stripe.org.x.refunds.create"], + }, + + // --- Stripe subscriptions β€” queries that rely on rich description + input facets --- + { + query: "start a free trial on a subscription", + expectedPath: "tools.stripe.org.x.subscriptions.create", + expectedTop5: ["tools.stripe.org.x.subscriptions.create"], + }, + { + query: "cancel a subscription at end of billing period", + expectedPath: "tools.stripe.org.x.subscriptions.create", + expectedTop5: ["tools.stripe.org.x.subscriptions.create"], + }, + { + query: "prorate a plan change mid-cycle", + expectedPath: "tools.stripe.org.x.subscriptions.create", + expectedTop5: ["tools.stripe.org.x.subscriptions.create"], + }, + + // --- Slack --- + { + query: "send a message to a Slack channel", + expectedPath: "tools.slack.org.x.chat.postMessage", + expectedTop5: ["tools.slack.org.x.chat.postMessage"], + }, + + // --- Linear --- + { + query: "create a new Linear issue", + expectedPath: "tools.linear.org.x.issues.create", + expectedTop5: ["tools.linear.org.x.issues.create", "tools.github.org.x.repos.issues.create"], + }, +]; diff --git a/packages/plugins/vectorize-search/src/sdk/eval.ts b/packages/plugins/vectorize-search/src/sdk/eval.ts new file mode 100644 index 0000000000..2e10b08d45 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/eval.ts @@ -0,0 +1,520 @@ +import { Cause, Effect, Exit, Layer } from "effect"; + +import { ChunkerService, facetChunkerLayer, wholeChunkerLayer } from "./chunker-service"; +import type { ToolDocumentInput } from "./chunker"; +import { + EmbedderService, + geminiEmbedderLayer, + hashEmbedderLayer, + openAiCompatibleEmbedderLayer, +} from "./embedding-service"; +import { VectorStoreService, zvecStoreLayer } from "./store-service"; +import { GOLDEN } from "./eval-golden"; + +// --------------------------------------------------------------------------- +// Relevance eval β€” exercises the facet chunker + embed + zvec pipeline locally +// to measure search quality before touching production. The embedder (EMBEDDER) +// and chunker (CHUNKER) are selectable from env; the local vector store is +// always zvec (ZVEC_PATH sets its location): +// +// # fully offline (no key, no network, default): +// EMBEDDER=hash ZVEC_PATH="$(mktemp -d)/eval.zvec" CHUNKER=facet \ +// bun run packages/plugins/vectorize-search/src/sdk/eval.ts +// +// # local OpenAI-compatible endpoint: +// EMBEDDER=openai OPENAI_BASE_URL=http://localhost:1234/v1 OPENAI_MODEL=… \ +// EMBEDDING_DIMENSIONS=1536 CHUNKER=facet bun run …/eval.ts +// +// # Gemini (production parity): +// EMBEDDER=gemini GEMINI_API_KEY=… CHUNKER=facet bun run …/eval.ts +// +// Precision@1 and precision@5 are printed against GOLDEN (eval-golden.ts). +// --------------------------------------------------------------------------- + +const NAMESPACE = "default"; +const DIMENSIONS = Number(process.env.EMBEDDING_DIMENSIONS ?? 1536); +const CHUNKER_KIND = process.env.CHUNKER ?? "facet"; + +// --------------------------------------------------------------------------- +// Sample catalog β€” realistic tool documents with input/output TypeScript so +// the facet chunker's input + output facets actually fire. +// --------------------------------------------------------------------------- + +const SUBSCRIPTION_DESC = [ + "Creates a new subscription on an existing customer. Each customer can have up to 500 active or scheduled subscriptions. A subscription ties a customer to a recurring price and bills them on a schedule (the billing cycle anchor).", + "When you create a subscription, Stripe attempts to bill the customer immediately for the first invoice, unless you configure otherwise. You can pass items, each referencing a price and an optional quantity, to build a multi-line subscription.", + "Trial periods: pass trial_period_days or trial_end to start the subscription in a trialing state. During a trial the customer is not charged, and at the end of the trial Stripe generates the first real invoice.", + "Proration: when a subscription's items or quantities change mid-cycle, Stripe by default prorates the difference, generating proration line items that credit unused time on the old price and charge for the new one. Set proration_behavior to 'none' to disable this.", + "Cancellation: set cancel_at_period_end to true to schedule the subscription to stop at the end of the current billing period rather than immediately; the customer keeps access until then and is not billed again.", +].join("\n\n"); + +const SAMPLE: readonly ToolDocumentInput[] = [ + { + path: "tools.google.org.x.calendar.events.insert", + name: "events.insert", + integration: "google", + description: + "Creates an event on the specified calendar. The event can include a title, description, start/end time, attendees, location, and recurrence rules.", + inputTypeScript: `{ + calendarId: string; // 'primary' or a calendar id + summary: string; // event title + description?: string; + start: { dateTime: string; timeZone?: string }; + end: { dateTime: string; timeZone?: string }; + attendees?: Array<{ email: string }>; + location?: string; + recurrence?: string[]; // RRULE strings +}`, + outputTypeScript: `{ + id: string; + htmlLink: string; + status: 'confirmed' | 'tentative' | 'cancelled'; + summary: string; + start: { dateTime: string }; + end: { dateTime: string }; +}`, + }, + { + path: "tools.google.org.x.calendar.events.quickAdd", + name: "events.quickAdd", + integration: "google", + description: + "Creates an event based on a simple text string such as 'Appointment at Somewhere on June 3rd 10am-10:25am'. Google parses the natural-language text to populate the event fields.", + inputTypeScript: `{ + calendarId: string; + text: string; // natural-language event description + sendUpdates?: 'all' | 'externalOnly' | 'none'; +}`, + outputTypeScript: `{ + id: string; + htmlLink: string; + summary: string; + start: { dateTime: string }; + end: { dateTime: string }; +}`, + }, + { + path: "tools.google.org.x.calendar.events.delete", + name: "events.delete", + integration: "google", + description: + "Deletes an event from a calendar. Once deleted the event is removed from the calendar view. For recurring events you can target a single instance or the entire series.", + inputTypeScript: `{ + calendarId: string; + eventId: string; + sendUpdates?: 'all' | 'externalOnly' | 'none'; +}`, + outputTypeScript: "void", + }, + { + path: "tools.google.org.x.gmail.users.messages.send", + name: "messages.send", + integration: "google", + description: + "Sends the specified message to the recipients in the To, Cc, and Bcc headers. The message body is supplied as a base64url-encoded RFC 2822 email string.", + inputTypeScript: `{ + userId: string; // 'me' for the authenticated user + raw: string; // base64url-encoded RFC 2822 message + threadId?: string; // supply to reply in an existing thread +}`, + outputTypeScript: `{ + id: string; + threadId: string; + labelIds: string[]; +}`, + }, + { + path: "tools.github.org.x.repos.issues.create", + name: "issues.create", + integration: "github", + description: + "Create an issue in a repository. Any user with pull access to a repository can create an issue. Issues can include a title, body, labels, assignees, and milestone.", + inputTypeScript: `{ + owner: string; + repo: string; + title: string; + body?: string; + labels?: string[]; + assignees?: string[]; + milestone?: number; +}`, + outputTypeScript: `{ + id: number; + number: number; + html_url: string; + state: 'open' | 'closed'; + title: string; +}`, + }, + { + path: "tools.github.org.x.repos.pulls.create", + name: "pulls.create", + integration: "github", + description: + "Create a pull request to propose and collaborate on changes to a repository. The pull request includes a title, body, head branch, and base branch to merge into.", + inputTypeScript: `{ + owner: string; + repo: string; + title: string; + head: string; // branch with the changes + base: string; // branch to merge into + body?: string; + draft?: boolean; +}`, + outputTypeScript: `{ + id: number; + number: number; + html_url: string; + state: 'open' | 'closed' | 'merged'; + title: string; + mergeable: boolean | null; +}`, + }, + { + path: "tools.github.org.x.repos.get", + name: "repos.get", + integration: "github", + description: + "Gets a repository's metadata, including its description, default branch, visibility, star count, and clone URL.", + inputTypeScript: `{ owner: string; repo: string }`, + outputTypeScript: `{ + id: number; + name: string; + full_name: string; + private: boolean; + description: string | null; + default_branch: string; + stargazers_count: number; +}`, + }, + { + path: "tools.stripe.org.x.charges.create", + name: "charges.create", + integration: "stripe", + description: + "To charge a credit or debit card, you create a Charge object. Stripe attempts to immediately collect funds from the customer's payment source. Pass amount in the smallest currency unit (e.g. cents for USD).", + inputTypeScript: `{ + amount: number; // in smallest currency unit (e.g. cents) + currency: string; // ISO 4217 code, e.g. 'usd' + source?: string; // card token or payment method id + customer?: string; // existing customer id + description?: string; + metadata?: Record; + capture?: boolean; // default true; false for auth-only +}`, + outputTypeScript: `{ + id: string; + amount: number; + currency: string; + status: 'succeeded' | 'pending' | 'failed'; + paid: boolean; + balance_transaction: string; +}`, + }, + { + path: "tools.stripe.org.x.refunds.create", + name: "refunds.create", + integration: "stripe", + description: + "Refund a charge that has previously been created but not yet refunded. Funds will be refunded to the credit or debit card that was originally charged. You can refund the full amount or a partial amount.", + inputTypeScript: `{ + charge?: string; // charge id to refund + payment_intent?: string; // or payment_intent id + amount?: number; // omit to refund the full charge + reason?: 'duplicate' | 'fraudulent' | 'requested_by_customer'; + metadata?: Record; +}`, + outputTypeScript: `{ + id: string; + amount: number; + currency: string; + status: 'pending' | 'succeeded' | 'failed' | 'canceled'; + charge: string; +}`, + }, + { + path: "tools.stripe.org.x.customers.create", + name: "customers.create", + integration: "stripe", + description: + "Creates a new customer object you can later charge or attach payment methods to. Customer objects allow you to perform recurring charges, track multiple payments, and store payment details.", + inputTypeScript: `{ + email?: string; + name?: string; + phone?: string; + description?: string; + payment_method?: string; + metadata?: Record; +}`, + outputTypeScript: `{ + id: string; + email: string | null; + name: string | null; + created: number; + default_source: string | null; +}`, + }, + { + path: "tools.slack.org.x.chat.postMessage", + name: "chat.postMessage", + integration: "slack", + description: + "Sends a message to a channel. The channel can be a public channel, private channel, or direct message. Supports rich formatting via Block Kit or simple markdown text.", + inputTypeScript: `{ + channel: string; // channel ID or name + text?: string; // plain text fallback + blocks?: unknown[]; // Block Kit blocks + thread_ts?: string; // reply in a thread + mrkdwn?: boolean; +}`, + outputTypeScript: `{ + ok: boolean; + channel: string; + ts: string; // message timestamp (also the message id) + message: { text: string; ts: string }; +}`, + }, + { + path: "tools.linear.org.x.issues.create", + name: "issues.create", + integration: "linear", + description: + "Create a new issue in a Linear team, with title, description, assignee, and priority. Issues track work items and bugs within a team's backlog.", + inputTypeScript: `{ + teamId: string; + title: string; + description?: string; // markdown + priority?: 0 | 1 | 2 | 3 | 4; // 0=no priority, 1=urgent, 4=low + assigneeId?: string; + labelIds?: string[]; + stateId?: string; +}`, + outputTypeScript: `{ + id: string; + identifier: string; // e.g. 'ENG-123' + title: string; + url: string; + priority: number; + state: { name: string }; +}`, + }, + { + path: "tools.stripe.org.x.subscriptions.create", + name: "subscriptions.create", + integration: "stripe", + description: SUBSCRIPTION_DESC, + inputTypeScript: `{ + customer: string; + items: Array<{ price: string; quantity?: number }>; + trial_period_days?: number; + trial_end?: number | 'now'; + proration_behavior?: 'create_prorations' | 'none' | 'always_invoice'; + cancel_at_period_end?: boolean; + cancel_at?: number; + default_payment_method?: string; + billing_cycle_anchor?: number | 'now'; + metadata?: Record; +}`, + outputTypeScript: `{ + id: string; + status: 'active' | 'trialing' | 'past_due' | 'canceled' | 'unpaid' | 'incomplete'; + current_period_start: number; + current_period_end: number; + trial_start: number | null; + trial_end: number | null; + cancel_at_period_end: boolean; + items: { data: Array<{ price: { id: string }; quantity: number }> }; +}`, + }, +]; + +// --------------------------------------------------------------------------- +// Precision metrics +// --------------------------------------------------------------------------- + +const computePrecision = ( + rankedPaths: readonly string[], + golden: readonly string[], + atK: number, +): number => { + const top = rankedPaths.slice(0, atK); + const hits = top.filter((p) => golden.includes(p)).length; + return hits / Math.min(atK, golden.length === 0 ? 1 : golden.length); +}; + +// --------------------------------------------------------------------------- +// Main program +// --------------------------------------------------------------------------- + +const program = Effect.gen(function* () { + const embedder = yield* EmbedderService; + const store = yield* VectorStoreService; + const chunker = yield* ChunkerService; + + // ------------------------------------------------------------------------- + // Phase 1 β€” Chunk + embed + upsert + // ------------------------------------------------------------------------- + const allChunks = SAMPLE.flatMap((doc) => chunker.chunk(NAMESPACE, doc)); + + yield* Effect.sync(() => { + console.log(`\n=== chunking (chunker=${CHUNKER_KIND}) ===`); + console.log(`tools=${SAMPLE.length} chunks=${allChunks.length}`); + // Report per-tool chunk breakdown. + for (const doc of SAMPLE) { + const cs = chunker.chunk(NAMESPACE, doc); + if (cs.length > 1) { + const facetSummary = cs.map((c) => c.facet).join(", "); + console.log(` multi-chunk: ${doc.path} -> ${cs.length} chunks [${facetSummary}]`); + } + } + }); + + const vectors = yield* embedder.embedDocuments(allChunks.map((c) => c.embeddingText)); + + yield* store.upsert( + allChunks.map((c, i) => ({ + id: c.id, + values: [...vectors[i]!], + namespace: NAMESPACE, + metadata: { + path: c.path, + name: c.name, + integration: c.integration, + facet: c.facet, + chunkIndex: c.chunkIndex, + }, + })), + ); + + // ------------------------------------------------------------------------- + // Phase 2 β€” Query, dedup by path (best score), top 5 + // ------------------------------------------------------------------------- + const allQueries = GOLDEN.map((g) => g.query); + + yield* Effect.sync(() => + console.log(`\n=== queries (embedder=${embedder.model}, dedup by path, top 5) ===`), + ); + + // Collect ranked results for precision computation. + const rankedByQuery = new Map(); + + for (const q of allQueries) { + const qv = yield* embedder.embedQuery(q); + // Over-fetch so dedup-by-path still yields a full top-5 window. + const matches = yield* store.query({ vector: qv, namespace: NAMESPACE, topK: 50 }); + + const best = new Map(); + for (const m of matches) { + const p = String(m.metadata?.path ?? ""); + if (p === "") continue; + const prev = best.get(p); + if (!prev || m.score > prev.score) { + best.set(p, { + score: m.score, + facet: String(m.metadata?.facet ?? ""), + chunkIndex: Number(m.metadata?.chunkIndex ?? 0), + }); + } + } + + const ranked = [...best.entries()].sort((a, b) => b[1].score - a[1].score).slice(0, 5); + + rankedByQuery.set( + q, + ranked.map(([p]) => p), + ); + + yield* Effect.sync(() => { + console.log(`\nQ: "${q}"`); + for (const [p, { score, facet, chunkIndex }] of ranked) { + console.log(` ${score.toFixed(3)} ${p} (facet=${facet} chunk=${chunkIndex})`); + } + }); + } + + // ------------------------------------------------------------------------- + // Phase 3 β€” Precision@1 and precision@5 against the golden set + // ------------------------------------------------------------------------- + let sumP1 = 0; + let sumP5 = 0; + + yield* Effect.sync(() => { + console.log(`\n=== precision summary (chunker=${CHUNKER_KIND}) ===`); + }); + + for (const golden of GOLDEN) { + const ranked = rankedByQuery.get(golden.query) ?? []; + const p1 = ranked[0] === golden.expectedPath ? 1 : 0; + const p5 = computePrecision(ranked, golden.expectedTop5, 5); + sumP1 += p1; + sumP5 += p5; + + yield* Effect.sync(() => { + const hit1 = p1 === 1 ? "βœ“" : "βœ—"; + console.log(` ${hit1} [P@1=${p1} P@5=${p5.toFixed(2)}] "${golden.query}"`); + if (p1 === 0) { + console.log(` expected: ${golden.expectedPath}`); + console.log(` got: ${ranked[0] ?? "(none)"}`); + } + }); + } + + const n = GOLDEN.length; + const avgP1 = sumP1 / n; + const avgP5 = sumP5 / n; + + yield* Effect.sync(() => { + console.log( + `\nprecision@1=${avgP1.toFixed(3)} precision@5=${avgP5.toFixed(3)} (${n} queries, chunker=${CHUNKER_KIND}, embedder=${embedder.model})`, + ); + if (embedder.model.startsWith("hash")) { + console.log( + "(note: hash embedder is deterministic but weak β€” low precision is expected offline)", + ); + } + }); +}); + +// --------------------------------------------------------------------------- +// Layer selection from env +// --------------------------------------------------------------------------- + +const embedderLayer = (): Layer.Layer => { + const kind = process.env.EMBEDDER ?? "gemini"; + if (kind === "openai") { + return openAiCompatibleEmbedderLayer({ + baseUrl: process.env.OPENAI_BASE_URL ?? "http://localhost:1234/v1", + model: process.env.OPENAI_MODEL ?? "text-embedding-3-small", + dimensions: DIMENSIONS, + apiKey: process.env.OPENAI_API_KEY, + }); + } + if (kind === "hash") return hashEmbedderLayer(DIMENSIONS); + return geminiEmbedderLayer({ + apiKey: process.env.GEMINI_API_KEY?.trim() ?? "", + model: process.env.GEMINI_EMBEDDING_MODEL?.trim() || undefined, + dimensions: DIMENSIONS, + }); +}; + +const storeLayer = (): Layer.Layer => { + // Default to zvec (no in-memory store β€” removed in P0). + return zvecStoreLayer({ + path: process.env.ZVEC_PATH ?? "/tmp/vectorize-eval.zvec", + dimensions: DIMENSIONS, + }); +}; + +const chunkerLayer = (): Layer.Layer => { + if (CHUNKER_KIND === "whole") return wholeChunkerLayer(); + // Default: facet chunker. + return facetChunkerLayer(); +}; + +const exit = await Effect.runPromiseExit( + program.pipe(Effect.provide(Layer.mergeAll(embedderLayer(), storeLayer(), chunkerLayer()))), +); +if (Exit.isFailure(exit)) { + console.error(Cause.pretty(exit.cause)); + process.exit(1); +} diff --git a/packages/plugins/vectorize-search/src/sdk/harness.test.ts b/packages/plugins/vectorize-search/src/sdk/harness.test.ts new file mode 100644 index 0000000000..8725a1039d --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/harness.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; +import * as os from "node:os"; +import * as path from "node:path"; + +import { makeHashEmbedder } from "./embedder-hash"; +import { withCloudflareLimits } from "./store-cloudflare-limits"; +import { makeZVecStore } from "./store-zvec"; + +// --------------------------------------------------------------------------- +// The harness exercises the real chunkβ†’embedβ†’upsertβ†’queryβ†’dedup flow over a +// zvec-backed store (real local ANN index, not in-memory) and verifies that +// the Cloudflare Vectorize limits enforced by `withCloudflareLimits` fire +// correctly. Using zvec here gives a realistic backend β€” the in-memory index +// was simpler but lower fidelity, and we no longer need it. +// +// Each test uses an isolated OS temp directory so runs never share state. +// --------------------------------------------------------------------------- + +/** Return a non-existent temp path for zvec (it creates the directory itself). */ +const tempPath = (): string => + path.join(os.tmpdir(), `zvec-harness-${Date.now()}-${Math.random().toString(36).slice(2)}`); + +describe("harness β€” CF limits enforced by withCloudflareLimits over a zvec store", () => { + it.effect("upsert rejects an id over 64 bytes (VECTOR_UPSERT_ERROR / id-too-long)", () => + Effect.gen(function* () { + const dir = tempPath(); + const inner = makeZVecStore({ path: dir, dimensions: 3 }); + const store = withCloudflareLimits(inner); + // The shape `${namespace}#${path}` blows past 64 bytes for long OpenAPI paths. + const id = `default#${"cloudflare_api.org.aryalabs.aiGateway.aigBillingCreateTopup".repeat(2)}`; + const exit = yield* store.upsert([{ id, values: [1, 0, 0] }]).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + }), + ); + + it.effect("query rejects topK > 20 when returning full metadata (the topK cap)", () => + Effect.gen(function* () { + const dir = tempPath(); + const inner = makeZVecStore({ path: dir, dimensions: 3 }); + const store = withCloudflareLimits(inner); + const exit = yield* store + .query({ vector: [1, 0, 0], namespace: "default", topK: 50 }) + .pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + }), + ); + + it.effect("a within-limits query returns ranked, namespace-scoped matches", () => + Effect.gen(function* () { + const dir = tempPath(); + const embedder = makeHashEmbedder(); // default 256 dimensions + const inner = makeZVecStore({ path: dir, dimensions: 256 }); + // Wrap the inner store with CF limits (topK≀20, id≀64 bytes) to exercise + // the full production-shaped stack. The test uses topK=5 and short ids + // so both limits are respected and the query succeeds. + const store = withCloudflareLimits(inner); + + const docs = [ + { id: "t_a", text: "google calendar events insert create a calendar event" }, + { id: "t_b", text: "github repos issues create an issue" }, + { id: "t_c", text: "google calendar events delete remove a calendar event" }, + ]; + const vectors = yield* embedder.embedDocuments(docs.map((d) => d.text)); + yield* store.upsert( + docs.map((d, i) => ({ + id: d.id, + values: [...vectors[i]!], + namespace: "default", + metadata: { path: d.id, name: d.id, description: d.text, integration: "x" }, + })), + ); + const qv = yield* embedder.embedQuery("create a calendar event"); + const matches = yield* store.query({ vector: qv, namespace: "default", topK: 5 }); + // The two calendar docs should outrank the github one (token overlap). + expect(matches[0]!.id === "t_a" || matches[0]!.id === "t_c").toBe(true); + expect(matches.map((m) => m.id)).not.toContain(undefined); + }), + ); +}); diff --git a/packages/plugins/vectorize-search/src/sdk/store-service.ts b/packages/plugins/vectorize-search/src/sdk/store-service.ts new file mode 100644 index 0000000000..89194b2791 --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/store-service.ts @@ -0,0 +1,21 @@ +import { Context, Layer } from "effect"; + +import { makeZVecStore, type ZVecStoreOptions } from "./store-zvec"; +import { makeVectorizeStore, type VectorizeIndex, type VectorizeStore } from "./vectorize"; + +// --------------------------------------------------------------------------- +// Pluggable vector-store seam. The indexer + query provider depend on +// `VectorStoreService`; the host (or eval) supplies ONE of the layers β€” a local +// zvec ANN index for development or the Cloudflare Vectorize binding for +// production. +// --------------------------------------------------------------------------- + +export class VectorStoreService extends Context.Service()( + "@executor-js/plugin-vectorize-search/VectorStoreService", +) {} + +export const vectorizeStoreLayer = (index: VectorizeIndex): Layer.Layer => + Layer.succeed(VectorStoreService)(makeVectorizeStore(index)); + +export const zvecStoreLayer = (options: ZVecStoreOptions): Layer.Layer => + Layer.succeed(VectorStoreService)(makeZVecStore(options)); diff --git a/packages/plugins/vectorize-search/src/sdk/store-zvec.ts b/packages/plugins/vectorize-search/src/sdk/store-zvec.ts new file mode 100644 index 0000000000..e2f781cb5b --- /dev/null +++ b/packages/plugins/vectorize-search/src/sdk/store-zvec.ts @@ -0,0 +1,174 @@ +import { Effect } from "effect"; + +import { VectorizeSearchError } from "./errors"; +import type { VectorizeMatch, VectorizeStore, VectorizeVectorInput } from "./vectorize"; + +// --------------------------------------------------------------------------- +// zvec-backed VectorizeStore β€” a local, in-process, file-backed ANN index +// (@zvec/zvec, HNSW + cosine). A real local backend for developing + bench- +// marking the search logic without Cloudflare; the same `VectorizeStore` shape +// the production Vectorize binding satisfies. +// +// Two zvec specifics handled here: +// - metadata is stored as one JSON `fields.metadataJson` (the seam carries +// arbitrary metadata; zvec needs a fixed string schema), plus a `namespace` +// field for post-filtering. +// - zvec `score` is cosine DISTANCE (lower = nearer); we convert to a +// similarity (`1 - distance`) so it matches Vectorize's higher-is-better. +// +// `@zvec/zvec` is a native addon loaded via dynamic import, so the plugin's +// Cloudflare build never pulls it in. +// --------------------------------------------------------------------------- + +export interface ZVecStoreOptions { + /** Collection directory on disk. */ + readonly path: string; + readonly dimensions: number; + readonly hnswM?: number; + readonly hnswEfConstruction?: number; + readonly hnswEf?: number; +} + +interface ZVecQueryRow { + readonly id: string; + readonly score: number; + readonly fields?: Record; +} + +interface ZVecCollection { + upsertSync(docs: readonly unknown[]): unknown; + querySync(input: unknown): readonly ZVecQueryRow[]; + deleteSync(ids: readonly string[]): unknown; +} + +interface OpenedCollection { + readonly coll: ZVecCollection; + readonly ef: number; + readonly indexType: unknown; +} + +const openCollection = async (options: ZVecStoreOptions): Promise => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- native addon, no shipped types + const zvec = (await import("@zvec/zvec")) as any; + const schema = new zvec.ZVecCollectionSchema({ + name: "vectors", + vectors: { + name: "embedding", + dataType: zvec.ZVecDataType.VECTOR_FP32, + dimension: options.dimensions, + indexParams: { + indexType: zvec.ZVecIndexType.HNSW, + metricType: zvec.ZVecMetricType.COSINE, + m: options.hnswM ?? 32, + efConstruction: options.hnswEfConstruction ?? 200, + }, + }, + fields: [ + { name: "namespace", dataType: zvec.ZVecDataType.STRING }, + { name: "metadataJson", dataType: zvec.ZVecDataType.STRING }, + ], + }); + let coll: ZVecCollection; + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: @zvec/zvec native addon throws synchronously on open/create + try { + coll = zvec.ZVecOpen(options.path) as ZVecCollection; + } catch { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: @zvec/zvec native addon throws synchronously on open/create + try { + coll = zvec.ZVecCreateAndOpen(options.path, schema) as ZVecCollection; + } catch { + coll = zvec.ZVecOpen(options.path) as ZVecCollection; + } + } + return { coll, ef: options.hnswEf ?? 128, indexType: zvec.ZVecIndexType.HNSW }; +}; + +const toMatches = ( + rows: readonly ZVecQueryRow[], + namespace: string, + topK: number, +): readonly VectorizeMatch[] => { + const out: VectorizeMatch[] = []; + for (const row of rows) { + const ns = String(row.fields?.namespace ?? ""); + if (ns !== namespace) continue; + let metadata: Record | undefined; + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: decoding zvec's stored metadataJson string + try { + // oxlint-disable-next-line executor/no-json-parse -- adapter boundary: decoding zvec's stored metadataJson string + metadata = JSON.parse(String(row.fields?.metadataJson ?? "{}")) as Record; + } catch { + metadata = undefined; + } + out.push({ id: row.id, score: 1 - row.score, namespace: ns, metadata }); + if (out.length >= topK) break; + } + return out; +}; + +export const makeZVecStore = (options: ZVecStoreOptions): VectorizeStore => { + let cached: Promise | null = null; + const getCollection = Effect.tryPromise({ + try: () => (cached ??= openCollection(options)), + catch: (cause) => + new VectorizeSearchError({ message: `zvec open/create failed at ${options.path}.`, cause }), + }); + + return { + upsert: (vectors: readonly VectorizeVectorInput[]) => + vectors.length === 0 + ? Effect.void + : getCollection.pipe( + Effect.flatMap((opened) => + Effect.try({ + try: () => + opened.coll.upsertSync( + vectors.map((v) => ({ + id: v.id, + vectors: { embedding: [...v.values] }, + fields: { + namespace: v.namespace ?? "", + metadataJson: JSON.stringify(v.metadata ?? {}), + }, + })), + ), + catch: (cause) => + new VectorizeSearchError({ message: "zvec upsert failed.", cause }), + }), + ), + Effect.asVoid, + ), + + query: ({ vector, namespace, topK }) => + getCollection.pipe( + Effect.flatMap((opened) => + Effect.try({ + try: () => + opened.coll.querySync({ + fieldName: "embedding", + vector: [...vector], + // Over-fetch so the namespace post-filter still yields topK. + topk: Math.max(topK * 4, 40), + outputFields: ["namespace", "metadataJson"], + params: { indexType: opened.indexType, ef: opened.ef }, + }), + catch: (cause) => new VectorizeSearchError({ message: "zvec query failed.", cause }), + }).pipe(Effect.map((rows) => toMatches(rows, namespace, topK))), + ), + ), + + deleteByIds: (ids: readonly string[]) => + ids.length === 0 + ? Effect.void + : getCollection.pipe( + Effect.flatMap((opened) => + Effect.try({ + try: () => opened.coll.deleteSync([...ids]), + catch: (cause) => + new VectorizeSearchError({ message: "zvec delete failed.", cause }), + }), + ), + Effect.asVoid, + ), + }; +}; From 6cb665add4f5e510b4475b458982e6bffc3b6f62 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Tue, 16 Jun 2026 17:48:51 +0530 Subject: [PATCH 32/57] refactor: drop the underscore prefix from unused parameters (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Drops the `_`-prefix convention for unused function parameters. They now keep their real, descriptive names. - **Lint:** `no-unused-vars` is configured with `{ "args": "none", "caughtErrors": "none", "varsIgnorePattern": "^_" }`. Unused params + catch bindings are no longer flagged (no `_` needed); unused **variables** and **imports** are still caught. - **Sweep:** 37 unused params renamed across 23 files (`_args` β†’ `args`, `_event` β†’ `event`, `_cause` β†’ `cause`, …). Intentional `_`-names that are actually used (`_db`, `_open`), destructuring discards, codegen string-literals, and vendor code were left untouched. Pure rename β€” no behavior change. ## One thing to weigh in on Destructuring discards (`const { type: _type, ...rest } = obj`) **keep** the `_` via `varsIgnorePattern: "^_"`, because they structurally need a name to exclude a field while spreading the rest. If you'd rather those also drop the underscore (and find another way to omit the field), say so and I'll follow up. ## Verification - `bunx oxlint -c .oxlintrc.jsonc --deny-warnings packages apps` β†’ 0 warnings, 0 errors - `bunx turbo run typecheck` β†’ 43/43 pass --- .oxlintrc.jsonc | 10 ++++++++++ apps/cloud/src/api/protected-api-key-auth.node.test.ts | 2 +- apps/cloud/src/auth/api-keys.node.test.ts | 2 +- apps/cloud/src/auth/org-selector-auth.node.test.ts | 2 +- apps/cloud/src/org/handlers.test.ts | 4 ++-- apps/desktop/src/main/diagnostics.ts | 4 ++-- apps/desktop/src/main/index.ts | 6 +++--- apps/host-selfhost/src/auth/better-auth.ts | 2 +- packages/core/execution/src/engine-observer.test.ts | 2 +- packages/core/fumadb/src/query/aggregate.test.ts | 2 +- packages/core/sdk/src/core-tools.ts | 8 ++++---- packages/core/test-servers/src/worker.ts | 4 ++-- .../hosts/cloudflare/src/mcp/session-durable-object.ts | 2 +- packages/hosts/mcp/src/tool-server.test.ts | 10 +++++----- packages/plugins/desktop-settings/src/client.tsx | 2 +- packages/plugins/graphql/src/testing/index.ts | 4 ++-- packages/plugins/openapi/src/sdk/parse.ts | 4 ++-- .../plugins/openapi/src/sdk/upstream-failures.test.ts | 4 ++-- packages/react/src/api/analytics.test.ts | 2 +- packages/react/src/api/oauth-popup.test.ts | 2 +- .../react/src/components/add-account-modal.test.ts | 2 +- .../react/src/components/auth-method-list-editor.tsx | 2 +- packages/react/src/components/json-schema-form.tsx | 2 +- packages/react/src/components/placement-editor.tsx | 2 +- 24 files changed, 48 insertions(+), 38 deletions(-) diff --git a/.oxlintrc.jsonc b/.oxlintrc.jsonc index a117a09ec9..3c6810a79e 100644 --- a/.oxlintrc.jsonc +++ b/.oxlintrc.jsonc @@ -3,6 +3,16 @@ "jsPlugins": [{ "name": "executor", "specifier": "./scripts/oxlint-plugin-executor.js" }], "rules": { "typescript/no-explicit-any": "error", + // Unused function parameters are allowed with their real names (no `_` + // prefix required) β€” keep a descriptive name for params retained for + // signature/interface reasons. Unused *variables* and imports are still + // flagged. Same for unused catch bindings. Destructuring-skip variables + // (e.g. `{ type: _type, ...rest }`) keep the `_` convention since they + // cannot be omitted structurally. + "no-unused-vars": [ + "error", + { "args": "none", "caughtErrors": "none", "varsIgnorePattern": "^_" }, + ], "executor/no-vitest-import": "error", "executor/no-conditional-tests": "error", "executor/no-double-cast": "error", diff --git a/apps/cloud/src/api/protected-api-key-auth.node.test.ts b/apps/cloud/src/api/protected-api-key-auth.node.test.ts index c13ae96ee8..86b3332e8d 100644 --- a/apps/cloud/src/api/protected-api-key-auth.node.test.ts +++ b/apps/cloud/src/api/protected-api-key-auth.node.test.ts @@ -27,7 +27,7 @@ const stubApiKeys = Layer.succeed(ApiKeyService)({ const stubWorkOS = Layer.succeed( WorkOSClient, new Proxy({} as WorkOSClientService, { - get: (_target, prop) => { + get: (target, prop) => { if (prop === "listUserMemberships") { return (userId: string) => Effect.succeed({ diff --git a/apps/cloud/src/auth/api-keys.node.test.ts b/apps/cloud/src/auth/api-keys.node.test.ts index 6208dae37d..be41eb5931 100644 --- a/apps/cloud/src/auth/api-keys.node.test.ts +++ b/apps/cloud/src/auth/api-keys.node.test.ts @@ -8,7 +8,7 @@ const stubWorkOS = (overrides: Partial) => Layer.succeed( WorkOSClient, new Proxy({} as WorkOSClientService, { - get: (_target, prop) => { + get: (target, prop) => { if (prop in overrides) return overrides[prop as keyof WorkOSClientService]; return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); }, diff --git a/apps/cloud/src/auth/org-selector-auth.node.test.ts b/apps/cloud/src/auth/org-selector-auth.node.test.ts index ef27f68f77..54eebb104b 100644 --- a/apps/cloud/src/auth/org-selector-auth.node.test.ts +++ b/apps/cloud/src/auth/org-selector-auth.node.test.ts @@ -32,7 +32,7 @@ const stubApiKeys = Layer.succeed(ApiKeyService)({ const stubWorkOS = Layer.succeed( WorkOSClient, new Proxy({} as WorkOSClientService, { - get: (_t, prop) => { + get: (target, prop) => { if (prop === "authenticateRequest") { return () => Effect.succeed({ userId: MEMBER, email: "u@e2e.test", organizationId: SESSION_ORG }); diff --git a/apps/cloud/src/org/handlers.test.ts b/apps/cloud/src/org/handlers.test.ts index fc2b08dccc..3e0272b5e5 100644 --- a/apps/cloud/src/org/handlers.test.ts +++ b/apps/cloud/src/org/handlers.test.ts @@ -32,7 +32,7 @@ const stubWorkOS = (overrides: StubOverrides = {}) => Layer.succeed( WorkOSClient, new Proxy({} as WorkOSClientService, { - get: (_target, prop) => { + get: (target, prop) => { if (typeof prop === "string" && prop in overrides) { return overrides[prop as keyof StubOverrides]; } @@ -80,7 +80,7 @@ const requireAdmin = Effect.gen(function* () { }); const withCurrentMembership: StubOverrides = { - getUserOrgMembership: (_organizationId: string, userId: string) => + getUserOrgMembership: (organizationId: string, userId: string) => Effect.succeed( userId === "user_admin" ? { id: "mem_admin", userId, status: "active", role: { slug: "admin" } } diff --git a/apps/desktop/src/main/diagnostics.ts b/apps/desktop/src/main/diagnostics.ts index ea02eb2b4a..ee685dec7c 100644 --- a/apps/desktop/src/main/diagnostics.ts +++ b/apps/desktop/src/main/diagnostics.ts @@ -99,10 +99,10 @@ export const initErrorReporting = () => { // Persist process-death signals to main.log regardless of Sentry β€” these // are the events a "the app just disappeared" report hinges on. Sentry's // ChildProcess integration reports them upstream; this keeps a local copy. - app.on("child-process-gone", (_event, details) => { + app.on("child-process-gone", (event, details) => { log.error("[crash] child process gone", details); }); - app.on("render-process-gone", (_event, webContents, details) => { + app.on("render-process-gone", (event, webContents, details) => { log.error("[crash] render process gone", { url: webContents.getURL(), ...details }); }); diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 1ab260cde5..735a081f6a 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -315,7 +315,7 @@ const registerIpcHandlers = () => { ipcMain.handle("executor:settings:get", (): DesktopServerSettings => getServerSettings()); ipcMain.handle( "executor:settings:update", - (_evt, patch: Partial): DesktopServerSettings => + (evt, patch: Partial): DesktopServerSettings => updateServerSettings(patch), ); ipcMain.handle( @@ -323,7 +323,7 @@ const registerIpcHandlers = () => { (): DesktopServerSettings => regeneratePassword(), ); ipcMain.handle("executor:server-profiles:get", (): string | null => getServerProfiles()); - ipcMain.handle("executor:server-profiles:set", (_evt, value: unknown): void => { + ipcMain.handle("executor:server-profiles:set", (evt, value: unknown): void => { if (typeof value !== "string") return; setServerProfiles(value); }); @@ -348,7 +348,7 @@ const registerIpcHandlers = () => { await announceBackup(backupDir); return true; }); - ipcMain.handle("executor:shell:open-external", async (_evt, rawUrl: unknown) => { + ipcMain.handle("executor:shell:open-external", async (evt, rawUrl: unknown) => { if (typeof rawUrl !== "string") return; // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: untrusted renderer string, URL ctor throws on malformed input try { diff --git a/apps/host-selfhost/src/auth/better-auth.ts b/apps/host-selfhost/src/auth/better-auth.ts index 07acc4df70..61eac77a5d 100644 --- a/apps/host-selfhost/src/auth/better-auth.ts +++ b/apps/host-selfhost/src/auth/better-auth.ts @@ -116,7 +116,7 @@ const makeAuthOptions = (url: string, getOrganizationId: () => string, gate?: Si ? { user: { create: { - before: async (_user, context) => { + before: async (user, context) => { if (context?.path !== SIGNUP_PATH) return; if (await orgHasNoMembers(gate)) return; // first user claims the org const code = inviteCodeFrom(context); diff --git a/packages/core/execution/src/engine-observer.test.ts b/packages/core/execution/src/engine-observer.test.ts index 4c46ab8667..3a869a68dd 100644 --- a/packages/core/execution/src/engine-observer.test.ts +++ b/packages/core/execution/src/engine-observer.test.ts @@ -19,7 +19,7 @@ const makeExecutor = () => createExecutor(makeTestConfig({ plugins: [emptyPlugin // A code executor that issues one builtin tool call (tools.search) and then // completes β€” enough to exercise the full event sequence. const toolCallingExecutor: CodeExecutor = { - execute: (_code, invoker) => + execute: (code, invoker) => invoker .invoke({ path: "search", args: { query: "anything" } }) .pipe(Effect.as({ result: "ok", logs: [] } satisfies ExecuteResult), Effect.orDie), diff --git a/packages/core/fumadb/src/query/aggregate.test.ts b/packages/core/fumadb/src/query/aggregate.test.ts index cfd6b9eb37..45abe53328 100644 --- a/packages/core/fumadb/src/query/aggregate.test.ts +++ b/packages/core/fumadb/src/query/aggregate.test.ts @@ -294,7 +294,7 @@ describe("json filter LIKE-wildcard parity (memory vs sqlite)", () => { ["eq", statusCompare("=", "a_b%c")], ["starts with", statusCompare("starts with", "a_")], ["ends with", statusCompare("ends with", "%c")], - ] as const)("matches identically for %s", async (_label, filter) => { + ] as const)("matches identically for %s", async (label, filter) => { const fromMemory = await idsFor(memory.orm, filter); const fromSqlite = await idsFor(sqlite.orm, filter); expect(fromSqlite).toEqual(fromMemory); diff --git a/packages/core/sdk/src/core-tools.ts b/packages/core/sdk/src/core-tools.ts index 921402031d..cd5fc3dbc1 100644 --- a/packages/core/sdk/src/core-tools.ts +++ b/packages/core/sdk/src/core-tools.ts @@ -400,7 +400,7 @@ export const coreToolsPlugin = definePlugin((options: CoreToolsPluginOptions = { description: "List integrations in the workspace catalog (slug, description, owning plugin kind). Connections authenticate against these.", outputSchema: IntegrationsListOutputStd, - execute: (_args, { ctx }) => + execute: (args, { ctx }) => Effect.map(ctx.core.integrations.list(), (integrations) => ({ integrations: integrations.map((i) => ({ slug: String(i.slug), @@ -507,7 +507,7 @@ export const coreToolsPlugin = definePlugin((options: CoreToolsPluginOptions = { description: "List registered credential provider keys (the storage backends, not API vendors). Use `providers.items` to browse a backend's entries.", outputSchema: ProvidersOutputStd, - execute: (_args, { ctx }) => + execute: (args, { ctx }) => Effect.map(ctx.providers.list(), (providers) => ({ providers: providers.map((p) => String(p)), })), @@ -529,7 +529,7 @@ export const coreToolsPlugin = definePlugin((options: CoreToolsPluginOptions = { description: "List registered OAuth clients visible to this executor. Returns metadata only; client secrets are never returned.", outputSchema: OAuthClientsListOutputStd, - execute: (_args, { ctx }) => + execute: (args, { ctx }) => Effect.map(ctx.oauth.listClients(), (clients) => ({ clients: clients.map((client) => ({ owner: client.owner, @@ -663,7 +663,7 @@ export const coreToolsPlugin = definePlugin((options: CoreToolsPluginOptions = { description: "List tool policies (approve / require_approval / block) for org and user owners, in evaluation order.", outputSchema: PoliciesListOutputStd, - execute: (_args, { ctx }) => + execute: (args, { ctx }) => Effect.map(ctx.core.policies.list(), (policies) => ({ policies: policies.map((p) => ({ id: String(p.id), diff --git a/packages/core/test-servers/src/worker.ts b/packages/core/test-servers/src/worker.ts index 3158b84d37..1427f39046 100644 --- a/packages/core/test-servers/src/worker.ts +++ b/packages/core/test-servers/src/worker.ts @@ -301,11 +301,11 @@ const yoga = createYoga({ `, resolvers: { Query: { - hello: (_source: unknown, args: { readonly name?: string }) => + hello: (source: unknown, args: { readonly name?: string }) => `Hello ${args.name ?? "world"}`, }, Mutation: { - setGreeting: (_source: unknown, args: { readonly message: string }) => args.message, + setGreeting: (source: unknown, args: { readonly message: string }) => args.message, }, }, }), diff --git a/packages/hosts/cloudflare/src/mcp/session-durable-object.ts b/packages/hosts/cloudflare/src/mcp/session-durable-object.ts index 6404a09441..80972e69ca 100644 --- a/packages/hosts/cloudflare/src/mcp/session-durable-object.ts +++ b/packages/hosts/cloudflare/src/mcp/session-durable-object.ts @@ -228,7 +228,7 @@ export abstract class McpSessionDOBase< } /** Optional error seam: report a fatal request cause (cloud β†’ Sentry). */ - protected captureCause(_cause: Cause.Cause): void {} + protected captureCause(cause: Cause.Cause): void {} /** Optional flush seam: force-export buffered spans before the DO RPC * settles. Default is a no-op; cloud overrides it with the tracer diff --git a/packages/hosts/mcp/src/tool-server.test.ts b/packages/hosts/mcp/src/tool-server.test.ts index 7d2edaf0c3..8f7048b9df 100644 --- a/packages/hosts/mcp/src/tool-server.test.ts +++ b/packages/hosts/mcp/src/tool-server.test.ts @@ -93,7 +93,7 @@ const makeElicitingEngine = ( ) => r.action, ): ExecutionEngine => makeStubEngine({ - execute: (_code, { onElicitation }) => + execute: (code, { onElicitation }) => Effect.gen(function* () { const response = yield* onElicitation({ address: STUB_TOOL_ADDRESS, @@ -619,7 +619,7 @@ describe("MCP host server β€” client without elicitation (pause/resume)", () => it("resume tool passes parsed content to engine", async () => { let receivedContent: Record | undefined; const engine = makeStubEngine({ - resume: (_id, response) => + resume: (id, response) => Effect.sync(() => { receivedContent = response.content; return { status: "completed", result: { result: "ok" } }; @@ -647,7 +647,7 @@ describe("MCP host server β€” client without elicitation (pause/resume)", () => it("resume with empty content passes undefined", async () => { let receivedContent: Record | undefined = { marker: true }; const engine = makeStubEngine({ - resume: (_id, response) => + resume: (id, response) => Effect.sync(() => { receivedContent = response.content; return { status: "completed", result: { result: "ok" } }; @@ -766,7 +766,7 @@ describe("MCP host server β€” resume content parsing", () => { const makeResumeEngine = () => { let receivedContent: Record | undefined = { marker: true }; const engine = makeStubEngine({ - resume: (_id, response) => + resume: (id, response) => Effect.sync(() => { receivedContent = response.content; return { status: "completed", result: { result: "ok" } }; @@ -822,7 +822,7 @@ describe("MCP host server β€” resume content parsing", () => { describe("MCP host server β€” multiple elicitations", () => { it("engine can elicit multiple times during a single execute call", async () => { const engine = makeStubEngine({ - execute: (_code, { onElicitation }) => + execute: (code, { onElicitation }) => Effect.gen(function* () { const r1 = yield* onElicitation({ address: STUB_TOOL_ADDRESS, diff --git a/packages/plugins/desktop-settings/src/client.tsx b/packages/plugins/desktop-settings/src/client.tsx index 05607f1450..b6e59e4636 100644 --- a/packages/plugins/desktop-settings/src/client.tsx +++ b/packages/plugins/desktop-settings/src/client.tsx @@ -74,7 +74,7 @@ const inDesktop = readBridge() !== null; // The renderer doesn't get typed errors back from Electron's invoke channel. // We don't pull `err.message` out β€” the structured error doesn't help the // user, only "save failed" matters. The main process logs the full error. -const describeIpcError = (_err: unknown): string => +const describeIpcError = (err: unknown): string => "Save failed β€” check the desktop console for details."; // --------------------------------------------------------------------------- diff --git a/packages/plugins/graphql/src/testing/index.ts b/packages/plugins/graphql/src/testing/index.ts index 8e972acb16..d43aef5f40 100644 --- a/packages/plugins/graphql/src/testing/index.ts +++ b/packages/plugins/graphql/src/testing/index.ts @@ -236,13 +236,13 @@ export const makeGreetingGraphqlSchema = ( `, resolvers: { Query: { - hello: (_source: unknown, args: Readonly>) => + hello: (source: unknown, args: Readonly>) => `Hello ${stringArgument(args, "name", "world")}`, }, ...(includeMutation ? { Mutation: { - setGreeting: (_source: unknown, args: Readonly>) => + setGreeting: (source: unknown, args: Readonly>) => stringArgument(args, "message", ""), }, } diff --git a/packages/plugins/openapi/src/sdk/parse.ts b/packages/plugins/openapi/src/sdk/parse.ts index 1eb621dc49..e3d85ba07d 100644 --- a/packages/plugins/openapi/src/sdk/parse.ts +++ b/packages/plugins/openapi/src/sdk/parse.ts @@ -39,7 +39,7 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( const response = yield* client.execute(request).pipe( Effect.timeout(Duration.seconds(60)), Effect.mapError( - (_cause) => + (cause) => new OpenApiParseError({ message: "Failed to fetch OpenAPI document", }), @@ -52,7 +52,7 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( } return yield* response.text.pipe( Effect.mapError( - (_cause) => + (cause) => new OpenApiParseError({ message: "Failed to read OpenAPI document body", }), diff --git a/packages/plugins/openapi/src/sdk/upstream-failures.test.ts b/packages/plugins/openapi/src/sdk/upstream-failures.test.ts index ba23894239..9fb4deaca4 100644 --- a/packages/plugins/openapi/src/sdk/upstream-failures.test.ts +++ b/packages/plugins/openapi/src/sdk/upstream-failures.test.ts @@ -84,7 +84,7 @@ const startScriptedServer = (script: ResponseScript) => const startDroppingServer = () => Effect.acquireRelease( Effect.callback<{ baseUrl: string; close: () => void }>((resume) => { - const server = createServer((_req, res) => { + const server = createServer((req, res) => { res.writeHead(200, { "content-type": "application/json" }); res.write("partial"); res.destroy(); @@ -272,7 +272,7 @@ describe("OpenAPI upstream failure modes", () => { Effect.gen(function* () { const slowServer = Effect.acquireRelease( Effect.callback<{ baseUrl: string; close: () => void }>((resume) => { - const server = createServer((_req, res) => { + const server = createServer((req, res) => { setTimeout(() => { res.writeHead(200, { "content-type": "application/json" }); res.end("[]"); diff --git a/packages/react/src/api/analytics.test.ts b/packages/react/src/api/analytics.test.ts index 97809adf88..b2ed2986d3 100644 --- a/packages/react/src/api/analytics.test.ts +++ b/packages/react/src/api/analytics.test.ts @@ -34,7 +34,7 @@ describe("analytics seam", () => { it("sends an empty properties object when none are given", () => { const seen: Array = []; - setAnalyticsClient((_name, properties) => { + setAnalyticsClient((name, properties) => { seen.push(properties); }); diff --git a/packages/react/src/api/oauth-popup.test.ts b/packages/react/src/api/oauth-popup.test.ts index d27ec973c1..4c03d27087 100644 --- a/packages/react/src/api/oauth-popup.test.ts +++ b/packages/react/src/api/oauth-popup.test.ts @@ -55,7 +55,7 @@ describe("openOAuthPopup", () => { location: { origin: "https://app.example" }, addEventListener: () => {}, removeEventListener: () => {}, - open: (url: string, _name: string, requestedFeatures: string) => { + open: (url: string, name: string, requestedFeatures: string) => { opened = url; features = requestedFeatures; return popup; diff --git a/packages/react/src/components/add-account-modal.test.ts b/packages/react/src/components/add-account-modal.test.ts index 64ffd5290a..91cb02f53e 100644 --- a/packages/react/src/components/add-account-modal.test.ts +++ b/packages/react/src/components/add-account-modal.test.ts @@ -199,7 +199,7 @@ describe("runDcrConnect", () => { let registerArgs: RegisterArgs | null = null; let startArgs: StartArgs | null = null; - const probe = (_url: string): Promise => { + const probe = (url: string): Promise => { calls.push("probe"); return Promise.resolve({ authorizationUrl: "https://auth.example.com/authorize", diff --git a/packages/react/src/components/auth-method-list-editor.tsx b/packages/react/src/components/auth-method-list-editor.tsx index 49b6576292..42d6db6fcd 100644 --- a/packages/react/src/components/auth-method-list-editor.tsx +++ b/packages/react/src/components/auth-method-list-editor.tsx @@ -74,7 +74,7 @@ export function useAuthMethodList(seeds: readonly AuthMethodSeed[]): AuthMethodL const removeRowAt = useCallback((index: number) => { setRows((current: readonly AuthMethodRow[]) => - current.filter((_row: AuthMethodRow, i: number) => i !== index), + current.filter((row: AuthMethodRow, i: number) => i !== index), ); }, []); diff --git a/packages/react/src/components/json-schema-form.tsx b/packages/react/src/components/json-schema-form.tsx index de26787a6f..3bdd5a0c1a 100644 --- a/packages/react/src/components/json-schema-form.tsx +++ b/packages/react/src/components/json-schema-form.tsx @@ -612,7 +612,7 @@ function ArrayField(props: { const setItem = (index: number, next: unknown): void => { if (next === REMOVE) { - onChange(items.filter((_item: unknown, i: number) => i !== index)); + onChange(items.filter((item: unknown, i: number) => i !== index)); return; } onChange(items.map((item: unknown, i: number) => (i === index ? next : item))); diff --git a/packages/react/src/components/placement-editor.tsx b/packages/react/src/components/placement-editor.tsx index 50cbcae475..07b36143aa 100644 --- a/packages/react/src/components/placement-editor.tsx +++ b/packages/react/src/components/placement-editor.tsx @@ -45,7 +45,7 @@ export function PlacementEditor(props: { onChange(placements.map((p: Placement, j: number) => (j === index ? { ...p, ...patch } : p))); const remove = (index: number): void => - onChange(placements.filter((_p: Placement, j: number) => j !== index)); + onChange(placements.filter((p: Placement, j: number) => j !== index)); return (
From 0a8fd7dd82dc0f585fea2ddf5fd1600e3196be9d Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Wed, 17 Jun 2026 02:12:18 +0530 Subject: [PATCH 33/57] refactor(plugin-semantic-search): backend-agnostic rename + decoupled store + real-catalog eval (#30) Rename plugin-vectorize-search -> plugin-semantic-search and make it backend-agnostic: pluggable VectorStore (Vectorize, zvec, sqlite-vec) and FTS5 lexical store behind a shared namespace filter, transactional upserts, lazy native deps, and a real-catalog eval harness with a sample-catalog golden set for offline smoke. --- apps/host-cloudflare/package.json | 2 +- apps/host-cloudflare/src/config.ts | 6 +- apps/host-cloudflare/src/plugins.ts | 18 +- bun.lock | 75 ++-- .../CHANGELOG.md | 0 .../package.json | 7 +- .../src/api/group.ts | 10 +- .../src/api/handlers.ts | 26 +- .../plugins/semantic-search/src/api/index.ts | 20 + .../src/sdk/chunker-service.ts | 2 +- .../src/sdk/chunker.test.ts | 0 .../src/sdk/chunker.ts | 6 + .../src/sdk/collections.ts | 0 .../semantic-search/src/sdk/documents.ts | 129 ++++++ .../src/sdk/embedder-hash.ts | 0 .../src/sdk/embedder-openai.ts | 12 +- .../src/sdk/embedder.ts | 10 +- .../src/sdk/embedding-service.ts | 2 +- .../src/sdk/errors.ts | 4 +- .../semantic-search/src/sdk/eval-golden.ts | 302 +++++++++++++ .../src/sdk/eval.ts | 302 ++++++++++--- .../src/sdk/fingerprint.test.ts | 0 .../src/sdk/fingerprint.ts | 0 .../src/sdk/harness.test.ts | 0 .../src/sdk/hybrid.test.ts | 0 .../src/sdk/hybrid.ts | 0 .../src/sdk/index.ts | 40 +- .../src/sdk/indexer.test.ts | 9 +- .../src/sdk/indexer.ts | 20 +- .../src/sdk/plugin.ts | 68 ++- .../src/sdk/provider.ts | 33 +- .../src/sdk/search.test.ts | 37 +- .../src/sdk/store-cloudflare-limits.test.ts | 13 +- .../src/sdk/store-cloudflare-limits.ts | 19 +- .../src/sdk/store-cloudflare.ts} | 59 +-- .../src/sdk/store-fts-provider.test.ts | 65 +++ .../semantic-search/src/sdk/store-fts.ts | 413 ++++++++++++++++++ .../src/sdk/store-service.ts | 13 +- .../src/sdk/store-sqlite-vec.ts | 268 ++++++++++++ .../src/sdk/store-zvec.ts | 65 ++- .../plugins/semantic-search/src/sdk/store.ts | 35 ++ .../tsconfig.json | 0 .../tsup.config.ts | 0 .../vitest.config.ts | 0 .../plugins/vectorize-search/src/api/index.ts | 20 - .../vectorize-search/src/sdk/documents.ts | 69 --- .../vectorize-search/src/sdk/eval-golden.ts | 102 ----- 47 files changed, 1780 insertions(+), 501 deletions(-) rename packages/plugins/{vectorize-search => semantic-search}/CHANGELOG.md (100%) rename packages/plugins/{vectorize-search => semantic-search}/package.json (84%) rename packages/plugins/{vectorize-search => semantic-search}/src/api/group.ts (72%) rename packages/plugins/{vectorize-search => semantic-search}/src/api/handlers.ts (55%) create mode 100644 packages/plugins/semantic-search/src/api/index.ts rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/chunker-service.ts (94%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/chunker.test.ts (100%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/chunker.ts (97%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/collections.ts (100%) create mode 100644 packages/plugins/semantic-search/src/sdk/documents.ts rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/embedder-hash.ts (100%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/embedder-openai.ts (92%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/embedder.ts (90%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/embedding-service.ts (95%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/errors.ts (63%) create mode 100644 packages/plugins/semantic-search/src/sdk/eval-golden.ts rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/eval.ts (60%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/fingerprint.test.ts (100%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/fingerprint.ts (100%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/harness.test.ts (100%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/hybrid.test.ts (100%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/hybrid.ts (100%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/index.ts (50%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/indexer.test.ts (98%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/indexer.ts (94%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/plugin.ts (65%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/provider.ts (79%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/search.test.ts (86%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/store-cloudflare-limits.test.ts (93%) rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/store-cloudflare-limits.ts (81%) rename packages/plugins/{vectorize-search/src/sdk/vectorize.ts => semantic-search/src/sdk/store-cloudflare.ts} (57%) create mode 100644 packages/plugins/semantic-search/src/sdk/store-fts-provider.test.ts create mode 100644 packages/plugins/semantic-search/src/sdk/store-fts.ts rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/store-service.ts (65%) create mode 100644 packages/plugins/semantic-search/src/sdk/store-sqlite-vec.ts rename packages/plugins/{vectorize-search => semantic-search}/src/sdk/store-zvec.ts (74%) create mode 100644 packages/plugins/semantic-search/src/sdk/store.ts rename packages/plugins/{vectorize-search => semantic-search}/tsconfig.json (100%) rename packages/plugins/{vectorize-search => semantic-search}/tsup.config.ts (100%) rename packages/plugins/{vectorize-search => semantic-search}/vitest.config.ts (100%) delete mode 100644 packages/plugins/vectorize-search/src/api/index.ts delete mode 100644 packages/plugins/vectorize-search/src/sdk/documents.ts delete mode 100644 packages/plugins/vectorize-search/src/sdk/eval-golden.ts diff --git a/apps/host-cloudflare/package.json b/apps/host-cloudflare/package.json index 983f3d96bf..9f63d3d72d 100644 --- a/apps/host-cloudflare/package.json +++ b/apps/host-cloudflare/package.json @@ -28,8 +28,8 @@ "@executor-js/plugin-graphql": "workspace:*", "@executor-js/plugin-mcp": "workspace:*", "@executor-js/plugin-openapi": "workspace:*", + "@executor-js/plugin-semantic-search": "workspace:*", "@executor-js/plugin-service-tokens": "workspace:*", - "@executor-js/plugin-vectorize-search": "workspace:*", "@executor-js/react": "workspace:*", "@executor-js/runtime-quickjs": "workspace:*", "@executor-js/sdk": "workspace:*", diff --git a/apps/host-cloudflare/src/config.ts b/apps/host-cloudflare/src/config.ts index e718525df3..7c01d79afb 100644 --- a/apps/host-cloudflare/src/config.ts +++ b/apps/host-cloudflare/src/config.ts @@ -1,6 +1,6 @@ import type { D1Database, DurableObjectNamespace, R2Bucket } from "@cloudflare/workers-types"; import type { AnalyticsEngineDataset } from "@executor-js/plugin-execution-metrics/cloudflare"; -import type { VectorizeIndex } from "@executor-js/plugin-vectorize-search"; +import type { VectorizeIndex } from "@executor-js/plugin-semantic-search"; import { isValidOrgSlug } from "@executor-js/api"; @@ -25,12 +25,12 @@ export interface CloudflareEnv { * finished execution/tool call writes a data point; absent, metrics are off. */ readonly ANALYTICS?: AnalyticsEngineDataset; /** Vectorize index binding β€” opt-in semantic `tools.search`. When bound (add a - * `vectorize` binding in wrangler.jsonc) the vectorize-search plugin embeds + * `vectorize` binding in wrangler.jsonc) the semantic-search plugin embeds * the tool catalog and answers `tools.search` from it; absent, the engine * keeps its built-in lexical search. */ readonly VECTORIZE?: VectorizeIndex; /** Gemini API key (a `wrangler secret`) powering the embeddings for the - * Vectorize search. Absent β†’ vectorize search stays inert even if the index + * Vectorize search. Absent β†’ semantic search stays inert even if the index * is bound. */ readonly GEMINI_API_KEY?: string; /** MCP session Durable Object namespace β€” one addressable isolate per MCP diff --git a/apps/host-cloudflare/src/plugins.ts b/apps/host-cloudflare/src/plugins.ts index d1055d8f7e..98680c37b9 100644 --- a/apps/host-cloudflare/src/plugins.ts +++ b/apps/host-cloudflare/src/plugins.ts @@ -9,8 +9,12 @@ import { } from "@executor-js/plugin-execution-metrics/cloudflare"; import { noopExecutionObserver } from "@executor-js/sdk"; import { serviceTokensPlugin } from "@executor-js/plugin-service-tokens/server"; -import { vectorizeSearchHttpPlugin } from "@executor-js/plugin-vectorize-search/api"; -import type { VectorizeIndex } from "@executor-js/plugin-vectorize-search"; +import { semanticSearchHttpPlugin } from "@executor-js/plugin-semantic-search/api"; +import { + makeVectorizeStore, + withCloudflareLimits, + type VectorizeIndex, +} from "@executor-js/plugin-semantic-search"; // --------------------------------------------------------------------------- // The Cloudflare host's plugin list β€” the same protocol/provider plugins as @@ -30,7 +34,7 @@ import type { VectorizeIndex } from "@executor-js/plugin-vectorize-search"; // is the durable sink. To enable: uncomment `analytics_engine_datasets` in // wrangler.jsonc. // -// Vectorize search follows the same opt-in-by-binding shape: the plugin is +// Semantic search follows the same opt-in-by-binding shape: the plugin is // always in the tuple (its reindex route keeps the API shape stable), but it is // inert β€” the engine keeps its lexical `tools.search` β€” until BOTH a `vectorize` // binding and the `GEMINI_API_KEY` secret are present. To enable: create a @@ -43,8 +47,9 @@ export const makeCloudflarePlugins = ( vectorize?: VectorizeIndex, geminiApiKey?: string, searchNamespace?: string, -) => - [ +) => { + const store = vectorize ? withCloudflareLimits(makeVectorizeStore(vectorize)) : undefined; + return [ openApiHttpPlugin(), mcpHttpPlugin({ dangerouslyAllowStdioMCP: false }), graphqlHttpPlugin(), @@ -53,7 +58,8 @@ export const makeCloudflarePlugins = ( observer: () => (analytics ? createWaeMetricsObserver(analytics) : noopExecutionObserver), }), serviceTokensPlugin(), - vectorizeSearchHttpPlugin({ vectorize, geminiApiKey, namespace: searchNamespace }), + semanticSearchHttpPlugin({ store, geminiApiKey, namespace: searchNamespace }), ] as const; +}; export type CloudflarePlugins = ReturnType; diff --git a/bun.lock b/bun.lock index 3d9309b2bc..1b93b3a471 100644 --- a/bun.lock +++ b/bun.lock @@ -171,8 +171,8 @@ "@executor-js/plugin-graphql": "workspace:*", "@executor-js/plugin-mcp": "workspace:*", "@executor-js/plugin-openapi": "workspace:*", + "@executor-js/plugin-semantic-search": "workspace:*", "@executor-js/plugin-service-tokens": "workspace:*", - "@executor-js/plugin-vectorize-search": "workspace:*", "@executor-js/react": "workspace:*", "@executor-js/runtime-quickjs": "workspace:*", "@executor-js/sdk": "workspace:*", @@ -1081,63 +1081,68 @@ "react", ], }, - "packages/plugins/service-tokens": { - "name": "@executor-js/plugin-service-tokens", - "version": "1.5.3", + "packages/plugins/semantic-search": { + "name": "@executor-js/plugin-semantic-search", + "version": "0.0.0", "dependencies": { - "@effect/atom-react": "catalog:", + "@ai-sdk/google": "^3.0.80", "@executor-js/sdk": "workspace:*", + "ai": "^6.0.191", + "effect": "catalog:", }, "devDependencies": { + "@cloudflare/workers-types": "^4.20250620.0", "@effect/vitest": "catalog:", "@executor-js/api": "workspace:*", - "@executor-js/react": "workspace:*", + "@types/better-sqlite3": "^7.6.13", "@types/node": "catalog:", - "@types/react": "catalog:", + "@zvec/zvec": "^0.4.1", "bun-types": "catalog:", - "effect": "catalog:", - "react": "catalog:", "tsup": "catalog:", - "typescript": "catalog:", "vitest": "catalog:", }, + "optionalDependencies": { + "better-sqlite3": "^11.10.0", + "sqlite-vec": "^0.1.7", + }, "peerDependencies": { - "@effect/atom-react": "catalog:", "@executor-js/api": "workspace:*", - "@executor-js/react": "workspace:*", - "effect": "catalog:", - "react": "catalog:", }, "optionalPeers": [ - "@effect/atom-react", - "@executor-js/react", - "react", + "@executor-js/api", ], }, - "packages/plugins/vectorize-search": { - "name": "@executor-js/plugin-vectorize-search", - "version": "0.0.0", + "packages/plugins/service-tokens": { + "name": "@executor-js/plugin-service-tokens", + "version": "1.5.3", "dependencies": { - "@ai-sdk/google": "^3.0.80", + "@effect/atom-react": "catalog:", "@executor-js/sdk": "workspace:*", - "ai": "^6.0.191", - "effect": "catalog:", }, "devDependencies": { - "@cloudflare/workers-types": "^4.20250620.0", "@effect/vitest": "catalog:", "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", "@types/node": "catalog:", - "@zvec/zvec": "^0.4.1", + "@types/react": "catalog:", "bun-types": "catalog:", + "effect": "catalog:", + "react": "catalog:", "tsup": "catalog:", + "typescript": "catalog:", "vitest": "catalog:", }, "peerDependencies": { + "@effect/atom-react": "catalog:", "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", + "effect": "catalog:", + "react": "catalog:", }, "optionalPeers": [ - "@executor-js/api", + "@effect/atom-react", + "@executor-js/react", + "react", ], }, "packages/plugins/workos-vault": { @@ -1741,9 +1746,9 @@ "@executor-js/plugin-openapi": ["@executor-js/plugin-openapi@workspace:packages/plugins/openapi"], - "@executor-js/plugin-service-tokens": ["@executor-js/plugin-service-tokens@workspace:packages/plugins/service-tokens"], + "@executor-js/plugin-semantic-search": ["@executor-js/plugin-semantic-search@workspace:packages/plugins/semantic-search"], - "@executor-js/plugin-vectorize-search": ["@executor-js/plugin-vectorize-search@workspace:packages/plugins/vectorize-search"], + "@executor-js/plugin-service-tokens": ["@executor-js/plugin-service-tokens@workspace:packages/plugins/service-tokens"], "@executor-js/plugin-workos-vault": ["@executor-js/plugin-workos-vault@workspace:packages/plugins/workos-vault"], @@ -5145,6 +5150,18 @@ "sql-highlight": ["sql-highlight@6.1.0", "", {}, "sha512-ed7OK4e9ywpE7pgRMkMQmZDPKSVdm0oX5IEtZiKnFucSF0zu6c80GZBe38UqHuVhTWJ9xsKgSMjCG2bml86KvA=="], + "sqlite-vec": ["sqlite-vec@0.1.9", "", { "optionalDependencies": { "sqlite-vec-darwin-arm64": "0.1.9", "sqlite-vec-darwin-x64": "0.1.9", "sqlite-vec-linux-arm64": "0.1.9", "sqlite-vec-linux-x64": "0.1.9", "sqlite-vec-windows-x64": "0.1.9" } }, "sha512-L7XJWRIBNvR9O5+vh1FQ+IGkh/3D2AzVksW5gdtk28m78Hy8skFD0pqReKH1Yp0/BUKRGcffgKvyO/EON5JXpA=="], + + "sqlite-vec-darwin-arm64": ["sqlite-vec-darwin-arm64@0.1.9", "", { "os": "darwin", "cpu": "arm64" }, "sha512-jSsZpE42OfBkGL/ItyJTVCUwl6o6Ka3U5rc4j+UBDIQzC1ulSSKMEhQLthsOnF/MdAf1MuAkYhkdKmmcjaIZQg=="], + + "sqlite-vec-darwin-x64": ["sqlite-vec-darwin-x64@0.1.9", "", { "os": "darwin", "cpu": "x64" }, "sha512-KDlVyqQT7pnOhU1ymB9gs7dMbSoVmKHitT+k1/xkjarcX8bBqPxWrGlK/R+C5WmWkfvWwyq5FfXfiBYCBs6PlA=="], + + "sqlite-vec-linux-arm64": ["sqlite-vec-linux-arm64@0.1.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-5wXVJ9c9kR4CHm/wVqXb/R+XUHTdpZ4nWbPHlS+gc9qQFVHs92Km4bPnCKX4rtcPMzvNis+SIzMJR1SCEwpuUw=="], + + "sqlite-vec-linux-x64": ["sqlite-vec-linux-x64@0.1.9", "", { "os": "linux", "cpu": "x64" }, "sha512-w3tCH8xK2finW8fQJ/m8uqKodXUZ9KAuAar2UIhz4BHILfpE0WM/MTGCRfa7RjYbrYim5Luk3guvMOGI7T7JQA=="], + + "sqlite-vec-windows-x64": ["sqlite-vec-windows-x64@0.1.9", "", { "os": "win32", "cpu": "x64" }, "sha512-y3gEIyy/17bq2QFPQOWLE68TYWcRZkBQVA2XLrTPHNTOp55xJi/BBBmOm40tVMDMjtP+Elpk6UBUXdaq+46b0Q=="], + "srvx": ["srvx@0.11.15", "", { "bin": { "srvx": "bin/srvx.mjs" } }, "sha512-iXsux0UcOjdvs0LCMa2Ws3WwcDUozA3JN3BquNXkaFPP7TpRqgunKdEgoZ/uwb1J6xaYHfxtz9Twlh6yzwM6Tg=="], "stack-utils": ["stack-utils@2.0.6", "", { "dependencies": { "escape-string-regexp": "^2.0.0" } }, "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ=="], @@ -5651,6 +5668,8 @@ "@executor-js/motel/@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-r86ut4T1e8vNwB35CqCcKd45yzqH6/6Wzvpk2/cZB8PsPLlZFTvrh8yfOS3CYZYcUmAx4hHTZJ8AO8Dj8nrdhw=="], + "@executor-js/plugin-semantic-search/better-sqlite3": ["better-sqlite3@11.10.0", "", { "dependencies": { "bindings": "^1.5.0", "prebuild-install": "^7.1.1" } }, "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ=="], + "@fastify/otel/@opentelemetry/core": ["@opentelemetry/core@2.8.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww=="], "@fastify/otel/@opentelemetry/instrumentation": ["@opentelemetry/instrumentation@0.212.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.212.0", "import-in-the-middle": "^2.0.6", "require-in-the-middle": "^8.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-IyXmpNnifNouMOe0I/gX7ENfv2ZCNdYTF0FpCsoBcpbIHzk81Ww9rQTYTnvghszCg7qGrIhNvWC8dhEifgX9Jg=="], diff --git a/packages/plugins/vectorize-search/CHANGELOG.md b/packages/plugins/semantic-search/CHANGELOG.md similarity index 100% rename from packages/plugins/vectorize-search/CHANGELOG.md rename to packages/plugins/semantic-search/CHANGELOG.md diff --git a/packages/plugins/vectorize-search/package.json b/packages/plugins/semantic-search/package.json similarity index 84% rename from packages/plugins/vectorize-search/package.json rename to packages/plugins/semantic-search/package.json index d73c34a9f4..b79982a7f8 100644 --- a/packages/plugins/vectorize-search/package.json +++ b/packages/plugins/semantic-search/package.json @@ -1,5 +1,5 @@ { - "name": "@executor-js/plugin-vectorize-search", + "name": "@executor-js/plugin-semantic-search", "version": "0.0.0", "private": true, "type": "module", @@ -24,6 +24,7 @@ "@cloudflare/workers-types": "^4.20250620.0", "@effect/vitest": "catalog:", "@executor-js/api": "workspace:*", + "@types/better-sqlite3": "^7.6.13", "@types/node": "catalog:", "@zvec/zvec": "^0.4.1", "bun-types": "catalog:", @@ -37,5 +38,9 @@ "@executor-js/api": { "optional": true } + }, + "optionalDependencies": { + "better-sqlite3": "^11.10.0", + "sqlite-vec": "^0.1.7" } } diff --git a/packages/plugins/vectorize-search/src/api/group.ts b/packages/plugins/semantic-search/src/api/group.ts similarity index 72% rename from packages/plugins/vectorize-search/src/api/group.ts rename to packages/plugins/semantic-search/src/api/group.ts index 97dd70cdd5..94cd359e30 100644 --- a/packages/plugins/vectorize-search/src/api/group.ts +++ b/packages/plugins/semantic-search/src/api/group.ts @@ -4,14 +4,14 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; import { InternalError } from "@executor-js/api"; // --------------------------------------------------------------------------- -// HTTP surface for the Vectorize search plugin: a single explicit reindex. +// HTTP surface for the Semantic search plugin: a single explicit reindex. // -// The route is flat and plugin-id-prefixed (`/vectorize-search/reindex`), +// The route is flat and plugin-id-prefixed (`/semantic-search/reindex`), // matching the execution-history/graphql convention β€” the per-request executor // is already owner-scoped at the host edge, so there is no `:scopeId` segment. // // Reindex takes no body: it reconciles the whole tool catalog for the scoped -// tenant against Vectorize (incremental fingerprint diff). `VectorizeSearchError` +// tenant against Vectorize (incremental fingerprint diff). `SemanticSearchError` // on the extension flows through the typed channel and `capture` downgrades it // to `InternalError`. // --------------------------------------------------------------------------- @@ -25,8 +25,8 @@ export const ReindexResponse = Schema.Struct({ removedSkipped: Schema.Number, }); -export const VectorizeSearchGroup = HttpApiGroup.make("vectorizeSearch").add( - HttpApiEndpoint.post("reindex", "/vectorize-search/reindex", { +export const SemanticSearchGroup = HttpApiGroup.make("semanticSearch").add( + HttpApiEndpoint.post("reindex", "/semantic-search/reindex", { success: ReindexResponse, error: InternalError, }), diff --git a/packages/plugins/vectorize-search/src/api/handlers.ts b/packages/plugins/semantic-search/src/api/handlers.ts similarity index 55% rename from packages/plugins/vectorize-search/src/api/handlers.ts rename to packages/plugins/semantic-search/src/api/handlers.ts index 1675809f2b..af8ab9d1d7 100644 --- a/packages/plugins/vectorize-search/src/api/handlers.ts +++ b/packages/plugins/semantic-search/src/api/handlers.ts @@ -4,35 +4,35 @@ import { HttpApiBuilder } from "effect/unstable/httpapi"; import { addGroup, captureEngineError } from "@executor-js/api"; import { ExecutorService } from "@executor-js/api/server"; -import type { VectorizeSearchExtension } from "../sdk/plugin"; -import { VectorizeSearchGroup } from "./group"; +import type { SemanticSearchExtension } from "../sdk/plugin"; +import { SemanticSearchGroup } from "./group"; // --------------------------------------------------------------------------- // Service tag // -// Holds the `executor.vectorizeSearch` surface. The host provides an +// Holds the `executor.semanticSearch` surface. The host provides an // already-bound extension via -// `Layer.succeed(VectorizeSearchExtensionService, executor.vectorizeSearch)`. +// `Layer.succeed(SemanticSearchExtensionService, executor.semanticSearch)`. // The handler also yields the per-request `ExecutorService` (the scoped // executor) and hands it to `reindex`, since the catalog lives on the executor, // not the plugin ctx. // --------------------------------------------------------------------------- -export class VectorizeSearchExtensionService extends Context.Service< - VectorizeSearchExtensionService, - VectorizeSearchExtension ->()("VectorizeSearchExtensionService") {} +export class SemanticSearchExtensionService extends Context.Service< + SemanticSearchExtensionService, + SemanticSearchExtension +>()("SemanticSearchExtensionService") {} -const ExecutorApiWithVectorizeSearch = addGroup(VectorizeSearchGroup); +const ExecutorApiWithSemanticSearch = addGroup(SemanticSearchGroup); -export const VectorizeSearchHandlers = HttpApiBuilder.group( - ExecutorApiWithVectorizeSearch, - "vectorizeSearch", +export const SemanticSearchHandlers = HttpApiBuilder.group( + ExecutorApiWithSemanticSearch, + "semanticSearch", (handlers) => handlers.handle("reindex", () => captureEngineError( Effect.gen(function* () { - const search = yield* VectorizeSearchExtensionService; + const search = yield* SemanticSearchExtensionService; const executor = yield* ExecutorService; return yield* search.reindex(executor); }), diff --git a/packages/plugins/semantic-search/src/api/index.ts b/packages/plugins/semantic-search/src/api/index.ts new file mode 100644 index 0000000000..f93a7eb14f --- /dev/null +++ b/packages/plugins/semantic-search/src/api/index.ts @@ -0,0 +1,20 @@ +import { definePlugin } from "@executor-js/sdk/core"; + +import { semanticSearchPlugin, type SemanticSearchPluginOptions } from "../sdk/plugin"; +import { SemanticSearchGroup } from "./group"; +import { SemanticSearchExtensionService, SemanticSearchHandlers } from "./handlers"; + +export { SemanticSearchGroup, ReindexResponse } from "./group"; +export { SemanticSearchHandlers, SemanticSearchExtensionService } from "./handlers"; + +// HTTP-augmented variant of `semanticSearchPlugin`. The returned plugin carries +// the reindex `routes`/`handlers`/`extensionService` so a host can mount the +// reindex endpoint. Hosts that compose an `HttpApi` import this; SDK-only +// consumers stay on `@executor-js/plugin-semantic-search` and never load +// `@executor-js/api`. +export const semanticSearchHttpPlugin = definePlugin((options?: SemanticSearchPluginOptions) => ({ + ...semanticSearchPlugin(options), + routes: () => SemanticSearchGroup, + handlers: () => SemanticSearchHandlers, + extensionService: SemanticSearchExtensionService, +})); diff --git a/packages/plugins/vectorize-search/src/sdk/chunker-service.ts b/packages/plugins/semantic-search/src/sdk/chunker-service.ts similarity index 94% rename from packages/plugins/vectorize-search/src/sdk/chunker-service.ts rename to packages/plugins/semantic-search/src/sdk/chunker-service.ts index 7854b3d39b..60ab7d90ee 100644 --- a/packages/plugins/vectorize-search/src/sdk/chunker-service.ts +++ b/packages/plugins/semantic-search/src/sdk/chunker-service.ts @@ -14,7 +14,7 @@ import { // --------------------------------------------------------------------------- export class ChunkerService extends Context.Service()( - "@executor-js/plugin-vectorize-search/ChunkerService", + "@executor-js/plugin-semantic-search/ChunkerService", ) {} export const facetChunkerLayer = (options?: FacetChunkerOptions): Layer.Layer => diff --git a/packages/plugins/vectorize-search/src/sdk/chunker.test.ts b/packages/plugins/semantic-search/src/sdk/chunker.test.ts similarity index 100% rename from packages/plugins/vectorize-search/src/sdk/chunker.test.ts rename to packages/plugins/semantic-search/src/sdk/chunker.test.ts diff --git a/packages/plugins/vectorize-search/src/sdk/chunker.ts b/packages/plugins/semantic-search/src/sdk/chunker.ts similarity index 97% rename from packages/plugins/vectorize-search/src/sdk/chunker.ts rename to packages/plugins/semantic-search/src/sdk/chunker.ts index 54978240ce..c99ea42dc4 100644 --- a/packages/plugins/vectorize-search/src/sdk/chunker.ts +++ b/packages/plugins/semantic-search/src/sdk/chunker.ts @@ -44,6 +44,12 @@ export interface ToolDocumentInput { readonly inputTypeScript?: string; readonly outputTypeScript?: string; readonly typeScriptDefinitions?: Record; + /** + * Broad single-string representation of the tool for FTS indexing. + * Populated by the projector (`buildLexicalText` in documents.ts); absent + * when the input comes from a source that does not populate it. + */ + readonly lexicalText?: string; } /** Which semantic facet a chunk represents. */ diff --git a/packages/plugins/vectorize-search/src/sdk/collections.ts b/packages/plugins/semantic-search/src/sdk/collections.ts similarity index 100% rename from packages/plugins/vectorize-search/src/sdk/collections.ts rename to packages/plugins/semantic-search/src/sdk/collections.ts diff --git a/packages/plugins/semantic-search/src/sdk/documents.ts b/packages/plugins/semantic-search/src/sdk/documents.ts new file mode 100644 index 0000000000..e1d7ed012c --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/documents.ts @@ -0,0 +1,129 @@ +import type { Executor } from "@executor-js/sdk/core"; +import { Effect } from "effect"; + +import type { ToolDocumentInput } from "./chunker"; +import { SemanticSearchError } from "./errors"; + +const ADDRESS_PREFIX = "tools."; + +// --------------------------------------------------------------------------- +// HTML-stripping helper +// --------------------------------------------------------------------------- + +/** + * Strip HTML tags from a string, decode common HTML entities, and collapse + * runs of whitespace to a single space. Stripe (and other integrations) + * sometimes embed raw `

…

` markup in tool descriptions; stripping them + * gives clean text for both embedding and FTS indexing. + */ +export const stripHtml = (s: string): string => + s + // Remove all HTML tags. + .replace(/<[^>]*>/g, " ") + // Decode the handful of entities that appear in practice. + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/ /g, " ") + // Collapse whitespace (including newlines introduced by the tag removal). + .replace(/\s+/g, " ") + .trim(); + +// --------------------------------------------------------------------------- +// Lexical-text builder +// --------------------------------------------------------------------------- + +/** + * Build a single broad text string for FTS indexing from a `ToolDocumentInput`. + * Concatenates the most searchable fields in rough importance order: + * integration Β· path Β· name Β· clean description Β· key identifiers from schemas + * + * FTS5 already weights fields via BM25; this is the fallback `lexical_text` + * column that carries identifiers and schema keywords not present in the + * structured columns. + */ +export const buildLexicalText = (doc: ToolDocumentInput): string => { + const parts: string[] = [doc.integration, doc.path, doc.name, stripHtml(doc.description)]; + + if (doc.inputTypeScript !== undefined) { + parts.push(doc.inputTypeScript); + } + if (doc.outputTypeScript !== undefined) { + parts.push(doc.outputTypeScript); + } + if (doc.typeScriptDefinitions !== undefined) { + for (const def of Object.values(doc.typeScriptDefinitions)) { + parts.push(def); + } + } + + return parts.join(" Β· "); +}; + +/** Strip the proxy-root `tools.` prefix so the address becomes the + * sandbox-callable path the model writes after `tools.` β€” mirrors the engine's + * own `addressToPath` so the `path` we index matches what `tools.search` + * callers expect (and can pass back to `describe`/invoke). */ +export const addressToPath = (address: string): string => + address.startsWith(ADDRESS_PREFIX) ? address.slice(ADDRESS_PREFIX.length) : address; + +/** Collect the full `ToolDocumentInput` set from the live catalog. + * + * For each tool returned by `tools.list`, we attempt to fetch its schema via + * `tools.schema(address)` to populate the TypeScript facets (inputTypeScript, + * outputTypeScript, typeScriptDefinitions). If the schema fetch fails for any + * individual tool we degrade gracefully to an identity-only document (no TS + * fields) β€” never failing the whole collection for one tool. + * + * Bounded concurrency (16) keeps the catalog walk fast while avoiding + * unbounded fan-out. */ +export const collectToolDocumentInputs = ( + namespace: string, + executor: Executor, +): Effect.Effect => + executor.tools.list({ includeAnnotations: false }).pipe( + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to list tools for indexing.", cause }), + ), + Effect.flatMap((tools) => + Effect.forEach( + tools, + (tool) => { + const path = addressToPath(String(tool.address)); + const base: ToolDocumentInput = { + path, + name: String(tool.name), + integration: String(tool.integration), + description: stripHtml(String(tool.description ?? "")), + }; + // Attempt schema fetch; on any failure degrade to identity-only. + return executor.tools.schema(tool.address).pipe( + Effect.map((view): ToolDocumentInput => { + const doc: ToolDocumentInput = + view === null + ? base + : { + ...base, + ...(view.inputTypeScript !== undefined + ? { inputTypeScript: view.inputTypeScript } + : {}), + ...(view.outputTypeScript !== undefined + ? { outputTypeScript: view.outputTypeScript } + : {}), + ...(view.typeScriptDefinitions !== undefined + ? { typeScriptDefinitions: view.typeScriptDefinitions } + : {}), + }; + return { ...doc, lexicalText: buildLexicalText(doc) }; + }), + // Degrade: schema fetch failed β€” use identity-only document. + Effect.catch(() => Effect.succeed({ ...base, lexicalText: buildLexicalText(base) })), + ); + }, + { concurrency: 16 }, + ), + ), + ); diff --git a/packages/plugins/vectorize-search/src/sdk/embedder-hash.ts b/packages/plugins/semantic-search/src/sdk/embedder-hash.ts similarity index 100% rename from packages/plugins/vectorize-search/src/sdk/embedder-hash.ts rename to packages/plugins/semantic-search/src/sdk/embedder-hash.ts diff --git a/packages/plugins/vectorize-search/src/sdk/embedder-openai.ts b/packages/plugins/semantic-search/src/sdk/embedder-openai.ts similarity index 92% rename from packages/plugins/vectorize-search/src/sdk/embedder-openai.ts rename to packages/plugins/semantic-search/src/sdk/embedder-openai.ts index f4f99aa71c..7472061171 100644 --- a/packages/plugins/vectorize-search/src/sdk/embedder-openai.ts +++ b/packages/plugins/semantic-search/src/sdk/embedder-openai.ts @@ -1,7 +1,7 @@ import { Effect } from "effect"; import type { ToolEmbedder } from "./embedder"; -import { VectorizeSearchError } from "./errors"; +import { SemanticSearchError } from "./errors"; // --------------------------------------------------------------------------- // OpenAI-compatible embedder β€” POSTs to `${baseUrl}/embeddings` in the standard @@ -43,7 +43,7 @@ export const makeOpenAiCompatibleEmbedder = ( const embedBatch = ( texts: readonly string[], - ): Effect.Effect => { + ): Effect.Effect => { if (texts.length === 0) return Effect.succeed([]); return Effect.gen(function* () { const response = yield* Effect.tryPromise({ @@ -62,7 +62,7 @@ export const makeOpenAiCompatibleEmbedder = ( }), }), catch: (cause) => - new VectorizeSearchError({ + new SemanticSearchError({ message: `OpenAI-compatible embedding failed for ${options.model} at ${endpoint}.`, cause, }), @@ -74,21 +74,21 @@ export const makeOpenAiCompatibleEmbedder = ( const body = yield* Effect.tryPromise(() => response.text()).pipe( Effect.catch(() => Effect.succeed("")), ); - return yield* new VectorizeSearchError({ + return yield* new SemanticSearchError({ message: `embeddings HTTP ${response.status}: ${body.slice(0, 300)}`, }); } const json = yield* Effect.tryPromise({ try: () => response.json() as Promise, catch: (cause) => - new VectorizeSearchError({ + new SemanticSearchError({ message: `OpenAI-compatible embedding failed for ${options.model} at ${endpoint}.`, cause, }), }); const embeddings = json.data?.map((item) => item.embedding ?? []); if (!embeddings || embeddings.length !== texts.length) { - return yield* new VectorizeSearchError({ + return yield* new SemanticSearchError({ message: `embeddings response returned ${embeddings?.length ?? 0} vectors for ${texts.length} inputs`, }); } diff --git a/packages/plugins/vectorize-search/src/sdk/embedder.ts b/packages/plugins/semantic-search/src/sdk/embedder.ts similarity index 90% rename from packages/plugins/vectorize-search/src/sdk/embedder.ts rename to packages/plugins/semantic-search/src/sdk/embedder.ts index e044ca2bd8..fb22ec6db5 100644 --- a/packages/plugins/vectorize-search/src/sdk/embedder.ts +++ b/packages/plugins/semantic-search/src/sdk/embedder.ts @@ -2,7 +2,7 @@ import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { embed, embedMany } from "ai"; import { Effect } from "effect"; -import { VectorizeSearchError } from "./errors"; +import { SemanticSearchError } from "./errors"; /** Turns tool documents and queries into embedding vectors. Document and query * embeddings use different task types (Gemini distinguishes @@ -14,8 +14,8 @@ export interface ToolEmbedder { readonly dimensions: number; readonly embedDocuments: ( texts: readonly string[], - ) => Effect.Effect; - readonly embedQuery: (text: string) => Effect.Effect; + ) => Effect.Effect; + readonly embedQuery: (text: string) => Effect.Effect; } export interface GeminiEmbedderOptions { @@ -53,7 +53,7 @@ export const makeGeminiEmbedder = (options: GeminiEmbedderOptions): ToolEmbedder const embedTexts = ( texts: readonly string[], taskType: typeof DOCUMENT_TASK_TYPE | typeof QUERY_TASK_TYPE, - ): Effect.Effect => + ): Effect.Effect => Effect.tryPromise({ try: async () => { if (texts.length === 0) return []; @@ -76,7 +76,7 @@ export const makeGeminiEmbedder = (options: GeminiEmbedderOptions): ToolEmbedder return embeddings; }, catch: (cause) => - new VectorizeSearchError({ message: `Gemini embedding failed for ${model}.`, cause }), + new SemanticSearchError({ message: `Gemini embedding failed for ${model}.`, cause }), }); return { diff --git a/packages/plugins/vectorize-search/src/sdk/embedding-service.ts b/packages/plugins/semantic-search/src/sdk/embedding-service.ts similarity index 95% rename from packages/plugins/vectorize-search/src/sdk/embedding-service.ts rename to packages/plugins/semantic-search/src/sdk/embedding-service.ts index a9a7ef6bfd..7daad9fd48 100644 --- a/packages/plugins/vectorize-search/src/sdk/embedding-service.ts +++ b/packages/plugins/semantic-search/src/sdk/embedding-service.ts @@ -15,7 +15,7 @@ import { makeGeminiEmbedder, type GeminiEmbedderOptions, type ToolEmbedder } fro // --------------------------------------------------------------------------- export class EmbedderService extends Context.Service()( - "@executor-js/plugin-vectorize-search/EmbedderService", + "@executor-js/plugin-semantic-search/EmbedderService", ) {} export const geminiEmbedderLayer = (options: GeminiEmbedderOptions): Layer.Layer => diff --git a/packages/plugins/vectorize-search/src/sdk/errors.ts b/packages/plugins/semantic-search/src/sdk/errors.ts similarity index 63% rename from packages/plugins/vectorize-search/src/sdk/errors.ts rename to packages/plugins/semantic-search/src/sdk/errors.ts index f1a3de63db..e324a9dbeb 100644 --- a/packages/plugins/vectorize-search/src/sdk/errors.ts +++ b/packages/plugins/semantic-search/src/sdk/errors.ts @@ -1,10 +1,10 @@ import { Data } from "effect"; -/** A failure in the Vectorize search plugin's index or query path β€” an +/** A failure in the Semantic search plugin's index or query path β€” an * embedding call, the Vectorize binding, or a catalog read. The query path * maps this into the engine's `ExecutionToolError` contract; the reindex path * surfaces it directly. */ -export class VectorizeSearchError extends Data.TaggedError("VectorizeSearchError")<{ +export class SemanticSearchError extends Data.TaggedError("SemanticSearchError")<{ readonly message: string; readonly cause?: unknown; }> {} diff --git a/packages/plugins/semantic-search/src/sdk/eval-golden.ts b/packages/plugins/semantic-search/src/sdk/eval-golden.ts new file mode 100644 index 0000000000..f00d5ee1e7 --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/eval-golden.ts @@ -0,0 +1,302 @@ +// --------------------------------------------------------------------------- +// Eval golden set β€” expected top tool path per query. +// +// Used by eval.ts to compute precision@1 and precision@5 against the embedded +// + ranked results. Keep it small and representative: a few clear-intent +// queries and a few that require deeper semantic matching (e.g. sub-features +// hidden in long descriptions or input schemas). +// +// GOLDEN targets the real catalog (CATALOG=real): paths of the form +// "." matching the path field the catalog loader in eval.ts +// produces. SAMPLE_GOLDEN targets the small offline SAMPLE catalog (CATALOG=sample, +// the default), whose tools use the hand-crafted "tools..org.x.<...>" paths. +// eval.ts selects the matching set by CATALOG, so the default mode scores against +// queries the sample catalog can actually answer rather than reporting all-zeros. +// --------------------------------------------------------------------------- + +export interface GoldenQuery { + /** The natural-language query used for retrieval. */ + readonly query: string; + /** The canonical tool path that should rank in the top result. */ + readonly expectedPath: string; + /** The top-5 set that ALL contain relevant results (for precision@5). */ + readonly expectedTop5: readonly string[]; +} + +export const GOLDEN: readonly GoldenQuery[] = [ + // --------------------------------------------------------------------------- + // GitHub (real catalog: integration=github) + // --------------------------------------------------------------------------- + { + query: "create a new GitHub repository for the authenticated user", + expectedPath: "github.repos.createForAuthenticatedUser", + expectedTop5: [ + "github.repos.createForAuthenticatedUser", + "github.repos.createInOrg", + "github.repos.createUsingTemplate", + ], + }, + { + query: "read the contents of a file in a GitHub repository", + expectedPath: "github.repos.getContent", + expectedTop5: ["github.repos.getContent", "github.repos.getReadme"], + }, + { + query: "create or update a file in a GitHub repo", + expectedPath: "github.repos.createOrUpdateFileContents", + expectedTop5: ["github.repos.createOrUpdateFileContents", "github.repos.getContent"], + }, + { + query: "publish a new release on GitHub", + expectedPath: "github.repos.createRelease", + expectedTop5: [ + "github.repos.createRelease", + "github.repos.listReleases", + "github.repos.getLatestRelease", + ], + }, + { + query: "register a webhook on a GitHub repository", + expectedPath: "github.repos.createWebhook", + expectedTop5: ["github.repos.createWebhook"], + }, + { + query: "look up a GitHub user's profile by username", + expectedPath: "github.users.getByUsername", + expectedTop5: ["github.users.getByUsername", "github.users.getById"], + }, + { + query: "trigger a GitHub Actions workflow run manually", + expectedPath: "github.actions.createWorkflowDispatch", + expectedTop5: ["github.actions.createWorkflowDispatch"], + }, + { + query: "cancel a running GitHub Actions workflow", + expectedPath: "github.actions.cancelWorkflowRun", + expectedTop5: ["github.actions.cancelWorkflowRun", "github.actions.forceCancelWorkflowRun"], + }, + + // --------------------------------------------------------------------------- + // Stripe API (real catalog: integration=stripe_api) + // --------------------------------------------------------------------------- + { + query: "create a new Stripe customer", + expectedPath: "stripe_api.customers.postCustomers", + expectedTop5: ["stripe_api.customers.postCustomers"], + }, + { + query: "create a payment intent to charge a customer", + expectedPath: "stripe_api.paymentIntents.postPaymentIntents", + expectedTop5: [ + "stripe_api.paymentIntents.postPaymentIntents", + "stripe_api.customers.postCustomers", + ], + }, + { + query: "set up a recurring subscription for a customer", + expectedPath: "stripe_api.subscriptions.postSubscriptions", + expectedTop5: ["stripe_api.subscriptions.postSubscriptions"], + }, + { + query: "issue a refund for a Stripe payment", + expectedPath: "stripe_api.refunds.postRefunds", + expectedTop5: ["stripe_api.refunds.postRefunds"], + }, + { + query: "create a draft invoice for a customer", + expectedPath: "stripe_api.invoices.postInvoices", + expectedTop5: ["stripe_api.invoices.postInvoices", "stripe_api.customers.postCustomers"], + }, + + // --------------------------------------------------------------------------- + // Sentry (real catalog: integration=sentry) + // --------------------------------------------------------------------------- + { + query: "bulk resolve or ignore a list of Sentry issues", + expectedPath: "sentry.events.bulkMutateAListOfIssues", + expectedTop5: [ + "sentry.events.bulkMutateAListOfIssues", + "sentry.events.bulkMutateAnOrganizationSIssues", + ], + }, + { + query: "save a Discover query in Sentry", + expectedPath: "sentry.discover.createANewSavedQuery", + expectedTop5: ["sentry.discover.createANewSavedQuery"], + }, + { + query: "set up a cron job monitor in Sentry", + expectedPath: "sentry.crons.createAMonitor", + expectedTop5: ["sentry.crons.createAMonitor", "sentry.crons.updateAMonitor"], + }, + + // --------------------------------------------------------------------------- + // PostHog (real catalog: integration=posthog) + // --------------------------------------------------------------------------- + { + query: "create a user survey in PostHog", + expectedPath: "posthog.survey_create", + expectedTop5: ["posthog.survey_create"], + }, + { + query: "run a HogQL analytics query against PostHog data", + expectedPath: "posthog.query_run", + expectedTop5: ["posthog.query_run", "posthog.query_validate"], + }, + { + query: "get A/B test experiment results in PostHog", + expectedPath: "posthog.experiment_results_get", + expectedTop5: ["posthog.experiment_results_get"], + }, + + // --------------------------------------------------------------------------- + // Gmail (real catalog: integration=gmail) + // --------------------------------------------------------------------------- + { + query: "send an email via Gmail", + expectedPath: "gmail.users.messages.send", + expectedTop5: ["gmail.users.messages.send", "gmail.users.drafts.send"], + }, + { + query: "list messages in a Gmail mailbox", + expectedPath: "gmail.users.messages.list", + expectedTop5: ["gmail.users.messages.list"], + }, + { + query: "create a Gmail label", + expectedPath: "gmail.users.labels.create", + expectedTop5: ["gmail.users.labels.create"], + }, + { + query: "save an email as a draft in Gmail", + expectedPath: "gmail.users.drafts.create", + expectedTop5: ["gmail.users.drafts.create", "gmail.users.drafts.update"], + }, + + // --------------------------------------------------------------------------- + // Spotify (real catalog: integration=spotify_web_api) + // --------------------------------------------------------------------------- + { + query: "start playing music on a Spotify device", + expectedPath: "spotify_web_api.player.startAUsersPlayback", + expectedTop5: [ + "spotify_web_api.player.startAUsersPlayback", + "spotify_web_api.player.resumePlayback", + ], + }, + { + query: "search for tracks or artists on Spotify", + expectedPath: "spotify_web_api.search.getOperation", + expectedTop5: ["spotify_web_api.search.getOperation"], + }, + { + query: "create a new Spotify playlist", + expectedPath: "spotify_web_api.playlists.createPlaylist", + expectedTop5: ["spotify_web_api.playlists.createPlaylist"], + }, + + // --------------------------------------------------------------------------- + // Neon (real catalog: integration=neon) + // --------------------------------------------------------------------------- + { + query: "create a new Neon database project", + expectedPath: "neon.create_project", + expectedTop5: ["neon.create_project"], + }, + { + query: "branch a Neon database for a feature or migration", + expectedPath: "neon.create_branch", + expectedTop5: ["neon.create_branch", "neon.prepare_database_migration"], + }, + { + query: "run a SQL query against a Neon database", + expectedPath: "neon.run_sql", + expectedTop5: ["neon.run_sql", "neon.run_sql_transaction"], + }, + + // --------------------------------------------------------------------------- + // Axiom (real catalog: integration=axiom) + // --------------------------------------------------------------------------- + { + query: "query log events with APL in Axiom", + expectedPath: "axiom.querydataset", + expectedTop5: ["axiom.querydataset", "axiom.querymetrics"], + }, + + // --------------------------------------------------------------------------- + // Exa (real catalog: integration=exa) + // --------------------------------------------------------------------------- + { + query: "search the web for recent information", + expectedPath: "exa.web_search_exa", + expectedTop5: ["exa.web_search_exa", "exa.web_fetch_exa"], + }, +]; + +// --------------------------------------------------------------------------- +// SAMPLE_GOLDEN β€” queries for the offline SAMPLE catalog (CATALOG=sample, default). +// Paths match the hand-crafted "tools..org.x.<...>" sample tools in eval.ts. +// With the weak hash embedder these scores stay modest; the point is an offline, +// no-data, no-network smoke that the chunk β†’ embed β†’ store β†’ rank pipeline runs +// and surfaces relevant tools β€” not a quality benchmark (use CATALOG=real for that). +// --------------------------------------------------------------------------- +export const SAMPLE_GOLDEN: readonly GoldenQuery[] = [ + { + query: "schedule a meeting on my Google calendar", + expectedPath: "tools.google.org.x.calendar.events.insert", + expectedTop5: [ + "tools.google.org.x.calendar.events.insert", + "tools.google.org.x.calendar.events.quickAdd", + ], + }, + { + query: "delete an event from my calendar", + expectedPath: "tools.google.org.x.calendar.events.delete", + expectedTop5: ["tools.google.org.x.calendar.events.delete"], + }, + { + query: "send an email through Gmail", + expectedPath: "tools.google.org.x.gmail.users.messages.send", + expectedTop5: ["tools.google.org.x.gmail.users.messages.send"], + }, + { + query: "open a new issue on a GitHub repository", + expectedPath: "tools.github.org.x.repos.issues.create", + expectedTop5: ["tools.github.org.x.repos.issues.create"], + }, + { + query: "open a pull request on GitHub", + expectedPath: "tools.github.org.x.repos.pulls.create", + expectedTop5: ["tools.github.org.x.repos.pulls.create"], + }, + { + query: "look up a GitHub repository's metadata", + expectedPath: "tools.github.org.x.repos.get", + expectedTop5: ["tools.github.org.x.repos.get"], + }, + { + query: "charge a customer's card with Stripe", + expectedPath: "tools.stripe.org.x.charges.create", + expectedTop5: ["tools.stripe.org.x.charges.create"], + }, + { + query: "refund a Stripe payment", + expectedPath: "tools.stripe.org.x.refunds.create", + expectedTop5: ["tools.stripe.org.x.refunds.create"], + }, + { + query: "start a recurring subscription in Stripe", + expectedPath: "tools.stripe.org.x.subscriptions.create", + expectedTop5: ["tools.stripe.org.x.subscriptions.create"], + }, + { + query: "post a message to a Slack channel", + expectedPath: "tools.slack.org.x.chat.postMessage", + expectedTop5: ["tools.slack.org.x.chat.postMessage"], + }, + { + query: "create an issue in Linear", + expectedPath: "tools.linear.org.x.issues.create", + expectedTop5: ["tools.linear.org.x.issues.create"], + }, +]; diff --git a/packages/plugins/vectorize-search/src/sdk/eval.ts b/packages/plugins/semantic-search/src/sdk/eval.ts similarity index 60% rename from packages/plugins/vectorize-search/src/sdk/eval.ts rename to packages/plugins/semantic-search/src/sdk/eval.ts index 2e10b08d45..c57d770d21 100644 --- a/packages/plugins/vectorize-search/src/sdk/eval.ts +++ b/packages/plugins/semantic-search/src/sdk/eval.ts @@ -1,15 +1,21 @@ +import { createRequire } from "node:module"; +import { homedir } from "node:os"; import { Cause, Effect, Exit, Layer } from "effect"; import { ChunkerService, facetChunkerLayer, wholeChunkerLayer } from "./chunker-service"; import type { ToolDocumentInput } from "./chunker"; +import { stripHtml, buildLexicalText } from "./documents"; import { EmbedderService, geminiEmbedderLayer, hashEmbedderLayer, openAiCompatibleEmbedderLayer, } from "./embedding-service"; -import { VectorStoreService, zvecStoreLayer } from "./store-service"; -import { GOLDEN } from "./eval-golden"; +import { VectorStoreService, zvecStoreLayer, sqliteVecStoreLayer } from "./store-service"; +import { GOLDEN, SAMPLE_GOLDEN } from "./eval-golden"; +import { makeFtsLexicalStore, makeFtsLexicalProvider } from "./store-fts"; +import { makeVectorToolDiscoveryProvider } from "./provider"; +import { makeHybridToolDiscoveryProvider } from "./hybrid"; // --------------------------------------------------------------------------- // Relevance eval β€” exercises the facet chunker + embed + zvec pipeline locally @@ -28,12 +34,23 @@ import { GOLDEN } from "./eval-golden"; // # Gemini (production parity): // EMBEDDER=gemini GEMINI_API_KEY=… CHUNKER=facet bun run …/eval.ts // -// Precision@1 and precision@5 are printed against GOLDEN (eval-golden.ts). +// Precision@1 and precision@5 are printed against the catalog-matched golden set +// (GOLDEN for CATALOG=real, SAMPLE_GOLDEN for CATALOG=sample; eval-golden.ts). +// +// CATALOG env controls which tool catalog is loaded: +// CATALOG=sample (default) β€” 13 hand-written sample tools +// CATALOG=real β€” all tools from ~/.executor/data.db (5 505 tools) +// +// CATALOG_LIMIT=N caps the real catalog to the first N rows (after always +// including every tool referenced by the golden set). // --------------------------------------------------------------------------- const NAMESPACE = "default"; const DIMENSIONS = Number(process.env.EMBEDDING_DIMENSIONS ?? 1536); const CHUNKER_KIND = process.env.CHUNKER ?? "facet"; +/** LEXICAL=none (default) β€” vector-only; LEXICAL=fts β€” FTS-hybrid via RRF. */ +const LEXICAL_KIND = (process.env.LEXICAL ?? "none") as "none" | "fts"; +const FTS_PATH = process.env.FTS_PATH ?? "/tmp/vectorize-eval-fts.sqlite"; // --------------------------------------------------------------------------- // Sample catalog β€” realistic tool documents with input/output TypeScript so @@ -328,6 +345,109 @@ const SAMPLE: readonly ToolDocumentInput[] = [ }, ]; +// --------------------------------------------------------------------------- +// Real catalog loader β€” reads from ~/.executor/data.db via bun:sqlite. +// +// path convention: "." so golden expectedPath values +// like "github.repos.createForAuthenticatedUser" resolve unambiguously. +// +// CATALOG_LIMIT caps the number of tools loaded, but always includes the +// tools referenced by the golden set (so precision metrics are valid). +// --------------------------------------------------------------------------- + +interface DbToolRow { + integration: string; + name: string; + description: string; + // bun:sqlite returns BLOB columns as Buffer; TEXT columns come as string. + input_schema: string | Buffer | null; + output_schema: string | Buffer | null; +} + +/** Coerce a bun:sqlite column value (string | Buffer | null) to string | undefined. */ +const colToString = (v: string | Buffer | null): string | undefined => { + if (v === null || v === undefined) return undefined; + if (typeof v === "string") return v || undefined; + // Buffer / Uint8Array β€” decode as UTF-8. + return new TextDecoder().decode(v) || undefined; +}; + +const loadRealCatalog = (): readonly ToolDocumentInput[] => { + // Dynamic require so the sqlite module is only loaded when CATALOG=real. + // Supports both Bun (bun:sqlite) and Node.js (better-sqlite3). + const isBun = typeof Bun !== "undefined"; + + const dbPath = (process.env.DATA_DB_PATH ?? `${homedir()}/.executor/data.db`).replace( + /^~/, + homedir(), + ); + + let rows: DbToolRow[]; + + if (isBun) { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { Database } = require("bun:sqlite") as typeof import("bun:sqlite"); + const db = new Database(dbPath, { readonly: true }); + rows = db + .query( + "SELECT integration, name, description, input_schema, output_schema FROM tool", + ) + .all(); + db.close(); + } else { + // Node.js path: use better-sqlite3 via createRequire (works in ESM). + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary: better-sqlite3 not available in Bun + const NodeDb = createRequire(import.meta.url)("better-sqlite3") as any; + const db = new NodeDb(dbPath, { readonly: true }); + rows = db + .prepare("SELECT integration, name, description, input_schema, output_schema FROM tool") + .all() as DbToolRow[]; + db.close(); + } + + // Build a set of paths that the golden set demands are always present. + const goldenPaths = new Set(GOLDEN.map((g) => g.expectedPath)); + + const tools: ToolDocumentInput[] = []; + const limit = process.env.CATALOG_LIMIT ? Number(process.env.CATALOG_LIMIT) : Infinity; + let nonGoldenCount = 0; + + for (const row of rows) { + const path = `${row.integration}.${row.name}`; + const isGolden = goldenPaths.has(path); + + if (!isGolden) { + if (nonGoldenCount >= limit) continue; + nonGoldenCount++; + } + + const inputTs = colToString(row.input_schema); + const outputTs = colToString(row.output_schema); + + const doc: ToolDocumentInput = { + path, + name: row.name, + integration: row.integration, + description: stripHtml((colToString(row.description as string | Buffer | null) ?? "").trim()), + inputTypeScript: inputTs ? inputTs.slice(0, 1500) : undefined, + outputTypeScript: outputTs ? outputTs.slice(0, 1500) : undefined, + }; + + tools.push({ ...doc, lexicalText: buildLexicalText(doc) }); + } + + return tools; +}; + +// --------------------------------------------------------------------------- +// Active catalog β€” chosen by CATALOG env (default: sample) +// --------------------------------------------------------------------------- + +const CATALOG_KIND = process.env.CATALOG ?? "sample"; +const CATALOG: readonly ToolDocumentInput[] = CATALOG_KIND === "real" ? loadRealCatalog() : SAMPLE; +// Golden set must match the active catalog's path format, or precision is all-zeros. +const ACTIVE_GOLDEN = CATALOG_KIND === "real" ? GOLDEN : SAMPLE_GOLDEN; + // --------------------------------------------------------------------------- // Precision metrics // --------------------------------------------------------------------------- @@ -352,15 +472,18 @@ const program = Effect.gen(function* () { const chunker = yield* ChunkerService; // ------------------------------------------------------------------------- - // Phase 1 β€” Chunk + embed + upsert + // Phase 1 β€” Chunk + embed + upsert (vector index) // ------------------------------------------------------------------------- - const allChunks = SAMPLE.flatMap((doc) => chunker.chunk(NAMESPACE, doc)); + const allChunks = CATALOG.flatMap((doc) => chunker.chunk(NAMESPACE, doc)); yield* Effect.sync(() => { - console.log(`\n=== chunking (chunker=${CHUNKER_KIND}) ===`); - console.log(`tools=${SAMPLE.length} chunks=${allChunks.length}`); - // Report per-tool chunk breakdown. - for (const doc of SAMPLE) { + console.log( + `\n=== chunking (catalog=${CATALOG_KIND}, chunker=${CHUNKER_KIND}, lexical=${LEXICAL_KIND}) ===`, + ); + console.log(`tools=${CATALOG.length} chunks=${allChunks.length}`); + // Report per-tool chunk breakdown (skip for large catalogs to avoid noise). + const multiChunkDocs = CATALOG_KIND === "real" ? [] : CATALOG; + for (const doc of multiChunkDocs) { const cs = chunker.chunk(NAMESPACE, doc); if (cs.length > 1) { const facetSummary = cs.map((c) => c.facet).join(", "); @@ -371,25 +494,52 @@ const program = Effect.gen(function* () { const vectors = yield* embedder.embedDocuments(allChunks.map((c) => c.embeddingText)); - yield* store.upsert( - allChunks.map((c, i) => ({ - id: c.id, - values: [...vectors[i]!], - namespace: NAMESPACE, - metadata: { - path: c.path, - name: c.name, - integration: c.integration, - facet: c.facet, - chunkIndex: c.chunkIndex, - }, - })), - ); + // Batch upserts to respect zvec's 1 024-doc-per-call limit. + const UPSERT_BATCH = 1024; + const allVectors = allChunks.map((c, i) => ({ + id: c.id, + values: [...vectors[i]!], + namespace: NAMESPACE, + metadata: { + path: c.path, + name: c.name, + integration: c.integration, + facet: c.facet, + chunkIndex: c.chunkIndex, + }, + })); + for (let batchStart = 0; batchStart < allVectors.length; batchStart += UPSERT_BATCH) { + yield* store.upsert(allVectors.slice(batchStart, batchStart + UPSERT_BATCH)); + } + + // ------------------------------------------------------------------------- + // Phase 1b β€” Build FTS index when LEXICAL=fts + // ------------------------------------------------------------------------- + const ftsProvider = + LEXICAL_KIND === "fts" + ? yield* Effect.gen(function* () { + const ftsStore = makeFtsLexicalStore({ path: FTS_PATH }); + const ftsDocs = CATALOG.map((doc) => ({ + id: doc.path, + namespace: NAMESPACE, + path: doc.path, + name: doc.name, + description: doc.description, + integration: doc.integration, + lexicalText: doc.lexicalText ?? buildLexicalText(doc), + })); + yield* ftsStore.upsert(ftsDocs); + yield* Effect.sync(() => + console.log(`FTS index built: ${ftsDocs.length} docs at ${FTS_PATH}`), + ); + return makeFtsLexicalProvider(ftsStore, NAMESPACE); + }) + : null; // ------------------------------------------------------------------------- // Phase 2 β€” Query, dedup by path (best score), top 5 // ------------------------------------------------------------------------- - const allQueries = GOLDEN.map((g) => g.query); + const allQueries = ACTIVE_GOLDEN.map((g) => g.query); yield* Effect.sync(() => console.log(`\n=== queries (embedder=${embedder.model}, dedup by path, top 5) ===`), @@ -398,42 +548,75 @@ const program = Effect.gen(function* () { // Collect ranked results for precision computation. const rankedByQuery = new Map(); + // Build vector-only provider for use in hybrid mode. + const vectorProvider = makeVectorToolDiscoveryProvider({ + embedder, + store, + namespace: NAMESPACE, + }); + + // If FTS provider is available, build the hybrid provider (RRF). + const activeProvider = + ftsProvider !== null + ? makeHybridToolDiscoveryProvider({ lexical: ftsProvider, vector: vectorProvider }) + : null; + for (const q of allQueries) { - const qv = yield* embedder.embedQuery(q); - // Over-fetch so dedup-by-path still yields a full top-5 window. - const matches = yield* store.query({ vector: qv, namespace: NAMESPACE, topK: 50 }); - - const best = new Map(); - for (const m of matches) { - const p = String(m.metadata?.path ?? ""); - if (p === "") continue; - const prev = best.get(p); - if (!prev || m.score > prev.score) { - best.set(p, { - score: m.score, - facet: String(m.metadata?.facet ?? ""), - chunkIndex: Number(m.metadata?.chunkIndex ?? 0), - }); + let ranked: readonly string[]; + + if (activeProvider !== null) { + // Hybrid path: use the fused provider. + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- eval-only: providers don't use the executor + const page = yield* activeProvider.searchTools({ + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- eval-only: providers ignore executor + executor: null as any, + query: q, + // Do not pass namespace as a text filter: the vector provider interprets + // ToolDiscoveryInput.namespace as an integration-prefix filter, not a + // storage partition key. The FTS store's partition is baked into the + // store itself (all docs upserted under NAMESPACE = "default"). + namespace: undefined, + limit: 5, + offset: 0, + }); + ranked = page.items.map((r) => r.path); + } else { + // Vector-only path (original). + const qv = yield* embedder.embedQuery(q); + // Over-fetch so dedup-by-path still yields a full top-5 window. + const matches = yield* store.query({ vector: qv, namespace: NAMESPACE, topK: 50 }); + + const best = new Map(); + for (const m of matches) { + const p = String(m.metadata?.path ?? ""); + if (p === "") continue; + const prev = best.get(p); + if (!prev || m.score > prev.score) { + best.set(p, { + score: m.score, + facet: String(m.metadata?.facet ?? ""), + chunkIndex: Number(m.metadata?.chunkIndex ?? 0), + }); + } } + ranked = [...best.entries()] + .sort((a, b) => b[1].score - a[1].score) + .slice(0, 5) + .map(([p]) => p); } - const ranked = [...best.entries()].sort((a, b) => b[1].score - a[1].score).slice(0, 5); - - rankedByQuery.set( - q, - ranked.map(([p]) => p), - ); + rankedByQuery.set(q, ranked); yield* Effect.sync(() => { console.log(`\nQ: "${q}"`); - for (const [p, { score, facet, chunkIndex }] of ranked) { - console.log(` ${score.toFixed(3)} ${p} (facet=${facet} chunk=${chunkIndex})`); + for (const p of ranked) { + console.log(` ${p}`); } }); } // ------------------------------------------------------------------------- - // Phase 3 β€” Precision@1 and precision@5 against the golden set + // Phase 3 β€” Precision@1 and recall@5 against the golden set // ------------------------------------------------------------------------- let sumP1 = 0; let sumP5 = 0; @@ -442,16 +625,17 @@ const program = Effect.gen(function* () { console.log(`\n=== precision summary (chunker=${CHUNKER_KIND}) ===`); }); - for (const golden of GOLDEN) { + for (const golden of ACTIVE_GOLDEN) { const ranked = rankedByQuery.get(golden.query) ?? []; const p1 = ranked[0] === golden.expectedPath ? 1 : 0; - const p5 = computePrecision(ranked, golden.expectedTop5, 5); + // recall@5: is the single expected tool in the top 5 (1/0) + const p5 = computePrecision(ranked, [golden.expectedPath], 5); sumP1 += p1; sumP5 += p5; yield* Effect.sync(() => { const hit1 = p1 === 1 ? "βœ“" : "βœ—"; - console.log(` ${hit1} [P@1=${p1} P@5=${p5.toFixed(2)}] "${golden.query}"`); + console.log(` ${hit1} [P@1=${p1} R@5=${p5}] "${golden.query}"`); if (p1 === 0) { console.log(` expected: ${golden.expectedPath}`); console.log(` got: ${ranked[0] ?? "(none)"}`); @@ -459,13 +643,16 @@ const program = Effect.gen(function* () { }); } - const n = GOLDEN.length; + const n = ACTIVE_GOLDEN.length; const avgP1 = sumP1 / n; const avgP5 = sumP5 / n; yield* Effect.sync(() => { console.log( - `\nprecision@1=${avgP1.toFixed(3)} precision@5=${avgP5.toFixed(3)} (${n} queries, chunker=${CHUNKER_KIND}, embedder=${embedder.model})`, + `\nprecision@1=${avgP1.toFixed(3)} recall@5=${avgP5.toFixed(3)} (${n} queries, chunker=${CHUNKER_KIND}, embedder=${embedder.model}, lexical=${LEXICAL_KIND})`, + ); + console.log( + `catalog=${CATALOG_KIND}${CATALOG_KIND === "real" && process.env.CATALOG_LIMIT ? ` limit=${process.env.CATALOG_LIMIT}` : ""}`, ); if (embedder.model.startsWith("hash")) { console.log( @@ -498,7 +685,14 @@ const embedderLayer = (): Layer.Layer => { }; const storeLayer = (): Layer.Layer => { - // Default to zvec (no in-memory store β€” removed in P0). + const kind = process.env.STORE ?? "zvec"; + if (kind === "sqlite-vec") { + return sqliteVecStoreLayer({ + path: process.env.SQLITE_VEC_PATH ?? "/tmp/vectorize-eval.sqlite", + dimensions: DIMENSIONS, + }); + } + // Default: zvec. return zvecStoreLayer({ path: process.env.ZVEC_PATH ?? "/tmp/vectorize-eval.zvec", dimensions: DIMENSIONS, diff --git a/packages/plugins/vectorize-search/src/sdk/fingerprint.test.ts b/packages/plugins/semantic-search/src/sdk/fingerprint.test.ts similarity index 100% rename from packages/plugins/vectorize-search/src/sdk/fingerprint.test.ts rename to packages/plugins/semantic-search/src/sdk/fingerprint.test.ts diff --git a/packages/plugins/vectorize-search/src/sdk/fingerprint.ts b/packages/plugins/semantic-search/src/sdk/fingerprint.ts similarity index 100% rename from packages/plugins/vectorize-search/src/sdk/fingerprint.ts rename to packages/plugins/semantic-search/src/sdk/fingerprint.ts diff --git a/packages/plugins/vectorize-search/src/sdk/harness.test.ts b/packages/plugins/semantic-search/src/sdk/harness.test.ts similarity index 100% rename from packages/plugins/vectorize-search/src/sdk/harness.test.ts rename to packages/plugins/semantic-search/src/sdk/harness.test.ts diff --git a/packages/plugins/vectorize-search/src/sdk/hybrid.test.ts b/packages/plugins/semantic-search/src/sdk/hybrid.test.ts similarity index 100% rename from packages/plugins/vectorize-search/src/sdk/hybrid.test.ts rename to packages/plugins/semantic-search/src/sdk/hybrid.test.ts diff --git a/packages/plugins/vectorize-search/src/sdk/hybrid.ts b/packages/plugins/semantic-search/src/sdk/hybrid.ts similarity index 100% rename from packages/plugins/vectorize-search/src/sdk/hybrid.ts rename to packages/plugins/semantic-search/src/sdk/hybrid.ts diff --git a/packages/plugins/vectorize-search/src/sdk/index.ts b/packages/plugins/semantic-search/src/sdk/index.ts similarity index 50% rename from packages/plugins/vectorize-search/src/sdk/index.ts rename to packages/plugins/semantic-search/src/sdk/index.ts index 12b14fd279..1830bf9b93 100644 --- a/packages/plugins/vectorize-search/src/sdk/index.ts +++ b/packages/plugins/semantic-search/src/sdk/index.ts @@ -1,9 +1,9 @@ export { - vectorizeSearchPlugin, - type VectorizeSearchPluginOptions, - type VectorizeSearchExtension, + semanticSearchPlugin, + type SemanticSearchPluginOptions, + type SemanticSearchExtension, } from "./plugin"; -export { makeVectorizeToolDiscoveryProvider, MAX_METADATA_TOP_K } from "./provider"; +export { makeVectorToolDiscoveryProvider } from "./provider"; export { makeGeminiEmbedder, DEFAULT_GEMINI_MODEL, @@ -11,17 +11,10 @@ export { type ToolEmbedder, type GeminiEmbedderOptions, } from "./embedder"; -export { - makeVectorizeStore, - type VectorizeIndex, - type VectorizeStore, - type VectorizeMatch, - type VectorizeMatches, - type VectorizeVectorInput, - MAX_TOP_K, -} from "./vectorize"; +export { type VectorStore, type VectorMatch, type VectorMatches, type VectorInput } from "./store"; +export { makeVectorizeStore, type VectorizeIndex, MAX_TOP_K } from "./store-cloudflare"; export { reconcileToolCatalog, type ReconcileResult } from "./indexer"; -export { VectorizeSearchError } from "./errors"; +export { SemanticSearchError } from "./errors"; // Chunker export { @@ -46,3 +39,22 @@ export { toolFingerprints, FingerprintRow } from "./collections"; // Store limit decorator export { withCloudflareLimits } from "./store-cloudflare-limits"; + +// zvec store (local / dev) +export { makeZVecStore, type ZVecStoreOptions } from "./store-zvec"; + +// sqlite-vec store (local / dev) +export { makeSqliteVecStore, type SqliteVecStoreOptions } from "./store-sqlite-vec"; + +// FTS5 lexical store + provider (local / dev) +export { + makeFtsLexicalStore, + makeFtsLexicalProvider, + type FtsLexicalStore, + type FtsDocumentInput, + type FtsSearchInput, + type FtsSearchResult, +} from "./store-fts"; + +// Documents projector helpers +export { stripHtml, buildLexicalText } from "./documents"; diff --git a/packages/plugins/vectorize-search/src/sdk/indexer.test.ts b/packages/plugins/semantic-search/src/sdk/indexer.test.ts similarity index 98% rename from packages/plugins/vectorize-search/src/sdk/indexer.test.ts rename to packages/plugins/semantic-search/src/sdk/indexer.test.ts index acfacd8323..ff3facedf3 100644 --- a/packages/plugins/vectorize-search/src/sdk/indexer.test.ts +++ b/packages/plugins/semantic-search/src/sdk/indexer.test.ts @@ -12,7 +12,7 @@ import type { FingerprintRow } from "./collections"; import { toolFingerprints } from "./collections"; import type { ToolEmbedder } from "./embedder"; import { reconcileToolCatalog } from "./indexer"; -import type { VectorizeStore, VectorizeVectorInput } from "./vectorize"; +import type { VectorStore, VectorInput } from "./store"; // --------------------------------------------------------------------------- // Test helpers @@ -57,11 +57,12 @@ const makeExecutor = ( } as unknown as Executor; }; -/** Build a fake VectorizeStore that records upserts and deletes. */ +/** Build a fake VectorStore that records upserts and deletes. */ const makeStore = () => { - const upserted: VectorizeVectorInput[] = []; + const upserted: VectorInput[] = []; const deleted: string[] = []; - const store: VectorizeStore = { + const store: VectorStore = { + maxTopK: 200, query: () => Effect.succeed([]), upsert: (vectors) => Effect.sync(() => { diff --git a/packages/plugins/vectorize-search/src/sdk/indexer.ts b/packages/plugins/semantic-search/src/sdk/indexer.ts similarity index 94% rename from packages/plugins/vectorize-search/src/sdk/indexer.ts rename to packages/plugins/semantic-search/src/sdk/indexer.ts index ff2270e055..bfaa32c802 100644 --- a/packages/plugins/vectorize-search/src/sdk/indexer.ts +++ b/packages/plugins/semantic-search/src/sdk/indexer.ts @@ -7,9 +7,9 @@ import type { FingerprintRow } from "./collections"; import { toolFingerprints } from "./collections"; import { collectToolDocumentInputs } from "./documents"; import type { ToolEmbedder } from "./embedder"; -import { VectorizeSearchError } from "./errors"; +import { SemanticSearchError } from "./errors"; import { fingerprintTool } from "./fingerprint"; -import type { VectorizeStore, VectorizeVectorInput } from "./vectorize"; +import type { VectorStore, VectorInput } from "./store"; // --------------------------------------------------------------------------- // ReconcileResult @@ -50,11 +50,11 @@ export const reconcileToolCatalog = (input: { readonly namespace: string; readonly executor: Executor; readonly embedder: ToolEmbedder; - readonly store: VectorizeStore; + readonly store: VectorStore; readonly chunker: Chunker; readonly fingerprints: PluginStorageCollectionFacade; readonly owner: Owner; -}): Effect.Effect => +}): Effect.Effect => Effect.gen(function* () { const { namespace, executor, embedder, store, chunker, fingerprints, owner } = input; @@ -81,7 +81,7 @@ export const reconcileToolCatalog = (input: { const storedEntries = yield* fingerprints.list().pipe( Effect.mapError( (cause) => - new VectorizeSearchError({ + new SemanticSearchError({ message: "Failed to load stored fingerprints for reconcile.", cause, }), @@ -154,9 +154,9 @@ export const reconcileToolCatalog = (input: { // Single batched embedding call over all changed chunks. const allVectors = yield* embedder.embedDocuments(allTexts); - // Build VectorizeVectorInput records and collect new chunkIds per tool. + // Build VectorInput records and collect new chunkIds per tool. let vectorOffset = 0; - const records: VectorizeVectorInput[] = []; + const records: VectorInput[] = []; const fingerprintUpdates: Array<{ path: string; row: FingerprintRow }> = []; for (const group of chunkedGroups) { @@ -166,7 +166,7 @@ export const reconcileToolCatalog = (input: { for (const chunkItem of chunks) { const vec = allVectors[vectorOffset++]; if (vec === undefined) { - return yield* new VectorizeSearchError({ + return yield* new SemanticSearchError({ message: `reconcileToolCatalog: embedding vector missing at offset ${vectorOffset - 1}`, }); } @@ -202,7 +202,7 @@ export const reconcileToolCatalog = (input: { } } - // Upsert all new vectors in one call. `store.upsert` (makeVectorizeStore) + // Upsert all new vectors in one call. `store.upsert` (makeVectorStore) // chunks internally into UPSERT_BATCH_SIZE (50) sequential batches, so the // underlying Vectorize binding never receives more than 50 vectors per // call β€” well under Cloudflare's 1,000-vector / 2 MB per-upsert caps β€” @@ -216,7 +216,7 @@ export const reconcileToolCatalog = (input: { fingerprints.put({ owner, key: path, data: row }).pipe( Effect.mapError( (cause) => - new VectorizeSearchError({ + new SemanticSearchError({ message: `Failed to persist fingerprint row for tool "${path}".`, cause, }), diff --git a/packages/plugins/vectorize-search/src/sdk/plugin.ts b/packages/plugins/semantic-search/src/sdk/plugin.ts similarity index 65% rename from packages/plugins/vectorize-search/src/sdk/plugin.ts rename to packages/plugins/semantic-search/src/sdk/plugin.ts index a29253dfcf..629b600842 100644 --- a/packages/plugins/vectorize-search/src/sdk/plugin.ts +++ b/packages/plugins/semantic-search/src/sdk/plugin.ts @@ -4,18 +4,18 @@ import { Effect } from "effect"; import { type Chunker, makeFacetChunker } from "./chunker"; import { toolFingerprints } from "./collections"; import { makeGeminiEmbedder, type ToolEmbedder } from "./embedder"; -import { VectorizeSearchError } from "./errors"; +import { SemanticSearchError } from "./errors"; import { makeHybridToolDiscoveryProvider } from "./hybrid"; import { reconcileToolCatalog, type ReconcileResult } from "./indexer"; -import { makeVectorizeToolDiscoveryProvider } from "./provider"; -import { withCloudflareLimits } from "./store-cloudflare-limits"; -import { makeVectorizeStore, type VectorizeIndex, type VectorizeStore } from "./vectorize"; +import { makeVectorToolDiscoveryProvider } from "./provider"; +import type { VectorStore } from "./store"; -export interface VectorizeSearchPluginOptions { - /** The Cloudflare Vectorize binding the tool catalog is indexed into. Absent - * (binding unbound) β†’ the plugin is inert and the engine keeps its built-in - * lexical search, mirroring how the metrics plugin no-ops without its binding. */ - readonly vectorize?: VectorizeIndex; +export interface SemanticSearchPluginOptions { + /** A vector store β€” construct one with `makeVectorizeStore` (Cloudflare), + * `makeZVecStore` (local/dev), or any other `VectorStore` implementation. + * Absent β†’ the plugin is inert and the engine keeps its built-in lexical + * search, mirroring how the metrics plugin no-ops without its binding. */ + readonly store?: VectorStore; /** Gemini API key (a wrangler secret on the Cloudflare host). Absent β†’ inert, * unless a custom `embedder` is supplied. */ readonly geminiApiKey?: string; @@ -25,7 +25,7 @@ export interface VectorizeSearchPluginOptions { readonly namespace?: string; /** Gemini embedding model id. Defaults to the v2 model (see embedder). */ readonly model?: string; - /** Embedding dimensionality β€” MUST equal the Vectorize index's dimensions. */ + /** Embedding dimensionality β€” MUST equal the vector index's dimensions. */ readonly dimensions?: number; /** Inject a custom embedder (tests). Overrides `model`/`geminiApiKey`/`dimensions`. */ readonly embedder?: ToolEmbedder; @@ -41,29 +41,28 @@ export interface VectorizeSearchPluginOptions { readonly lexical?: ToolDiscoveryProvider; } -const notConfigured = (): Effect.Effect => +const notConfigured = (): Effect.Effect => Effect.fail( - new VectorizeSearchError({ - message: - "Vectorize search is not configured (missing the Vectorize binding or Gemini API key).", + new SemanticSearchError({ + message: "Semantic search is not configured (missing the vector store or Gemini API key).", }), ); -/** Build the `executor.vectorizeSearch` surface. `reindex` reconciles the +/** Build the `executor.semanticSearch` surface. `reindex` reconciles the * current tool catalog incrementally (fingerprint diff) and upserts changed - * tools into Vectorize; it takes the scoped executor as an argument because - * only the request/API layer holds it (the plugin ctx does not expose the - * catalog). Inert β€” `reindex` fails clearly β€” until both a Vectorize binding + * tools into the vector store; it takes the scoped executor as an argument + * because only the request/API layer holds it (the plugin ctx does not expose + * the catalog). Inert β€” `reindex` fails clearly β€” until both a vector store * and an embedder are present. */ -const makeVectorizeSearchExtension = (deps: { +const makeSemanticSearchExtension = (deps: { readonly namespace: string; readonly embedder: ToolEmbedder | undefined; - readonly store: VectorizeStore | undefined; + readonly store: VectorStore | undefined; readonly chunker: Chunker; readonly fingerprints: Parameters[0]["fingerprints"] | undefined; readonly owner: Parameters[0]["owner"] | undefined; }) => ({ - reindex: (executor: Executor): Effect.Effect => + reindex: (executor: Executor): Effect.Effect => deps.embedder && deps.store && deps.fingerprints && deps.owner ? reconcileToolCatalog({ namespace: deps.namespace, @@ -77,22 +76,22 @@ const makeVectorizeSearchExtension = (deps: { : notConfigured(), }); -/** The `executor.vectorizeSearch` surface, derived from its factory. */ -export type VectorizeSearchExtension = ReturnType; +/** The `executor.semanticSearch` surface, derived from its factory. */ +export type SemanticSearchExtension = ReturnType; /** - * Semantic `tools.search` backed by Cloudflare Vectorize + Gemini embeddings. + * Semantic `tools.search` backed by a vector store + Gemini embeddings. * Supplies a `runtime.toolDiscoveryProvider`, so it supersedes the engine's * built-in lexical scorer wherever it is registered. The tool catalog is * indexed explicitly via the `reindex` extension method (see the `/api` subpath * for the HTTP route). * - * When both an embedder and Vectorize binding are present, the plugin exposes a + * When both an embedder and a vector store are present, the plugin exposes a * discovery provider. If the host also supplies `lexical`, that provider and the * vector provider are fused with Reciprocal Rank Fusion (hybrid search); * otherwise the vector provider answers alone. */ -export const vectorizeSearchPlugin = definePlugin((options?: VectorizeSearchPluginOptions) => { +export const semanticSearchPlugin = definePlugin((options?: SemanticSearchPluginOptions) => { const namespace = options?.namespace ?? "default"; const embedder = options?.embedder ?? @@ -103,20 +102,15 @@ export const vectorizeSearchPlugin = definePlugin((options?: VectorizeSearchPlug dimensions: options.dimensions, }) : undefined); - // Inert without both a Vectorize binding and an embedder β€” the engine then + // Inert without both a vector store and an embedder β€” the engine then // keeps its built-in lexical search (mirrors the metrics plugin's no-op). - // The store is wrapped so Cloudflare's hard limits (64-byte ids, topK ≀ 20 - // with full metadata) fail loudly at the boundary rather than as opaque - // Vectorize 4xx errors deep in a reindex. - const store = options?.vectorize - ? withCloudflareLimits(makeVectorizeStore(options.vectorize)) - : undefined; + const store = options?.store; const chunker = options?.chunker ?? makeFacetChunker(); const lexical = options?.lexical; return { - id: "vectorizeSearch" as const, - packageName: "@executor-js/plugin-vectorize-search", + id: "semanticSearch" as const, + packageName: "@executor-js/plugin-semantic-search", pluginStorage: { toolFingerprints }, storage: (deps) => ({ fingerprints: deps.pluginStorage.collection(toolFingerprints), @@ -127,7 +121,7 @@ export const vectorizeSearchPlugin = definePlugin((options?: VectorizeSearchPlug owner: "org" as const, }), extension: (ctx) => - makeVectorizeSearchExtension({ + makeSemanticSearchExtension({ namespace, embedder, store, @@ -138,7 +132,7 @@ export const vectorizeSearchPlugin = definePlugin((options?: VectorizeSearchPlug runtime: { toolDiscoveryProvider: () => { if (!embedder || !store) return undefined; - const vector = makeVectorizeToolDiscoveryProvider({ embedder, store, namespace }); + const vector = makeVectorToolDiscoveryProvider({ embedder, store, namespace }); // Fuse with the host-supplied lexical provider (RRF) when present; else // the vector provider answers alone. return lexical ? makeHybridToolDiscoveryProvider({ lexical, vector }) : vector; diff --git a/packages/plugins/vectorize-search/src/sdk/provider.ts b/packages/plugins/semantic-search/src/sdk/provider.ts similarity index 79% rename from packages/plugins/vectorize-search/src/sdk/provider.ts rename to packages/plugins/semantic-search/src/sdk/provider.ts index 8d51012c6b..365d7ffb1d 100644 --- a/packages/plugins/vectorize-search/src/sdk/provider.ts +++ b/packages/plugins/semantic-search/src/sdk/provider.ts @@ -6,16 +6,11 @@ import { import { Effect } from "effect"; import type { ToolEmbedder } from "./embedder"; -import type { VectorizeMatch, VectorizeStore } from "./vectorize"; - -/** Cloudflare caps `topK` at 20 when `returnMetadata:"all"` is set. - * The vector store always queries with full metadata, so the provider must - * never exceed this limit on the Vectorize path. */ -export const MAX_METADATA_TOP_K = 20; +import type { VectorMatch, VectorStore } from "./store"; const asString = (value: unknown): string => (typeof value === "string" ? value : ""); -const toResult = (match: VectorizeMatch): ToolDiscoveryResult => { +const toResult = (match: VectorMatch): ToolDiscoveryResult => { const description = asString(match.metadata?.description); return { path: asString(match.metadata?.path), @@ -26,11 +21,14 @@ const toResult = (match: VectorizeMatch): ToolDiscoveryResult => { }; }; -/** Narrow results to a search `namespace` (an integration/path prefix), - * mirroring the lexical provider's `matchesNamespace`. Applied to the fetched - * page, so it best-effort narrows within the topK window rather than across the - * whole index. */ -const matchesNamespace = (result: ToolDiscoveryResult, namespace: string | undefined): boolean => { +/** Narrow results to a search `namespace` (an integration/path prefix). Shared by + * the vector and FTS lexical providers so the two paths filter identically in + * hybrid search. Applied to the fetched page, so it best-effort narrows within + * the topK window rather than across the whole index. */ +export const matchesNamespace = ( + result: ToolDiscoveryResult, + namespace: string | undefined, +): boolean => { if (namespace === undefined) return true; const ns = namespace.trim().toLowerCase(); if (ns.length === 0) return true; @@ -55,9 +53,9 @@ const matchesNamespace = (result: ToolDiscoveryResult, namespace: string | undef * `input.executor` is unused β€” results come from the index, not a live catalog * scan. `input.namespace` narrows the page to an integration/path prefix. */ -export const makeVectorizeToolDiscoveryProvider = (deps: { +export const makeVectorToolDiscoveryProvider = (deps: { readonly embedder: ToolEmbedder; - readonly store: VectorizeStore; + readonly store: VectorStore; readonly namespace: string; }): ToolDiscoveryProvider => ({ searchTools: ({ query, namespace, limit, offset }) => @@ -72,10 +70,9 @@ export const makeVectorizeToolDiscoveryProvider = (deps: { const matches = yield* deps.store.query({ vector, namespace: deps.namespace, - // Cloudflare caps topK at 20 when returnMetadata:"all" is used (the - // store always sets it). MAX_TOP_K (100) is kept for non-metadata - // paths; the metadata path must respect MAX_METADATA_TOP_K. - topK: MAX_METADATA_TOP_K, + // Use the store's own cap β€” each backend (Vectorize, zvec, etc.) + // declares its maximum via `maxTopK` on the VectorStore interface. + topK: deps.store.maxTopK, }); const mapped = matches .filter((match) => asString(match.metadata?.path).length > 0) diff --git a/packages/plugins/vectorize-search/src/sdk/search.test.ts b/packages/plugins/semantic-search/src/sdk/search.test.ts similarity index 86% rename from packages/plugins/vectorize-search/src/sdk/search.test.ts rename to packages/plugins/semantic-search/src/sdk/search.test.ts index d9256ed65c..f3e5f1fa8e 100644 --- a/packages/plugins/vectorize-search/src/sdk/search.test.ts +++ b/packages/plugins/semantic-search/src/sdk/search.test.ts @@ -2,14 +2,10 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect } from "effect"; import type { ToolEmbedder } from "./embedder"; -import { makeVectorizeToolDiscoveryProvider } from "./provider"; -import { makeVectorizeStore } from "./vectorize"; -import type { - VectorizeIndex, - VectorizeMatch, - VectorizeStore, - VectorizeVectorInput, -} from "./vectorize"; +import { makeVectorToolDiscoveryProvider } from "./provider"; +import { makeVectorizeStore } from "./store-cloudflare"; +import type { VectorizeIndex } from "./store-cloudflare"; +import type { VectorMatch, VectorStore, VectorInput } from "./store"; const fakeEmbedder: ToolEmbedder = { model: "test", @@ -20,24 +16,25 @@ const fakeEmbedder: ToolEmbedder = { // The fake store honours topK (slicing to it) so pagination is exercised against // a realistic window β€” a store that ignored topK would mask hasMore bugs. -const makeQueryStore = (matches: readonly VectorizeMatch[]): VectorizeStore => ({ +const makeQueryStore = (matches: readonly VectorMatch[]): VectorStore => ({ + maxTopK: 20, query: ({ topK }) => Effect.succeed(matches.slice(0, topK)), upsert: () => Effect.void, deleteByIds: () => Effect.void, }); -const matchIn = (path: string, integration: string, score: number): VectorizeMatch => ({ +const matchIn = (path: string, integration: string, score: number): VectorMatch => ({ id: `org#${path}`, score, metadata: { path, name: path, description: `desc ${path}`, integration }, }); -const match = (path: string, score: number): VectorizeMatch => matchIn(path, "github", score); +const match = (path: string, score: number): VectorMatch => matchIn(path, "github", score); -describe("makeVectorizeToolDiscoveryProvider", () => { +describe("makeVectorToolDiscoveryProvider", () => { it.effect("maps Vectorize matches to ToolDiscoveryResult", () => Effect.gen(function* () { - const provider = makeVectorizeToolDiscoveryProvider({ + const provider = makeVectorToolDiscoveryProvider({ embedder: fakeEmbedder, store: makeQueryStore([match("repos.get", 0.9), match("repos.list", 0.8)]), namespace: "org", @@ -64,7 +61,7 @@ describe("makeVectorizeToolDiscoveryProvider", () => { it.effect("paginates within the returned matches", () => Effect.gen(function* () { - const provider = makeVectorizeToolDiscoveryProvider({ + const provider = makeVectorToolDiscoveryProvider({ embedder: fakeEmbedder, store: makeQueryStore([match("a", 0.9), match("b", 0.8), match("c", 0.7)]), namespace: "org", @@ -83,7 +80,7 @@ describe("makeVectorizeToolDiscoveryProvider", () => { it.effect("reports hasMore on a full first page", () => Effect.gen(function* () { - const provider = makeVectorizeToolDiscoveryProvider({ + const provider = makeVectorToolDiscoveryProvider({ embedder: fakeEmbedder, store: makeQueryStore([match("a", 0.9), match("b", 0.8), match("c", 0.7)]), namespace: "org", @@ -102,7 +99,7 @@ describe("makeVectorizeToolDiscoveryProvider", () => { it.effect("narrows results to input.namespace (integration prefix)", () => Effect.gen(function* () { - const provider = makeVectorizeToolDiscoveryProvider({ + const provider = makeVectorToolDiscoveryProvider({ embedder: fakeEmbedder, store: makeQueryStore([ matchIn("github.repos.get", "github", 0.9), @@ -129,7 +126,7 @@ describe("makeVectorizeToolDiscoveryProvider", () => { it.effect("keeps hasMore correct when the namespace filter drops interspersed items", () => Effect.gen(function* () { // Greptile scenario: a non-matching item sits where a tight probe would be. - const provider = makeVectorizeToolDiscoveryProvider({ + const provider = makeVectorToolDiscoveryProvider({ embedder: fakeEmbedder, store: makeQueryStore([ matchIn("github.a", "github", 0.9), @@ -154,7 +151,7 @@ describe("makeVectorizeToolDiscoveryProvider", () => { it.effect("returns empty for a blank query (no embedding call)", () => Effect.gen(function* () { - const provider = makeVectorizeToolDiscoveryProvider({ + const provider = makeVectorToolDiscoveryProvider({ embedder: { ...fakeEmbedder, embedQuery: () => Effect.die("embedQuery must not be called for a blank query") as never, @@ -177,7 +174,7 @@ describe("makeVectorizeToolDiscoveryProvider", () => { Effect.gen(function* () { // The facet chunker indexes a tool as several chunks, so the same path can // come back more than once β€” the provider must collapse to the best score. - const provider = makeVectorizeToolDiscoveryProvider({ + const provider = makeVectorToolDiscoveryProvider({ embedder: fakeEmbedder, store: makeQueryStore([ match("repos.get", 0.6), // identity-facet chunk @@ -212,7 +209,7 @@ describe("makeVectorizeStore", () => { deleteByIds: () => Promise.resolve({}), }; const store = makeVectorizeStore(index); - const vectors: readonly VectorizeVectorInput[] = Array.from({ length: 120 }, (_, i) => ({ + const vectors: readonly VectorInput[] = Array.from({ length: 120 }, (_, i) => ({ id: `v${i}`, values: [0, 0, 0], })); diff --git a/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.test.ts b/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.test.ts similarity index 93% rename from packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.test.ts rename to packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.test.ts index 1ae1d4647e..8cc8334b97 100644 --- a/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.test.ts +++ b/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.test.ts @@ -1,27 +1,28 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit } from "effect"; -import type { VectorizeMatch, VectorizeStore, VectorizeVectorInput } from "./vectorize"; +import type { VectorMatch, VectorStore, VectorInput } from "./store"; import { withCloudflareLimits } from "./store-cloudflare-limits"; // --------------------------------------------------------------------------- -// Inline fake VectorizeStore β€” records every call it receives so assertions +// Inline fake VectorStore β€” records every call it receives so assertions // can inspect exactly what the decorator forwarded (or withheld). // --------------------------------------------------------------------------- interface FakeStoreState { - upsertCalls: ReadonlyArray; + upsertCalls: ReadonlyArray; queryCalls: ReadonlyArray<{ vector: readonly number[]; namespace: string; topK: number }>; deleteByIdsCalls: ReadonlyArray; } -const makeFakeStore = (): { store: VectorizeStore; state: FakeStoreState } => { +const makeFakeStore = (): { store: VectorStore; state: FakeStoreState } => { const state: FakeStoreState = { upsertCalls: [], queryCalls: [], deleteByIdsCalls: [], }; - const store: VectorizeStore = { + const store: VectorStore = { + maxTopK: 20, upsert: (vectors) => Effect.sync(() => { // @ts-expect-error -- mutating a readonly array tracked for assertions @@ -31,7 +32,7 @@ const makeFakeStore = (): { store: VectorizeStore; state: FakeStoreState } => { Effect.sync(() => { // @ts-expect-error -- mutating a readonly array tracked for assertions state.queryCalls.push(input); - return [] as readonly VectorizeMatch[]; + return [] as readonly VectorMatch[]; }), deleteByIds: (ids) => Effect.sync(() => { diff --git a/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.ts b/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.ts similarity index 81% rename from packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.ts rename to packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.ts index 7aea860619..8126e87468 100644 --- a/packages/plugins/vectorize-search/src/sdk/store-cloudflare-limits.ts +++ b/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.ts @@ -1,7 +1,7 @@ import { Effect } from "effect"; -import { VectorizeSearchError } from "./errors"; -import type { VectorizeStore, VectorizeVectorInput } from "./vectorize"; +import { SemanticSearchError } from "./errors"; +import type { VectorStore, VectorInput } from "./store"; // --------------------------------------------------------------------------- // Cloudflare Vectorize runtime limits β€” enforced as a decorator so the same @@ -27,14 +27,17 @@ const MAX_QUERY_TOP_K = 20; const utf8ByteLength = (value: string): number => new TextEncoder().encode(value).length; /** Wraps `inner` with Cloudflare Vectorize runtime-limit validation. Any call - * that would violate a limit fails immediately with `VectorizeSearchError` - * before the inner store is touched. All other calls pass through unchanged. */ -export const withCloudflareLimits = (inner: VectorizeStore): VectorizeStore => ({ - upsert: (vectors: readonly VectorizeVectorInput[]) => { + * that would violate a limit fails immediately with `SemanticSearchError` + * before the inner store is touched. All other calls pass through unchanged. + * The returned store's `maxTopK` is `Math.min(inner.maxTopK, 20)`. */ +export const withCloudflareLimits = (inner: VectorStore): VectorStore => ({ + maxTopK: Math.min(inner.maxTopK, MAX_QUERY_TOP_K), + + upsert: (vectors: readonly VectorInput[]) => { const offending = vectors.find((v) => utf8ByteLength(v.id) > MAX_ID_BYTES); if (offending !== undefined) { return Effect.fail( - new VectorizeSearchError({ + new SemanticSearchError({ message: `Vector id exceeds the Cloudflare Vectorize 64-byte limit (${utf8ByteLength(offending.id)} bytes): "${offending.id}".`, }), ); @@ -49,7 +52,7 @@ export const withCloudflareLimits = (inner: VectorizeStore): VectorizeStore => ( }) => { if (input.topK > MAX_QUERY_TOP_K) { return Effect.fail( - new VectorizeSearchError({ + new SemanticSearchError({ message: `topK ${input.topK} exceeds the Cloudflare Vectorize limit of ${MAX_QUERY_TOP_K} when returnMetadata is "all". Lower topK or switch to a metadata strategy that supports a higher cap.`, }), ); diff --git a/packages/plugins/vectorize-search/src/sdk/vectorize.ts b/packages/plugins/semantic-search/src/sdk/store-cloudflare.ts similarity index 57% rename from packages/plugins/vectorize-search/src/sdk/vectorize.ts rename to packages/plugins/semantic-search/src/sdk/store-cloudflare.ts index 72159928a7..9ce11e203f 100644 --- a/packages/plugins/vectorize-search/src/sdk/vectorize.ts +++ b/packages/plugins/semantic-search/src/sdk/store-cloudflare.ts @@ -1,6 +1,7 @@ import { Effect } from "effect"; -import { VectorizeSearchError } from "./errors"; +import { SemanticSearchError } from "./errors"; +import type { VectorInput, VectorMatches, VectorStore } from "./store"; /** Minimal structural view of a Cloudflare Vectorize binding (the subset this * plugin uses). A real `Vectorize` / `VectorizeIndex` binding from @@ -16,34 +17,19 @@ export interface VectorizeIndex { readonly returnMetadata?: boolean | "all" | "indexed" | "none"; readonly returnValues?: boolean; }, - ): Promise; - upsert(vectors: readonly VectorizeVectorInput[]): Promise; + ): Promise; + upsert(vectors: readonly VectorInput[]): Promise; deleteByIds(ids: readonly string[]): Promise; } -export interface VectorizeMatches { - readonly matches: readonly VectorizeMatch[]; - readonly count?: number; -} - -export interface VectorizeMatch { - readonly id: string; - readonly score: number; - readonly namespace?: string; - readonly metadata?: Record; -} - -export interface VectorizeVectorInput { - readonly id: string; - readonly values: readonly number[]; - readonly namespace?: string; - readonly metadata?: Record; -} - -/** Vectorize caps `topK` at 100 (with metadata). The provider fetches this full - * window and paginates within it, so the store clamps any request to it. */ +/** Vectorize caps `topK` at 100 (without metadata). With `returnMetadata:"all"` + * the effective cap is 20 β€” enforced separately by `withCloudflareLimits`. */ export const MAX_TOP_K = 100; +/** Vectorize caps `topK` at 20 when `returnMetadata:"all"` is set. This is the + * per-store cap exposed on the `VectorStore` interface for Cloudflare. */ +export const MAX_METADATA_TOP_K = 20; + /** Vectorize caps a single upsert at ~1000 vectors / ~2 MB. At 1536-d a few * dozen vectors already approach the size cap, so chunk conservatively well * under both limits and upsert the batches sequentially. */ @@ -58,20 +44,10 @@ const chunk = (items: readonly A[], size: number): readonly (readonly A[])[] return out; }; -/** Effect-wrapped, plugin-facing view over a Vectorize binding. */ -export interface VectorizeStore { - readonly query: (input: { - readonly vector: readonly number[]; - readonly namespace: string; - readonly topK: number; - }) => Effect.Effect; - readonly upsert: ( - vectors: readonly VectorizeVectorInput[], - ) => Effect.Effect; - readonly deleteByIds: (ids: readonly string[]) => Effect.Effect; -} - -export const makeVectorizeStore = (index: VectorizeIndex): VectorizeStore => ({ +/** Effect-wrapped `VectorStore` backed by a Cloudflare Vectorize binding. + * `maxTopK` reflects the `returnMetadata:"all"` cap (20). */ +export const makeVectorizeStore = (index: VectorizeIndex): VectorStore => ({ + maxTopK: MAX_METADATA_TOP_K, query: ({ vector, namespace, topK }) => Effect.tryPromise({ try: () => @@ -80,7 +56,7 @@ export const makeVectorizeStore = (index: VectorizeIndex): VectorizeStore => ({ namespace, returnMetadata: "all", }), - catch: (cause) => new VectorizeSearchError({ message: "Vectorize query failed.", cause }), + catch: (cause) => new SemanticSearchError({ message: "Vectorize query failed.", cause }), }).pipe(Effect.map((result) => result.matches)), upsert: (vectors) => vectors.length === 0 @@ -91,7 +67,7 @@ export const makeVectorizeStore = (index: VectorizeIndex): VectorizeStore => ({ Effect.tryPromise({ try: () => index.upsert(batch), catch: (cause) => - new VectorizeSearchError({ message: "Vectorize upsert failed.", cause }), + new SemanticSearchError({ message: "Vectorize upsert failed.", cause }), }), { concurrency: 1, discard: true }, ), @@ -100,7 +76,6 @@ export const makeVectorizeStore = (index: VectorizeIndex): VectorizeStore => ({ ? Effect.void : Effect.tryPromise({ try: () => index.deleteByIds([...ids]), - catch: (cause) => - new VectorizeSearchError({ message: "Vectorize delete failed.", cause }), + catch: (cause) => new SemanticSearchError({ message: "Vectorize delete failed.", cause }), }).pipe(Effect.asVoid), }); diff --git a/packages/plugins/semantic-search/src/sdk/store-fts-provider.test.ts b/packages/plugins/semantic-search/src/sdk/store-fts-provider.test.ts new file mode 100644 index 0000000000..dda9598a55 --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/store-fts-provider.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { type FtsLexicalStore, type FtsSearchResult, makeFtsLexicalProvider } from "./store-fts"; + +const row = (path: string, integration: string): FtsSearchResult => ({ + path, + name: path, + description: "", + integration, + score: 1, +}); + +/** A store that returns the same cross-integration rows regardless of query. */ +const mockStore = (rows: readonly FtsSearchResult[]): FtsLexicalStore => ({ + upsert: () => Effect.void, + deleteByIds: () => Effect.void, + search: () => Effect.succeed(rows), +}); + +describe("makeFtsLexicalProvider β€” namespace filtering", () => { + const rows = [ + row("github.repos.get", "github"), + row("github.issues.create", "github"), + row("stripe.charges.create", "stripe"), + row("slack.chat.postMessage", "slack"), + ]; + + it.effect("returns all results when no namespace is given", () => + Effect.gen(function* () { + const provider = makeFtsLexicalProvider(mockStore(rows), "default"); + const page = yield* provider.searchTools({ + query: "anything", + limit: 10, + offset: 0, + executor: undefined as never, + }); + expect(page.items.map((i) => i.path)).toEqual([ + "github.repos.get", + "github.issues.create", + "stripe.charges.create", + "slack.chat.postMessage", + ]); + expect(page.total).toBe(4); + }), + ); + + it.effect( + "narrows to the integration prefix when namespace is set (matches the vector path)", + () => + Effect.gen(function* () { + const provider = makeFtsLexicalProvider(mockStore(rows), "default"); + const page = yield* provider.searchTools({ + query: "anything", + namespace: "github", + limit: 10, + offset: 0, + executor: undefined as never, + }); + expect(page.items.map((i) => i.path)).toEqual(["github.repos.get", "github.issues.create"]); + expect(page.total).toBe(2); + expect(page.hasMore).toBe(false); + }), + ); +}); diff --git a/packages/plugins/semantic-search/src/sdk/store-fts.ts b/packages/plugins/semantic-search/src/sdk/store-fts.ts new file mode 100644 index 0000000000..ae4c96ac68 --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/store-fts.ts @@ -0,0 +1,413 @@ +import { + ExecutionToolError, + type PagedResult, + type ToolDiscoveryProvider, + type ToolDiscoveryResult, +} from "@executor-js/sdk/core"; +import { Effect } from "effect"; + +import { SemanticSearchError } from "./errors"; +import { matchesNamespace } from "./provider"; + +// --------------------------------------------------------------------------- +// FTS5 lexical store β€” a local, in-process, file-backed full-text search store +// using SQLite's built-in FTS5 module (no sqlite-vec needed). +// +// Schema: +// `docs` table β€” id, namespace, path, name, description, integration, lexical_text +// `docs_fts` FTS5 virtual table β€” path, integration, name, description, lexical_text +// tokenize='porter unicode61' (same as Pi's search) +// +// BM25 weights (Pi reference parity): +// path=12, integration=8, name=10, description=5, lexical_text=3 +// +// Score convention: `-bm25(...)` so higher is better (BM25 is negative for matches). +// +// `better-sqlite3` is lazy dynamic imported (like store-sqlite-vec/store-zvec) so +// the Cloudflare bundle never pulls in the native addon. +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Internal types β€” normalized interface that bridges better-sqlite3 and bun:sqlite +// --------------------------------------------------------------------------- + +interface SqliteStatement { + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + run(...args: any[]): { readonly lastInsertRowid: bigint | number }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + get(...args: any[]): unknown; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + all(...args: any[]): unknown[]; +} + +/** Normalized DB handle β€” subset used by both better-sqlite3 and bun:sqlite. */ +interface SqliteDatabase { + exec(sql: string): void; + prepare(sql: string): SqliteStatement; + close(): void; +} + +/** + * Minimal `bun:sqlite` Database shape we use β€” declared locally so the package + * typechecks without `@types/bun`. Consumers (e.g. host-cloudflare) typecheck + * this source transitively and don't carry bun types. + */ +interface BunSqliteDatabase { + exec(sql: string): unknown; + prepare(sql: string): { + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + run(...a: any[]): { readonly lastInsertRowid: bigint | number }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + get(...a: any[]): unknown; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + all(...a: any[]): unknown[]; + }; + close(): void; +} + +// --------------------------------------------------------------------------- +// FTS query normalisation β€” mirrors Pi's `toFtsQuery`: +// camelCase-split β†’ lowercase β†’ filter stop-words β†’ `term* term* …` +// --------------------------------------------------------------------------- + +const normalizeFtsQuery = (query: string): string | null => { + const terms = query + // Split camelCase: "createSubscription" β†’ "create Subscription" + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + // Treat punctuation / separators as word boundaries + .replace(/[_./:-]+/g, " ") + .toLowerCase() + .trim() + .split(/[^a-z0-9]+/) + .map((t) => t.trim()) + .filter(Boolean); + + return terms.length === 0 ? null : terms.map((t) => `${t}*`).join(" "); +}; + +// --------------------------------------------------------------------------- +// Public types +// --------------------------------------------------------------------------- + +export interface FtsDocumentInput { + readonly id: string; + readonly namespace: string; + readonly path: string; + readonly name: string; + readonly description: string; + readonly integration: string; + /** Broad text for FTS: identifiers, schema snippets, etc. */ + readonly lexicalText: string; +} + +export interface FtsSearchInput { + readonly query: string; + readonly namespace: string; + readonly topK: number; +} + +export interface FtsSearchResult { + readonly path: string; + readonly name: string; + readonly description: string; + readonly integration: string; + /** -bm25 score; higher is better. */ + readonly score: number; +} + +export interface FtsLexicalStore { + readonly upsert: (docs: readonly FtsDocumentInput[]) => Effect.Effect; + readonly deleteByIds: (ids: readonly string[]) => Effect.Effect; + readonly search: ( + input: FtsSearchInput, + ) => Effect.Effect; +} + +// --------------------------------------------------------------------------- +// Database open + schema init +// --------------------------------------------------------------------------- + +const openDatabase = async (dbPath: string): Promise => { + let db: SqliteDatabase; + + // `globalThis.Bun` rather than the ambient `Bun` global so the package doesn't + // require `@types/bun` in consumers' typecheck. + if ((globalThis as { Bun?: unknown }).Bun !== undefined) { + // Bun runtime: use the built-in bun:sqlite (better-sqlite3 is not supported in Bun). + // The specifier is computed (not a literal) so neither tsgo nor esbuild tries to + // resolve the bun-only "bun:sqlite" builtin β€” it's absent under node, where the + // tests bundle, and only ever reached when actually running on Bun. + const bunSqliteSpecifier = ["bun", "sqlite"].join(":"); + const { Database } = (await import(/* @vite-ignore */ bunSqliteSpecifier)) as { + Database: new (path: string) => BunSqliteDatabase; + }; + const bunDb = new Database(dbPath); + // bun:sqlite exposes pragma as exec. + // Wrap to satisfy our normalized SqliteDatabase interface. + db = { + exec: (sql) => bunDb.exec(sql), + prepare: (sql) => { + const stmt = bunDb.prepare(sql); + return { + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + run: (...args: any[]) => stmt.run(...args), + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + get: (...args: any[]) => stmt.get(...args), + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + all: (...args: any[]) => stmt.all(...args) as any[], + }; + }, + close: () => bunDb.close(), + }; + bunDb.exec("PRAGMA journal_mode = WAL"); + bunDb.exec("PRAGMA foreign_keys = ON"); + } else { + // Node.js runtime: use better-sqlite3 (lazy import keeps Cloudflare bundle clean). + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary: default export shape differs across bundlers + const BetterSqlite3Mod = await import("better-sqlite3"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + const DatabaseCtor = (BetterSqlite3Mod as any).default ?? BetterSqlite3Mod; + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: better-sqlite3 throws synchronously on open + const nodeDb = new DatabaseCtor(dbPath) as SqliteDatabase & { + pragma(s: string): unknown; + }; + nodeDb.pragma("journal_mode = WAL"); + nodeDb.pragma("foreign_keys = ON"); + db = nodeDb; + } + + db.exec(` + CREATE TABLE IF NOT EXISTS fts_docs ( + id TEXT PRIMARY KEY, + namespace TEXT NOT NULL DEFAULT '', + path TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + integration TEXT NOT NULL, + lexical_text TEXT NOT NULL + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS fts_docs_fts USING fts5( + path, + integration, + name, + description, + lexical_text, + content='fts_docs', + content_rowid='rowid', + tokenize='porter unicode61' + ); + + -- Triggers to keep the FTS5 content table in sync with fts_docs. + CREATE TRIGGER IF NOT EXISTS fts_docs_ai AFTER INSERT ON fts_docs BEGIN + INSERT INTO fts_docs_fts(rowid, path, integration, name, description, lexical_text) + VALUES (new.rowid, new.path, new.integration, new.name, new.description, new.lexical_text); + END; + + CREATE TRIGGER IF NOT EXISTS fts_docs_ad AFTER DELETE ON fts_docs BEGIN + INSERT INTO fts_docs_fts(fts_docs_fts, rowid, path, integration, name, description, lexical_text) + VALUES ('delete', old.rowid, old.path, old.integration, old.name, old.description, old.lexical_text); + END; + + CREATE TRIGGER IF NOT EXISTS fts_docs_au AFTER UPDATE ON fts_docs BEGIN + INSERT INTO fts_docs_fts(fts_docs_fts, rowid, path, integration, name, description, lexical_text) + VALUES ('delete', old.rowid, old.path, old.integration, old.name, old.description, old.lexical_text); + INSERT INTO fts_docs_fts(rowid, path, integration, name, description, lexical_text) + VALUES (new.rowid, new.path, new.integration, new.name, new.description, new.lexical_text); + END; + `); + + return db; +}; + +// --------------------------------------------------------------------------- +// Public factory +// --------------------------------------------------------------------------- + +export const makeFtsLexicalStore = (options: { readonly path: string }): FtsLexicalStore => { + let cached: Promise | null = null; + const getDb = Effect.tryPromise({ + try: () => (cached ??= openDatabase(options.path)), + catch: (cause) => + new SemanticSearchError({ + message: `FTS5 store open/init failed at ${options.path}.`, + cause, + }), + }); + + return { + upsert: (docs) => + docs.length === 0 + ? Effect.void + : getDb.pipe( + Effect.flatMap((db) => + Effect.try({ + try: () => { + const del = db.prepare("DELETE FROM fts_docs WHERE id = ?"); + const ins = db.prepare( + `INSERT INTO fts_docs (id, namespace, path, name, description, integration, lexical_text) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ); + // Atomic batch: delete-then-insert fires FTS triggers per doc, so a + // mid-loop failure must not leave the FTS index partially erased. + db.exec("BEGIN"); + let committed = false; + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: roll the batch back on partial failure + try { + for (const doc of docs) { + // Delete-then-insert: the triggers handle FTS sync. + del.run(doc.id); + ins.run( + doc.id, + doc.namespace, + doc.path, + doc.name, + doc.description, + doc.integration, + doc.lexicalText, + ); + } + db.exec("COMMIT"); + committed = true; + } finally { + if (!committed) db.exec("ROLLBACK"); + } + }, + catch: (cause) => + new SemanticSearchError({ message: "FTS5 upsert failed.", cause }), + }), + ), + Effect.asVoid, + ), + + deleteByIds: (ids) => + ids.length === 0 + ? Effect.void + : getDb.pipe( + Effect.flatMap((db) => + Effect.try({ + try: () => { + const del = db.prepare("DELETE FROM fts_docs WHERE id = ?"); + for (const id of ids) { + del.run(id); + } + }, + catch: (cause) => + new SemanticSearchError({ message: "FTS5 deleteByIds failed.", cause }), + }), + ), + Effect.asVoid, + ), + + search: ({ query, namespace, topK }) => + getDb.pipe( + Effect.flatMap((db) => + Effect.try({ + try: () => { + const ftsQuery = normalizeFtsQuery(query); + if (ftsQuery === null) return [] as readonly FtsSearchResult[]; + + // BM25 weights: path=12, integration=8, name=10, description=5, lexical_text=3 + // `-bm25(...)` so higher = better. + const rows = db + .prepare( + `SELECT + d.path AS path, + d.name AS name, + d.description AS description, + d.integration AS integration, + 0 - bm25(fts_docs_fts, 12.0, 8.0, 10.0, 5.0, 3.0) AS score + FROM fts_docs_fts + JOIN fts_docs d ON d.rowid = fts_docs_fts.rowid + WHERE fts_docs_fts MATCH ? + AND d.namespace = ? + ORDER BY score DESC + LIMIT ?`, + ) + .all(ftsQuery, namespace, topK) as ReadonlyArray<{ + readonly path: string; + readonly name: string; + readonly description: string; + readonly integration: string; + readonly score: number; + }>; + + return rows.map( + (r): FtsSearchResult => ({ + path: r.path, + name: r.name, + description: r.description, + integration: r.integration, + score: r.score, + }), + ); + }, + catch: (cause) => new SemanticSearchError({ message: "FTS5 search failed.", cause }), + }), + ), + ), + }; +}; + +// --------------------------------------------------------------------------- +// Provider adapter +// --------------------------------------------------------------------------- + +/** + * Wraps an `FtsLexicalStore` as a `ToolDiscoveryProvider` so it can be passed + * to `makeHybridToolDiscoveryProvider` as the `lexical` source. + * + * `storageNamespace` is the storage partition key under which all documents were + * upserted (the `FtsDocumentInput.namespace` value used at index time). It is + * separate from `ToolDiscoveryInput.namespace`, which is an integration-prefix + * text filter used by the vector provider and intentionally ignored here. + */ +export const makeFtsLexicalProvider = ( + store: FtsLexicalStore, + storageNamespace: string, +): ToolDiscoveryProvider => ({ + searchTools: ({ query, namespace, limit, offset }) => + Effect.gen(function* () { + if (query.trim().length === 0) { + return { + items: [], + total: 0, + hasMore: false, + nextOffset: null, + } satisfies PagedResult; + } + + // Over-fetch so pagination after dedup is correct. + const topK = Math.max(limit + Math.max(offset, 0), 50); + // store.search partitions by the storage-level namespace; ToolDiscoveryInput + // .namespace is a separate integration-prefix filter applied to the results, + // shared with the vector provider so hybrid search narrows both paths alike. + const rows = yield* store.search({ query, namespace: storageNamespace, topK }); + + const items: readonly ToolDiscoveryResult[] = rows + .map((r) => ({ + path: r.path, + name: r.name, + description: r.description.length > 0 ? r.description : undefined, + integration: r.integration, + score: r.score, + })) + .filter((result) => matchesNamespace(result, namespace)); + + const safeOffset = Math.max(offset, 0); + const start = Math.min(safeOffset, items.length); + const page = items.slice(start, start + limit); + const hasMore = items.length > start + page.length; + + return { + items: page, + total: items.length, + hasMore, + nextOffset: hasMore ? start + page.length : null, + } satisfies PagedResult; + }).pipe( + Effect.mapError( + (cause) => new ExecutionToolError({ message: "FTS lexical tool search failed.", cause }), + ), + ), +}); diff --git a/packages/plugins/vectorize-search/src/sdk/store-service.ts b/packages/plugins/semantic-search/src/sdk/store-service.ts similarity index 65% rename from packages/plugins/vectorize-search/src/sdk/store-service.ts rename to packages/plugins/semantic-search/src/sdk/store-service.ts index 89194b2791..dfbe9c5b0c 100644 --- a/packages/plugins/vectorize-search/src/sdk/store-service.ts +++ b/packages/plugins/semantic-search/src/sdk/store-service.ts @@ -1,7 +1,9 @@ import { Context, Layer } from "effect"; import { makeZVecStore, type ZVecStoreOptions } from "./store-zvec"; -import { makeVectorizeStore, type VectorizeIndex, type VectorizeStore } from "./vectorize"; +import { makeSqliteVecStore, type SqliteVecStoreOptions } from "./store-sqlite-vec"; +import { makeVectorizeStore, type VectorizeIndex } from "./store-cloudflare"; +import type { VectorStore } from "./store"; // --------------------------------------------------------------------------- // Pluggable vector-store seam. The indexer + query provider depend on @@ -10,8 +12,8 @@ import { makeVectorizeStore, type VectorizeIndex, type VectorizeStore } from "./ // production. // --------------------------------------------------------------------------- -export class VectorStoreService extends Context.Service()( - "@executor-js/plugin-vectorize-search/VectorStoreService", +export class VectorStoreService extends Context.Service()( + "@executor-js/plugin-semantic-search/VectorStoreService", ) {} export const vectorizeStoreLayer = (index: VectorizeIndex): Layer.Layer => @@ -19,3 +21,8 @@ export const vectorizeStoreLayer = (index: VectorizeIndex): Layer.Layer => Layer.succeed(VectorStoreService)(makeZVecStore(options)); + +export const sqliteVecStoreLayer = ( + options: SqliteVecStoreOptions, +): Layer.Layer => + Layer.succeed(VectorStoreService)(makeSqliteVecStore(options)); diff --git a/packages/plugins/semantic-search/src/sdk/store-sqlite-vec.ts b/packages/plugins/semantic-search/src/sdk/store-sqlite-vec.ts new file mode 100644 index 0000000000..f5418b7bd5 --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/store-sqlite-vec.ts @@ -0,0 +1,268 @@ +import { Effect } from "effect"; + +import { SemanticSearchError } from "./errors"; +import type { VectorInput, VectorMatch, VectorStore } from "./store"; + +// --------------------------------------------------------------------------- +// sqlite-vec-backed VectorStore β€” a local, in-process, file-backed vector +// similarity store using SQLite + the sqlite-vec extension (L2/cosine KNN). +// +// Schema: +// vec0 virtual table `embedding_vectors` β€” float[dimensions] embedding col +// bridge table `vectors` β€” id, vector_rowid, namespace, metadata_json +// +// Two implementation notes: +// - metadata is stored as JSON in `metadata_json`; namespace lives as its +// own column so the post-filter join is cheap. +// - sqlite-vec KNN score is L2 distance (lower = nearer); we convert to a +// similarity via `1 / (1 + distance)` so the API matches Vectorize's +// higher-is-better convention. +// - `better-sqlite3` + `sqlite-vec` are loaded lazily via dynamic import so +// the plugin's Cloudflare build never pulls in native Node addons. +// --------------------------------------------------------------------------- + +export interface SqliteVecStoreOptions { + /** Absolute path to the SQLite database file. Created if absent. */ + readonly path: string; + readonly dimensions: number; + /** Maximum number of results a single `query` can return. Default 200. */ + readonly maxTopK?: number; +} + +// --------------------------------------------------------------------------- +// Internal types β€” typed just enough to satisfy the adapter without shipping +// full better-sqlite3 types as a runtime dep (they're devDependencies only). +// --------------------------------------------------------------------------- + +interface SqliteStatement { + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + run(...args: any[]): { readonly lastInsertRowid: bigint | number }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + get(...args: any[]): unknown; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + all(...args: any[]): unknown[]; +} + +interface SqliteDatabase { + exec(sql: string): void; + prepare(sql: string): SqliteStatement; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + pragma(pragma: string): any; + close(): void; +} + +interface SqliteVec { + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- native addon + load(db: any): void; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** JSON-encode a numeric array for the MATCH param sqlite-vec expects. */ +const vectorToSql = (vector: readonly number[]): string => JSON.stringify(Array.from(vector)); + +const openDatabase = async (options: SqliteVecStoreOptions): Promise => { + // Dynamic imports keep native addons out of the Cloudflare bundle. + const [BetterSqlite3Mod, sqliteVecMod] = await Promise.all([ + import("better-sqlite3"), + import("sqlite-vec"), + ]); + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary: default export shape differs across bundlers + const DatabaseCtor = (BetterSqlite3Mod as any).default ?? BetterSqlite3Mod; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary: sqlite-vec ships no compatible TS types for this usage + // oxlint-disable-next-line executor/no-double-cast -- adapter boundary: sqlite-vec ships no TS types compatible with this usage; intermediate `any` is the only escape hatch + const sqliteVec = sqliteVecMod as any as SqliteVec; + + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: better-sqlite3 throws synchronously + const db: SqliteDatabase = new DatabaseCtor(options.path) as SqliteDatabase; + + db.pragma("journal_mode = WAL"); + db.pragma("foreign_keys = ON"); + + sqliteVec.load(db); + + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS embedding_vectors USING vec0( + embedding float[${options.dimensions}] + ); + + CREATE TABLE IF NOT EXISTS vectors ( + id TEXT PRIMARY KEY, + vector_rowid INTEGER NOT NULL UNIQUE, + namespace TEXT NOT NULL DEFAULT '', + metadata_json TEXT NOT NULL DEFAULT '{}' + ); + `); + + return db; +}; + +// --------------------------------------------------------------------------- +// Upsert helpers β€” separated so they can be composed inside a transaction +// --------------------------------------------------------------------------- + +const buildUpsertStatements = (db: SqliteDatabase) => { + const rowForId = db.prepare("SELECT vector_rowid AS vectorRowid FROM vectors WHERE id = ?"); + const deleteVector = db.prepare("DELETE FROM embedding_vectors WHERE rowid = ?"); + const deleteBridge = db.prepare("DELETE FROM vectors WHERE id = ?"); + const insertVector = db.prepare("INSERT INTO embedding_vectors(embedding) VALUES (?)"); + const insertBridge = db.prepare( + "INSERT INTO vectors (id, vector_rowid, namespace, metadata_json) VALUES (?, ?, ?, ?)", + ); + return { rowForId, deleteVector, deleteBridge, insertVector, insertBridge }; +}; + +// --------------------------------------------------------------------------- +// Public factory +// --------------------------------------------------------------------------- + +export const makeSqliteVecStore = (options: SqliteVecStoreOptions): VectorStore => { + const resolvedMaxTopK = options.maxTopK ?? 200; + let cached: Promise | null = null; + const getDb = Effect.tryPromise({ + try: () => (cached ??= openDatabase(options)), + catch: (cause) => + new SemanticSearchError({ + message: `sqlite-vec open/init failed at ${options.path}.`, + cause, + }), + }); + + return { + maxTopK: resolvedMaxTopK, + + upsert: (vectors: readonly VectorInput[]) => + vectors.length === 0 + ? Effect.void + : getDb.pipe( + Effect.flatMap((db) => + Effect.try({ + try: () => { + const stmts = buildUpsertStatements(db); + // Atomic batch: each vector is delete+delete+insert+insert across two + // tables, so a mid-loop failure must not leave the index half-written. + db.exec("BEGIN"); + let committed = false; + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: roll the batch back on partial failure + try { + for (const v of vectors) { + // Delete existing row if present (vector + bridge). + const existing = stmts.rowForId.get(v.id) as + | { readonly vectorRowid?: unknown } + | undefined; + if (typeof existing?.vectorRowid === "number") { + stmts.deleteVector.run(existing.vectorRowid); + } + stmts.deleteBridge.run(v.id); + + // Insert new vector and capture its rowid. + const result = stmts.insertVector.run(vectorToSql(v.values)); + const vectorRowid = Number(result.lastInsertRowid); + + // Insert bridge row. + stmts.insertBridge.run( + v.id, + vectorRowid, + v.namespace ?? "", + JSON.stringify(v.metadata ?? {}), + ); + } + db.exec("COMMIT"); + committed = true; + } finally { + if (!committed) db.exec("ROLLBACK"); + } + }, + catch: (cause) => + new SemanticSearchError({ message: "sqlite-vec upsert failed.", cause }), + }), + ), + Effect.asVoid, + ), + + query: ({ vector, namespace, topK }) => + getDb.pipe( + Effect.flatMap((db) => + Effect.try({ + try: () => { + // Over-fetch so namespace post-filter still yields topK results. + const fetchK = Math.max(topK * 4, resolvedMaxTopK); + const rows = db + .prepare( + `SELECT + b.id AS id, + b.namespace AS namespace, + b.metadata_json AS metadataJson, + v.distance AS distance + FROM embedding_vectors v + JOIN vectors b ON b.vector_rowid = v.rowid + WHERE v.embedding MATCH ? + AND k = ? + ORDER BY v.distance ASC`, + ) + .all(vectorToSql(vector), fetchK) as ReadonlyArray<{ + readonly id: string; + readonly namespace: string; + readonly metadataJson: string; + readonly distance: number; + }>; + + const out: VectorMatch[] = []; + for (const row of rows) { + if (row.namespace !== namespace) continue; + let metadata: Record | undefined; + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: decoding stored metadataJson string + try { + // oxlint-disable-next-line executor/no-json-parse -- adapter boundary: decoding stored metadataJson string + metadata = JSON.parse(row.metadataJson) as Record; + } catch { + metadata = undefined; + } + out.push({ + id: row.id, + score: 1 / (1 + row.distance), + namespace: row.namespace, + metadata, + }); + if (out.length >= topK) break; + } + return out as readonly VectorMatch[]; + }, + catch: (cause) => + new SemanticSearchError({ message: "sqlite-vec query failed.", cause }), + }), + ), + ), + + deleteByIds: (ids: readonly string[]) => + ids.length === 0 + ? Effect.void + : getDb.pipe( + Effect.flatMap((db) => + Effect.try({ + try: () => { + const rowForId = db.prepare( + "SELECT vector_rowid AS vectorRowid FROM vectors WHERE id = ?", + ); + const deleteVector = db.prepare("DELETE FROM embedding_vectors WHERE rowid = ?"); + const deleteBridge = db.prepare("DELETE FROM vectors WHERE id = ?"); + for (const id of ids) { + const existing = rowForId.get(id) as + | { readonly vectorRowid?: unknown } + | undefined; + if (typeof existing?.vectorRowid === "number") { + deleteVector.run(existing.vectorRowid); + } + deleteBridge.run(id); + } + }, + catch: (cause) => + new SemanticSearchError({ message: "sqlite-vec delete failed.", cause }), + }), + ), + Effect.asVoid, + ), + }; +}; diff --git a/packages/plugins/vectorize-search/src/sdk/store-zvec.ts b/packages/plugins/semantic-search/src/sdk/store-zvec.ts similarity index 74% rename from packages/plugins/vectorize-search/src/sdk/store-zvec.ts rename to packages/plugins/semantic-search/src/sdk/store-zvec.ts index e2f781cb5b..9a30a03c55 100644 --- a/packages/plugins/vectorize-search/src/sdk/store-zvec.ts +++ b/packages/plugins/semantic-search/src/sdk/store-zvec.ts @@ -1,12 +1,12 @@ import { Effect } from "effect"; -import { VectorizeSearchError } from "./errors"; -import type { VectorizeMatch, VectorizeStore, VectorizeVectorInput } from "./vectorize"; +import { SemanticSearchError } from "./errors"; +import type { VectorMatch, VectorStore, VectorInput } from "./store"; // --------------------------------------------------------------------------- -// zvec-backed VectorizeStore β€” a local, in-process, file-backed ANN index +// zvec-backed VectorStore β€” a local, in-process, file-backed ANN index // (@zvec/zvec, HNSW + cosine). A real local backend for developing + bench- -// marking the search logic without Cloudflare; the same `VectorizeStore` shape +// marking the search logic without Cloudflare; the same `VectorStore` shape // the production Vectorize binding satisfies. // // Two zvec specifics handled here: @@ -24,6 +24,11 @@ export interface ZVecStoreOptions { /** Collection directory on disk. */ readonly path: string; readonly dimensions: number; + /** Index type. "flat" = exact brute-force (lossless, fast for catalogs up to + * ~tens of thousands of vectors β€” the right default for tool search). "hnsw" = + * approximate ANN, only worth it past ~100k vectors (and can lose recall at + * small scale). Default "flat". */ + readonly index?: "flat" | "hnsw"; readonly hnswM?: number; readonly hnswEfConstruction?: number; readonly hnswEf?: number; @@ -43,25 +48,33 @@ interface ZVecCollection { interface OpenedCollection { readonly coll: ZVecCollection; - readonly ef: number; - readonly indexType: unknown; + /** Query-time params for HNSW (indexType + ef); undefined for FLAT (exact β€” + * zvec rejects a FLAT indexType in QueryParams). */ + readonly queryParams: Record | undefined; } const openCollection = async (options: ZVecStoreOptions): Promise => { // eslint-disable-next-line @typescript-eslint/no-explicit-any -- native addon, no shipped types const zvec = (await import("@zvec/zvec")) as any; + const useHnsw = options.index === "hnsw"; + const indexParams = useHnsw + ? { + indexType: zvec.ZVecIndexType.HNSW, + metricType: zvec.ZVecMetricType.COSINE, + m: options.hnswM ?? 32, + efConstruction: options.hnswEfConstruction ?? 200, + } + : { + indexType: zvec.ZVecIndexType.FLAT, + metricType: zvec.ZVecMetricType.COSINE, + }; const schema = new zvec.ZVecCollectionSchema({ name: "vectors", vectors: { name: "embedding", dataType: zvec.ZVecDataType.VECTOR_FP32, dimension: options.dimensions, - indexParams: { - indexType: zvec.ZVecIndexType.HNSW, - metricType: zvec.ZVecMetricType.COSINE, - m: options.hnswM ?? 32, - efConstruction: options.hnswEfConstruction ?? 200, - }, + indexParams, }, fields: [ { name: "namespace", dataType: zvec.ZVecDataType.STRING }, @@ -80,15 +93,20 @@ const openCollection = async (options: ZVecStoreOptions): Promise { - const out: VectorizeMatch[] = []; +): readonly VectorMatch[] => { + const out: VectorMatch[] = []; for (const row of rows) { const ns = String(row.fields?.namespace ?? ""); if (ns !== namespace) continue; @@ -106,16 +124,18 @@ const toMatches = ( return out; }; -export const makeZVecStore = (options: ZVecStoreOptions): VectorizeStore => { +export const makeZVecStore = (options: ZVecStoreOptions): VectorStore => { let cached: Promise | null = null; const getCollection = Effect.tryPromise({ try: () => (cached ??= openCollection(options)), catch: (cause) => - new VectorizeSearchError({ message: `zvec open/create failed at ${options.path}.`, cause }), + new SemanticSearchError({ message: `zvec open/create failed at ${options.path}.`, cause }), }); return { - upsert: (vectors: readonly VectorizeVectorInput[]) => + // zvec/HNSW has no hard metadata-fetch cap β€” expose a generous limit. + maxTopK: 200, + upsert: (vectors: readonly VectorInput[]) => vectors.length === 0 ? Effect.void : getCollection.pipe( @@ -133,7 +153,7 @@ export const makeZVecStore = (options: ZVecStoreOptions): VectorizeStore => { })), ), catch: (cause) => - new VectorizeSearchError({ message: "zvec upsert failed.", cause }), + new SemanticSearchError({ message: "zvec upsert failed.", cause }), }), ), Effect.asVoid, @@ -150,9 +170,10 @@ export const makeZVecStore = (options: ZVecStoreOptions): VectorizeStore => { // Over-fetch so the namespace post-filter still yields topK. topk: Math.max(topK * 4, 40), outputFields: ["namespace", "metadataJson"], - params: { indexType: opened.indexType, ef: opened.ef }, + // FLAT (exact) takes no query params; HNSW takes indexType + ef. + ...(opened.queryParams ? { params: opened.queryParams } : {}), }), - catch: (cause) => new VectorizeSearchError({ message: "zvec query failed.", cause }), + catch: (cause) => new SemanticSearchError({ message: "zvec query failed.", cause }), }).pipe(Effect.map((rows) => toMatches(rows, namespace, topK))), ), ), @@ -165,7 +186,7 @@ export const makeZVecStore = (options: ZVecStoreOptions): VectorizeStore => { Effect.try({ try: () => opened.coll.deleteSync([...ids]), catch: (cause) => - new VectorizeSearchError({ message: "zvec delete failed.", cause }), + new SemanticSearchError({ message: "zvec delete failed.", cause }), }), ), Effect.asVoid, diff --git a/packages/plugins/semantic-search/src/sdk/store.ts b/packages/plugins/semantic-search/src/sdk/store.ts new file mode 100644 index 0000000000..3afcd039af --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/store.ts @@ -0,0 +1,35 @@ +import { Effect } from "effect"; + +import { SemanticSearchError } from "./errors"; + +export interface VectorMatches { + readonly matches: readonly VectorMatch[]; + readonly count?: number; +} + +export interface VectorMatch { + readonly id: string; + readonly score: number; + readonly namespace?: string; + readonly metadata?: Record; +} + +export interface VectorInput { + readonly id: string; + readonly values: readonly number[]; + readonly namespace?: string; + readonly metadata?: Record; +} + +/** Effect-wrapped, plugin-facing view over a vector store backend. */ +export interface VectorStore { + /** Maximum number of results a single `query` can return for this backend. */ + readonly maxTopK: number; + readonly query: (input: { + readonly vector: readonly number[]; + readonly namespace: string; + readonly topK: number; + }) => Effect.Effect; + readonly upsert: (vectors: readonly VectorInput[]) => Effect.Effect; + readonly deleteByIds: (ids: readonly string[]) => Effect.Effect; +} diff --git a/packages/plugins/vectorize-search/tsconfig.json b/packages/plugins/semantic-search/tsconfig.json similarity index 100% rename from packages/plugins/vectorize-search/tsconfig.json rename to packages/plugins/semantic-search/tsconfig.json diff --git a/packages/plugins/vectorize-search/tsup.config.ts b/packages/plugins/semantic-search/tsup.config.ts similarity index 100% rename from packages/plugins/vectorize-search/tsup.config.ts rename to packages/plugins/semantic-search/tsup.config.ts diff --git a/packages/plugins/vectorize-search/vitest.config.ts b/packages/plugins/semantic-search/vitest.config.ts similarity index 100% rename from packages/plugins/vectorize-search/vitest.config.ts rename to packages/plugins/semantic-search/vitest.config.ts diff --git a/packages/plugins/vectorize-search/src/api/index.ts b/packages/plugins/vectorize-search/src/api/index.ts deleted file mode 100644 index 2d0d6474eb..0000000000 --- a/packages/plugins/vectorize-search/src/api/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { definePlugin } from "@executor-js/sdk/core"; - -import { vectorizeSearchPlugin, type VectorizeSearchPluginOptions } from "../sdk/plugin"; -import { VectorizeSearchGroup } from "./group"; -import { VectorizeSearchExtensionService, VectorizeSearchHandlers } from "./handlers"; - -export { VectorizeSearchGroup, ReindexResponse } from "./group"; -export { VectorizeSearchHandlers, VectorizeSearchExtensionService } from "./handlers"; - -// HTTP-augmented variant of `vectorizeSearchPlugin`. The returned plugin carries -// the reindex `routes`/`handlers`/`extensionService` so a host can mount the -// reindex endpoint. Hosts that compose an `HttpApi` import this; SDK-only -// consumers stay on `@executor-js/plugin-vectorize-search` and never load -// `@executor-js/api`. -export const vectorizeSearchHttpPlugin = definePlugin((options?: VectorizeSearchPluginOptions) => ({ - ...vectorizeSearchPlugin(options), - routes: () => VectorizeSearchGroup, - handlers: () => VectorizeSearchHandlers, - extensionService: VectorizeSearchExtensionService, -})); diff --git a/packages/plugins/vectorize-search/src/sdk/documents.ts b/packages/plugins/vectorize-search/src/sdk/documents.ts deleted file mode 100644 index fcd9026388..0000000000 --- a/packages/plugins/vectorize-search/src/sdk/documents.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { Executor } from "@executor-js/sdk/core"; -import { Effect } from "effect"; - -import type { ToolDocumentInput } from "./chunker"; -import { VectorizeSearchError } from "./errors"; - -const ADDRESS_PREFIX = "tools."; - -/** Strip the proxy-root `tools.` prefix so the address becomes the - * sandbox-callable path the model writes after `tools.` β€” mirrors the engine's - * own `addressToPath` so the `path` we index matches what `tools.search` - * callers expect (and can pass back to `describe`/invoke). */ -export const addressToPath = (address: string): string => - address.startsWith(ADDRESS_PREFIX) ? address.slice(ADDRESS_PREFIX.length) : address; - -/** Collect the full `ToolDocumentInput` set from the live catalog. - * - * For each tool returned by `tools.list`, we attempt to fetch its schema via - * `tools.schema(address)` to populate the TypeScript facets (inputTypeScript, - * outputTypeScript, typeScriptDefinitions). If the schema fetch fails for any - * individual tool we degrade gracefully to an identity-only document (no TS - * fields) β€” never failing the whole collection for one tool. - * - * Bounded concurrency (16) keeps the catalog walk fast while avoiding - * unbounded fan-out. */ -export const collectToolDocumentInputs = ( - namespace: string, - executor: Executor, -): Effect.Effect => - executor.tools.list({ includeAnnotations: false }).pipe( - Effect.mapError( - (cause) => new VectorizeSearchError({ message: "Failed to list tools for indexing.", cause }), - ), - Effect.flatMap((tools) => - Effect.forEach( - tools, - (tool) => { - const path = addressToPath(String(tool.address)); - const base: ToolDocumentInput = { - path, - name: String(tool.name), - integration: String(tool.integration), - description: tool.description, - }; - // Attempt schema fetch; on any failure degrade to identity-only. - return executor.tools.schema(tool.address).pipe( - Effect.map((view): ToolDocumentInput => { - if (view === null) return base; - return { - ...base, - ...(view.inputTypeScript !== undefined - ? { inputTypeScript: view.inputTypeScript } - : {}), - ...(view.outputTypeScript !== undefined - ? { outputTypeScript: view.outputTypeScript } - : {}), - ...(view.typeScriptDefinitions !== undefined - ? { typeScriptDefinitions: view.typeScriptDefinitions } - : {}), - }; - }), - // Degrade: schema fetch failed β€” use identity-only document. - Effect.catch(() => Effect.succeed(base)), - ); - }, - { concurrency: 16 }, - ), - ), - ); diff --git a/packages/plugins/vectorize-search/src/sdk/eval-golden.ts b/packages/plugins/vectorize-search/src/sdk/eval-golden.ts deleted file mode 100644 index 8782a41689..0000000000 --- a/packages/plugins/vectorize-search/src/sdk/eval-golden.ts +++ /dev/null @@ -1,102 +0,0 @@ -// --------------------------------------------------------------------------- -// Eval golden set β€” expected top tool path per query. -// -// Used by eval.ts to compute precision@1 and precision@5 against the embedded -// + ranked results. Keep it small and representative: a few clear-intent -// queries and a few that require deeper semantic matching (e.g. sub-features -// hidden in long descriptions or input schemas). -// --------------------------------------------------------------------------- - -export interface GoldenQuery { - /** The natural-language query used for retrieval. */ - readonly query: string; - /** The canonical tool path that should rank in the top result. */ - readonly expectedPath: string; - /** The top-5 set that ALL contain relevant results (for precision@5). */ - readonly expectedTop5: readonly string[]; -} - -export const GOLDEN: readonly GoldenQuery[] = [ - // --- Google Calendar --- - { - query: "create a calendar event", - expectedPath: "tools.google.org.x.calendar.events.insert", - expectedTop5: [ - "tools.google.org.x.calendar.events.insert", - "tools.google.org.x.calendar.events.quickAdd", - ], - }, - { - query: "delete a calendar event", - expectedPath: "tools.google.org.x.calendar.events.delete", - expectedTop5: ["tools.google.org.x.calendar.events.delete"], - }, - - // --- Gmail --- - { - query: "send an email to recipients", - expectedPath: "tools.google.org.x.gmail.users.messages.send", - expectedTop5: ["tools.google.org.x.gmail.users.messages.send"], - }, - - // --- GitHub --- - { - query: "open a bug report on a repository", - expectedPath: "tools.github.org.x.repos.issues.create", - expectedTop5: [ - "tools.github.org.x.repos.issues.create", - "tools.github.org.x.repos.pulls.create", - ], - }, - { - query: "make a pull request", - expectedPath: "tools.github.org.x.repos.pulls.create", - expectedTop5: [ - "tools.github.org.x.repos.pulls.create", - "tools.github.org.x.repos.issues.create", - ], - }, - - // --- Stripe charges + refunds --- - { - query: "charge a credit card", - expectedPath: "tools.stripe.org.x.charges.create", - expectedTop5: ["tools.stripe.org.x.charges.create", "tools.stripe.org.x.customers.create"], - }, - { - query: "give a refund for a payment", - expectedPath: "tools.stripe.org.x.refunds.create", - expectedTop5: ["tools.stripe.org.x.refunds.create"], - }, - - // --- Stripe subscriptions β€” queries that rely on rich description + input facets --- - { - query: "start a free trial on a subscription", - expectedPath: "tools.stripe.org.x.subscriptions.create", - expectedTop5: ["tools.stripe.org.x.subscriptions.create"], - }, - { - query: "cancel a subscription at end of billing period", - expectedPath: "tools.stripe.org.x.subscriptions.create", - expectedTop5: ["tools.stripe.org.x.subscriptions.create"], - }, - { - query: "prorate a plan change mid-cycle", - expectedPath: "tools.stripe.org.x.subscriptions.create", - expectedTop5: ["tools.stripe.org.x.subscriptions.create"], - }, - - // --- Slack --- - { - query: "send a message to a Slack channel", - expectedPath: "tools.slack.org.x.chat.postMessage", - expectedTop5: ["tools.slack.org.x.chat.postMessage"], - }, - - // --- Linear --- - { - query: "create a new Linear issue", - expectedPath: "tools.linear.org.x.issues.create", - expectedTop5: ["tools.linear.org.x.issues.create", "tools.github.org.x.repos.issues.create"], - }, -]; From d5ce1d91cb74819360433a6a4ed0e72648cfa1fd Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Wed, 17 Jun 2026 02:12:42 +0530 Subject: [PATCH 34/57] test(host-selfhost): quarantine flaky concurrent-identity isolation test in CI (#31) Skip the timing-sensitive scope-isolation concurrency test under CI via test.skipIf(!!process.env.CI); it intermittently exceeds the 15s budget on shared runners and reds CI for unrelated PRs. Still runs locally, so the cross-fiber identity-leak guarantee stays covered. --- .../host-selfhost/src/scope-isolation.test.ts | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/apps/host-selfhost/src/scope-isolation.test.ts b/apps/host-selfhost/src/scope-isolation.test.ts index 4880c0c258..47acf636fd 100644 --- a/apps/host-selfhost/src/scope-isolation.test.ts +++ b/apps/host-selfhost/src/scope-isolation.test.ts @@ -91,35 +91,44 @@ const listConnectionAddresses = async ( return { status: res.status, addresses: body.map((c) => c.address) }; }; -test("concurrent requests with distinct identities get disjoint, correct executor bindings", async () => { - // 6 identities, each its own (org, user). Seed each sequentially, then fire 48 - // interleaved reads over the one long-lived SQLite handle. - const identities = Array.from({ length: 6 }, (_, i) => ({ - userId: `user-${i}`, - organizationId: `org-${i}`, - })); +// Quarantined in CI: seeding 6 identities sequentially and firing 48 interleaved +// reads over one long-lived SQLite handle is timing-sensitive and intermittently +// blows the 15s budget on shared CI runners (the assertions never fail β€” it just +// runs out of wall clock). It still runs locally, so the cross-fiber identity-leak +// guarantee stays covered; only the CI gate is relaxed. +test.skipIf(!!process.env.CI)( + "concurrent requests with distinct identities get disjoint, correct executor bindings", + async () => { + // 6 identities, each its own (org, user). Seed each sequentially, then fire 48 + // interleaved reads over the one long-lived SQLite handle. + const identities = Array.from({ length: 6 }, (_, i) => ({ + userId: `user-${i}`, + organizationId: `org-${i}`, + })); - for (const id of identities) { - await seedIdentity(id.userId, id.organizationId); - } + for (const id of identities) { + await seedIdentity(id.userId, id.organizationId); + } - const requests = Array.from({ length: 48 }, (_, i) => identities[i % identities.length]); - const results = await Promise.all( - requests.map((id) => listConnectionAddresses(id.userId, id.organizationId)), - ); + const requests = Array.from({ length: 48 }, (_, i) => identities[i % identities.length]); + const results = await Promise.all( + requests.map((id) => listConnectionAddresses(id.userId, id.organizationId)), + ); - results.forEach((result, i) => { - const { userId } = requests[i]; - // Each response reflects ONLY its own request's identity β€” no bleed. The - // subject's own user connection is present, and no OTHER subject's is. - expect(result.status).toBe(200); - expect(result.addresses.some((a) => a.includes(connectionNameForUser(userId)))).toBe(true); - const otherUsers = identities.map((id) => id.userId).filter((u) => u !== userId); - for (const other of otherUsers) { - expect(result.addresses.some((a) => a.includes(connectionNameForUser(other)))).toBe(false); - } - }); -}, 15_000); + results.forEach((result, i) => { + const { userId } = requests[i]; + // Each response reflects ONLY its own request's identity β€” no bleed. The + // subject's own user connection is present, and no OTHER subject's is. + expect(result.status).toBe(200); + expect(result.addresses.some((a) => a.includes(connectionNameForUser(userId)))).toBe(true); + const otherUsers = identities.map((id) => id.userId).filter((u) => u !== userId); + for (const other of otherUsers) { + expect(result.addresses.some((a) => a.includes(connectionNameForUser(other)))).toBe(false); + } + }); + }, + 15_000, +); test("a request with no identity is rejected", async () => { const res = await handler(new Request("http://localhost/api/connections")); From 723cf0f372d810d41b7f183c72c9508c6320b93f Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Wed, 17 Jun 2026 03:11:53 +0530 Subject: [PATCH 35/57] fix(package): remove duplicate entry for 'effect' in dependencies and peerDependencies --- packages/plugins/semantic-search/package.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/plugins/semantic-search/package.json b/packages/plugins/semantic-search/package.json index b79982a7f8..d64e95c9b4 100644 --- a/packages/plugins/semantic-search/package.json +++ b/packages/plugins/semantic-search/package.json @@ -17,8 +17,7 @@ "dependencies": { "@ai-sdk/google": "^3.0.80", "@executor-js/sdk": "workspace:*", - "ai": "^6.0.191", - "effect": "catalog:" + "ai": "^6.0.191" }, "devDependencies": { "@cloudflare/workers-types": "^4.20250620.0", @@ -28,11 +27,13 @@ "@types/node": "catalog:", "@zvec/zvec": "^0.4.1", "bun-types": "catalog:", + "effect": "catalog:", "tsup": "catalog:", "vitest": "catalog:" }, "peerDependencies": { - "@executor-js/api": "workspace:*" + "@executor-js/api": "workspace:*", + "effect": "catalog:" }, "peerDependenciesMeta": { "@executor-js/api": { From c3e48e01fe59004c69efd9552ed41bf26a80f7a5 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Wed, 17 Jun 2026 15:54:01 +0530 Subject: [PATCH 36/57] feat(plugin-semantic-search): D1-backed FTS5 lexical store + hybrid reindex population (#32) Add makeD1FtsLexicalStore (async FTS5 over Cloudflare D1, sharing the schema/queries with the sync better-sqlite3/bun:sqlite adapter) and wire reconcileToolCatalog to populate an optional lexicalStore, so a Cloudflare host gets real Vectorize + FTS5/BM25 hybrid tools.search. Lexical doc ids are namespace-scoped; the lexical index is rebuilt in full per reindex (complete even when attached to an already-indexed deployment); D1 schema init is tracked via a WeakSet so a transient failure is never cached. --- bun.lock | 3 +- .../plugins/semantic-search/src/sdk/index.ts | 3 + .../semantic-search/src/sdk/indexer.test.ts | 83 +++++ .../semantic-search/src/sdk/indexer.ts | 30 +- .../plugins/semantic-search/src/sdk/plugin.ts | 24 +- .../src/sdk/store-fts-d1.test.ts | 347 ++++++++++++++++++ .../semantic-search/src/sdk/store-fts-d1.ts | 158 ++++++++ .../semantic-search/src/sdk/store-fts.ts | 134 ++++--- 8 files changed, 716 insertions(+), 66 deletions(-) create mode 100644 packages/plugins/semantic-search/src/sdk/store-fts-d1.test.ts create mode 100644 packages/plugins/semantic-search/src/sdk/store-fts-d1.ts diff --git a/bun.lock b/bun.lock index 1b93b3a471..b1e92c21f7 100644 --- a/bun.lock +++ b/bun.lock @@ -1088,7 +1088,6 @@ "@ai-sdk/google": "^3.0.80", "@executor-js/sdk": "workspace:*", "ai": "^6.0.191", - "effect": "catalog:", }, "devDependencies": { "@cloudflare/workers-types": "^4.20250620.0", @@ -1098,6 +1097,7 @@ "@types/node": "catalog:", "@zvec/zvec": "^0.4.1", "bun-types": "catalog:", + "effect": "catalog:", "tsup": "catalog:", "vitest": "catalog:", }, @@ -1107,6 +1107,7 @@ }, "peerDependencies": { "@executor-js/api": "workspace:*", + "effect": "catalog:", }, "optionalPeers": [ "@executor-js/api", diff --git a/packages/plugins/semantic-search/src/sdk/index.ts b/packages/plugins/semantic-search/src/sdk/index.ts index 1830bf9b93..68325518d0 100644 --- a/packages/plugins/semantic-search/src/sdk/index.ts +++ b/packages/plugins/semantic-search/src/sdk/index.ts @@ -56,5 +56,8 @@ export { type FtsSearchResult, } from "./store-fts"; +// FTS5 lexical store backed by Cloudflare D1 +export { makeD1FtsLexicalStore, type D1Database, type D1PreparedStatement } from "./store-fts-d1"; + // Documents projector helpers export { stripHtml, buildLexicalText } from "./documents"; diff --git a/packages/plugins/semantic-search/src/sdk/indexer.test.ts b/packages/plugins/semantic-search/src/sdk/indexer.test.ts index ff3facedf3..d1c5498eae 100644 --- a/packages/plugins/semantic-search/src/sdk/indexer.test.ts +++ b/packages/plugins/semantic-search/src/sdk/indexer.test.ts @@ -13,6 +13,7 @@ import { toolFingerprints } from "./collections"; import type { ToolEmbedder } from "./embedder"; import { reconcileToolCatalog } from "./indexer"; import type { VectorStore, VectorInput } from "./store"; +import type { FtsDocumentInput, FtsLexicalStore } from "./store-fts"; // --------------------------------------------------------------------------- // Test helpers @@ -76,6 +77,24 @@ const makeStore = () => { return { store, upserted, deleted }; }; +/** Build a fake FtsLexicalStore that records upserts/deletes. */ +const makeFakeLexical = () => { + const upserted: FtsDocumentInput[] = []; + const deleted: string[] = []; + const store: FtsLexicalStore = { + upsert: (docs) => + Effect.sync(() => { + upserted.push(...docs); + }), + deleteByIds: (ids) => + Effect.sync(() => { + deleted.push(...ids); + }), + search: () => Effect.succeed([]), + }; + return { store, upserted, deleted }; +}; + /** Build a fake PluginStorageCollectionFacade backed by an in-test Map. */ const makeFingerprints = ( initial: Map = new Map(), @@ -366,4 +385,68 @@ describe("reconcileToolCatalog", () => { expect(deleted2).toHaveLength(0); }), ); + + it.effect( + "populates the lexical store with every live tool, keyed by namespace-prefixed id", + () => + Effect.gen(function* () { + const fingerprints = makeFingerprints(); + const tools = [ + { + address: "tools.github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + }, + { + address: "tools.calendar.events.create", + name: "events.create", + integration: "calendar", + description: "Create a calendar event", + }, + ]; + + const lexical = makeFakeLexical(); + yield* reconcileToolCatalog({ + namespace, + executor: makeExecutor(tools), + embedder: fakeEmbedder, + store: makeStore().store, + chunker, + fingerprints, + owner, + lexicalStore: lexical.store, + }); + + // One lexical doc per tool, id namespace-prefixed so paths can't collide + // across namespaces in a shared store. + expect(lexical.upserted.map((d) => d.id).sort()).toEqual([ + `${namespace}:calendar.events.create`, + `${namespace}:github.repos.get`, + ]); + const repo = lexical.upserted.find((d) => d.id === `${namespace}:github.repos.get`); + expect(repo?.path).toBe("github.repos.get"); + expect(repo?.integration).toBe("github"); + expect(repo?.namespace).toBe(namespace); + expect((repo?.lexicalText ?? "").length).toBeGreaterThan(0); + + // A SECOND reindex with identical tools re-embeds nothing (fingerprints + // match) yet still fully populates the lexical store β€” so attaching a + // lexical store to an already-indexed deployment is not silently empty. + const lexical2 = makeFakeLexical(); + const result2 = yield* reconcileToolCatalog({ + namespace, + executor: makeExecutor(tools), + embedder: fakeEmbedder, + store: makeStore().store, + chunker, + fingerprints, + owner, + lexicalStore: lexical2.store, + }); + expect(result2.reembedded).toBe(0); + expect(result2.unchanged).toBe(2); + expect(lexical2.upserted).toHaveLength(2); + }), + ); }); diff --git a/packages/plugins/semantic-search/src/sdk/indexer.ts b/packages/plugins/semantic-search/src/sdk/indexer.ts index bfaa32c802..7f7087bab1 100644 --- a/packages/plugins/semantic-search/src/sdk/indexer.ts +++ b/packages/plugins/semantic-search/src/sdk/indexer.ts @@ -5,11 +5,12 @@ import { Effect } from "effect"; import type { Chunker } from "./chunker"; import type { FingerprintRow } from "./collections"; import { toolFingerprints } from "./collections"; -import { collectToolDocumentInputs } from "./documents"; +import { buildLexicalText, collectToolDocumentInputs } from "./documents"; import type { ToolEmbedder } from "./embedder"; import { SemanticSearchError } from "./errors"; import { fingerprintTool } from "./fingerprint"; import type { VectorStore, VectorInput } from "./store"; +import type { FtsDocumentInput, FtsLexicalStore } from "./store-fts"; // --------------------------------------------------------------------------- // ReconcileResult @@ -54,6 +55,10 @@ export const reconcileToolCatalog = (input: { readonly chunker: Chunker; readonly fingerprints: PluginStorageCollectionFacade; readonly owner: Owner; + /** Optional FTS5 lexical store. When present, each reindexed tool is also + * written here (one doc per tool) so a hybrid lexical index stays in lockstep + * with the vector index. */ + readonly lexicalStore?: FtsLexicalStore; }): Effect.Effect => Effect.gen(function* () { const { namespace, executor, embedder, store, chunker, fingerprints, owner } = input; @@ -68,6 +73,29 @@ export const reconcileToolCatalog = (input: { return { namespace, total: 0, reembedded: 0, unchanged: 0, removedSkipped: 0 }; } + // ------------------------------------------------------------------------- + // Step 1b β€” Populate the FTS5 lexical store with EVERY live tool, when one + // is configured. Unlike the vector index (gated below by the fingerprint + // diff to avoid re-embedding), the lexical store carries no embedding cost, + // so it is rebuilt in full on each reindex. That keeps it complete even + // when the vector index is unchanged or the lexical store was attached to a + // deployment whose vectors were already indexed. The id is namespace- + // prefixed so tools sharing a path across namespaces never collide in a + // shared D1 database. + // ------------------------------------------------------------------------- + if (input.lexicalStore) { + const lexicalDocs: readonly FtsDocumentInput[] = docs.map((doc) => ({ + id: `${namespace}:${doc.path}`, + namespace, + path: doc.path, + name: doc.name, + description: doc.description, + integration: doc.integration, + lexicalText: buildLexicalText(doc), + })); + yield* input.lexicalStore.upsert(lexicalDocs); + } + // ------------------------------------------------------------------------- // Step 2 β€” Fingerprint each live tool. // ------------------------------------------------------------------------- diff --git a/packages/plugins/semantic-search/src/sdk/plugin.ts b/packages/plugins/semantic-search/src/sdk/plugin.ts index 629b600842..c103e76662 100644 --- a/packages/plugins/semantic-search/src/sdk/plugin.ts +++ b/packages/plugins/semantic-search/src/sdk/plugin.ts @@ -9,6 +9,7 @@ import { makeHybridToolDiscoveryProvider } from "./hybrid"; import { reconcileToolCatalog, type ReconcileResult } from "./indexer"; import { makeVectorToolDiscoveryProvider } from "./provider"; import type { VectorStore } from "./store"; +import { type FtsLexicalStore, makeFtsLexicalProvider } from "./store-fts"; export interface SemanticSearchPluginOptions { /** A vector store β€” construct one with `makeVectorizeStore` (Cloudflare), @@ -39,6 +40,13 @@ export interface SemanticSearchPluginOptions { * lexical scorer does NOT run, because a plugin provider supersedes it). Supply * it to get hybrid lexical+vector search. */ readonly lexical?: ToolDiscoveryProvider; + /** A populated FTS5 lexical store β€” e.g. `makeD1FtsLexicalStore(env.DB)` on + * Cloudflare, or `makeFtsLexicalStore({ path })` locally. When supplied, + * `reindex` ALSO writes each tool's lexical document here, and the store is + * wrapped as the hybrid `lexical` provider β€” giving real FTS5/BM25 + vector + * RRF without the host owning lexical indexing. Takes precedence over + * `lexical`. */ + readonly lexicalStore?: FtsLexicalStore; } const notConfigured = (): Effect.Effect => @@ -61,6 +69,7 @@ const makeSemanticSearchExtension = (deps: { readonly chunker: Chunker; readonly fingerprints: Parameters[0]["fingerprints"] | undefined; readonly owner: Parameters[0]["owner"] | undefined; + readonly lexicalStore: FtsLexicalStore | undefined; }) => ({ reindex: (executor: Executor): Effect.Effect => deps.embedder && deps.store && deps.fingerprints && deps.owner @@ -72,6 +81,7 @@ const makeSemanticSearchExtension = (deps: { chunker: deps.chunker, fingerprints: deps.fingerprints, owner: deps.owner, + lexicalStore: deps.lexicalStore, }) : notConfigured(), }); @@ -107,6 +117,7 @@ export const semanticSearchPlugin = definePlugin((options?: SemanticSearchPlugin const store = options?.store; const chunker = options?.chunker ?? makeFacetChunker(); const lexical = options?.lexical; + const lexicalStore = options?.lexicalStore; return { id: "semanticSearch" as const, @@ -128,14 +139,21 @@ export const semanticSearchPlugin = definePlugin((options?: SemanticSearchPlugin chunker, fingerprints: ctx.storage.fingerprints, owner: ctx.storage.owner, + lexicalStore, }), runtime: { toolDiscoveryProvider: () => { if (!embedder || !store) return undefined; const vector = makeVectorToolDiscoveryProvider({ embedder, store, namespace }); - // Fuse with the host-supplied lexical provider (RRF) when present; else - // the vector provider answers alone. - return lexical ? makeHybridToolDiscoveryProvider({ lexical, vector }) : vector; + // Resolve the lexical side: a populated FTS5 store (wrapped as a provider) + // takes precedence over a host-supplied provider (e.g. the engine's token + // scorer). Fuse with the vector provider via RRF when either is present. + const lexicalProvider = lexicalStore + ? makeFtsLexicalProvider(lexicalStore, namespace) + : lexical; + return lexicalProvider + ? makeHybridToolDiscoveryProvider({ lexical: lexicalProvider, vector }) + : vector; }, }, }; diff --git a/packages/plugins/semantic-search/src/sdk/store-fts-d1.test.ts b/packages/plugins/semantic-search/src/sdk/store-fts-d1.test.ts new file mode 100644 index 0000000000..dea75ab933 --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/store-fts-d1.test.ts @@ -0,0 +1,347 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit } from "effect"; + +import { makeD1FtsLexicalStore, type D1Database, type D1PreparedStatement } from "./store-fts-d1"; +import type { FtsDocumentInput } from "./store-fts"; + +// --------------------------------------------------------------------------- +// D1 shim backed by better-sqlite3 +// +// Implements the minimal D1Database / D1PreparedStatement async API as a thin +// wrapper over an in-memory better-sqlite3 Database. The shim lets us run the +// exact adapter code against real SQLite FTS5 without needing a real D1 binding. +// --------------------------------------------------------------------------- + +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary +type BetterSqliteDb = any; + +const makeD1Shim = (db: BetterSqliteDb): D1Database => { + const makeStmt = (sql: string, boundValues: readonly unknown[]): D1PreparedStatement => ({ + bind(...values: unknown[]): D1PreparedStatement { + return makeStmt(sql, [...boundValues, ...values]); + }, + run(): Promise { + return Promise.resolve(db.prepare(sql).run(...boundValues)); + }, + all(): Promise<{ readonly results: readonly T[] }> { + const results: readonly T[] = db.prepare(sql).all(...boundValues) as readonly T[]; + return Promise.resolve({ results }); + }, + }); + + return { + prepare(sql: string): D1PreparedStatement { + return makeStmt(sql, []); + }, + + async batch(statements: readonly D1PreparedStatement[]): Promise { + db.exec("BEGIN"); + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: need rollback on batch failure + try { + const results: unknown[] = []; + for (const stmt of statements) { + results.push(await stmt.run()); + } + db.exec("COMMIT"); + return results; + } catch (err) { + db.exec("ROLLBACK"); + // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: re-throw after rollback + throw err; + } + }, + }; +}; + +// --------------------------------------------------------------------------- +// Test fixtures +// --------------------------------------------------------------------------- + +const doc = ( + id: string, + integration: string, + path: string, + name: string, + description: string, + lexicalText: string, + namespace = "default", +): FtsDocumentInput => ({ + id, + namespace, + integration, + path, + name, + description, + lexicalText, +}); + +// --------------------------------------------------------------------------- +// Helper: create a fresh in-memory D1-shimmed store per test +// --------------------------------------------------------------------------- + +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary +const makeTestStore = async (): Promise<{ + store: ReturnType; + db: any; +}> => { + const BetterSqlite3Mod = await import("better-sqlite3"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + const DatabaseCtor = (BetterSqlite3Mod as any).default ?? BetterSqlite3Mod; + const db = new DatabaseCtor(":memory:"); + const d1: D1Database = makeD1Shim(db); + const store = makeD1FtsLexicalStore(d1); + return { store, db }; +}; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("makeD1FtsLexicalStore", () => { + it.effect("upserts and searches β€” github doc ranks for 'create repo'", () => + Effect.gen(function* () { + const { store } = yield* Effect.promise(() => makeTestStore()); + + yield* store.upsert([ + doc( + "github:repos.create", + "github", + "github.repos.create", + "Create a repository", + "Creates a new GitHub repository for the authenticated user.", + "github repos create repository owner name private", + ), + doc( + "calendar:events.create", + "calendar", + "calendar.events.create", + "Create a calendar event", + "Creates a new event on the user's calendar.", + "calendar events create event start end summary", + ), + ]); + + const results = yield* store.search({ + query: "create repo", + namespace: "default", + topK: 10, + }); + + expect(results.length).toBeGreaterThan(0); + expect(results[0]!.path).toBe("github.repos.create"); + expect(results[0]!.integration).toBe("github"); + expect(typeof results[0]!.score).toBe("number"); + expect(results[0]!.score).toBeGreaterThan(0); + }), + ); + + it.effect("namespace partitioning β€” doc in namespace 'a' not returned for namespace 'b'", () => + Effect.gen(function* () { + const { store } = yield* Effect.promise(() => makeTestStore()); + + yield* store.upsert([ + doc( + "org-a:repos.list", + "github", + "github.repos.list", + "List repositories", + "Lists repositories for a user.", + "github repos list repository", + "a", + ), + doc( + "org-b:repos.create", + "github", + "github.repos.create", + "Create a repository", + "Creates a new GitHub repository.", + "github repos create repository", + "b", + ), + ]); + + const resultsA = yield* store.search({ query: "repos", namespace: "a", topK: 10 }); + const resultsB = yield* store.search({ query: "repos", namespace: "b", topK: 10 }); + + expect(resultsA.map((r) => r.path)).toEqual(["github.repos.list"]); + expect(resultsB.map((r) => r.path)).toEqual(["github.repos.create"]); + }), + ); + + it.effect("deleteByIds β€” removes a doc; subsequent search omits it", () => + Effect.gen(function* () { + const { store } = yield* Effect.promise(() => makeTestStore()); + + yield* store.upsert([ + doc( + "github:repos.create", + "github", + "github.repos.create", + "Create a repository", + "Creates a new GitHub repository.", + "github repos create", + ), + doc( + "github:repos.list", + "github", + "github.repos.list", + "List repositories", + "Lists repositories.", + "github repos list", + ), + ]); + + yield* store.deleteByIds(["github:repos.create"]); + + const results = yield* store.search({ query: "repos", namespace: "default", topK: 10 }); + const paths = results.map((r) => r.path); + + expect(paths).not.toContain("github.repos.create"); + expect(paths).toContain("github.repos.list"); + }), + ); + + it.effect("prefix matching β€” query 'repo' matches 'repos'", () => + Effect.gen(function* () { + const { store } = yield* Effect.promise(() => makeTestStore()); + + yield* store.upsert([ + doc( + "github:repos.get", + "github", + "github.repos.get", + "Get a repository", + "Returns information about a GitHub repository.", + "github repos get repository", + ), + ]); + + const results = yield* store.search({ query: "repo", namespace: "default", topK: 10 }); + + // prefix query 'repo*' must match 'repos' via FTS5 prefix matching + expect(results.length).toBeGreaterThan(0); + expect(results[0]!.path).toBe("github.repos.get"); + }), + ); + + it.effect("empty upsert is a no-op and does not fail", () => + Effect.gen(function* () { + const { store } = yield* Effect.promise(() => makeTestStore()); + yield* store.upsert([]); + const results = yield* store.search({ query: "anything", namespace: "default", topK: 10 }); + expect(results).toHaveLength(0); + }), + ); + + it.effect("empty deleteByIds is a no-op and does not fail", () => + Effect.gen(function* () { + const { store } = yield* Effect.promise(() => makeTestStore()); + yield* store.deleteByIds([]); + }), + ); + + it.effect("search with blank-ish query returns empty without error", () => + Effect.gen(function* () { + const { store } = yield* Effect.promise(() => makeTestStore()); + yield* store.upsert([ + doc( + "github:repos.get", + "github", + "github.repos.get", + "Get a repository", + "Returns information about a GitHub repository.", + "github repos get", + ), + ]); + // normalizeFtsQuery("") returns null β†’ should short-circuit to [] + const results = yield* store.search({ query: "", namespace: "default", topK: 10 }); + expect(results).toHaveLength(0); + }), + ); + + it.effect("upsert is idempotent β€” re-upsert same id replaces the doc", () => + Effect.gen(function* () { + const { store } = yield* Effect.promise(() => makeTestStore()); + + yield* store.upsert([ + doc( + "github:repos.create", + "github", + "github.repos.create", + "Create a repository", + "Creates a new GitHub repository.", + "github repos create repository", + ), + ]); + // Re-upsert with updated description + yield* store.upsert([ + doc( + "github:repos.create", + "github", + "github.repos.create", + "Create a repository (updated)", + "Updated description.", + "github repos create repository updated", + ), + ]); + + const results = yield* store.search({ query: "create repo", namespace: "default", topK: 10 }); + // Should still be exactly one result (not two) + const paths = results.map((r) => r.path); + expect(paths.filter((p) => p === "github.repos.create")).toHaveLength(1); + }), + ); + + it.effect("retries schema init after a transient failure (the rejection is not cached)", () => + Effect.gen(function* () { + const BetterSqlite3Mod = yield* Effect.promise(() => import("better-sqlite3")); + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- adapter boundary + const DatabaseCtor = (BetterSqlite3Mod as any).default ?? BetterSqlite3Mod; + const db = new DatabaseCtor(":memory:"); + + // A D1 shim whose very first run() rejects (a transient init failure), + // then behaves normally β€” exercises the "don't cache the rejection" path. + const base = makeD1Shim(db); + let injectedFailure = false; + const flaky: D1Database = { + prepare(sql: string): D1PreparedStatement { + const stmt = base.prepare(sql); + return { + bind: stmt.bind, + all: stmt.all, + run(): Promise { + if (!injectedFailure) { + injectedFailure = true; + // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- test shim: simulate a transient D1 runtime failure + return Promise.reject(new Error("transient D1 failure")); + } + return stmt.run(); + }, + }; + }, + batch: base.batch, + }; + const store = makeD1FtsLexicalStore(flaky); + + // First use fails because schema init hits the injected failure. + const firstExit = yield* Effect.exit( + store.search({ query: "repo", namespace: "default", topK: 10 }), + ); + expect(Exit.isFailure(firstExit)).toBe(true); + + // A subsequent use retries schema init (not cached as failed) and works. + yield* store.upsert([ + doc( + "github:repos.get", + "github", + "github.repos.get", + "Get a repository", + "Returns a repository.", + "github repos get repository", + ), + ]); + const results = yield* store.search({ query: "repo", namespace: "default", topK: 10 }); + expect(results.map((r) => r.path)).toContain("github.repos.get"); + }), + ); +}); diff --git a/packages/plugins/semantic-search/src/sdk/store-fts-d1.ts b/packages/plugins/semantic-search/src/sdk/store-fts-d1.ts new file mode 100644 index 0000000000..a5da3d0ef1 --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/store-fts-d1.ts @@ -0,0 +1,158 @@ +import { Effect } from "effect"; + +import { SemanticSearchError } from "./errors"; +import { + FTS_DELETE_BY_ID_SQL, + FTS_INSERT_SQL, + FTS_SCHEMA_STATEMENTS, + FTS_SEARCH_SQL, + type FtsDocumentInput, + type FtsLexicalStore, + type FtsSearchInput, + type FtsSearchResult, + normalizeFtsQuery, +} from "./store-fts"; + +// --------------------------------------------------------------------------- +// Minimal D1 binding types β€” declared locally so the package needs no +// `@cloudflare/workers-types` dependency. The shapes mirror the subset of +// the D1 API that this adapter uses. +// --------------------------------------------------------------------------- + +interface D1PreparedStatement { + bind(...values: unknown[]): D1PreparedStatement; + run(): Promise; + all(): Promise<{ readonly results: readonly T[] }>; +} + +interface D1Database { + prepare(sql: string): D1PreparedStatement; + batch(statements: readonly D1PreparedStatement[]): Promise; +} + +// Re-export the interface so hosts can type their binding without importing +// from `@cloudflare/workers-types`. +export type { D1Database, D1PreparedStatement }; + +// --------------------------------------------------------------------------- +// Schema init β€” applied once per D1Database instance. DDL runs sequentially +// (not via batch). Tracked in a WeakSet of already-initialised bindings rather +// than a cached init Promise, so a transient init failure is NEVER cached: the +// next call retries instead of leaving the store permanently unrecoverable for +// the isolate's lifetime. The statements are `CREATE … IF NOT EXISTS`, so the +// rare concurrent first-call re-run is idempotent. +// --------------------------------------------------------------------------- + +const initialized = new WeakSet(); + +// --------------------------------------------------------------------------- +// Public factory +// --------------------------------------------------------------------------- + +/** + * Creates an `FtsLexicalStore` backed by a Cloudflare D1 database. + * + * The D1 binding uses the same SQLite FTS5 schema as `makeFtsLexicalStore` + * (better-sqlite3/bun:sqlite variant), so index data is fully interchangeable. + * Schema DDL is applied lazily on first use, once per binding instance. + */ +export const makeD1FtsLexicalStore = (d1: D1Database): FtsLexicalStore => { + const ensureSchema = Effect.suspend(() => + initialized.has(d1) + ? Effect.void + : Effect.tryPromise({ + try: async () => { + for (const stmt of FTS_SCHEMA_STATEMENTS) { + await d1.prepare(stmt).run(); + } + }, + catch: (cause) => + new SemanticSearchError({ message: "D1 FTS5 schema init failed.", cause }), + }).pipe(Effect.tap(() => Effect.sync(() => initialized.add(d1)))), + ); + + return { + upsert: (docs: readonly FtsDocumentInput[]) => + docs.length === 0 + ? Effect.void + : ensureSchema.pipe( + Effect.flatMap(() => + Effect.tryPromise({ + try: async () => { + const stmts: D1PreparedStatement[] = []; + for (const doc of docs) { + stmts.push(d1.prepare(FTS_DELETE_BY_ID_SQL).bind(doc.id)); + stmts.push( + d1 + .prepare(FTS_INSERT_SQL) + .bind( + doc.id, + doc.namespace, + doc.path, + doc.name, + doc.description, + doc.integration, + doc.lexicalText, + ), + ); + } + await d1.batch(stmts); + }, + catch: (cause) => + new SemanticSearchError({ message: "D1 FTS5 upsert failed.", cause }), + }), + ), + Effect.asVoid, + ), + + deleteByIds: (ids: readonly string[]) => + ids.length === 0 + ? Effect.void + : ensureSchema.pipe( + Effect.flatMap(() => + Effect.tryPromise({ + try: async () => { + await d1.batch(ids.map((id) => d1.prepare(FTS_DELETE_BY_ID_SQL).bind(id))); + }, + catch: (cause) => + new SemanticSearchError({ message: "D1 FTS5 deleteByIds failed.", cause }), + }), + ), + Effect.asVoid, + ), + + search: ({ query, namespace, topK }: FtsSearchInput) => + ensureSchema.pipe( + Effect.flatMap(() => + Effect.tryPromise({ + try: async () => { + const ftsQuery = normalizeFtsQuery(query); + if (ftsQuery === null) return [] as readonly FtsSearchResult[]; + + const { results } = await d1 + .prepare(FTS_SEARCH_SQL) + .bind(ftsQuery, namespace, topK) + .all<{ + readonly path: string; + readonly name: string; + readonly description: string; + readonly integration: string; + readonly score: number; + }>(); + + return results.map( + (r): FtsSearchResult => ({ + path: r.path, + name: r.name, + description: r.description, + integration: r.integration, + score: r.score, + }), + ); + }, + catch: (cause) => new SemanticSearchError({ message: "D1 FTS5 search failed.", cause }), + }), + ), + ), + }; +}; diff --git a/packages/plugins/semantic-search/src/sdk/store-fts.ts b/packages/plugins/semantic-search/src/sdk/store-fts.ts index ae4c96ac68..91b75500ce 100644 --- a/packages/plugins/semantic-search/src/sdk/store-fts.ts +++ b/packages/plugins/semantic-search/src/sdk/store-fts.ts @@ -65,12 +65,79 @@ interface BunSqliteDatabase { close(): void; } +// --------------------------------------------------------------------------- +// Shared SQL constants β€” imported by D1 and other adapters so the schema and +// query strings stay in exactly one place. +// --------------------------------------------------------------------------- + +/** Schema statements in dependency order: content table β†’ FTS5 virtual table β†’ 3 triggers. */ +export const FTS_SCHEMA_STATEMENTS: readonly string[] = [ + `CREATE TABLE IF NOT EXISTS fts_docs ( + id TEXT PRIMARY KEY, + namespace TEXT NOT NULL DEFAULT '', + path TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + integration TEXT NOT NULL, + lexical_text TEXT NOT NULL + )`, + `CREATE VIRTUAL TABLE IF NOT EXISTS fts_docs_fts USING fts5( + path, + integration, + name, + description, + lexical_text, + content='fts_docs', + content_rowid='rowid', + tokenize='porter unicode61' + )`, + `CREATE TRIGGER IF NOT EXISTS fts_docs_ai AFTER INSERT ON fts_docs BEGIN + INSERT INTO fts_docs_fts(rowid, path, integration, name, description, lexical_text) + VALUES (new.rowid, new.path, new.integration, new.name, new.description, new.lexical_text); + END`, + `CREATE TRIGGER IF NOT EXISTS fts_docs_ad AFTER DELETE ON fts_docs BEGIN + INSERT INTO fts_docs_fts(fts_docs_fts, rowid, path, integration, name, description, lexical_text) + VALUES ('delete', old.rowid, old.path, old.integration, old.name, old.description, old.lexical_text); + END`, + `CREATE TRIGGER IF NOT EXISTS fts_docs_au AFTER UPDATE ON fts_docs BEGIN + INSERT INTO fts_docs_fts(fts_docs_fts, rowid, path, integration, name, description, lexical_text) + VALUES ('delete', old.rowid, old.path, old.integration, old.name, old.description, old.lexical_text); + INSERT INTO fts_docs_fts(rowid, path, integration, name, description, lexical_text) + VALUES (new.rowid, new.path, new.integration, new.name, new.description, new.lexical_text); + END`, +]; + +/** DELETE a single document by id. */ +export const FTS_DELETE_BY_ID_SQL = "DELETE FROM fts_docs WHERE id = ?"; + +/** INSERT a document (delete-then-insert upsert pattern). */ +export const FTS_INSERT_SQL = `INSERT INTO fts_docs (id, namespace, path, name, description, integration, lexical_text) + VALUES (?, ?, ?, ?, ?, ?, ?)`; + +/** + * BM25 search query β€” weights: path=12, integration=8, name=10, description=5, lexical_text=3. + * Binds: (ftsQuery TEXT, namespace TEXT, topK INTEGER). + * Returns columns: path, name, description, integration, score (higher = better). + */ +export const FTS_SEARCH_SQL = `SELECT + d.path AS path, + d.name AS name, + d.description AS description, + d.integration AS integration, + 0 - bm25(fts_docs_fts, 12.0, 8.0, 10.0, 5.0, 3.0) AS score + FROM fts_docs_fts + JOIN fts_docs d ON d.rowid = fts_docs_fts.rowid + WHERE fts_docs_fts MATCH ? + AND d.namespace = ? + ORDER BY score DESC + LIMIT ?`; + // --------------------------------------------------------------------------- // FTS query normalisation β€” mirrors Pi's `toFtsQuery`: // camelCase-split β†’ lowercase β†’ filter stop-words β†’ `term* term* …` // --------------------------------------------------------------------------- -const normalizeFtsQuery = (query: string): string | null => { +export const normalizeFtsQuery = (query: string): string | null => { const terms = query // Split camelCase: "createSubscription" β†’ "create Subscription" .replace(/([a-z0-9])([A-Z])/g, "$1 $2") @@ -176,46 +243,7 @@ const openDatabase = async (dbPath: string): Promise => { db = nodeDb; } - db.exec(` - CREATE TABLE IF NOT EXISTS fts_docs ( - id TEXT PRIMARY KEY, - namespace TEXT NOT NULL DEFAULT '', - path TEXT NOT NULL, - name TEXT NOT NULL, - description TEXT NOT NULL, - integration TEXT NOT NULL, - lexical_text TEXT NOT NULL - ); - - CREATE VIRTUAL TABLE IF NOT EXISTS fts_docs_fts USING fts5( - path, - integration, - name, - description, - lexical_text, - content='fts_docs', - content_rowid='rowid', - tokenize='porter unicode61' - ); - - -- Triggers to keep the FTS5 content table in sync with fts_docs. - CREATE TRIGGER IF NOT EXISTS fts_docs_ai AFTER INSERT ON fts_docs BEGIN - INSERT INTO fts_docs_fts(rowid, path, integration, name, description, lexical_text) - VALUES (new.rowid, new.path, new.integration, new.name, new.description, new.lexical_text); - END; - - CREATE TRIGGER IF NOT EXISTS fts_docs_ad AFTER DELETE ON fts_docs BEGIN - INSERT INTO fts_docs_fts(fts_docs_fts, rowid, path, integration, name, description, lexical_text) - VALUES ('delete', old.rowid, old.path, old.integration, old.name, old.description, old.lexical_text); - END; - - CREATE TRIGGER IF NOT EXISTS fts_docs_au AFTER UPDATE ON fts_docs BEGIN - INSERT INTO fts_docs_fts(fts_docs_fts, rowid, path, integration, name, description, lexical_text) - VALUES ('delete', old.rowid, old.path, old.integration, old.name, old.description, old.lexical_text); - INSERT INTO fts_docs_fts(rowid, path, integration, name, description, lexical_text) - VALUES (new.rowid, new.path, new.integration, new.name, new.description, new.lexical_text); - END; - `); + db.exec(FTS_SCHEMA_STATEMENTS.join(";\n") + ";"); return db; }; @@ -243,11 +271,8 @@ export const makeFtsLexicalStore = (options: { readonly path: string }): FtsLexi Effect.flatMap((db) => Effect.try({ try: () => { - const del = db.prepare("DELETE FROM fts_docs WHERE id = ?"); - const ins = db.prepare( - `INSERT INTO fts_docs (id, namespace, path, name, description, integration, lexical_text) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - ); + const del = db.prepare(FTS_DELETE_BY_ID_SQL); + const ins = db.prepare(FTS_INSERT_SQL); // Atomic batch: delete-then-insert fires FTS triggers per doc, so a // mid-loop failure must not leave the FTS index partially erased. db.exec("BEGIN"); @@ -287,7 +312,7 @@ export const makeFtsLexicalStore = (options: { readonly path: string }): FtsLexi Effect.flatMap((db) => Effect.try({ try: () => { - const del = db.prepare("DELETE FROM fts_docs WHERE id = ?"); + const del = db.prepare(FTS_DELETE_BY_ID_SQL); for (const id of ids) { del.run(id); } @@ -310,20 +335,7 @@ export const makeFtsLexicalStore = (options: { readonly path: string }): FtsLexi // BM25 weights: path=12, integration=8, name=10, description=5, lexical_text=3 // `-bm25(...)` so higher = better. const rows = db - .prepare( - `SELECT - d.path AS path, - d.name AS name, - d.description AS description, - d.integration AS integration, - 0 - bm25(fts_docs_fts, 12.0, 8.0, 10.0, 5.0, 3.0) AS score - FROM fts_docs_fts - JOIN fts_docs d ON d.rowid = fts_docs_fts.rowid - WHERE fts_docs_fts MATCH ? - AND d.namespace = ? - ORDER BY score DESC - LIMIT ?`, - ) + .prepare(FTS_SEARCH_SQL) .all(ftsQuery, namespace, topK) as ReadonlyArray<{ readonly path: string; readonly name: string; From a8f624a66959aaabc1a3e799423391d67f78ae22 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Wed, 17 Jun 2026 17:46:05 +0530 Subject: [PATCH 37/57] feat(plugin-semantic-search): operator Search console page + live search/status API (#33) Adds the Search sidebar tab (live tools.search over the hybrid provider, index status, reindex), the ./api search+status endpoints, the shared search/status extension methods, and FtsLexicalStore.count. Greptile 5/5. --- bun.lock | 13 ++ packages/plugins/semantic-search/package.json | 27 ++- .../plugins/semantic-search/src/api/group.ts | 80 ++++++-- .../semantic-search/src/api/handlers.ts | 38 +++- .../plugins/semantic-search/src/api/index.ts | 8 +- .../semantic-search/src/react/SearchPage.tsx | 188 ++++++++++++++++++ .../semantic-search/src/react/atoms.ts | 28 +++ .../semantic-search/src/react/client.ts | 16 ++ .../semantic-search/src/react/index.ts | 3 + .../src/react/plugin-client.tsx | 19 ++ .../semantic-search/src/sdk/indexer.test.ts | 1 + .../plugins/semantic-search/src/sdk/plugin.ts | 121 +++++++++-- .../src/sdk/store-fts-d1.test.ts | 38 ++++ .../semantic-search/src/sdk/store-fts-d1.ts | 17 ++ .../src/sdk/store-fts-provider.test.ts | 1 + .../semantic-search/src/sdk/store-fts.ts | 20 ++ .../plugins/semantic-search/tsconfig.json | 5 +- 17 files changed, 576 insertions(+), 47 deletions(-) create mode 100644 packages/plugins/semantic-search/src/react/SearchPage.tsx create mode 100644 packages/plugins/semantic-search/src/react/atoms.ts create mode 100644 packages/plugins/semantic-search/src/react/client.ts create mode 100644 packages/plugins/semantic-search/src/react/index.ts create mode 100644 packages/plugins/semantic-search/src/react/plugin-client.tsx diff --git a/bun.lock b/bun.lock index b1e92c21f7..fbcb1ca2b6 100644 --- a/bun.lock +++ b/bun.lock @@ -1091,13 +1091,18 @@ }, "devDependencies": { "@cloudflare/workers-types": "^4.20250620.0", + "@effect/atom-react": "catalog:", "@effect/vitest": "catalog:", "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", "@types/better-sqlite3": "^7.6.13", "@types/node": "catalog:", + "@types/react": "catalog:", "@zvec/zvec": "^0.4.1", "bun-types": "catalog:", "effect": "catalog:", + "lucide-react": "^1.7.0", + "react": "catalog:", "tsup": "catalog:", "vitest": "catalog:", }, @@ -1106,11 +1111,19 @@ "sqlite-vec": "^0.1.7", }, "peerDependencies": { + "@effect/atom-react": "catalog:", "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", "effect": "catalog:", + "lucide-react": "^1.7.0", + "react": "catalog:", }, "optionalPeers": [ + "@effect/atom-react", "@executor-js/api", + "@executor-js/react", + "lucide-react", + "react", ], }, "packages/plugins/service-tokens": { diff --git a/packages/plugins/semantic-search/package.json b/packages/plugins/semantic-search/package.json index d64e95c9b4..67a4d4e9b4 100644 --- a/packages/plugins/semantic-search/package.json +++ b/packages/plugins/semantic-search/package.json @@ -5,7 +5,9 @@ "type": "module", "exports": { ".": "./src/sdk/index.ts", - "./api": "./src/api/index.ts" + "./api": "./src/api/index.ts", + "./react": "./src/react/index.ts", + "./client": "./src/react/plugin-client.tsx" }, "scripts": { "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src || true)", @@ -21,23 +23,44 @@ }, "devDependencies": { "@cloudflare/workers-types": "^4.20250620.0", + "@effect/atom-react": "catalog:", "@effect/vitest": "catalog:", "@executor-js/api": "workspace:*", + "@executor-js/react": "workspace:*", "@types/better-sqlite3": "^7.6.13", "@types/node": "catalog:", + "@types/react": "catalog:", "@zvec/zvec": "^0.4.1", "bun-types": "catalog:", "effect": "catalog:", + "lucide-react": "^1.7.0", + "react": "catalog:", "tsup": "catalog:", "vitest": "catalog:" }, "peerDependencies": { + "@effect/atom-react": "catalog:", "@executor-js/api": "workspace:*", - "effect": "catalog:" + "@executor-js/react": "workspace:*", + "effect": "catalog:", + "lucide-react": "^1.7.0", + "react": "catalog:" }, "peerDependenciesMeta": { + "@effect/atom-react": { + "optional": true + }, "@executor-js/api": { "optional": true + }, + "@executor-js/react": { + "optional": true + }, + "lucide-react": { + "optional": true + }, + "react": { + "optional": true } }, "optionalDependencies": { diff --git a/packages/plugins/semantic-search/src/api/group.ts b/packages/plugins/semantic-search/src/api/group.ts index 94cd359e30..869ca19afc 100644 --- a/packages/plugins/semantic-search/src/api/group.ts +++ b/packages/plugins/semantic-search/src/api/group.ts @@ -4,16 +4,20 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; import { InternalError } from "@executor-js/api"; // --------------------------------------------------------------------------- -// HTTP surface for the Semantic search plugin: a single explicit reindex. +// HTTP surface for the Semantic search plugin: reindex + operator search/status. // -// The route is flat and plugin-id-prefixed (`/semantic-search/reindex`), -// matching the execution-history/graphql convention β€” the per-request executor -// is already owner-scoped at the host edge, so there is no `:scopeId` segment. +// Routes are flat and plugin-id-prefixed (`/semantic-search/...`), matching the +// execution-history/graphql convention β€” the per-request executor is already +// owner-scoped at the host edge, so there is no `:scopeId` segment. // -// Reindex takes no body: it reconciles the whole tool catalog for the scoped -// tenant against Vectorize (incremental fingerprint diff). `SemanticSearchError` -// on the extension flows through the typed channel and `capture` downgrades it -// to `InternalError`. +// - reindex (POST) reconciles the whole tool catalog for the scoped tenant +// against Vectorize + the lexical store (incremental fingerprint diff). +// - search (GET) runs a live `tools.search` through the SAME provider the +// engine uses, so the operator console sees what the agent would. +// - status (GET) reports index population (vector fingerprints + lexical docs). +// +// `SemanticSearchError` on the extension flows through the typed channel and +// `capture` downgrades it to `InternalError`. // --------------------------------------------------------------------------- /** Result of a reindex reconcile: counts for each category of tool processed. */ @@ -25,9 +29,57 @@ export const ReindexResponse = Schema.Struct({ removedSkipped: Schema.Number, }); -export const SemanticSearchGroup = HttpApiGroup.make("semanticSearch").add( - HttpApiEndpoint.post("reindex", "/semantic-search/reindex", { - success: ReindexResponse, - error: InternalError, - }), -); +/** One live `tools.search` match (fused vector/lexical score; higher is better). */ +export const SearchResultItem = Schema.Struct({ + path: Schema.String, + name: Schema.String, + description: Schema.optional(Schema.String), + integration: Schema.String, + score: Schema.Number, +}); + +/** A page of live search results for the operator console. */ +export const SearchResponse = Schema.Struct({ + namespace: Schema.String, + query: Schema.String, + items: Schema.Array(SearchResultItem), +}); + +/** URL query for the live search endpoint (`limit` arrives as a string). */ +const SearchQuery = Schema.Struct({ + q: Schema.String, + namespace: Schema.optional(Schema.String), + limit: Schema.optional(Schema.NumberFromString), +}); + +/** Operator index status β€” vector (fingerprint) + lexical document counts. */ +export const StatusResponse = Schema.Struct({ + namespace: Schema.String, + indexed: Schema.Number, + lexical: Schema.NullOr(Schema.Number), +}); + +export type SearchResultItemType = typeof SearchResultItem.Type; +export type SearchResponseType = typeof SearchResponse.Type; +export type StatusResponseType = typeof StatusResponse.Type; + +export const SemanticSearchGroup = HttpApiGroup.make("semanticSearch") + .add( + HttpApiEndpoint.post("reindex", "/semantic-search/reindex", { + success: ReindexResponse, + error: InternalError, + }), + ) + .add( + HttpApiEndpoint.get("search", "/semantic-search/search", { + query: SearchQuery, + success: SearchResponse, + error: InternalError, + }), + ) + .add( + HttpApiEndpoint.get("status", "/semantic-search/status", { + success: StatusResponse, + error: InternalError, + }), + ); diff --git a/packages/plugins/semantic-search/src/api/handlers.ts b/packages/plugins/semantic-search/src/api/handlers.ts index af8ab9d1d7..688298287c 100644 --- a/packages/plugins/semantic-search/src/api/handlers.ts +++ b/packages/plugins/semantic-search/src/api/handlers.ts @@ -29,13 +29,35 @@ export const SemanticSearchHandlers = HttpApiBuilder.group( ExecutorApiWithSemanticSearch, "semanticSearch", (handlers) => - handlers.handle("reindex", () => - captureEngineError( - Effect.gen(function* () { - const search = yield* SemanticSearchExtensionService; - const executor = yield* ExecutorService; - return yield* search.reindex(executor); - }), + handlers + .handle("reindex", () => + captureEngineError( + Effect.gen(function* () { + const search = yield* SemanticSearchExtensionService; + const executor = yield* ExecutorService; + return yield* search.reindex(executor); + }), + ), + ) + .handle("search", ({ query }) => + captureEngineError( + Effect.gen(function* () { + const search = yield* SemanticSearchExtensionService; + const executor = yield* ExecutorService; + return yield* search.search(executor, { + query: query.q, + namespace: query.namespace, + limit: query.limit, + }); + }), + ), + ) + .handle("status", () => + captureEngineError( + Effect.gen(function* () { + const search = yield* SemanticSearchExtensionService; + return yield* search.status(); + }), + ), ), - ), ); diff --git a/packages/plugins/semantic-search/src/api/index.ts b/packages/plugins/semantic-search/src/api/index.ts index f93a7eb14f..17e9611769 100644 --- a/packages/plugins/semantic-search/src/api/index.ts +++ b/packages/plugins/semantic-search/src/api/index.ts @@ -4,7 +4,13 @@ import { semanticSearchPlugin, type SemanticSearchPluginOptions } from "../sdk/p import { SemanticSearchGroup } from "./group"; import { SemanticSearchExtensionService, SemanticSearchHandlers } from "./handlers"; -export { SemanticSearchGroup, ReindexResponse } from "./group"; +export { + SemanticSearchGroup, + ReindexResponse, + SearchResultItem, + SearchResponse, + StatusResponse, +} from "./group"; export { SemanticSearchHandlers, SemanticSearchExtensionService } from "./handlers"; // HTTP-augmented variant of `semanticSearchPlugin`. The returned plugin carries diff --git a/packages/plugins/semantic-search/src/react/SearchPage.tsx b/packages/plugins/semantic-search/src/react/SearchPage.tsx new file mode 100644 index 0000000000..9dc567d7e5 --- /dev/null +++ b/packages/plugins/semantic-search/src/react/SearchPage.tsx @@ -0,0 +1,188 @@ +import { useState } from "react"; +import { useAtomRefresh, useAtomSet, useAtomValue } from "@effect/atom-react"; +import * as Exit from "effect/Exit"; +import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; +import { RefreshCw, Search } from "lucide-react"; +import { Button } from "@executor-js/react/components/button"; +import { Input } from "@executor-js/react/components/input"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@executor-js/react/components/table"; + +import type { SearchResponseType, StatusResponseType } from "../api/group"; +import { reindexMutation, searchAtom, statusAtom } from "./atoms"; + +const SEARCH_LIMIT = 25; + +// Operator/debug page for the semantic-search plugin: a live `tools.search` box +// over the engine's discovery provider (Vectorize + FTS5/BM25 hybrid), an index +// status line, and an explicit reindex trigger. Read-only against the catalog β€” +// the only mutation is reindex. +export function SearchPage() { + const [input, setInput] = useState(""); + const [submitted, setSubmitted] = useState(""); + const [reindexing, setReindexing] = useState(false); + const [notice, setNotice] = useState(null); + + const searchResult = useAtomValue( + searchAtom({ q: submitted, limit: SEARCH_LIMIT }), + ) as AsyncResult.AsyncResult; + const statusResult = useAtomValue(statusAtom) as AsyncResult.AsyncResult< + StatusResponseType, + unknown + >; + const refreshStatus = useAtomRefresh(statusAtom); + const doReindex = useAtomSet(reindexMutation, { mode: "promiseExit" }); + + const items = AsyncResult.match(searchResult, { + onInitial: () => [] as SearchResponseType["items"], + onFailure: () => [] as SearchResponseType["items"], + onSuccess: ({ value }) => value.items, + }); + const searching = AsyncResult.match(searchResult, { + onInitial: () => submitted.length > 0, + onFailure: () => false, + onSuccess: () => false, + }); + const searchFailed = AsyncResult.match(searchResult, { + onInitial: () => false, + onFailure: () => true, + onSuccess: () => false, + }); + const status = AsyncResult.match(statusResult, { + onInitial: () => null, + onFailure: () => null, + onSuccess: ({ value }) => value, + }); + + const runSearch = () => setSubmitted(input.trim()); + + const handleReindex = async () => { + setReindexing(true); + setNotice(null); + const exit = await doReindex({ reactivityKeys: [] }); + setReindexing(false); + if (Exit.isFailure(exit)) { + setNotice("Reindex failed β€” a full-catalog reconcile can exceed the Worker CPU limit."); + return; + } + setNotice("Reindex complete."); + refreshStatus(); + }; + + return ( +
+
+
+
+

+ Search +

+

+ Run a live tools.search through the + engine's discovery provider β€” semantic (Vectorize) fused with lexical (FTS5/BM25) when + both are indexed. This is what the agent sees. +

+
+ +
+ +
+
+ + setInput((e.target as HTMLInputElement).value)} + className="h-9 w-full pl-9 text-[13px]" + onKeyDown={(e) => { + if (e.key === "Enter") runSearch(); + }} + /> +
+ +
+ +
+ {status ? ( + <> + namespace {status.namespace} Β· indexed{" "} + {status.indexed.toLocaleString()} + {status.lexical !== null ? ` Β· lexical ${status.lexical.toLocaleString()}` : ""} + + ) : ( + "index status unavailable" + )} + {notice ? {notice} : null} +
+ + {searchFailed ? ( +
+ Search failed. +
+ ) : submitted.length === 0 ? ( +
+ Enter a query to search the tool catalog. +
+ ) : searching ? ( +
Searching…
+ ) : items.length === 0 ? ( +
+ No tools matched {submitted}. The index may be empty + β€” run a reindex. +
+ ) : ( + + + + # + Tool + Integration + Score + + + + {items.map((item, i) => ( + + + {i + 1} + + +
{item.path}
+ {item.description ? ( +
+ {item.description} +
+ ) : null} + + + {item.integration} + + + {item.score.toFixed(4)} + + + ))} + +
+ )} +
+
+ ); +} diff --git a/packages/plugins/semantic-search/src/react/atoms.ts b/packages/plugins/semantic-search/src/react/atoms.ts new file mode 100644 index 0000000000..6843a6ca69 --- /dev/null +++ b/packages/plugins/semantic-search/src/react/atoms.ts @@ -0,0 +1,28 @@ +import * as Atom from "effect/unstable/reactivity/Atom"; + +import { SemanticSearchClient } from "./client"; + +export interface SearchInput { + readonly q: string; + readonly limit: number; +} + +// One cached result per (query, limit). An empty `q` is a cheap no-op on the +// server (the provider short-circuits before embedding), so the page can read +// this atom unconditionally and gate on the submitted query in the view. +export const searchAtom = Atom.family((input: SearchInput) => + SemanticSearchClient.query("semanticSearch", "search", { + query: { q: input.q, limit: input.limit }, + timeToLive: "30 seconds", + reactivityKeys: [], + }), +); + +// Index status β€” vector (fingerprint) + lexical document counts. +export const statusAtom = SemanticSearchClient.query("semanticSearch", "status", { + timeToLive: "10 seconds", + reactivityKeys: [], +}); + +// Explicit reindex (reconcile the whole catalog into the vector + lexical index). +export const reindexMutation = SemanticSearchClient.mutation("semanticSearch", "reindex"); diff --git a/packages/plugins/semantic-search/src/react/client.ts b/packages/plugins/semantic-search/src/react/client.ts new file mode 100644 index 0000000000..d6ba77b5b4 --- /dev/null +++ b/packages/plugins/semantic-search/src/react/client.ts @@ -0,0 +1,16 @@ +import { createPluginAtomClient } from "@executor-js/sdk/client"; +import { + getExecutorApiBaseUrl, + getExecutorServerAuthorizationHeader, +} from "@executor-js/react/api/server-connection"; + +import { SemanticSearchGroup } from "../api/group"; + +// Typed reactive client for the semantic-search plugin's HTTP surface +// (search / status / reindex). Mirrors execution-history's `ExecutionHistoryClient`: +// it binds to the active Executor Server Connection's base URL + auth header so +// the page works against whichever host the console is pointed at. +export const SemanticSearchClient = createPluginAtomClient(SemanticSearchGroup, { + baseUrl: getExecutorApiBaseUrl, + authorizationHeader: getExecutorServerAuthorizationHeader, +}); diff --git a/packages/plugins/semantic-search/src/react/index.ts b/packages/plugins/semantic-search/src/react/index.ts new file mode 100644 index 0000000000..73331a3ebb --- /dev/null +++ b/packages/plugins/semantic-search/src/react/index.ts @@ -0,0 +1,3 @@ +export { SearchPage } from "./SearchPage"; +export { SemanticSearchClient } from "./client"; +export { searchAtom, statusAtom, reindexMutation, type SearchInput } from "./atoms"; diff --git a/packages/plugins/semantic-search/src/react/plugin-client.tsx b/packages/plugins/semantic-search/src/react/plugin-client.tsx new file mode 100644 index 0000000000..f902492462 --- /dev/null +++ b/packages/plugins/semantic-search/src/react/plugin-client.tsx @@ -0,0 +1,19 @@ +import { defineClientPlugin } from "@executor-js/sdk/client"; +import { lazy } from "react"; + +const SearchPage = lazy(() => + import("./SearchPage").then((module) => ({ default: module.SearchPage })), +); + +// Registers the "Search" sidebar tab + its page. The host mounts this under +// `/plugins/semanticSearch`. Mirrors execution-history's `./client` entry. +export default defineClientPlugin({ + id: "semanticSearch" as const, + pages: [ + { + path: "/", + component: SearchPage, + nav: { label: "Search" }, + }, + ], +}); diff --git a/packages/plugins/semantic-search/src/sdk/indexer.test.ts b/packages/plugins/semantic-search/src/sdk/indexer.test.ts index d1c5498eae..d07bc94075 100644 --- a/packages/plugins/semantic-search/src/sdk/indexer.test.ts +++ b/packages/plugins/semantic-search/src/sdk/indexer.test.ts @@ -91,6 +91,7 @@ const makeFakeLexical = () => { deleted.push(...ids); }), search: () => Effect.succeed([]), + count: () => Effect.succeed(upserted.length), }; return { store, upserted, deleted }; }; diff --git a/packages/plugins/semantic-search/src/sdk/plugin.ts b/packages/plugins/semantic-search/src/sdk/plugin.ts index c103e76662..47a3dc8f52 100644 --- a/packages/plugins/semantic-search/src/sdk/plugin.ts +++ b/packages/plugins/semantic-search/src/sdk/plugin.ts @@ -1,4 +1,9 @@ -import { definePlugin, type Executor, type ToolDiscoveryProvider } from "@executor-js/sdk/core"; +import { + definePlugin, + type Executor, + type ToolDiscoveryProvider, + type ToolDiscoveryResult, +} from "@executor-js/sdk/core"; import { Effect } from "effect"; import { type Chunker, makeFacetChunker } from "./chunker"; @@ -56,12 +61,31 @@ const notConfigured = (): Effect.Effect => }), ); -/** Build the `executor.semanticSearch` surface. `reindex` reconciles the - * current tool catalog incrementally (fingerprint diff) and upserts changed - * tools into the vector store; it takes the scoped executor as an argument - * because only the request/API layer holds it (the plugin ctx does not expose - * the catalog). Inert β€” `reindex` fails clearly β€” until both a vector store - * and an embedder are present. */ +/** Default page size for the operator `search` surface. */ +const DEFAULT_SEARCH_LIMIT = 20; + +/** A live `tools.search` result page from the operator search surface. */ +export interface SemanticSearchResultPage { + readonly namespace: string; + readonly query: string; + readonly items: readonly ToolDiscoveryResult[]; +} + +/** Operator-facing index status: indexed (vector) + lexical document counts. */ +export interface SemanticSearchStatus { + readonly namespace: string; + /** Tools with a stored fingerprint β€” the vector-indexed count. */ + readonly indexed: number; + /** FTS5 lexical documents, or `null` when no lexical store is configured. */ + readonly lexical: number | null; +} + +/** Build the `executor.semanticSearch` surface: `reindex` (reconcile the catalog + * into the vector + lexical index), `search` (live `tools.search` through the + * shared provider), and `status` (index counts). `reindex`/`search` take the + * scoped executor because only the request/API layer holds it. `reindex`/`search` + * are inert β€” failing clearly β€” until both a vector store and an embedder are + * present. */ const makeSemanticSearchExtension = (deps: { readonly namespace: string; readonly embedder: ToolEmbedder | undefined; @@ -70,6 +94,7 @@ const makeSemanticSearchExtension = (deps: { readonly fingerprints: Parameters[0]["fingerprints"] | undefined; readonly owner: Parameters[0]["owner"] | undefined; readonly lexicalStore: FtsLexicalStore | undefined; + readonly provider: ToolDiscoveryProvider | undefined; }) => ({ reindex: (executor: Executor): Effect.Effect => deps.embedder && deps.store && deps.fingerprints && deps.owner @@ -84,6 +109,55 @@ const makeSemanticSearchExtension = (deps: { lexicalStore: deps.lexicalStore, }) : notConfigured(), + + /** Run a live `tools.search` through the same provider the engine uses, so the + * operator console sees exactly what the agent would. Inert until configured. */ + search: ( + executor: Executor, + input: { readonly query: string; readonly namespace?: string; readonly limit?: number }, + ): Effect.Effect => { + const namespace = input.namespace ?? deps.namespace; + return deps.provider + ? deps.provider + .searchTools({ + executor, + query: input.query, + namespace, + limit: input.limit ?? DEFAULT_SEARCH_LIMIT, + offset: 0, + }) + .pipe( + Effect.map((page) => ({ namespace, query: input.query, items: page.items })), + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Semantic search query failed.", cause }), + ), + ) + : notConfigured(); + }, + + /** Index status for the operator console: vector (fingerprint) + lexical counts. */ + status: (): Effect.Effect => + Effect.gen(function* () { + const indexed = deps.fingerprints + ? yield* deps.fingerprints + .count() + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Failed to count indexed tools.", cause }), + ), + ) + : 0; + // A transient lexical-count failure must not mask the (already-fetched) + // vector count: degrade `lexical` to null and still return `indexed`. + const lexical = deps.lexicalStore + ? yield* deps.lexicalStore + .count(deps.namespace) + .pipe(Effect.catch(() => Effect.succeed(null))) + : null; + return { namespace: deps.namespace, indexed, lexical }; + }), }); /** The `executor.semanticSearch` surface, derived from its factory. */ @@ -119,6 +193,24 @@ export const semanticSearchPlugin = definePlugin((options?: SemanticSearchPlugin const lexical = options?.lexical; const lexicalStore = options?.lexicalStore; + // Build the discovery provider once and share it between the engine's + // `tools.search` (runtime) and the operator `search` extension surface, so + // both answer through exactly the same vector/hybrid path. A populated FTS5 + // store (wrapped as a provider) takes precedence over a host-supplied lexical + // provider; either side fuses with the vector provider via RRF. + const provider: ToolDiscoveryProvider | undefined = + !embedder || !store + ? undefined + : (() => { + const vector = makeVectorToolDiscoveryProvider({ embedder, store, namespace }); + const lexicalProvider = lexicalStore + ? makeFtsLexicalProvider(lexicalStore, namespace) + : lexical; + return lexicalProvider + ? makeHybridToolDiscoveryProvider({ lexical: lexicalProvider, vector }) + : vector; + })(); + return { id: "semanticSearch" as const, packageName: "@executor-js/plugin-semantic-search", @@ -140,21 +232,10 @@ export const semanticSearchPlugin = definePlugin((options?: SemanticSearchPlugin fingerprints: ctx.storage.fingerprints, owner: ctx.storage.owner, lexicalStore, + provider, }), runtime: { - toolDiscoveryProvider: () => { - if (!embedder || !store) return undefined; - const vector = makeVectorToolDiscoveryProvider({ embedder, store, namespace }); - // Resolve the lexical side: a populated FTS5 store (wrapped as a provider) - // takes precedence over a host-supplied provider (e.g. the engine's token - // scorer). Fuse with the vector provider via RRF when either is present. - const lexicalProvider = lexicalStore - ? makeFtsLexicalProvider(lexicalStore, namespace) - : lexical; - return lexicalProvider - ? makeHybridToolDiscoveryProvider({ lexical: lexicalProvider, vector }) - : vector; - }, + toolDiscoveryProvider: () => provider, }, }; }); diff --git a/packages/plugins/semantic-search/src/sdk/store-fts-d1.test.ts b/packages/plugins/semantic-search/src/sdk/store-fts-d1.test.ts index dea75ab933..a467e2bf6d 100644 --- a/packages/plugins/semantic-search/src/sdk/store-fts-d1.test.ts +++ b/packages/plugins/semantic-search/src/sdk/store-fts-d1.test.ts @@ -292,6 +292,44 @@ describe("makeD1FtsLexicalStore", () => { }), ); + it.effect("count β€” reports indexed docs per namespace", () => + Effect.gen(function* () { + const { store } = yield* Effect.promise(() => makeTestStore()); + + yield* store.upsert([ + doc( + "default:repos.get", + "github", + "github.repos.get", + "Get a repository", + "Returns a repository.", + "github repos get", + ), + doc( + "default:repos.list", + "github", + "github.repos.list", + "List repositories", + "Lists repositories.", + "github repos list", + ), + doc( + "other:events.create", + "calendar", + "calendar.events.create", + "Create an event", + "Creates an event.", + "calendar events create", + "other", + ), + ]); + + expect(yield* store.count("default")).toBe(2); + expect(yield* store.count("other")).toBe(1); + expect(yield* store.count("empty")).toBe(0); + }), + ); + it.effect("retries schema init after a transient failure (the rejection is not cached)", () => Effect.gen(function* () { const BetterSqlite3Mod = yield* Effect.promise(() => import("better-sqlite3")); diff --git a/packages/plugins/semantic-search/src/sdk/store-fts-d1.ts b/packages/plugins/semantic-search/src/sdk/store-fts-d1.ts index a5da3d0ef1..f841345be5 100644 --- a/packages/plugins/semantic-search/src/sdk/store-fts-d1.ts +++ b/packages/plugins/semantic-search/src/sdk/store-fts-d1.ts @@ -2,6 +2,7 @@ import { Effect } from "effect"; import { SemanticSearchError } from "./errors"; import { + FTS_COUNT_SQL, FTS_DELETE_BY_ID_SQL, FTS_INSERT_SQL, FTS_SCHEMA_STATEMENTS, @@ -154,5 +155,21 @@ export const makeD1FtsLexicalStore = (d1: D1Database): FtsLexicalStore => { }), ), ), + + count: (namespace: string) => + ensureSchema.pipe( + Effect.flatMap(() => + Effect.tryPromise({ + try: async () => { + const { results } = await d1 + .prepare(FTS_COUNT_SQL) + .bind(namespace) + .all<{ readonly n: number }>(); + return results[0]?.n ?? 0; + }, + catch: (cause) => new SemanticSearchError({ message: "D1 FTS5 count failed.", cause }), + }), + ), + ), }; }; diff --git a/packages/plugins/semantic-search/src/sdk/store-fts-provider.test.ts b/packages/plugins/semantic-search/src/sdk/store-fts-provider.test.ts index dda9598a55..6772271cff 100644 --- a/packages/plugins/semantic-search/src/sdk/store-fts-provider.test.ts +++ b/packages/plugins/semantic-search/src/sdk/store-fts-provider.test.ts @@ -16,6 +16,7 @@ const mockStore = (rows: readonly FtsSearchResult[]): FtsLexicalStore => ({ upsert: () => Effect.void, deleteByIds: () => Effect.void, search: () => Effect.succeed(rows), + count: () => Effect.succeed(rows.length), }); describe("makeFtsLexicalProvider β€” namespace filtering", () => { diff --git a/packages/plugins/semantic-search/src/sdk/store-fts.ts b/packages/plugins/semantic-search/src/sdk/store-fts.ts index 91b75500ce..22e65863eb 100644 --- a/packages/plugins/semantic-search/src/sdk/store-fts.ts +++ b/packages/plugins/semantic-search/src/sdk/store-fts.ts @@ -132,6 +132,9 @@ export const FTS_SEARCH_SQL = `SELECT ORDER BY score DESC LIMIT ?`; +/** COUNT of indexed lexical documents in a namespace. Binds: (namespace TEXT). */ +export const FTS_COUNT_SQL = "SELECT count(*) AS n FROM fts_docs WHERE namespace = ?"; + // --------------------------------------------------------------------------- // FTS query normalisation β€” mirrors Pi's `toFtsQuery`: // camelCase-split β†’ lowercase β†’ filter stop-words β†’ `term* term* …` @@ -188,6 +191,8 @@ export interface FtsLexicalStore { readonly search: ( input: FtsSearchInput, ) => Effect.Effect; + /** Count of indexed lexical documents in `namespace` (operator status). */ + readonly count: (namespace: string) => Effect.Effect; } // --------------------------------------------------------------------------- @@ -358,6 +363,21 @@ export const makeFtsLexicalStore = (options: { readonly path: string }): FtsLexi }), ), ), + + count: (namespace) => + getDb.pipe( + Effect.flatMap((db) => + Effect.try({ + try: () => { + const row = db.prepare(FTS_COUNT_SQL).get(namespace) as + | { readonly n: number } + | undefined; + return row?.n ?? 0; + }, + catch: (cause) => new SemanticSearchError({ message: "FTS5 count failed.", cause }), + }), + ), + ), }; }; diff --git a/packages/plugins/semantic-search/tsconfig.json b/packages/plugins/semantic-search/tsconfig.json index eebc1e6f14..c495442f10 100644 --- a/packages/plugins/semantic-search/tsconfig.json +++ b/packages/plugins/semantic-search/tsconfig.json @@ -5,10 +5,11 @@ "moduleResolution": "Bundler", "strict": true, "skipLibCheck": true, - "lib": ["ES2022"], + "lib": ["ES2022", "DOM"], "types": ["bun-types", "node"], "noUnusedLocals": true, "noImplicitOverride": true, + "jsx": "react-jsx", "plugins": [ { "name": "@effect/language-service", @@ -20,5 +21,5 @@ } ] }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts", "src/**/*.tsx"] } From f1f0f5c48b743098c85642d045b9cdd73132dc18 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Wed, 17 Jun 2026 18:37:33 +0530 Subject: [PATCH 38/57] =?UTF-8?q?feat(plugin-semantic-search):=20paged=20r?= =?UTF-8?q?eindex=20=E2=80=94=20reconcileToolCatalogPage=20+=20sweepRemove?= =?UTF-8?q?d=20(#34)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits the full-catalog reconcile into resumable, engine-agnostic primitives (page + sweep) so a durable driver can run each within its own CPU budget. Removed tools are now actually swept. Greptile 5/5. --- .../plugins/semantic-search/src/api/group.ts | 2 +- .../semantic-search/src/sdk/documents.ts | 121 +++--- .../plugins/semantic-search/src/sdk/index.ts | 10 +- .../semantic-search/src/sdk/indexer.test.ts | 268 +++++++++--- .../semantic-search/src/sdk/indexer.ts | 390 ++++++++++++------ .../plugins/semantic-search/src/sdk/plugin.ts | 45 +- 6 files changed, 604 insertions(+), 232 deletions(-) diff --git a/packages/plugins/semantic-search/src/api/group.ts b/packages/plugins/semantic-search/src/api/group.ts index 869ca19afc..9271ae3fe9 100644 --- a/packages/plugins/semantic-search/src/api/group.ts +++ b/packages/plugins/semantic-search/src/api/group.ts @@ -26,7 +26,7 @@ export const ReindexResponse = Schema.Struct({ total: Schema.Number, reembedded: Schema.Number, unchanged: Schema.Number, - removedSkipped: Schema.Number, + removed: Schema.Number, }); /** One live `tools.search` match (fused vector/lexical score; higher is better). */ diff --git a/packages/plugins/semantic-search/src/sdk/documents.ts b/packages/plugins/semantic-search/src/sdk/documents.ts index e1d7ed012c..9019e8cd12 100644 --- a/packages/plugins/semantic-search/src/sdk/documents.ts +++ b/packages/plugins/semantic-search/src/sdk/documents.ts @@ -1,4 +1,4 @@ -import type { Executor } from "@executor-js/sdk/core"; +import type { Executor, Tool } from "@executor-js/sdk/core"; import { Effect } from "effect"; import type { ToolDocumentInput } from "./chunker"; @@ -70,60 +70,81 @@ export const buildLexicalText = (doc: ToolDocumentInput): string => { export const addressToPath = (address: string): string => address.startsWith(ADDRESS_PREFIX) ? address.slice(ADDRESS_PREFIX.length) : address; -/** Collect the full `ToolDocumentInput` set from the live catalog. +/** List the live tool descriptors, stably sorted by address. * - * For each tool returned by `tools.list`, we attempt to fetch its schema via - * `tools.schema(address)` to populate the TypeScript facets (inputTypeScript, - * outputTypeScript, typeScriptDefinitions). If the schema fetch fails for any - * individual tool we degrade gracefully to an identity-only document (no TS - * fields) β€” never failing the whole collection for one tool. - * - * Bounded concurrency (16) keeps the catalog walk fast while avoiding - * unbounded fan-out. */ -export const collectToolDocumentInputs = ( - namespace: string, + * Cheap β€” descriptors only, NO per-tool schema fetch β€” so it is safe to call + * once per reindex page (to slice deterministically) and in the removal sweep + * (for liveness). The stable sort gives paged reindex consistent cursor indices + * even though `tools.list` has no native pagination. */ +export const listToolDescriptors = ( executor: Executor, -): Effect.Effect => +): Effect.Effect => executor.tools.list({ includeAnnotations: false }).pipe( Effect.mapError( (cause) => new SemanticSearchError({ message: "Failed to list tools for indexing.", cause }), ), - Effect.flatMap((tools) => - Effect.forEach( - tools, - (tool) => { - const path = addressToPath(String(tool.address)); - const base: ToolDocumentInput = { - path, - name: String(tool.name), - integration: String(tool.integration), - description: stripHtml(String(tool.description ?? "")), - }; - // Attempt schema fetch; on any failure degrade to identity-only. - return executor.tools.schema(tool.address).pipe( - Effect.map((view): ToolDocumentInput => { - const doc: ToolDocumentInput = - view === null - ? base - : { - ...base, - ...(view.inputTypeScript !== undefined - ? { inputTypeScript: view.inputTypeScript } - : {}), - ...(view.outputTypeScript !== undefined - ? { outputTypeScript: view.outputTypeScript } - : {}), - ...(view.typeScriptDefinitions !== undefined - ? { typeScriptDefinitions: view.typeScriptDefinitions } - : {}), - }; - return { ...doc, lexicalText: buildLexicalText(doc) }; - }), - // Degrade: schema fetch failed β€” use identity-only document. - Effect.catch(() => Effect.succeed({ ...base, lexicalText: buildLexicalText(base) })), - ); - }, - { concurrency: 16 }, - ), + Effect.map((tools) => + [...tools].sort((a, b) => String(a.address).localeCompare(String(b.address))), ), ); + +/** Collect `ToolDocumentInput` for a specific set of tool descriptors. + * + * For each tool we attempt to fetch its schema via `tools.schema(address)` to + * populate the TypeScript facets; on any per-tool failure we degrade gracefully + * to an identity-only document (never failing the whole batch for one tool). + * This per-tool schema β†’ TypeScript codegen is the CPU-heavy part of indexing, + * so callers bound the input (one page) to stay within a single invocation's + * CPU budget. + * + * Bounded concurrency (16) keeps the walk fast while avoiding unbounded fan-out. */ +export const collectDocsForTools = ( + executor: Executor, + tools: readonly Tool[], +): Effect.Effect => + Effect.forEach( + tools, + (tool) => { + const path = addressToPath(String(tool.address)); + const base: ToolDocumentInput = { + path, + name: String(tool.name), + integration: String(tool.integration), + description: stripHtml(String(tool.description ?? "")), + }; + // Attempt schema fetch; on any failure degrade to identity-only. + return executor.tools.schema(tool.address).pipe( + Effect.map((view): ToolDocumentInput => { + const doc: ToolDocumentInput = + view === null + ? base + : { + ...base, + ...(view.inputTypeScript !== undefined + ? { inputTypeScript: view.inputTypeScript } + : {}), + ...(view.outputTypeScript !== undefined + ? { outputTypeScript: view.outputTypeScript } + : {}), + ...(view.typeScriptDefinitions !== undefined + ? { typeScriptDefinitions: view.typeScriptDefinitions } + : {}), + }; + return { ...doc, lexicalText: buildLexicalText(doc) }; + }), + // Degrade: schema fetch failed β€” use identity-only document. + Effect.catch(() => Effect.succeed({ ...base, lexicalText: buildLexicalText(base) })), + ); + }, + { concurrency: 16 }, + ); + +/** Collect the full `ToolDocumentInput` set from the live catalog (list + schema). + * Convenience for non-paged callers; paged reindex uses `listToolDescriptors` + * to slice, then `collectDocsForTools` on each slice. */ +export const collectToolDocumentInputs = ( + executor: Executor, +): Effect.Effect => + listToolDescriptors(executor).pipe( + Effect.flatMap((tools) => collectDocsForTools(executor, tools)), + ); diff --git a/packages/plugins/semantic-search/src/sdk/index.ts b/packages/plugins/semantic-search/src/sdk/index.ts index 68325518d0..92e72c44ef 100644 --- a/packages/plugins/semantic-search/src/sdk/index.ts +++ b/packages/plugins/semantic-search/src/sdk/index.ts @@ -13,7 +13,15 @@ export { } from "./embedder"; export { type VectorStore, type VectorMatch, type VectorMatches, type VectorInput } from "./store"; export { makeVectorizeStore, type VectorizeIndex, MAX_TOP_K } from "./store-cloudflare"; -export { reconcileToolCatalog, type ReconcileResult } from "./indexer"; +export { + reconcileToolCatalog, + reconcileToolCatalogPage, + sweepRemoved, + DEFAULT_REINDEX_PAGE_SIZE, + type ReconcileResult, + type ReconcilePageResult, + type SweepResult, +} from "./indexer"; export { SemanticSearchError } from "./errors"; // Chunker diff --git a/packages/plugins/semantic-search/src/sdk/indexer.test.ts b/packages/plugins/semantic-search/src/sdk/indexer.test.ts index d07bc94075..ad2005ffdc 100644 --- a/packages/plugins/semantic-search/src/sdk/indexer.test.ts +++ b/packages/plugins/semantic-search/src/sdk/indexer.test.ts @@ -11,7 +11,7 @@ import { makeFacetChunker } from "./chunker"; import type { FingerprintRow } from "./collections"; import { toolFingerprints } from "./collections"; import type { ToolEmbedder } from "./embedder"; -import { reconcileToolCatalog } from "./indexer"; +import { reconcileToolCatalog, reconcileToolCatalogPage, sweepRemoved } from "./indexer"; import type { VectorStore, VectorInput } from "./store"; import type { FtsDocumentInput, FtsLexicalStore } from "./store-fts"; @@ -197,7 +197,7 @@ describe("reconcileToolCatalog", () => { expect(result.total).toBe(2); expect(result.reembedded).toBe(2); expect(result.unchanged).toBe(0); - expect(result.removedSkipped).toBe(0); + expect(result.removed).toBe(0); // Each tool produces at least 1 identity chunk. expect(upserted.length).toBeGreaterThanOrEqual(2); expect(deleted).toHaveLength(0); @@ -208,6 +208,42 @@ describe("reconcileToolCatalog", () => { }), ); + it.effect("reads the catalog once for the whole in-process reconcile (no N+1 tools.list)", () => + Effect.gen(function* () { + let listCalls = 0; + const inner = makeExecutor([ + { address: "tools.a.one", name: "one", integration: "a", description: "1" }, + { address: "tools.a.two", name: "two", integration: "a", description: "2" }, + { address: "tools.a.three", name: "three", integration: "a", description: "3" }, + ]); + // oxlint-disable-next-line executor/no-double-cast -- test stub: counting wrapper over the fake executor + const executor = { + tools: { + list: (...args: never[]) => { + listCalls++; + return inner.tools.list(...args); + }, + schema: inner.tools.schema, + }, + } as unknown as Executor; + + // pageSize 1 over 3 tools would re-list per page (3) + sweep (1) = 4 the old + // way; a single shared snapshot must collapse that to exactly one read. + yield* reconcileToolCatalog({ + namespace, + executor, + embedder: fakeEmbedder, + store: makeStore().store, + chunker, + fingerprints: makeFingerprints(), + owner, + pageSize: 1, + }); + + expect(listCalls).toBe(1); + }), + ); + it.effect("second run with no changes: reembeds nothing, unchanged equals total", () => Effect.gen(function* () { const executor = makeExecutor([ @@ -328,63 +364,65 @@ describe("reconcileToolCatalog", () => { }), ); - it.effect( - "a tool removed from the live list increments removedSkipped and is NOT deleted from the store", - () => - Effect.gen(function* () { - const executor1 = makeExecutor([ - { - address: "tools.github.repos.get", - name: "repos.get", - integration: "github", - description: "Get a repository", - }, - { - address: "tools.github.repos.delete", - name: "repos.delete", - integration: "github", - description: "Delete a repository", - }, - ]); + it.effect("a tool removed from the live list is swept: vector chunks + fingerprint deleted", () => + Effect.gen(function* () { + const executor1 = makeExecutor([ + { + address: "tools.github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + }, + { + address: "tools.github.repos.delete", + name: "repos.delete", + integration: "github", + description: "Delete a repository", + }, + ]); - // First run β€” both tools indexed. - const fingerprints = makeFingerprints(); - const { store: store1 } = makeStore(); - yield* reconcileToolCatalog({ - namespace, - executor: executor1, - embedder: fakeEmbedder, - store: store1, - chunker, - fingerprints, - owner, - }); + // First run β€” both tools indexed. + const fingerprints = makeFingerprints(); + const { store: store1 } = makeStore(); + yield* reconcileToolCatalog({ + namespace, + executor: executor1, + embedder: fakeEmbedder, + store: store1, + chunker, + fingerprints, + owner, + }); - // Second run β€” repos.delete is no longer in the catalog. - const executor2 = makeExecutor([ - { - address: "tools.github.repos.get", - name: "repos.get", - integration: "github", - description: "Get a repository", - }, - ]); - const { store: store2, deleted: deleted2 } = makeStore(); - const result2 = yield* reconcileToolCatalog({ - namespace, - executor: executor2, - embedder: fakeEmbedder, - store: store2, - chunker, - fingerprints, - owner, - }); + // Second run β€” repos.delete is no longer in the catalog. + const executor2 = makeExecutor([ + { + address: "tools.github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + }, + ]); + const { store: store2, deleted: deleted2 } = makeStore(); + const result2 = yield* reconcileToolCatalog({ + namespace, + executor: executor2, + embedder: fakeEmbedder, + store: store2, + chunker, + fingerprints, + owner, + }); - expect(result2.total).toBe(1); - expect(result2.removedSkipped).toBe(1); - // repos.delete's chunk ids must NOT appear in the delete list. - expect(deleted2).toHaveLength(0); - }), + expect(result2.total).toBe(1); + expect(result2.unchanged).toBe(1); + expect(result2.removed).toBe(1); + // repos.delete's vector chunks are swept (repos.get is unchanged, so it + // contributes no deletions); its fingerprint row is removed. + expect(deleted2.length).toBeGreaterThan(0); + const remaining = yield* fingerprints.list(); + expect(remaining.map((e) => e.key)).toEqual(["github.repos.get"]); + }), ); it.effect( @@ -451,3 +489,121 @@ describe("reconcileToolCatalog", () => { }), ); }); + +describe("reconcileToolCatalogPage", () => { + it.effect("pages through the catalog one slice at a time, advancing the cursor", () => + Effect.gen(function* () { + const executor = makeExecutor([ + { address: "tools.a.one", name: "one", integration: "a", description: "1" }, + { address: "tools.a.two", name: "two", integration: "a", description: "2" }, + { address: "tools.a.three", name: "three", integration: "a", description: "3" }, + ]); + const fingerprints = makeFingerprints(); + const { store } = makeStore(); + const base = { + namespace, + executor, + embedder: fakeEmbedder, + store, + chunker, + fingerprints, + owner, + pageSize: 1, + }; + + const p0 = yield* reconcileToolCatalogPage({ ...base, cursor: 0 }); + expect(p0.total).toBe(3); + expect(p0.processed).toBe(1); + expect(p0.reembedded).toBe(1); + expect(p0.nextCursor).toBe(1); + + const p1 = yield* reconcileToolCatalogPage({ ...base, cursor: 1 }); + expect(p1.processed).toBe(1); + expect(p1.nextCursor).toBe(2); + + const p2 = yield* reconcileToolCatalogPage({ ...base, cursor: 2 }); + expect(p2.processed).toBe(1); + expect(p2.nextCursor).toBe(null); + + // Every tool now has a fingerprint row (full coverage across pages). + const rows = yield* fingerprints.list(); + expect(rows.length).toBe(3); + }), + ); + + it.effect("a cursor past the end is an empty, terminal page", () => + Effect.gen(function* () { + const executor = makeExecutor([ + { address: "tools.a.one", name: "one", integration: "a", description: "1" }, + ]); + const page = yield* reconcileToolCatalogPage({ + namespace, + executor, + embedder: fakeEmbedder, + store: makeStore().store, + chunker, + fingerprints: makeFingerprints(), + owner, + cursor: 5, + }); + expect(page.total).toBe(1); + expect(page.processed).toBe(0); + expect(page.nextCursor).toBe(null); + }), + ); +}); + +describe("sweepRemoved", () => { + it.effect("deletes vector chunks + fingerprint for tools no longer in the catalog", () => + Effect.gen(function* () { + const fingerprints = makeFingerprints( + new Map([ + [ + "a.gone", + { path: "a.gone", integration: "a", fingerprint: "x", chunkIds: ["a.gone#0"] }, + ], + [ + "a.live", + { path: "a.live", integration: "a", fingerprint: "y", chunkIds: ["a.live#0"] }, + ], + ]), + ); + const { store, deleted } = makeStore(); + const executor = makeExecutor([ + { address: "tools.a.live", name: "live", integration: "a", description: "" }, + ]); + + const result = yield* sweepRemoved({ namespace, executor, store, fingerprints, owner }); + + expect(result.removed).toBe(1); + expect(deleted).toContain("a.gone#0"); + expect(deleted).not.toContain("a.live#0"); + const rows = yield* fingerprints.list(); + expect(rows.map((e) => e.key)).toEqual(["a.live"]); + }), + ); + + it.effect("an empty live catalog is a no-op (never wipes the index)", () => + Effect.gen(function* () { + const fingerprints = makeFingerprints( + new Map([ + ["a.x", { path: "a.x", integration: "a", fingerprint: "x", chunkIds: ["a.x#0"] }], + ]), + ); + const { store, deleted } = makeStore(); + + const result = yield* sweepRemoved({ + namespace, + executor: makeExecutor([]), + store, + fingerprints, + owner, + }); + + expect(result.removed).toBe(0); + expect(deleted).toHaveLength(0); + const rows = yield* fingerprints.list(); + expect(rows.length).toBe(1); + }), + ); +}); diff --git a/packages/plugins/semantic-search/src/sdk/indexer.ts b/packages/plugins/semantic-search/src/sdk/indexer.ts index 7f7087bab1..c4885ceb61 100644 --- a/packages/plugins/semantic-search/src/sdk/indexer.ts +++ b/packages/plugins/semantic-search/src/sdk/indexer.ts @@ -1,11 +1,16 @@ -import type { Executor } from "@executor-js/sdk/core"; +import type { Executor, Tool } from "@executor-js/sdk/core"; import type { Owner, PluginStorageCollectionFacade } from "@executor-js/sdk/core"; import { Effect } from "effect"; import type { Chunker } from "./chunker"; import type { FingerprintRow } from "./collections"; import { toolFingerprints } from "./collections"; -import { buildLexicalText, collectToolDocumentInputs } from "./documents"; +import { + addressToPath, + buildLexicalText, + collectDocsForTools, + listToolDescriptors, +} from "./documents"; import type { ToolEmbedder } from "./embedder"; import { SemanticSearchError } from "./errors"; import { fingerprintTool } from "./fingerprint"; @@ -13,7 +18,7 @@ import type { VectorStore, VectorInput } from "./store"; import type { FtsDocumentInput, FtsLexicalStore } from "./store-fts"; // --------------------------------------------------------------------------- -// ReconcileResult +// Results // --------------------------------------------------------------------------- export interface ReconcileResult { @@ -23,66 +28,99 @@ export interface ReconcileResult { readonly total: number; /** Number of tools whose embeddings were refreshed (new + changed). */ readonly reembedded: number; - /** Number of tools whose stored fingerprint matched β€” skipped entirely. */ + /** Number of tools whose stored fingerprint matched β€” skipped re-embedding. */ readonly unchanged: number; - /** Number of tools present in storage but absent from the current catalog - * (not deleted in v1 β€” deletion is unsafe across per-isolate catalog variance; - * logged only). */ - readonly removedSkipped: number; + /** Number of tools removed from the index because they left the catalog. */ + readonly removed: number; } -// --------------------------------------------------------------------------- -// reconcileToolCatalog β€” incremental fingerprint-based reindex -// --------------------------------------------------------------------------- +export interface ReconcilePageResult { + readonly namespace: string; + /** Total tools in the live catalog (size of the sorted descriptor list). */ + readonly total: number; + /** Tools handled in this page. */ + readonly processed: number; + readonly reembedded: number; + readonly unchanged: number; + /** Index of the next page, or `null` when the catalog is exhausted. */ + readonly nextCursor: number | null; +} -/** Reconcile the live tool catalog against the Vectorize index. - * - * Algorithm: - * 1. Collect `ToolDocumentInput` for every tool (tools.list + tools.schema). - * 2. Fingerprint each tool. - * 3. Load stored `FingerprintRow`s; diff: new βˆͺ changed β†’ re-embed + upsert. - * For CHANGED tools, delete old chunk ids before upserting new. - * 4. Tools in storage but absent from the live catalog β†’ skip deletion (v1), - * count them into `removedSkipped`. - * 5. Unchanged tools β†’ skip entirely. - * - * The owner parameter is the org-level scope used for all storage writes. */ -export const reconcileToolCatalog = (input: { +export interface SweepResult { + readonly namespace: string; + /** Tools deleted from the index (vector chunks + lexical doc + fingerprint). */ + readonly removed: number; +} + +/** Default tools-per-page. Bounds the CPU-heavy schemaβ†’TS codegen + embedding so + * one page fits within a single Worker invocation's CPU budget. */ +export const DEFAULT_REINDEX_PAGE_SIZE = 200; + +interface ReconcileStores { readonly namespace: string; readonly executor: Executor; - readonly embedder: ToolEmbedder; readonly store: VectorStore; - readonly chunker: Chunker; readonly fingerprints: PluginStorageCollectionFacade; readonly owner: Owner; - /** Optional FTS5 lexical store. When present, each reindexed tool is also - * written here (one doc per tool) so a hybrid lexical index stays in lockstep - * with the vector index. */ + /** Optional FTS5 lexical store kept in lockstep with the vector index. */ readonly lexicalStore?: FtsLexicalStore; -}): Effect.Effect => +} + +interface ReconcileDeps extends ReconcileStores { + readonly embedder: ToolEmbedder; + readonly chunker: Chunker; +} + +// --------------------------------------------------------------------------- +// reconcileToolCatalogPage β€” reconcile ONE bounded slice of the catalog +// --------------------------------------------------------------------------- + +/** Reconcile a single page of the live tool catalog into the vector + lexical + * index, returning the cursor for the next page. + * + * `tools.list` has no native pagination, so a page re-lists the (stably sorted) + * catalog β€” cheap, descriptors only β€” and slices `[cursor, cursor + pageSize)`. + * Only the slice pays the CPU-heavy `tools.schema` codegen + embedding, so each + * page stays within one invocation's CPU budget. This is the unit a durable + * driver (e.g. a Cloudflare Workflow step) calls in a loop; removal of tools + * that left the catalog is handled separately by `sweepRemoved`. + * + * Pass a pre-listed `descriptors` snapshot to reuse one catalog read across an + * in-process page loop (see `reconcileToolCatalog`); omit it and the page reads + * the catalog itself (the durable-step path, where each step is its own read). + * + * NOTE: the schema β†’ TS codegen runs for EVERY tool in the slice, including + * fingerprint-unchanged ones, because the fingerprint is derived from the + * schema β€” a schema-only change can't be detected without fetching it. So + * per-page CPU is dominated by codegen and is similar on re-runs; the + * incremental win is skipping the EMBEDDING of unchanged tools, not their + * schema fetch. */ +export const reconcileToolCatalogPage = ( + input: ReconcileDeps & { + readonly cursor: number; + readonly pageSize?: number; + readonly descriptors?: readonly Tool[]; + }, +): Effect.Effect => Effect.gen(function* () { const { namespace, executor, embedder, store, chunker, fingerprints, owner } = input; + const pageSize = input.pageSize ?? DEFAULT_REINDEX_PAGE_SIZE; + const cursor = Math.max(0, input.cursor); - // ------------------------------------------------------------------------- - // Step 1 β€” Collect tool documents (list + schema). - // ------------------------------------------------------------------------- - const docs = yield* collectToolDocumentInputs(namespace, executor); - const total = docs.length; + const all = input.descriptors ?? (yield* listToolDescriptors(executor)); + const total = all.length; + const slice = all.slice(cursor, cursor + pageSize); + const nextCursor = cursor + slice.length < total ? cursor + slice.length : null; - if (total === 0) { - return { namespace, total: 0, reembedded: 0, unchanged: 0, removedSkipped: 0 }; + if (slice.length === 0) { + return { namespace, total, processed: 0, reembedded: 0, unchanged: 0, nextCursor }; } - // ------------------------------------------------------------------------- - // Step 1b β€” Populate the FTS5 lexical store with EVERY live tool, when one - // is configured. Unlike the vector index (gated below by the fingerprint - // diff to avoid re-embedding), the lexical store carries no embedding cost, - // so it is rebuilt in full on each reindex. That keeps it complete even - // when the vector index is unchanged or the lexical store was attached to a - // deployment whose vectors were already indexed. The id is namespace- - // prefixed so tools sharing a path across namespaces never collide in a - // shared D1 database. - // ------------------------------------------------------------------------- + // Collect docs (schema β†’ TS) for THIS slice only. + const docs = yield* collectDocsForTools(executor, slice); + + // Lexical store carries no embedding cost, so rebuild it in full for the + // slice every page (delete-then-insert keyed by namespace-prefixed id). if (input.lexicalStore) { const lexicalDocs: readonly FtsDocumentInput[] = docs.map((doc) => ({ id: `${namespace}:${doc.path}`, @@ -96,82 +134,207 @@ export const reconcileToolCatalog = (input: { yield* input.lexicalStore.upsert(lexicalDocs); } - // ------------------------------------------------------------------------- - // Step 2 β€” Fingerprint each live tool. - // ------------------------------------------------------------------------- - const liveByPath = new Map( - docs.map((doc) => [doc.path, { doc, fingerprint: fingerprintTool(doc) }]), - ); - - // ------------------------------------------------------------------------- - // Step 3 β€” Load all stored fingerprints. - // ------------------------------------------------------------------------- - const storedEntries = yield* fingerprints.list().pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ - message: "Failed to load stored fingerprints for reconcile.", - cause, - }), - ), - ); - const storedByPath = new Map(storedEntries.map((entry) => [entry.key, entry.data])); + // Load stored fingerprints for just this slice's paths (not the whole index). + const slicePaths = docs.map((doc) => doc.path); + const storedByPath = yield* loadFingerprintsForPaths(fingerprints, owner, slicePaths); - // ------------------------------------------------------------------------- - // Step 4 β€” Diff: classify each live tool as new, changed, or unchanged. - // ------------------------------------------------------------------------- + // Diff: classify each tool in the slice as new/changed (re-embed) or unchanged. const toEmbed: Array<{ doc: (typeof docs)[number]; fingerprint: string; oldChunkIds: readonly string[] | null; }> = []; let unchanged = 0; - - for (const [path, { doc, fingerprint }] of liveByPath) { - const stored = storedByPath.get(path); + for (const doc of docs) { + const fingerprint = fingerprintTool(doc); + const stored = storedByPath.get(doc.path); if (stored !== undefined && stored.fingerprint === fingerprint) { unchanged++; } else { - toEmbed.push({ - doc, - fingerprint, - oldChunkIds: stored?.chunkIds ?? null, - }); + toEmbed.push({ doc, fingerprint, oldChunkIds: stored?.chunkIds ?? null }); } } - // ------------------------------------------------------------------------- - // Step 5 β€” Count removed (stored but no longer live) β€” skip deletion in v1. - // ------------------------------------------------------------------------- - let removedSkipped = 0; - for (const storedPath of storedByPath.keys()) { - if (!liveByPath.has(storedPath)) { - removedSkipped++; - // Intentionally NOT deleting: per-isolate catalog variance makes - // deletion unsafe in v1 β€” log the count so the operator is aware. - } + if (toEmbed.length === 0) { + return { namespace, total, processed: slice.length, reembedded: 0, unchanged, nextCursor }; } - if (removedSkipped > 0) { - yield* Effect.logWarning( - `reconcileToolCatalog [${namespace}]: ${removedSkipped} tool(s) present in ` + - `storage but absent from the live catalog β€” skipping deletion (v1).`, - ); + + yield* embedAndUpsert({ namespace, embedder, store, chunker, fingerprints, owner }, toEmbed); + + return { + namespace, + total, + processed: slice.length, + reembedded: toEmbed.length, + unchanged, + nextCursor, + }; + }); + +// --------------------------------------------------------------------------- +// sweepRemoved β€” delete tools that left the catalog +// --------------------------------------------------------------------------- + +/** Delete index entries for tools no longer present in the live catalog. + * + * Uses the cheap `listToolDescriptors` (paths only β€” no schema codegen) for + * liveness and diffs it against all stored fingerprints, so it fits one + * invocation regardless of catalog size. For each removed tool it deletes the + * vector chunks, the lexical document, and the fingerprint row. Guards against + * an empty live list (a transient `tools.list` failure must not wipe the index). + * Accepts a pre-listed `descriptors` snapshot to share one catalog read with a + * preceding in-process page loop. */ +export const sweepRemoved = ( + input: ReconcileStores & { readonly descriptors?: readonly Tool[] }, +): Effect.Effect => + Effect.gen(function* () { + const { namespace, executor, store, fingerprints, owner } = input; + + const live = input.descriptors ?? (yield* listToolDescriptors(executor)); + // Safety: never treat an empty live catalog as "everything removed". + if (live.length === 0) { + return { namespace, removed: 0 }; } + const livePaths = new Set(live.map((tool) => addressToPath(String(tool.address)))); - // ------------------------------------------------------------------------- - // Step 6 β€” Re-embed and upsert changed/new tools. - // ------------------------------------------------------------------------- - if (toEmbed.length === 0) { - return { namespace, total, reembedded: 0, unchanged, removedSkipped }; + const storedEntries = yield* fingerprints.list().pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to load stored fingerprints for sweep.", + cause, + }), + ), + ); + const removed = storedEntries.filter((entry) => !livePaths.has(entry.key)); + if (removed.length === 0) { + return { namespace, removed: 0 }; } - // Chunk all tools to re-embed. + yield* Effect.forEach( + removed, + (entry) => + Effect.gen(function* () { + const { key: path, data } = entry; + if (data.chunkIds.length > 0) { + yield* store.deleteByIds([...data.chunkIds]); + } + if (input.lexicalStore) { + yield* input.lexicalStore.deleteByIds([`${namespace}:${path}`]); + } + yield* fingerprints.remove({ owner, key: path }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to delete fingerprint row for removed tool "${path}".`, + cause, + }), + ), + ); + }), + { concurrency: 8, discard: true }, + ); + + return { namespace, removed: removed.length }; + }); + +// --------------------------------------------------------------------------- +// reconcileToolCatalog β€” full reconcile (loop pages + sweep) +// --------------------------------------------------------------------------- + +/** Reconcile the entire live tool catalog into the vector + lexical index. + * + * Drives `reconcileToolCatalogPage` over every page in-process, then runs + * `sweepRemoved`. This is the convenience entry for hosts that can complete the + * whole reconcile in one execution (local / self-host, or small catalogs). At + * large catalog sizes on a CPU-bounded host (a Worker), drive the page + + * sweep primitives from a durable scheduler instead so each step gets its own + * CPU budget. */ +export const reconcileToolCatalog = ( + input: ReconcileDeps & { readonly pageSize?: number }, +): Effect.Effect => + Effect.gen(function* () { + // One catalog snapshot for the whole in-process run: avoids N+1 `tools.list` + // round-trips across pages + sweep, and gives a consistent `total` and diff + // even if the catalog changes mid-run. (The durable-step path re-lists per + // step by design, since each step is a separate invocation.) + const descriptors = yield* listToolDescriptors(input.executor); + const total = descriptors.length; + let cursor = 0; + let reembedded = 0; + let unchanged = 0; + + for (;;) { + const page = yield* reconcileToolCatalogPage({ ...input, cursor, descriptors }); + reembedded += page.reembedded; + unchanged += page.unchanged; + if (page.nextCursor === null) break; + cursor = page.nextCursor; + } + + const swept = yield* sweepRemoved({ ...input, descriptors }); + + return { namespace: input.namespace, total, reembedded, unchanged, removed: swept.removed }; + }); + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +/** Load stored fingerprint rows for a specific set of paths (one page), keyed by + * path. Avoids loading the whole fingerprint store per page. */ +const loadFingerprintsForPaths = ( + fingerprints: PluginStorageCollectionFacade, + owner: Owner, + paths: readonly string[], +): Effect.Effect, SemanticSearchError> => + Effect.forEach( + paths, + (path) => + fingerprints + .getForOwner({ owner, key: path }) + .pipe(Effect.map((entry) => [path, entry?.data ?? null] as const)), + { concurrency: 16 }, + ).pipe( + Effect.map( + (pairs) => + new Map(pairs.flatMap(([path, data]) => (data === null ? [] : [[path, data] as const]))), + ), + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to load stored fingerprints.", cause }), + ), + ); + +/** Embed the new/changed tools of a page, upsert their vectors (deleting any old + * chunks first), and persist their fingerprint rows. */ +const embedAndUpsert = ( + deps: { + readonly namespace: string; + readonly embedder: ToolEmbedder; + readonly store: VectorStore; + readonly chunker: Chunker; + readonly fingerprints: PluginStorageCollectionFacade; + readonly owner: Owner; + }, + toEmbed: ReadonlyArray<{ + readonly doc: { + readonly path: string; + readonly name: string; + readonly description: string; + readonly integration: string; + }; + readonly fingerprint: string; + readonly oldChunkIds: readonly string[] | null; + }>, +): Effect.Effect => + Effect.gen(function* () { + const { namespace, embedder, store, chunker, fingerprints, owner } = deps; + const chunkedGroups = toEmbed.map((item) => ({ ...item, chunks: chunker.chunk(namespace, item.doc), })); - // Gather all embedding texts, preserving their group + chunk positions. const allTexts: string[] = []; for (const group of chunkedGroups) { for (const chunk of group.chunks) { @@ -179,10 +342,8 @@ export const reconcileToolCatalog = (input: { } } - // Single batched embedding call over all changed chunks. const allVectors = yield* embedder.embedDocuments(allTexts); - // Build VectorInput records and collect new chunkIds per tool. let vectorOffset = 0; const records: VectorInput[] = []; const fingerprintUpdates: Array<{ path: string; row: FingerprintRow }> = []; @@ -195,7 +356,7 @@ export const reconcileToolCatalog = (input: { const vec = allVectors[vectorOffset++]; if (vec === undefined) { return yield* new SemanticSearchError({ - message: `reconcileToolCatalog: embedding vector missing at offset ${vectorOffset - 1}`, + message: `reconcileToolCatalogPage: embedding vector missing at offset ${vectorOffset - 1}`, }); } records.push({ @@ -216,12 +377,7 @@ export const reconcileToolCatalog = (input: { fingerprintUpdates.push({ path: doc.path, - row: { - path: doc.path, - integration: doc.integration, - fingerprint, - chunkIds: newChunkIds, - }, + row: { path: doc.path, integration: doc.integration, fingerprint, chunkIds: newChunkIds }, }); // Delete OLD chunk ids for changed tools before upserting new ones. @@ -230,14 +386,10 @@ export const reconcileToolCatalog = (input: { } } - // Upsert all new vectors in one call. `store.upsert` (makeVectorStore) - // chunks internally into UPSERT_BATCH_SIZE (50) sequential batches, so the - // underlying Vectorize binding never receives more than 50 vectors per - // call β€” well under Cloudflare's 1,000-vector / 2 MB per-upsert caps β€” - // regardless of how many tools changed. + // `store.upsert` chunks internally (≀50/batch) so the Vectorize binding never + // exceeds Cloudflare's per-upsert caps regardless of page size. yield* store.upsert(records); - // Persist updated fingerprint rows. yield* Effect.forEach( fingerprintUpdates, ({ path, row }) => @@ -253,12 +405,4 @@ export const reconcileToolCatalog = (input: { ), { concurrency: 8, discard: true }, ); - - return { - namespace, - total, - reembedded: toEmbed.length, - unchanged, - removedSkipped, - }; }); diff --git a/packages/plugins/semantic-search/src/sdk/plugin.ts b/packages/plugins/semantic-search/src/sdk/plugin.ts index 47a3dc8f52..ef4314ba9a 100644 --- a/packages/plugins/semantic-search/src/sdk/plugin.ts +++ b/packages/plugins/semantic-search/src/sdk/plugin.ts @@ -11,7 +11,14 @@ import { toolFingerprints } from "./collections"; import { makeGeminiEmbedder, type ToolEmbedder } from "./embedder"; import { SemanticSearchError } from "./errors"; import { makeHybridToolDiscoveryProvider } from "./hybrid"; -import { reconcileToolCatalog, type ReconcileResult } from "./indexer"; +import { + reconcileToolCatalog, + reconcileToolCatalogPage, + sweepRemoved, + type ReconcilePageResult, + type ReconcileResult, + type SweepResult, +} from "./indexer"; import { makeVectorToolDiscoveryProvider } from "./provider"; import type { VectorStore } from "./store"; import { type FtsLexicalStore, makeFtsLexicalProvider } from "./store-fts"; @@ -110,6 +117,42 @@ const makeSemanticSearchExtension = (deps: { }) : notConfigured(), + /** Reconcile ONE bounded page of the catalog. The unit a durable driver (a + * Cloudflare Workflow step) calls in a loop so each step gets its own CPU + * budget. Pair with `sweep` after the last page. */ + reindexPage: ( + executor: Executor, + input: { readonly cursor: number; readonly pageSize?: number }, + ): Effect.Effect => + deps.embedder && deps.store && deps.fingerprints && deps.owner + ? reconcileToolCatalogPage({ + namespace: deps.namespace, + executor, + embedder: deps.embedder, + store: deps.store, + chunker: deps.chunker, + fingerprints: deps.fingerprints, + owner: deps.owner, + lexicalStore: deps.lexicalStore, + cursor: input.cursor, + pageSize: input.pageSize, + }) + : notConfigured(), + + /** Delete index entries for tools that left the catalog. Run as the terminal + * step after a paged reindex (or standalone). Needs no embedder. */ + sweep: (executor: Executor): Effect.Effect => + deps.store && deps.fingerprints && deps.owner + ? sweepRemoved({ + namespace: deps.namespace, + executor, + store: deps.store, + fingerprints: deps.fingerprints, + owner: deps.owner, + lexicalStore: deps.lexicalStore, + }) + : notConfigured(), + /** Run a live `tools.search` through the same provider the engine uses, so the * operator console sees exactly what the agent would. Inert until configured. */ search: ( From 046b6a8a48b71eddd76539976c676327917d1ffc Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Thu, 18 Jun 2026 17:38:25 +0530 Subject: [PATCH 39/57] feat(sdk): cache TypeScript schema previews (#35) ## Summary - add an Effect KeyValueStore-backed cache primitive for derived executor artifacts - let hosts provide a durable KeyValueStore and wire Cloudflare KV through a generic optional `CACHE` binding - use that generic cache from the TypeScript schema-preview path, keyed by reachable schema roots/definitions so unused pathological definitions do not poison the preview path ## Validation - bun run --filter @executor-js/sdk test -- src/executor.test.ts - bun run --filter @executor-js/sdk typecheck - bun run --filter @executor-js/api typecheck - bun run --filter @executor-js/cloudflare typecheck - dogfood host: bun run typecheck ## Notes - `apps/host-cloudflare` typecheck still fails in this checkout because `@executor-js/plugin-semantic-search` is not resolvable in the current package graph; the same dogfood host typechecks when linked to the live package workspace. --- apps/host-cloudflare/src/app.ts | 9 +- apps/host-cloudflare/src/config.ts | 9 +- apps/host-cloudflare/wrangler.jsonc | 6 ++ packages/core/api/src/server/executor-app.ts | 7 +- .../core/api/src/server/scoped-executor.ts | 6 ++ packages/core/sdk/src/executor.test.ts | 34 +++++- packages/core/sdk/src/executor.ts | 102 ++++++++++++++---- packages/core/sdk/src/test-config.ts | 3 + .../sdk/src/tool-typescript-preview-cache.ts | 60 +++++++++++ packages/hosts/cloudflare/package.json | 4 + .../cloudflare/src/key-value-store.test.ts | 93 ++++++++++++++++ .../hosts/cloudflare/src/key-value-store.ts | 66 ++++++++++++ 12 files changed, 373 insertions(+), 26 deletions(-) create mode 100644 packages/core/sdk/src/tool-typescript-preview-cache.ts create mode 100644 packages/hosts/cloudflare/src/key-value-store.test.ts create mode 100644 packages/hosts/cloudflare/src/key-value-store.ts diff --git a/apps/host-cloudflare/src/app.ts b/apps/host-cloudflare/src/app.ts index 9efb71facf..b6056505c5 100644 --- a/apps/host-cloudflare/src/app.ts +++ b/apps/host-cloudflare/src/app.ts @@ -1,6 +1,7 @@ -import { Effect } from "effect"; +import { Effect, Layer } from "effect"; import { dbProviderLayer, ExecutorApp, textFailureStrategy } from "@executor-js/api/server"; +import { layerCloudflareKeyValueStore } from "@executor-js/cloudflare/key-value-store"; import { loadConfig, type CloudflareEnv } from "./config"; import { makeCloudflarePlugins } from "./plugins"; @@ -55,6 +56,10 @@ export const makeCloudflareApp = async (env: CloudflareEnv) => { // all three agree on identity. Reads the long-lived `dbHandle`. const aliasLookup = makeServiceTokenAliasLookup(dbHandle, config.organizationId); const identityLayer = cloudflareAccessIdentityLayer(config, aliasLookup); + const bootLayer = + env.CACHE === undefined + ? identityLayer + : Layer.mergeAll(identityLayer, layerCloudflareKeyValueStore(env.CACHE)); // MCP runs through the `MCP_SESSION` Durable Object (cross-isolate sessions); // each session DO opens its own D1 handle, so it takes `env`, not `dbHandle`. // The `/mcp` GATE auth runs in the Worker, so it shares the alias lookup. @@ -80,7 +85,7 @@ export const makeCloudflareApp = async (env: CloudflareEnv) => { mcp: { auth: mcp.auth, sessions: mcp.sessions, reporter: mcp.reporter }, }, config: { mountPrefix: "/api", failure: textFailureStrategy }, - boot: identityLayer, + boot: bootLayer, }); return { appLayer, toWebHandler }; diff --git a/apps/host-cloudflare/src/config.ts b/apps/host-cloudflare/src/config.ts index 7c01d79afb..a4b8a04ffc 100644 --- a/apps/host-cloudflare/src/config.ts +++ b/apps/host-cloudflare/src/config.ts @@ -1,4 +1,9 @@ -import type { D1Database, DurableObjectNamespace, R2Bucket } from "@cloudflare/workers-types"; +import type { + D1Database, + DurableObjectNamespace, + KVNamespace, + R2Bucket, +} from "@cloudflare/workers-types"; import type { AnalyticsEngineDataset } from "@executor-js/plugin-execution-metrics/cloudflare"; import type { VectorizeIndex } from "@executor-js/plugin-semantic-search"; @@ -20,6 +25,8 @@ export interface CloudflareEnv { readonly DB: D1Database; /** R2 bucket binding β€” holds values too large for a D1 row (~1-2MB cap). */ readonly BLOBS?: R2Bucket; + /** KV namespace binding β€” durable cache for derived executor artifacts. */ + readonly CACHE?: KVNamespace; /** Workers Analytics Engine binding β€” opt-in sink for execution metrics. When * bound (uncomment `analytics_engine_datasets` in wrangler.jsonc), each * finished execution/tool call writes a data point; absent, metrics are off. */ diff --git a/apps/host-cloudflare/wrangler.jsonc b/apps/host-cloudflare/wrangler.jsonc index c7eee826a1..f4ec274806 100644 --- a/apps/host-cloudflare/wrangler.jsonc +++ b/apps/host-cloudflare/wrangler.jsonc @@ -33,6 +33,12 @@ "bucket_name": "executor-blobs", }, ], + // Optional durable cache for derived executor artifacts. Create with + // `wrangler kv namespace create executor-cache` + // and paste the returned id here. + // "kv_namespaces": [ + // { "binding": "CACHE", "id": "..." }, + // ], // Workers Analytics Engine β€” the opt-in sink for execution metrics. Uncomment // to enable: each finished execution/tool call writes a data point (queryable // via Cloudflare's GraphQL Analytics API). The plugin is inert until this diff --git a/packages/core/api/src/server/executor-app.ts b/packages/core/api/src/server/executor-app.ts index 68dbabccf2..b81adb14ef 100644 --- a/packages/core/api/src/server/executor-app.ts +++ b/packages/core/api/src/server/executor-app.ts @@ -295,6 +295,7 @@ export interface ExecutorAppOptions< McpExport, RIdentity = never, RMcpAuth = never, + RBootExtra = never, > { /** The host's plugin tuple (drives the API + per-request extension Services). */ readonly plugins: TPlugins; @@ -309,7 +310,7 @@ export interface ExecutorAppOptions< * handle, the resolved identity, the router config). Satisfies the residual * `RDb | RAcct` left by the seams. */ - readonly boot: Layer.Layer; + readonly boot: Layer.Layer; /** Optional per-request Layer (cloud's request-scoped postgres socket). */ readonly requestScoped?: Layer.Layer; } @@ -355,6 +356,7 @@ export const make = < McpExport = undefined, RIdentity = never, RMcpAuth = never, + RBootExtra = never, >( options: ExecutorAppOptions< TPlugins, @@ -365,7 +367,8 @@ export const make = < RReq, McpExport, RIdentity, - RMcpAuth + RMcpAuth, + RBootExtra >, ): ExecutorApp => { const { plugins, providers, config } = options; diff --git a/packages/core/api/src/server/scoped-executor.ts b/packages/core/api/src/server/scoped-executor.ts index a92735fb9a..f99f919299 100644 --- a/packages/core/api/src/server/scoped-executor.ts +++ b/packages/core/api/src/server/scoped-executor.ts @@ -32,6 +32,7 @@ // --------------------------------------------------------------------------- import { Context, Effect, Option } from "effect"; +import * as KeyValueStore from "effect/unstable/persistence/KeyValueStore"; import { createExecutor, @@ -181,6 +182,7 @@ export const makeScopedExecutor = < const { db, blobs } = yield* DbProvider; const { plugins: pluginsFactory } = yield* PluginsProvider; const config = yield* HostConfig; + const keyValueStore = yield* Effect.serviceOption(KeyValueStore.KeyValueStore); // Explicit config wins; otherwise fall back to the request origin if a host // provided one (HTTP middleware / MCP session DO). Stays `undefined` for // non-request callers β€” `coreTools.webBaseUrl` is optional and only the @@ -220,6 +222,10 @@ export const makeScopedExecutor = < subject: Subject.make(accountId), db, blobs, + ...Option.match(keyValueStore, { + onNone: () => ({}), + onSome: (store) => ({ keyValueStore: store }), + }), plugins, httpClientLayer, onElicitation: "accept-all", diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 2c68fe5296..4198314a5b 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; import { Data, Effect, Predicate, Result } from "effect"; +import * as KeyValueStore from "effect/unstable/persistence/KeyValueStore"; import { ToolNotFoundError } from "./errors"; import { @@ -17,6 +18,7 @@ import type { CredentialProvider } from "./provider"; import { IntegrationDetectionResult } from "./types"; import { makeTestExecutor } from "./testing"; import { serveOAuthTestServer } from "./testing/oauth-test-server"; +import { toolTypeScriptPreviewCacheKey } from "./tool-typescript-preview-cache"; // removed: v1 secret browser-handoff, source.configure, case-insensitive tool-id // resolution, secrets/sources/scope-stack. The integration coverage below is @@ -83,7 +85,10 @@ const demoPlugin = definePlugin(() => ({ Cat: { type: "object", properties: { lives: { type: "number" } } }, Collar: { type: "object", properties: { id: { type: "string" } } }, Owner: { type: "object", properties: { pet: { $ref: "#/$defs/Pet" } } }, - Unused: { type: "object", properties: { value: { type: "string" } } }, + // Regression fixture: unused provider definitions can be malformed. + // Tool previews must compile only the definitions reachable from the + // requested tool's schema roots. + Unused: { type: "object", properties: { broken: { $ref: "#/$defs/Missing" } } }, }, }), invokeTool: ({ toolRow }) => Effect.succeed({ ran: toolRow.name }), @@ -383,8 +388,25 @@ describe("createExecutor", () => { it.effect("tools.schema returns roots with shared reachable definitions", () => Effect.gen(function* () { + const cacheRows = new Map(); + const keyValueStore = KeyValueStore.makeStringOnly({ + get: (key) => Effect.sync(() => cacheRows.get(key)), + set: (key, value) => + Effect.sync(() => { + cacheRows.set(key, value); + }), + remove: (key) => + Effect.sync(() => { + cacheRows.delete(key); + }), + clear: Effect.sync(() => { + cacheRows.clear(); + }), + size: Effect.sync(() => cacheRows.size), + }); const executor = yield* makeTestExecutor({ plugins: [demoPlugin] as const, + keyValueStore, }); yield* executor.demo.seed(); yield* executor.connections.create({ @@ -403,6 +425,16 @@ describe("createExecutor", () => { const defs = schema?.schemaDefinitions ?? {}; // Reachable defs from inspect's input/output are attached; Unused is not. expect(Object.keys(defs).sort()).toEqual(["Cat", "Collar", "Dog", "Owner", "Pet"]); + expect(schema?.inputTypeScript).not.toBe("unknown"); + expect(schema?.outputTypeScript).not.toBe("unknown"); + + const cacheKey = yield* toolTypeScriptPreviewCacheKey({ + inputSchema: schema?.inputSchema, + outputSchema: schema?.outputSchema, + definitions: defs, + }); + const cached = yield* keyValueStore.get(cacheKey); + expect(cached).toContain("preview"); }), ); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 7aa192d9d5..fc23b9991d 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -1,4 +1,5 @@ -import { Effect, Layer, Option, Predicate, Schema } from "effect"; +import { Cache, Duration, Effect, Layer, Option, Predicate, Schema } from "effect"; +import * as KeyValueStore from "effect/unstable/persistence/KeyValueStore"; import { FetchHttpClient, type HttpClient } from "effect/unstable/http"; import { fumadb } from "@executor-js/fumadb"; import { memoryAdapter } from "@executor-js/fumadb/adapters/memory"; @@ -143,8 +144,14 @@ import { } from "./owner-policy"; import { ToolSchemaView, type IntegrationDetectionResult } from "./types"; import { type Tool, type ToolAnnotations, type ToolDef, type ToolListFilter } from "./tool"; -import { buildToolTypeScriptPreview } from "./schema-types"; +import { buildToolTypeScriptPreview, type ToolTypeScriptPreview } from "./schema-types"; import { collectReferencedDefinitions } from "./schema-refs"; +import { + TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, + TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, + ToolTypeScriptPreviewCacheEntry, + toolTypeScriptPreviewCacheKey, +} from "./tool-typescript-preview-cache"; import { refreshAccessToken, exchangeClientCredentials, @@ -355,6 +362,13 @@ export interface ExecutorConfig + Effect.gen(function* () { + if (typeScriptPreviewStore === undefined) return null; + + const entry = yield* typeScriptPreviewStore + .get(key) + .pipe(Effect.catch(() => Effect.succeed(Option.none()))); + if (Option.isNone(entry)) return null; + + return entry.value.preview; + }), + }); const transaction = (effect: Effect.Effect) => fuma.transaction(effect); // Populated once, never mutated after startup. @@ -1615,6 +1648,44 @@ export const createExecutor = ; + }): Effect.Effect => + Effect.gen(function* () { + const definitions = Object.fromEntries(input.defs); + const key = yield* toolTypeScriptPreviewCacheKey({ + inputSchema: input.inputSchema, + outputSchema: input.outputSchema, + definitions, + }); + const cached = yield* Cache.get(typeScriptPreviewCache, key); + if (cached !== null) return cached; + + const preview = yield* Effect.tryPromise({ + try: () => + buildToolTypeScriptPreview({ + inputSchema: input.inputSchema, + outputSchema: input.outputSchema, + defs: input.defs, + }), + catch: (cause) => + storageFailureFromUnknown("Failed to build tool TypeScript preview", cause), + }); + yield* Cache.set(typeScriptPreviewCache, key, preview); + if (typeScriptPreviewStore !== undefined) { + yield* typeScriptPreviewStore + .set(key, { + version: TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, + compilerVersion: TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, + preview, + }) + .pipe(Effect.ignore); + } + return preview; + }); + const staticSourceToIntegration = (source: StaticSourceDecl): Integration => ({ slug: IntegrationSlug.make(source.id), description: source.name, @@ -2726,15 +2797,10 @@ export const createExecutor = - buildToolTypeScriptPreview({ - inputSchema: tool.inputSchema, - outputSchema: tool.outputSchema, - defs: new Map(), - }), - catch: (cause) => - storageFailureFromUnknown("Failed to build static tool TypeScript preview", cause), + const preview = yield* buildCachedToolTypeScriptPreview({ + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + defs: new Map(), }).pipe(Effect.option); return ToolSchemaView.make({ address, @@ -2777,15 +2843,11 @@ export const createExecutor = - buildToolTypeScriptPreview({ - inputSchema: tool.inputSchema, - outputSchema: tool.outputSchema, - defs, - }), - catch: (cause) => - storageFailureFromUnknown("Failed to build tool TypeScript preview", cause), + const referencedDefs = new Map(Object.entries(referenced)); + const preview = yield* buildCachedToolTypeScriptPreview({ + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + defs: referencedDefs, }).pipe(Effect.option); const view = preview; diff --git a/packages/core/sdk/src/test-config.ts b/packages/core/sdk/src/test-config.ts index 66948f5c3d..f599c1370d 100644 --- a/packages/core/sdk/src/test-config.ts +++ b/packages/core/sdk/src/test-config.ts @@ -1,4 +1,5 @@ import { Context, Effect, Layer } from "effect"; +import type * as KeyValueStore from "effect/unstable/persistence/KeyValueStore"; import { withQueryContext } from "@executor-js/fumadb/query"; import { collectTables, createExecutor, type Executor, type ExecutorConfig } from "./executor"; import type { FumaDb } from "./fuma-runtime"; @@ -123,6 +124,7 @@ export type TestConfigOptions["coreTools"]; + readonly keyValueStore?: KeyValueStore.KeyValueStore; /** Override the OAuth callback URL. Pass `null` to construct an executor with * no OAuth callback (exercises the fail-loud redirect path). */ readonly redirectUri?: string | null; @@ -160,6 +162,7 @@ export const makeTestConfig = >; + readonly options?: TypeScriptRenderOptions; +} + +const normalizeForHash = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(normalizeForHash); + if (value === null || typeof value !== "object") return value; + + const out: Record = {}; + for (const key of Object.keys(value as Record).sort()) { + out[key] = normalizeForHash((value as Record)[key]); + } + return out; +}; + +const cachePayload = (input: ToolTypeScriptPreviewCacheKeyInput): string => + JSON.stringify( + normalizeForHash({ + version: TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, + compilerVersion: TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, + inputSchema: input.inputSchema, + outputSchema: input.outputSchema, + definitions: input.definitions, + options: input.options ?? {}, + }), + ); + +export const toolTypeScriptPreviewCacheKey = ( + input: ToolTypeScriptPreviewCacheKeyInput, +): Effect.Effect => + sha256Hex(cachePayload(input)).pipe( + Effect.map( + (hash) => + `${TOOL_TYPESCRIPT_PREVIEW_CACHE_PREFIX}${TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION}/${TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION}/${hash}`, + ), + ); diff --git a/packages/hosts/cloudflare/package.json b/packages/hosts/cloudflare/package.json index b622812da2..3928e4df2c 100644 --- a/packages/hosts/cloudflare/package.json +++ b/packages/hosts/cloudflare/package.json @@ -8,6 +8,10 @@ "types": "./src/blob-store.ts", "default": "./src/blob-store.ts" }, + "./key-value-store": { + "types": "./src/key-value-store.ts", + "default": "./src/key-value-store.ts" + }, "./mcp/worker-transport": { "types": "./src/mcp/worker-transport.ts", "default": "./src/mcp/worker-transport.ts" diff --git a/packages/hosts/cloudflare/src/key-value-store.test.ts b/packages/hosts/cloudflare/src/key-value-store.test.ts new file mode 100644 index 0000000000..7f0382aabc --- /dev/null +++ b/packages/hosts/cloudflare/src/key-value-store.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import type { KVNamespace } from "@cloudflare/workers-types"; + +import { makeCloudflareKeyValueStore } from "./key-value-store"; + +const makeFakeKv = ( + pageSize: number, +): { + readonly kv: KVNamespace; + readonly values: Map; + readonly maxConcurrentDeletes: () => number; +} => { + const values = new Map(); + let activeDeletes = 0; + let maxActiveDeletes = 0; + + // oxlint-disable-next-line executor/no-double-cast -- test double: only the KV slice the adapter calls is implemented + const kv = { + get: async (key: string) => values.get(key) ?? null, + put: async (key: string, value: string) => { + values.set(key, value); + }, + delete: async (key: string) => { + activeDeletes += 1; + maxActiveDeletes = Math.max(maxActiveDeletes, activeDeletes); + await new Promise((resolve) => setTimeout(resolve, 0)); + values.delete(key); + activeDeletes -= 1; + }, + list: async (options?: { readonly cursor?: string }) => { + const offset = options?.cursor === undefined ? 0 : Number(options.cursor); + const keys = [...values.keys()].sort().slice(offset, offset + pageSize); + const nextOffset = offset + pageSize; + const listComplete = nextOffset >= values.size; + + return { + keys: keys.map((name) => ({ name })), + list_complete: listComplete, + cursor: listComplete ? "" : String(nextOffset), + }; + }, + } as unknown as KVNamespace; + + return { + kv, + values, + maxConcurrentDeletes: () => maxActiveDeletes, + }; +}; + +describe("makeCloudflareKeyValueStore", () => { + it.effect("round-trips string values", () => + Effect.gen(function* () { + const { kv } = makeFakeKv(10); + const store = makeCloudflareKeyValueStore(kv); + + yield* store.set("a", "value"); + expect(yield* store.get("a")).toBe("value"); + + yield* store.remove("a"); + expect(yield* store.get("a")).toBeUndefined(); + }), + ); + + it.effect("counts paginated keys", () => + Effect.gen(function* () { + const { values, kv } = makeFakeKv(2); + values.set("a", "1"); + values.set("b", "2"); + values.set("c", "3"); + + const store = makeCloudflareKeyValueStore(kv); + expect(yield* store.size).toBe(3); + }), + ); + + it.effect("clears paginated keys in bounded parallel batches", () => + Effect.gen(function* () { + const { values, kv, maxConcurrentDeletes } = makeFakeKv(25); + for (let index = 0; index < 75; index += 1) { + values.set(`key-${index.toString().padStart(2, "0")}`, String(index)); + } + + const store = makeCloudflareKeyValueStore(kv); + yield* store.clear; + + expect(values.size).toBe(0); + expect(maxConcurrentDeletes()).toBeGreaterThan(1); + expect(maxConcurrentDeletes()).toBeLessThanOrEqual(50); + }), + ); +}); diff --git a/packages/hosts/cloudflare/src/key-value-store.ts b/packages/hosts/cloudflare/src/key-value-store.ts new file mode 100644 index 0000000000..9737d82731 --- /dev/null +++ b/packages/hosts/cloudflare/src/key-value-store.ts @@ -0,0 +1,66 @@ +import { Effect, Layer } from "effect"; +import * as KeyValueStore from "effect/unstable/persistence/KeyValueStore"; +import type { KVNamespace } from "@cloudflare/workers-types"; + +const KV_DELETE_CONCURRENCY = 50; + +const storeError = (method: string, key: string | undefined, cause: unknown) => + new KeyValueStore.KeyValueStoreError({ + method, + key, + message: `Cloudflare KV ${method} failed`, + cause, + }); + +const listKeys = async (kv: KVNamespace): Promise> => { + const keys: Array = []; + let cursor: string | undefined; + + do { + const page = await kv.list({ cursor }); + keys.push(...page.keys.map((key) => key.name)); + cursor = page.list_complete ? undefined : page.cursor; + } while (cursor !== undefined); + + return keys; +}; + +const deleteKeys = async (kv: KVNamespace, keys: ReadonlyArray): Promise => { + for (let index = 0; index < keys.length; index += KV_DELETE_CONCURRENCY) { + await Promise.all( + keys.slice(index, index + KV_DELETE_CONCURRENCY).map((key) => kv.delete(key)), + ); + } +}; + +export const makeCloudflareKeyValueStore = (kv: KVNamespace): KeyValueStore.KeyValueStore => + KeyValueStore.makeStringOnly({ + get: (key) => + Effect.tryPromise({ + try: async () => (await kv.get(key)) ?? undefined, + catch: (cause) => storeError("get", key, cause), + }), + set: (key, value) => + Effect.tryPromise({ + try: () => kv.put(key, value), + catch: (cause) => storeError("set", key, cause), + }), + remove: (key) => + Effect.tryPromise({ + try: () => kv.delete(key), + catch: (cause) => storeError("remove", key, cause), + }), + clear: Effect.tryPromise({ + try: async () => deleteKeys(kv, await listKeys(kv)), + catch: (cause) => storeError("clear", undefined, cause), + }), + size: Effect.tryPromise({ + try: async () => (await listKeys(kv)).length, + catch: (cause) => storeError("size", undefined, cause), + }), + }); + +export const layerCloudflareKeyValueStore = ( + kv: KVNamespace, +): Layer.Layer => + Layer.succeed(KeyValueStore.KeyValueStore, makeCloudflareKeyValueStore(kv)); From a2c1c6d69ed650e4bda47fed2cf38cc4a72dc5ad Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Fri, 19 Jun 2026 00:07:20 +0530 Subject: [PATCH 40/57] fix(cloudflare): use resumable MCP streams (#38) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Use the current agents MCP transport primitives for Cloudflare MCP sessions and persist resumable events in Durable Object storage. This removes the local reflection shim around private transport internals. ## Changes - Upgrade `agents` to `^0.16.2` in the Cloudflare host. - Pass `DurableObjectEventStore` into `WorkerTransport` for MCP session resumability. - Delegate request handling directly through the JSON-RPC request-id queue. ## Tests - `bun run --cwd packages/hosts/cloudflare typecheck` - `bun run --cwd packages/hosts/cloudflare test` - `bun run typecheck` - `bun run test` - `bunx oxfmt --check ` - `bunx oxlint -c .oxlintrc.jsonc --deny-warnings` ### [Stack](https://github.com/kitlangton/stack) 1. **#38** πŸ‘ˆ current 2. #39 3. #40 --- bun.lock | 266 +++++++++++++++++- packages/hosts/cloudflare/package.json | 2 +- .../src/mcp/session-durable-object.ts | 21 +- .../cloudflare/src/mcp/worker-transport.ts | 64 +---- 4 files changed, 283 insertions(+), 70 deletions(-) diff --git a/bun.lock b/bun.lock index fbcb1ca2b6..0509cc8701 100644 --- a/bun.lock +++ b/bun.lock @@ -644,7 +644,7 @@ "@executor-js/host-mcp": "workspace:*", "@executor-js/sdk": "workspace:*", "@modelcontextprotocol/sdk": "^1.29.0", - "agents": "^0.10.0", + "agents": "^0.16.2", }, "devDependencies": { "@cloudflare/workers-types": "^4.20250620.0", @@ -1394,7 +1394,7 @@ "@babel/helpers": ["@babel/helpers@7.29.2", "", { "dependencies": { "@babel/template": "^7.28.6", "@babel/types": "^7.29.0" } }, "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw=="], - "@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@babel/parser": ["@babel/parser@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" }, "bin": "./bin/babel-parser.js" }, "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg=="], "@babel/plugin-proposal-decorators": ["@babel/plugin-proposal-decorators@7.29.0", "", { "dependencies": { "@babel/helper-create-class-features-plugin": "^7.28.6", "@babel/helper-plugin-utils": "^7.28.6", "@babel/plugin-syntax-decorators": "^7.28.6" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-CVBVv3VY/XRMxRYq5dwr2DS7/MvqPm23cOCjbwNnVrfOqcWlnefua1uUs0sjdKOGjvPUG633o07uWzJq4oI6dA=="], @@ -1444,6 +1444,8 @@ "@better-fetch/fetch": ["@better-fetch/fetch@1.1.21", "", {}, "sha512-/ImESw0sskqlVR94jB+5+Pxjf+xBwDZF/N5+y2/q4EqD7IARUTSpPfIo8uf39SYpCxyOCtbyYpUrZ3F/k0zT4A=="], + "@borewit/text-codec": ["@borewit/text-codec@0.2.2", "", {}, "sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ=="], + "@braintree/sanitize-url": ["@braintree/sanitize-url@7.1.2", "", {}, "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA=="], "@capsizecss/unpack": ["@capsizecss/unpack@4.0.0", "", { "dependencies": { "fontkitten": "^1.0.0" } }, "sha512-VERIM64vtTP1C4mxQ5thVT9fK0apjPFobqybMtA1UdUujWka24ERHbRHFGmpbbhp73MhV+KSsHQH9C6uOTdEQA=="], @@ -1502,6 +1504,8 @@ "@clack/prompts": ["@clack/prompts@1.3.0", "", { "dependencies": { "@clack/core": "1.3.0", "fast-string-width": "^3.0.2", "fast-wrap-ansi": "^0.2.0", "sisteransi": "^1.0.5" } }, "sha512-GgcWwRCs/xPtaqlMy8qRhPnZf9vlWcWZNHAitnVQ3yk7JmSralSiq5q07yaffYE8SogtDm7zFeKccx1QNVARpw=="], + "@cloudflare/codemode": ["@cloudflare/codemode@0.4.1", "", { "dependencies": { "@types/json-schema": "^7.0.15", "acorn": "^8.17.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.0", "@tanstack/ai": ">=0.8.0 <1.0.0", "ai": "^6.0.0", "zod": "^4.0.0" }, "optionalPeers": ["@modelcontextprotocol/sdk", "@tanstack/ai", "ai", "zod"] }, "sha512-i73cD7GeEzJVxcjqb4yPDCXXq7/h/Le/Vvz/Z3Xxf5qFiiw9G75fd1+KGWvZ/3/Gk6bgL7eUnnwMhByK3b5EUg=="], + "@cloudflare/kv-asset-handler": ["@cloudflare/kv-asset-handler@0.5.0", "", {}, "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg=="], "@cloudflare/unenv-preset": ["@cloudflare/unenv-preset@2.16.1", "", { "peerDependencies": { "unenv": "2.0.0-rc.24", "workerd": ">1.20260305.0 <2.0.0-0" }, "optionalPeers": ["workerd"] }, "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw=="], @@ -2062,8 +2066,12 @@ "@mishieck/ink-titled-box": ["@mishieck/ink-titled-box@0.3.0", "", { "peerDependencies": { "ink": "^6.0.0", "react": "^19.1.0", "typescript": "^5" } }, "sha512-ugzVH9hixp3hwKfQ8On/qnsrdAxS3y9rTu/aGOFed4zVUvtZyGZNIR4rxAwXult8HKI4vJEh0OM8wib9NPrwUg=="], + "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="], + "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="], + "@mongodb-js/zstd": ["@mongodb-js/zstd@7.0.0", "", { "dependencies": { "node-addon-api": "^8.5.0", "prebuild-install": "^7.1.3" } }, "sha512-mQ2s0pYYiav+tzCDR05Zptem8Ey2v8s11lri5RKGhTtL4COVCvVCk5vtyRYNT+9L8qSfyOqqefF9UtnW8mC5jA=="], + "@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="], "@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="], @@ -2110,6 +2118,8 @@ "@noble/hashes": ["@noble/hashes@2.2.0", "", {}, "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg=="], + "@nodable/entities": ["@nodable/entities@2.2.0", "", {}, "sha512-9uGyhaQavEUMC8AIddIjau4NsnsXhou+j5sBAGojCM1oxmQpVKTWR/9JxABD6UAv12vpIms55fPZKFQEhG6uBg=="], + "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], @@ -2902,6 +2912,8 @@ "@tanstack/virtual-file-routes": ["@tanstack/virtual-file-routes@1.162.0", "", {}, "sha512-uhOeFyxLcU41HzvrxsGpiWdcMbScY1EDgbZ5K7DVRMYInbLYWAC0EA/kx9wXAoSM8q82bUG2hRl8+EAjE6XAbA=="], + "@tokenizer/inflate": ["@tokenizer/inflate@0.4.1", "", { "dependencies": { "debug": "^4.4.3", "token-types": "^6.1.1" } }, "sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA=="], + "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], "@turbo/darwin-64": ["@turbo/darwin-64@2.9.6", "", { "os": "darwin", "cpu": "x64" }, "sha512-X/56SnVXIQZBLKwniGTwEQTGmtE5brSACnKMBWpY3YafuxVYefrC2acamfjgxP7BG5w3I+6jf0UrLoSzgPcSJg=="], @@ -3186,6 +3198,8 @@ "anymatch": ["anymatch@3.1.3", "", { "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" } }, "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw=="], + "anynum": ["anynum@1.0.1", "", {}, "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A=="], + "app-builder-bin": ["app-builder-bin@5.0.0-alpha.12", "", {}, "sha512-j87o0j6LqPL3QRr8yid6c+Tt5gC7xNfYo6uQIQkorAC6MpeayVMZrEDzKmJJ/Hlv7EnOQpaRm53k6ktDYZyB6w=="], "app-builder-lib": ["app-builder-lib@26.8.1", "", { "dependencies": { "@develar/schema-utils": "~2.6.5", "@electron/asar": "3.4.1", "@electron/fuses": "^1.8.0", "@electron/get": "^3.0.0", "@electron/notarize": "2.5.0", "@electron/osx-sign": "1.3.3", "@electron/rebuild": "^4.0.3", "@electron/universal": "2.0.3", "@malept/flatpak-bundler": "^0.4.0", "@types/fs-extra": "9.0.13", "async-exit-hook": "^2.0.1", "builder-util": "26.8.1", "builder-util-runtime": "9.5.1", "chromium-pickle-js": "^0.2.0", "ci-info": "4.3.1", "debug": "^4.3.4", "dotenv": "^16.4.5", "dotenv-expand": "^11.0.6", "ejs": "^3.1.8", "electron-publish": "26.8.1", "fs-extra": "^10.1.0", "hosted-git-info": "^4.1.0", "isbinaryfile": "^5.0.0", "jiti": "^2.4.2", "js-yaml": "^4.1.0", "json5": "^2.2.3", "lazy-val": "^1.0.5", "minimatch": "^10.0.3", "plist": "3.1.0", "proper-lockfile": "^4.1.2", "resedit": "^1.7.0", "semver": "~7.7.3", "tar": "^7.5.7", "temp-file": "^3.4.0", "tiny-async-pool": "1.3.0", "which": "^5.0.0" }, "peerDependencies": { "dmg-builder": "26.8.1", "electron-builder-squirrel-windows": "26.8.1" } }, "sha512-p0Im/Dx5C4tmz8QEE1Yn4MkuPC8PrnlRneMhWJj7BBXQfNTJUshM/bp3lusdEsDbvvfJZpXWnYesgSLvwtM2Zw=="], @@ -3828,6 +3842,10 @@ "fast-wrap-ansi": ["fast-wrap-ansi@0.2.0", "", { "dependencies": { "fast-string-width": "^3.0.2" } }, "sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w=="], + "fast-xml-builder": ["fast-xml-builder@1.2.0", "", { "dependencies": { "path-expression-matcher": "^1.5.0", "xml-naming": "^0.1.0" } }, "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q=="], + + "fast-xml-parser": ["fast-xml-parser@5.9.2", "", { "dependencies": { "@nodable/entities": "^2.2.0", "fast-xml-builder": "^1.2.0", "is-unsafe": "^1.0.1", "path-expression-matcher": "^1.5.0", "strnum": "^2.4.0", "xml-naming": "^0.1.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-DYPkXnVSJHAGAkSBeVYhEo/seIpz2SLr9OQcX7m6lXaX3gvoB+DCKzFdZIEhZGI3I1DUhObBAUOT/v2xfoXz/w=="], + "fastq": ["fastq@1.20.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw=="], "fd-package-json": ["fd-package-json@2.0.0", "", { "dependencies": { "walk-up-path": "^4.0.0" } }, "sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ=="], @@ -3844,7 +3862,7 @@ "file-selector": ["file-selector@0.5.0", "", { "dependencies": { "tslib": "^2.0.3" } }, "sha512-s8KNnmIDTBoD0p9uJ9uD0XY38SCeBOtj0UMXyQSLg1Ypfrfj8+dAvwsLjYQkQ2GjhVtp2HrnF5cJzMhBjfD8HA=="], - "file-type": ["file-type@16.5.4", "", { "dependencies": { "readable-web-to-node-stream": "^3.0.0", "strtok3": "^6.2.4", "token-types": "^4.1.1" } }, "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw=="], + "file-type": ["file-type@21.3.4", "", { "dependencies": { "@tokenizer/inflate": "^0.4.1", "strtok3": "^10.3.4", "token-types": "^6.1.1", "uint8array-extras": "^1.4.0" } }, "sha512-Ievi/yy8DS3ygGvT47PjSfdFoX+2isQueoYP1cntFW1JLYAuS4GD7NUPGg4zv2iZfV52uDyk5w5Z0TdpRS6Q1g=="], "file-uri-to-path": ["file-uri-to-path@1.0.0", "", {}, "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="], @@ -4144,6 +4162,8 @@ "is-unicode-supported": ["is-unicode-supported@2.1.0", "", {}, "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ=="], + "is-unsafe": ["is-unsafe@1.0.1", "", {}, "sha512-CLK2+VdgERgD96EYm5lUQssZYlRg2tkZnbsxZoacmSiRxiFJ4Nk4SzjCl+Ur+v3kXIY9dTIdb3IH22y1mZ56LA=="], + "is-wayland": ["is-wayland@0.1.0", "", {}, "sha512-QkbMsWkIfkrzOPxenwye0h56iAXirZYHG9eHVPb22fO9y+wPbaX/CHacOWBa/I++4ohTcByimhM1/nyCsH8KNA=="], "is-windows": ["is-windows@1.0.2", "", {}, "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA=="], @@ -4210,6 +4230,8 @@ "jsonwebtoken": ["jsonwebtoken@9.0.3", "", { "dependencies": { "jws": "^4.0.1", "lodash.includes": "^4.3.0", "lodash.isboolean": "^3.0.3", "lodash.isinteger": "^4.0.4", "lodash.isnumber": "^3.0.3", "lodash.isplainobject": "^4.0.6", "lodash.isstring": "^4.0.1", "lodash.once": "^4.0.0", "ms": "^2.1.1", "semver": "^7.5.4" } }, "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g=="], + "just-bash": ["just-bash@3.0.1", "", { "dependencies": { "diff": "^8.0.2", "fast-xml-parser": "^5.7.3", "file-type": "^21.2.0", "ini": "^6.0.0", "minimatch": "^10.1.1", "modern-tar": "^0.7.3", "papaparse": "^5.5.3", "quickjs-emscripten": "^0.32.0", "re2js": "^1.2.1", "seek-bzip": "^2.0.0", "smol-toml": "^1.6.0", "sprintf-js": "^1.1.3", "sql.js": "^1.13.0", "turndown": "^7.2.2", "yaml": "^2.8.2" }, "optionalDependencies": { "@mongodb-js/zstd": "^7.0.0", "node-liblzma": "^2.0.3" }, "bin": { "just-bash": "dist/bin/just-bash.js", "just-bash-shell": "dist/bin/shell/shell.js" } }, "sha512-YVyzCN08fKarUnwqy7rKOAcX+2MLYLnYInuowmUXn3mqhrtd4ieZNBuzdQG+qYV9DqnIWuv9Whiph0WRIWsBtw=="], + "jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], "jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="], @@ -4506,6 +4528,8 @@ "mlly": ["mlly@1.8.2", "", { "dependencies": { "acorn": "^8.16.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "ufo": "^1.6.3" } }, "sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA=="], + "modern-tar": ["modern-tar@0.7.6", "", {}, "sha512-sweCIVXzx1aIGTCdzcMlSZt1h8k5Tmk08VNAuRk3IU28XamGiOH5ypi11g6De2CH7PhYqSSnGy2A/EFhbWnVKg=="], + "module-details-from-path": ["module-details-from-path@1.0.4", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="], "monaco-editor": ["monaco-editor@0.55.1", "", { "dependencies": { "dompurify": "3.2.7", "marked": "14.0.0" } }, "sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A=="], @@ -4546,7 +4570,7 @@ "node-abi": ["node-abi@3.89.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA=="], - "node-addon-api": ["node-addon-api@1.7.2", "", {}, "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg=="], + "node-addon-api": ["node-addon-api@8.8.0", "", {}, "sha512-c5Ko1fZJIJmzhFIkhRN76WTq+fC6tWnGy9CXA0fA+XygsWZmEwG8vmbkNqxMyoaa0Tin4djul49NzdVcJJcjeA=="], "node-api-version": ["node-api-version@0.2.1", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-2xP/IGGMmmSQpI1+O/k72jF/ykvZ89JeuKX3TLJAYPDVLUalrshrLHkeVcCCZqG/eEa635cr8IBYzgnDvM2O8Q=="], @@ -4558,8 +4582,12 @@ "node-gyp": ["node-gyp@12.3.0", "", { "dependencies": { "env-paths": "^2.2.0", "exponential-backoff": "^3.1.1", "graceful-fs": "^4.2.6", "nopt": "^9.0.0", "proc-log": "^6.0.0", "semver": "^7.3.5", "tar": "^7.5.4", "tinyglobby": "^0.2.12", "undici": "^6.25.0", "which": "^6.0.0" }, "bin": { "node-gyp": "bin/node-gyp.js" } }, "sha512-QNcUWM+HgJplcPzBvFBZ9VXacyGZ4+VTOb80PwWR+TlVzoHbRKULNEzpRsnaoxG3Wzr7Qh7BYxGDU3CbKib2Yg=="], + "node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="], + "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="], + "node-liblzma": ["node-liblzma@2.2.0", "", { "dependencies": { "node-addon-api": "^8.5.0", "node-gyp-build": "^4.8.4" }, "bin": { "nxz": "lib/cli/nxz.js" } }, "sha512-s0KzNOWwOJJgPG6wxg6cKohnAl9Wk/oW1KrQaVzJBjQwVcUGPQCzpR46Ximygjqj/3KhOrtJXnYMp/xYAXp75g=="], + "node-mock-http": ["node-mock-http@1.0.4", "", {}, "sha512-8DY+kFsDkNXy1sJglUfuODx1/opAGJGyrTuFqEoN90oRc2Vk0ZbD4K2qmKXBBEhZQzdKHIVfEJpDU8Ak2NJEvQ=="], "node-releases": ["node-releases@2.0.37", "", {}, "sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg=="], @@ -4644,6 +4672,8 @@ "pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="], + "papaparse": ["papaparse@5.5.3", "", {}, "sha512-5QvjGxYVjxO59MGU2lHVYpRWBBtKHnlIAcSe1uNFCkkptUh63NFRj0FJQm7nR67puEruUci/ZkjmEFrjCAyP4A=="], + "parent-module": ["parent-module@1.0.1", "", { "dependencies": { "callsites": "^3.0.0" } }, "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g=="], "parse-bmfont-ascii": ["parse-bmfont-ascii@1.0.6", "", {}, "sha512-U4RrVsUFCleIOBsIGYOMKjn9PavsGOXxbvYGtMOEfnId0SVNsgehXh1DxUdVPLoxd5mvcEtvmKs2Mmf0Mpa1ZA=="], @@ -4678,6 +4708,8 @@ "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], + "path-expression-matcher": ["path-expression-matcher@1.5.0", "", {}, "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ=="], + "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], @@ -4858,6 +4890,8 @@ "re-resizable": ["re-resizable@6.11.2", "", { "peerDependencies": { "react": "^16.13.1 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.13.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-2xI2P3OHs5qw7K0Ud1aLILK6MQxW50TcO+DetD9eIV58j84TqYeHoZcL9H4GXFXXIh7afhH8mv5iUCXII7OW7A=="], + "re2js": ["re2js@1.4.0", "", {}, "sha512-KTOIcZTSOpOxbu3i0+T6mFQ6tkxXKlTxfcMFs1trQbsMnG84qNq+DjXr8Afu+FEFjvF1NNlldpC7roPyazFI8g=="], + "react": ["react@19.2.5", "", {}, "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA=="], "react-avatar-editor": ["react-avatar-editor@15.1.0", "", { "peerDependencies": { "react": "^0.14.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^0.14.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Zto7u9l6Wd5LPPtjeFJ+7uwoT4bs01OSgkN2kxD18lWl8IiZ0GY3nWCbKPx4qIU7Au1vENsMJm19rfVWHHayaQ=="], @@ -5072,6 +5106,8 @@ "scroll-into-view-if-needed": ["scroll-into-view-if-needed@3.1.0", "", { "dependencies": { "compute-scroll-into-view": "^3.0.2" } }, "sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ=="], + "seek-bzip": ["seek-bzip@2.0.0", "", { "dependencies": { "commander": "^6.0.0" }, "bin": { "seek-bunzip": "bin/seek-bunzip", "seek-table": "bin/seek-bzip-table" } }, "sha512-SMguiTnYrhpLdk3PwfzHeotrcwi8bNV4iemL9tx9poR/yeaMYwB9VzR1w7b57DuWpuqR8n6oZboi0hj3AxZxQg=="], + "semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], "semver-compare": ["semver-compare@1.0.0", "", {}, "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow=="], @@ -5164,6 +5200,8 @@ "sql-highlight": ["sql-highlight@6.1.0", "", {}, "sha512-ed7OK4e9ywpE7pgRMkMQmZDPKSVdm0oX5IEtZiKnFucSF0zu6c80GZBe38UqHuVhTWJ9xsKgSMjCG2bml86KvA=="], + "sql.js": ["sql.js@1.14.1", "", {}, "sha512-gcj8zBWU5cFsi9WUP+4bFNXAyF1iRpA3LLyS/DP5xlrNzGmPIizUeBggKa8DbDwdqaKwUcTEnChtd2grWo/x/A=="], + "sqlite-vec": ["sqlite-vec@0.1.9", "", { "optionalDependencies": { "sqlite-vec-darwin-arm64": "0.1.9", "sqlite-vec-darwin-x64": "0.1.9", "sqlite-vec-linux-arm64": "0.1.9", "sqlite-vec-linux-x64": "0.1.9", "sqlite-vec-windows-x64": "0.1.9" } }, "sha512-L7XJWRIBNvR9O5+vh1FQ+IGkh/3D2AzVksW5gdtk28m78Hy8skFD0pqReKH1Yp0/BUKRGcffgKvyO/EON5JXpA=="], "sqlite-vec-darwin-arm64": ["sqlite-vec-darwin-arm64@0.1.9", "", { "os": "darwin", "cpu": "arm64" }, "sha512-jSsZpE42OfBkGL/ItyJTVCUwl6o6Ka3U5rc4j+UBDIQzC1ulSSKMEhQLthsOnF/MdAf1MuAkYhkdKmmcjaIZQg=="], @@ -5216,7 +5254,9 @@ "strip-json-comments": ["strip-json-comments@5.0.3", "", {}, "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw=="], - "strtok3": ["strtok3@6.3.0", "", { "dependencies": { "@tokenizer/token": "^0.3.0", "peek-readable": "^4.1.0" } }, "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw=="], + "strnum": ["strnum@2.4.1", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg=="], + + "strtok3": ["strtok3@10.3.5", "", { "dependencies": { "@tokenizer/token": "^0.3.0" } }, "sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA=="], "stubborn-fs": ["stubborn-fs@2.0.0", "", { "dependencies": { "stubborn-utils": "^1.0.1" } }, "sha512-Y0AvSwDw8y+nlSNFXMm2g6L51rBGdAQT20J3YSOqxC53Lo3bjWRtr2BKcfYoAf352WYpsZSTURrA0tqhfgudPA=="], @@ -5316,7 +5356,7 @@ "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], - "token-types": ["token-types@4.2.1", "", { "dependencies": { "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ=="], + "token-types": ["token-types@6.1.2", "", { "dependencies": { "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww=="], "toml": ["toml@4.1.1", "", {}, "sha512-EBJnVBr3dTXdA89WVFoAIPUqkBjxPMwRqsfuo1r240tKFHXv3zgca4+NJib/h6TyvGF7vOawz0jGuryJCdNHrw=="], @@ -5348,6 +5388,8 @@ "turbo": ["turbo@2.9.6", "", { "optionalDependencies": { "@turbo/darwin-64": "2.9.6", "@turbo/darwin-arm64": "2.9.6", "@turbo/linux-64": "2.9.6", "@turbo/linux-arm64": "2.9.6", "@turbo/windows-64": "2.9.6", "@turbo/windows-arm64": "2.9.6" }, "bin": { "turbo": "bin/turbo" } }, "sha512-+v2QJey7ZUeUiuigkU+uFfklvNUyPI2VO2vBpMYJA+a1hKFLFiKtUYlRHdb3P9CrAvMzi0upbjI4WT+zKtqkBg=="], + "turndown": ["turndown@7.2.4", "", { "dependencies": { "@mixmark-io/domino": "^2.2.0" } }, "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ=="], + "type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="], "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], @@ -5518,6 +5560,8 @@ "wsl-utils": ["wsl-utils@0.1.0", "", { "dependencies": { "is-wsl": "^3.1.0" } }, "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw=="], + "xml-naming": ["xml-naming@0.1.0", "", {}, "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw=="], + "xml-parse-from-string": ["xml-parse-from-string@1.0.1", "", {}, "sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g=="], "xml2js": ["xml2js@0.5.0", "", { "dependencies": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" } }, "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA=="], @@ -5534,7 +5578,7 @@ "yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="], - "yaml": ["yaml@2.8.3", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg=="], + "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], "yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], @@ -5580,20 +5624,32 @@ "@azure/msal-node/uuid": ["uuid@8.3.2", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="], + "@babel/core/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@babel/core/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + "@babel/generator/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@babel/helper-compilation-targets/lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], "@babel/helper-compilation-targets/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], "@babel/helper-create-class-features-plugin/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + "@babel/parser/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + + "@babel/template/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + + "@babel/traverse/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@better-auth/core/jose": ["jose@6.2.2", "", {}, "sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ=="], "@changesets/apply-release-plan/prettier": ["prettier@2.8.8", "", { "bin": { "prettier": "bin-prettier.js" } }, "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q=="], "@changesets/write/prettier": ["prettier@2.8.8", "", { "bin": { "prettier": "bin-prettier.js" } }, "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q=="], + "@cloudflare/codemode/acorn": ["acorn@8.17.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg=="], + "@cloudflare/vite-plugin/@cloudflare/unenv-preset": ["@cloudflare/unenv-preset@2.16.0", "", { "peerDependencies": { "unenv": "2.0.0-rc.24", "workerd": "1.20260301.1 || ~1.20260302.1 || ~1.20260303.1 || ~1.20260304.1 || >1.20260305.0 <2.0.0-0" }, "optionalPeers": ["workerd"] }, "sha512-8ovsRpwzPoEqPUzoErAYVv8l3FMZNeBVQfJTvtzP4AgLSRGZISRfuChFxHWUQd3n6cnrwkuTGxT+2cGo8EsyYg=="], "@cloudflare/vite-plugin/miniflare": ["miniflare@4.20260415.0", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "^0.34.5", "undici": "7.24.8", "workerd": "1.20260415.1", "ws": "8.18.0", "youch": "4.1.0-beta.10" }, "bin": { "miniflare": "bootstrap.js" } }, "sha512-JoExRWN4YBI2luA5BoSMFEgi8rQWXUGzo3mtE+58VXCLV3jj/Xnk5Yeqs/IXWz8Es5GJIaq6BtsixDvAxXSIng=="], @@ -5662,12 +5718,16 @@ "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], + "@executor-js/cloudflare/agents": ["agents@0.16.2", "", { "dependencies": { "@babel/plugin-proposal-decorators": "^7.29.7", "@cfworker/json-schema": "^4.1.1", "@cloudflare/codemode": "^0.4.1", "@modelcontextprotocol/sdk": "1.29.0", "@rolldown/plugin-babel": "^0.2.3", "cron-schedule": "^6.0.0", "esbuild": "^0.28.1", "just-bash": "^3.0.1", "mimetext": "^3.0.28", "nanoid": "^5.1.11", "partyserver": "^0.5.8", "partysocket": "1.2.0", "yaml": "^2.9.0", "yargs": "^18.0.0" }, "peerDependencies": { "@cloudflare/ai-chat": ">=0.8.5 <1.0.0", "@tanstack/ai": ">=0.10.2 <1.0.0", "@x402/core": "^2.0.0", "@x402/evm": "^2.0.0", "ai": "^6.0.0", "chat": "^4.29.0", "react": "^19.0.0", "vite": ">=6.0.0 <9.0.0", "zod": "^4.0.0" }, "optionalPeers": ["@cloudflare/ai-chat", "@tanstack/ai", "@x402/core", "@x402/evm", "chat", "vite"], "bin": { "agents": "dist/cli/index.js" } }, "sha512-pSWRw4l5j14cGsCPLDhXGjEfv4RX/kUHdgYsqCohd3AfutPNUEyGhbqfNzg8HklfQdZLpjJj+X7vsRAect5/4Q=="], + "@executor-js/e2e/@types/node": ["@types/node@25.9.2", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-G05zqtJhcDLb8uslf5EjCxXg9G1KQxiV8OS0R26IC//Eoyitzqe8z37I7cqvnZlrlSfgocQRfSn/AHBZJJFyGw=="], "@executor-js/emulate/commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="], "@executor-js/emulate/jose": ["jose@6.2.2", "", {}, "sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ=="], + "@executor-js/emulate/yaml": ["yaml@2.8.3", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg=="], + "@executor-js/example-all-plugins/typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], "@executor-js/example-promise-sdk/typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], @@ -5702,6 +5762,8 @@ "@isaacs/cliui/wrap-ansi": ["wrap-ansi@8.1.0", "", { "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } }, "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ=="], + "@jimp/core/file-type": ["file-type@16.5.4", "", { "dependencies": { "readable-web-to-node-stream": "^3.0.0", "strtok3": "^6.2.4", "token-types": "^4.1.1" } }, "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw=="], + "@jimp/core/mime": ["mime@3.0.0", "", { "bin": { "mime": "cli.js" } }, "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="], "@jimp/plugin-blit/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], @@ -5772,6 +5834,8 @@ "@modelcontextprotocol/sdk/jose": ["jose@6.2.2", "", {}, "sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ=="], + "@opentelemetry/configuration/yaml": ["yaml@2.8.3", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg=="], + "@opentelemetry/exporter-logs-otlp-proto/@opentelemetry/resources": ["@opentelemetry/resources@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-lID/vxSuKWXM55XhAKNoYXu9Cutoq5hFdkbTdI/zDKQktXzcWBVhNsOkiZFTMU9UtEWuGRNe0HUgmsFldIdxVA=="], "@opentelemetry/exporter-logs-otlp-proto/@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-r86ut4T1e8vNwB35CqCcKd45yzqH6/6Wzvpk2/cZB8PsPLlZFTvrh8yfOS3CYZYcUmAx4hHTZJ8AO8Dj8nrdhw=="], @@ -6022,6 +6086,8 @@ "@tanstack/router-plugin/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "@tanstack/router-utils/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@tanstack/start-plugin-core/@babel/code-frame": ["@babel/code-frame@7.27.1", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg=="], "@tanstack/start-plugin-core/@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.40", "", {}, "sha512-s3GeJKSQOwBlzdUrj4ISjJj5SfSh+aqn0wjOar4Bx95iV1ETI7F6S/5hLcfAxZ9kXDcyrAkxPlqmd1ZITttf+w=="], @@ -6030,6 +6096,10 @@ "@tanstack/start-plugin-core/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "@types/babel__core/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + + "@types/babel__template/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@types/better-sqlite3/@types/node": ["@types/node@25.6.0", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ=="], "@types/cacheable-request/@types/node": ["@types/node@25.6.0", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ=="], @@ -6084,6 +6154,8 @@ "atmn/jiti": ["jiti@2.6.1", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="], + "babel-dead-code-elimination/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "better-auth/jose": ["jose@6.2.2", "", {}, "sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ=="], "better-call/rou3": ["rou3@0.7.12", "", {}, "sha512-iFE4hLDuloSWcD7mjdCDhx2bKcIsYbtOTpfH5MHHLSKMOUyjqQXTeZVa289uuwEGEKFoE/BAPbhaU4B774nceg=="], @@ -6138,6 +6210,8 @@ "drizzle-kit/esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], + "effect/yaml": ["yaml@2.8.3", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg=="], + "electron-builder/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], "electron-builder/fs-extra": ["fs-extra@10.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ=="], @@ -6186,6 +6260,8 @@ "iconv-corefoundation/cli-truncate": ["cli-truncate@2.1.0", "", { "dependencies": { "slice-ansi": "^3.0.0", "string-width": "^4.2.0" } }, "sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg=="], + "iconv-corefoundation/node-addon-api": ["node-addon-api@1.7.2", "", {}, "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg=="], + "image-q/@types/node": ["@types/node@16.9.1", "", {}, "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g=="], "import-fresh/resolve-from": ["resolve-from@4.0.0", "", {}, "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g=="], @@ -6202,12 +6278,18 @@ "ink-confirm-input/ink-text-input": ["ink-text-input@3.3.0", "", { "dependencies": { "chalk": "^3.0.0", "prop-types": "^15.5.10" }, "peerDependencies": { "ink": "^2.0.0", "react": "^16.5.2" } }, "sha512-gO4wrOf2ie3YuEARTIwGlw37lMjFn3Gk6CKIDrMlHb46WFMagZU7DplohjM24zynlqfnXA5UDEIfC2NBcvD8kg=="], + "just-bash/quickjs-emscripten": ["quickjs-emscripten@0.32.0", "", { "dependencies": { "@jitl/quickjs-wasmfile-debug-asyncify": "0.32.0", "@jitl/quickjs-wasmfile-debug-sync": "0.32.0", "@jitl/quickjs-wasmfile-release-asyncify": "0.32.0", "@jitl/quickjs-wasmfile-release-sync": "0.32.0", "quickjs-emscripten-core": "0.32.0" } }, "sha512-So0Sqw869y/S2oE3Nuc0uT3Dhqgvsj8FSrwBdsuTosVsG8ME5/OcudU1GxsrIFdFABgy17GHnTVO9TYV/bLQcA=="], + "katex/commander": ["commander@8.3.0", "", {}, "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww=="], "knip/jiti": ["jiti@2.6.1", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="], + "knip/yaml": ["yaml@2.8.3", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg=="], + "libsql/detect-libc": ["detect-libc@2.0.2", "", {}, "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw=="], + "magicast/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "matcher/escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], "mdast-util-find-and-replace/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], @@ -6300,6 +6382,8 @@ "run-jxa/type-fest": ["type-fest@2.19.0", "", {}, "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA=="], + "seek-bzip/commander": ["commander@6.2.1", "", {}, "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA=="], + "serialize-error/type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="], "set-value/is-extendable": ["is-extendable@0.1.1", "", {}, "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw=="], @@ -6364,6 +6448,10 @@ "@babel/helper-compilation-targets/lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], + "@babel/parser/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], + + "@babel/parser/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], + "@cloudflare/vite-plugin/miniflare/undici": ["undici@7.24.8", "", {}, "sha512-6KQ/+QxK49Z/p3HO6E5ZCZWNnCasyZLa5ExaVYyvPxUwKtbCPMKELJOqh7EqOle0t9cH/7d2TaaTRRa6Nhs4YQ=="], "@cloudflare/vite-plugin/miniflare/workerd": ["workerd@1.20260415.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20260415.1", "@cloudflare/workerd-darwin-arm64": "1.20260415.1", "@cloudflare/workerd-linux-64": "1.20260415.1", "@cloudflare/workerd-linux-arm64": "1.20260415.1", "@cloudflare/workerd-windows-64": "1.20260415.1" }, "bin": { "workerd": "bin/workerd" } }, "sha512-phyPjRnx+mQDfkhN9ENPioL1L0SdhYs4S0YmJK/xF9Oga+ykNfdSy1MHnsOj8yqnOV96zcVQMx32dJ0r3pq0jQ=="], @@ -6508,6 +6596,18 @@ "@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="], + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators": ["@babel/plugin-proposal-decorators@7.29.7", "", { "dependencies": { "@babel/helper-create-class-features-plugin": "^7.29.7", "@babel/helper-plugin-utils": "^7.29.7", "@babel/plugin-syntax-decorators": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-EtU0Hi3GvrTqD56xKmZvV/uCXK2ZbwVNPNLAquVItcAZpUhkXwWlo3Fmj0c2LxgSf2I8IDULeAepwNP1OefLXg=="], + + "@executor-js/cloudflare/agents/esbuild": ["esbuild@0.28.1", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.1", "@esbuild/android-arm": "0.28.1", "@esbuild/android-arm64": "0.28.1", "@esbuild/android-x64": "0.28.1", "@esbuild/darwin-arm64": "0.28.1", "@esbuild/darwin-x64": "0.28.1", "@esbuild/freebsd-arm64": "0.28.1", "@esbuild/freebsd-x64": "0.28.1", "@esbuild/linux-arm": "0.28.1", "@esbuild/linux-arm64": "0.28.1", "@esbuild/linux-ia32": "0.28.1", "@esbuild/linux-loong64": "0.28.1", "@esbuild/linux-mips64el": "0.28.1", "@esbuild/linux-ppc64": "0.28.1", "@esbuild/linux-riscv64": "0.28.1", "@esbuild/linux-s390x": "0.28.1", "@esbuild/linux-x64": "0.28.1", "@esbuild/netbsd-arm64": "0.28.1", "@esbuild/netbsd-x64": "0.28.1", "@esbuild/openbsd-arm64": "0.28.1", "@esbuild/openbsd-x64": "0.28.1", "@esbuild/openharmony-arm64": "0.28.1", "@esbuild/sunos-x64": "0.28.1", "@esbuild/win32-arm64": "0.28.1", "@esbuild/win32-ia32": "0.28.1", "@esbuild/win32-x64": "0.28.1" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw=="], + + "@executor-js/cloudflare/agents/nanoid": ["nanoid@5.1.11", "", { "bin": { "nanoid": "bin/nanoid.js" } }, "sha512-v+KEsUv2ps74PaSKv0gHTxTCgMXOIfBEbaqa6w6ISIGC7ZsvHN4N9oJ8d4cmf0n5oTzQz2SLmThbQWhjd/8eKg=="], + + "@executor-js/cloudflare/agents/partyserver": ["partyserver@0.5.8", "", { "dependencies": { "nanoid": "^5.1.9" }, "peerDependencies": { "@cloudflare/workers-types": "^4.20260424.1" } }, "sha512-htgSwiBcBu9zIYLrsxBAOvdkjukHvncbTk0nDrJgfruvZ08rxtEN1Ab4T7j9osykP80Bq3zA2oWFd3ngc4Z9uw=="], + + "@executor-js/cloudflare/agents/partysocket": ["partysocket@1.2.0", "", { "dependencies": { "event-target-polyfill": "^0.0.4" }, "peerDependencies": { "react": ">=17" }, "optionalPeers": ["react"] }, "sha512-xgXql4N0b3umN263lHkrZMvvtYC906h4YjY8l63LNcF0x1bfJmQtZLQGqNQWzFHAxLns/6h6/rjdLW4EdYqQwA=="], + + "@executor-js/cloudflare/agents/yargs": ["yargs@18.0.0", "", { "dependencies": { "cliui": "^9.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "string-width": "^7.2.0", "y18n": "^5.0.5", "yargs-parser": "^22.0.0" } }, "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg=="], + "@executor-js/e2e/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], "@executor-js/mcporter/rolldown/@oxc-project/types": ["@oxc-project/types@0.130.0", "", {}, "sha512-ibD2usx9JRu7f5pu2tMKMI4cpA4NgXJQoYRP4pQ7Pxmn1l6k/53qWtQWZayhYy3X4QZkt90Ot+mJEaeXouio6Q=="], @@ -6568,6 +6668,10 @@ "@isaacs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], + "@jimp/core/file-type/strtok3": ["strtok3@6.3.0", "", { "dependencies": { "@tokenizer/token": "^0.3.0", "peek-readable": "^4.1.0" } }, "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw=="], + + "@jimp/core/file-type/token-types": ["token-types@4.2.1", "", { "dependencies": { "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ=="], + "@libsql/kysely-libsql/@libsql/client/@libsql/core": ["@libsql/core@0.8.1", "", { "dependencies": { "js-base64": "^3.7.5" } }, "sha512-u6nrj6HZMTPsgJ9EBhLzO2uhqhlHQJQmVHV+0yFLvfGf3oSP8w7TjZCNUgu1G8jHISx6KFi7bmcrdXW9lRt++A=="], "@libsql/kysely-libsql/@libsql/client/@libsql/hrana-client": ["@libsql/hrana-client@0.6.2", "", { "dependencies": { "@libsql/isomorphic-fetch": "^0.2.1", "@libsql/isomorphic-ws": "^0.1.5", "js-base64": "^3.7.5", "node-fetch": "^3.3.2" } }, "sha512-MWxgD7mXLNf9FXXiM0bc90wCjZSpErWKr5mGza7ERy2FJNNMXd7JIOv+DepBA1FQTIfI8TFO4/QDYgaQC0goNw=="], @@ -6652,6 +6756,8 @@ "@tanstack/router-generator/@tanstack/router-core/seroval-plugins": ["seroval-plugins@1.5.4", "", { "peerDependencies": { "seroval": "^1.0" } }, "sha512-S0xQPhUTefAhNvNWFg0c1J8qJArHt5KdtJ/cFAofo06KD1MVSeFWyl4iiu+ApDIuw0WhjpOfCdgConOfAnLgkw=="], + "@tanstack/router-generator/@tanstack/router-utils/@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="], + "@tanstack/start-plugin-core/@tanstack/router-generator/@tanstack/virtual-file-routes": ["@tanstack/virtual-file-routes@1.161.7", "", { "bin": { "intent": "bin/intent.js" } }, "sha512-olW33+Cn+bsCsZKPwEGhlkqS6w3M2slFv11JIobdnCFKMLG97oAI2kWKdx5/zsywTL8flpnoIgaZZPlQTFYhdQ=="], "@types/better-sqlite3/@types/node/undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], @@ -6878,6 +6984,16 @@ "ink-confirm-input/ink-text-input/chalk": ["chalk@3.0.0", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg=="], + "just-bash/quickjs-emscripten/@jitl/quickjs-wasmfile-debug-asyncify": ["@jitl/quickjs-wasmfile-debug-asyncify@0.32.0", "", { "dependencies": { "@jitl/quickjs-ffi-types": "0.32.0" } }, "sha512-EX8zbXwGqCgAE764M+qvkHtyXDi/FUoMBea0JnES7vCM3P7a2+EOZOjGv85wtZ2sJhI1oJ+nekmqpOODFDY+hw=="], + + "just-bash/quickjs-emscripten/@jitl/quickjs-wasmfile-debug-sync": ["@jitl/quickjs-wasmfile-debug-sync@0.32.0", "", { "dependencies": { "@jitl/quickjs-ffi-types": "0.32.0" } }, "sha512-LeYWrPGC1uNCTBWvibo3ZLJj0CSVNYUXvJpXMCmuQ5Sap2cCACc3uvGvYV4homHHBAzfw5akoTqMMS4YFRtw+Q=="], + + "just-bash/quickjs-emscripten/@jitl/quickjs-wasmfile-release-asyncify": ["@jitl/quickjs-wasmfile-release-asyncify@0.32.0", "", { "dependencies": { "@jitl/quickjs-ffi-types": "0.32.0" } }, "sha512-3oSwPfja12ICz4aIblB58cuY8JlEq5Txt8Cut4VLo+LH47QN+mzCnSgnbB03hWzg1LBcc+VyyI9UOag7a1NF+Q=="], + + "just-bash/quickjs-emscripten/@jitl/quickjs-wasmfile-release-sync": ["@jitl/quickjs-wasmfile-release-sync@0.32.0", "", { "dependencies": { "@jitl/quickjs-ffi-types": "0.32.0" } }, "sha512-BKNDI/TPBfGlLNGYpLrhcDGXmIk4xHm4MRAisOBnOzpXVn9HZWsfmMAc9WMBrAHjvvds6HOikKeaOBKdPdpVrg=="], + + "just-bash/quickjs-emscripten/quickjs-emscripten-core": ["quickjs-emscripten-core@0.32.0", "", { "dependencies": { "@jitl/quickjs-ffi-types": "0.32.0" } }, "sha512-QFnPfjFey8EqknSrSxe1hZrf1/8z7/6s1QzGOmKo6++02r7QRRX7ZoyNaZh7JuVjWsVW87KnQrbZqnHkOAzUyg=="], + "mimetext/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], "node-gyp/which/isexe": ["isexe@4.0.0", "", {}, "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw=="], @@ -7080,6 +7196,68 @@ "@electron/universal/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin": ["@babel/helper-create-class-features-plugin@7.29.7", "", { "dependencies": { "@babel/helper-annotate-as-pure": "^7.29.7", "@babel/helper-member-expression-to-functions": "^7.29.7", "@babel/helper-optimise-call-expression": "^7.29.7", "@babel/helper-replace-supers": "^7.29.7", "@babel/helper-skip-transparent-expression-wrappers": "^7.29.7", "@babel/traverse": "^7.29.7", "semver": "^6.3.1" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-IY3ZD9Tmooqr3TUhc3DUWxiuo8xx1DWLhd5M7hQ+ZWJamqM2BbalrBJb2MisSLoYorOj75U03qULCxQTY9r3hg=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-plugin-utils": ["@babel/helper-plugin-utils@7.29.7", "", {}, "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/plugin-syntax-decorators": ["@babel/plugin-syntax-decorators@7.29.7", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-9MTTLbF39X6sqM92JPEsoI7++26hjZvzkxKZy64aMhWLH2mPkJ/Q3AV4QLmls3R14FpSpkOwQQfUh962JGQxxg=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.1", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.1", "", { "os": "android", "cpu": "arm" }, "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.1", "", { "os": "android", "cpu": "arm64" }, "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.1", "", { "os": "android", "cpu": "x64" }, "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.1", "", { "os": "linux", "cpu": "arm" }, "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.1", "", { "os": "linux", "cpu": "ia32" }, "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.1", "", { "os": "linux", "cpu": "x64" }, "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.1", "", { "os": "none", "cpu": "x64" }, "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.1", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.1", "", { "os": "sunos", "cpu": "x64" }, "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg=="], + + "@executor-js/cloudflare/agents/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.1", "", { "os": "win32", "cpu": "x64" }, "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A=="], + + "@executor-js/cloudflare/agents/yargs/cliui": ["cliui@9.0.1", "", { "dependencies": { "string-width": "^7.2.0", "strip-ansi": "^7.1.0", "wrap-ansi": "^9.0.0" } }, "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w=="], + + "@executor-js/cloudflare/agents/yargs/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], + "@executor-js/mcporter/rolldown/@rolldown/binding-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.10.0", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" } }, "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw=="], "@executor-js/mcporter/rolldown/@rolldown/binding-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], @@ -7152,6 +7330,16 @@ "ink-confirm-input/ink-text-input/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "just-bash/quickjs-emscripten/@jitl/quickjs-wasmfile-debug-asyncify/@jitl/quickjs-ffi-types": ["@jitl/quickjs-ffi-types@0.32.0", "", {}, "sha512-v9T+GQpmk43VDJ7d72sf0Nexhk+ArvtUihW27dy7lqAl0zBObFKtSBBIm5RBjwIhE8VwsPPm9PNuvPvNqLWUEg=="], + + "just-bash/quickjs-emscripten/@jitl/quickjs-wasmfile-debug-sync/@jitl/quickjs-ffi-types": ["@jitl/quickjs-ffi-types@0.32.0", "", {}, "sha512-v9T+GQpmk43VDJ7d72sf0Nexhk+ArvtUihW27dy7lqAl0zBObFKtSBBIm5RBjwIhE8VwsPPm9PNuvPvNqLWUEg=="], + + "just-bash/quickjs-emscripten/@jitl/quickjs-wasmfile-release-asyncify/@jitl/quickjs-ffi-types": ["@jitl/quickjs-ffi-types@0.32.0", "", {}, "sha512-v9T+GQpmk43VDJ7d72sf0Nexhk+ArvtUihW27dy7lqAl0zBObFKtSBBIm5RBjwIhE8VwsPPm9PNuvPvNqLWUEg=="], + + "just-bash/quickjs-emscripten/@jitl/quickjs-wasmfile-release-sync/@jitl/quickjs-ffi-types": ["@jitl/quickjs-ffi-types@0.32.0", "", {}, "sha512-v9T+GQpmk43VDJ7d72sf0Nexhk+ArvtUihW27dy7lqAl0zBObFKtSBBIm5RBjwIhE8VwsPPm9PNuvPvNqLWUEg=="], + + "just-bash/quickjs-emscripten/quickjs-emscripten-core/@jitl/quickjs-ffi-types": ["@jitl/quickjs-ffi-types@0.32.0", "", {}, "sha512-v9T+GQpmk43VDJ7d72sf0Nexhk+ArvtUihW27dy7lqAl0zBObFKtSBBIm5RBjwIhE8VwsPPm9PNuvPvNqLWUEg=="], + "ora/cli-cursor/restore-cursor/onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], "posthog-js/@opentelemetry/exporter-logs-otlp-http/@opentelemetry/otlp-transformer/@opentelemetry/resources": ["@opentelemetry/resources@2.2.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A=="], @@ -7164,6 +7352,26 @@ "rimraf/glob/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="], + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-annotate-as-pure": ["@babel/helper-annotate-as-pure@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" } }, "sha512-OoK6239jHPuSQOoS0kfTVKn0b/rVTk0seKq4Gd2UMLtmOVLjDC0ki3e+c90Trqv2gMfvJFqkiljrr568+qddiw=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-member-expression-to-functions": ["@babel/helper-member-expression-to-functions@7.29.7", "", { "dependencies": { "@babel/traverse": "^7.29.7", "@babel/types": "^7.29.7" } }, "sha512-j+7JYmk1JYDtACIGj0QJqqWZjoUpMoEikQGADMaHgCMCSDqd2+P32rfcibUNrGOMWrlzK1WJBdxrB3JJQZwWtg=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-optimise-call-expression": ["@babel/helper-optimise-call-expression@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" } }, "sha512-+kmGVjcT9RGYzoDwdwEqEvGgKe3BYq+O1iGzjFubaNgZHwYHP6lsF2Yghf4kEuv9BV7tYDZ913aBW9am6YKong=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-replace-supers": ["@babel/helper-replace-supers@7.29.7", "", { "dependencies": { "@babel/helper-member-expression-to-functions": "^7.29.7", "@babel/helper-optimise-call-expression": "^7.29.7", "@babel/traverse": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-atfGXWSeCiF4DnKZIfmJfQRkSw9b9gNNXR1kqKjbhG4pGYCOnkp8OcTB8E3NXjBu8NpheSnOeNKz8KT7UNFTmQ=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-skip-transparent-expression-wrappers": ["@babel/helper-skip-transparent-expression-wrappers@7.29.7", "", { "dependencies": { "@babel/traverse": "^7.29.7", "@babel/types": "^7.29.7" } }, "sha512-brcMGQaVzIeUb+6/bs1Av0f8YuNNjKY2JyvfRCsFuFsdKccEQ5Ges2y74D74NZ1Rz8lKJ9ksJkfqwQFJ/iNEyQ=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/traverse": ["@babel/traverse@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", "@babel/helper-globals": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/template": "^7.29.7", "@babel/types": "^7.29.7", "debug": "^4.3.1" } }, "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + + "@executor-js/cloudflare/agents/yargs/cliui/strip-ansi": ["strip-ansi@7.2.0", "", { "dependencies": { "ansi-regex": "^6.2.2" } }, "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w=="], + + "@executor-js/cloudflare/agents/yargs/string-width/emoji-regex": ["emoji-regex@10.6.0", "", {}, "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A=="], + + "@executor-js/cloudflare/agents/yargs/string-width/strip-ansi": ["strip-ansi@7.2.0", "", { "dependencies": { "ansi-regex": "^6.2.2" } }, "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w=="], + "@executor-js/motel/@opentelemetry/exporter-trace-otlp-http/@opentelemetry/otlp-transformer/protobufjs/@types/node": ["@types/node@25.9.2", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-G05zqtJhcDLb8uslf5EjCxXg9G1KQxiV8OS0R26IC//Eoyitzqe8z37I7cqvnZlrlSfgocQRfSn/AHBZJJFyGw=="], "@react-grab/cli/ora/cli-cursor/restore-cursor/onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], @@ -7174,6 +7382,50 @@ "rimraf/glob/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-annotate-as-pure/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-member-expression-to-functions/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-optimise-call-expression/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-skip-transparent-expression-wrappers/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/traverse/@babel/code-frame": ["@babel/code-frame@7.29.7", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/traverse/@babel/generator": ["@babel/generator@7.29.7", "", { "dependencies": { "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/traverse/@babel/helper-globals": ["@babel/helper-globals@7.29.7", "", {}, "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/traverse/@babel/template": ["@babel/template@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7" } }, "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/traverse/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + + "@executor-js/cloudflare/agents/yargs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], + + "@executor-js/cloudflare/agents/yargs/string-width/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], + "@executor-js/motel/@opentelemetry/exporter-trace-otlp-http/@opentelemetry/otlp-transformer/protobufjs/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-annotate-as-pure/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-annotate-as-pure/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-member-expression-to-functions/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-member-expression-to-functions/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-optimise-call-expression/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-optimise-call-expression/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-skip-transparent-expression-wrappers/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/helper-skip-transparent-expression-wrappers/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/traverse/@babel/code-frame/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/traverse/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], + + "@executor-js/cloudflare/agents/@babel/plugin-proposal-decorators/@babel/helper-create-class-features-plugin/@babel/traverse/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], } } diff --git a/packages/hosts/cloudflare/package.json b/packages/hosts/cloudflare/package.json index 3928e4df2c..94ec974294 100644 --- a/packages/hosts/cloudflare/package.json +++ b/packages/hosts/cloudflare/package.json @@ -48,7 +48,7 @@ "@executor-js/host-mcp": "workspace:*", "@executor-js/sdk": "workspace:*", "@modelcontextprotocol/sdk": "^1.29.0", - "agents": "^0.10.0" + "agents": "^0.16.2" }, "devDependencies": { "@cloudflare/workers-types": "^4.20250620.0", diff --git a/packages/hosts/cloudflare/src/mcp/session-durable-object.ts b/packages/hosts/cloudflare/src/mcp/session-durable-object.ts index 80972e69ca..357311b35a 100644 --- a/packages/hosts/cloudflare/src/mcp/session-durable-object.ts +++ b/packages/hosts/cloudflare/src/mcp/session-durable-object.ts @@ -14,7 +14,7 @@ import { DurableObject } from "cloudflare:workers"; import { Cause, Deferred, Effect } from "effect"; import type * as Tracer from "effect/Tracer"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; -import type { TransportState } from "agents/mcp"; +import { DurableObjectEventStore, type TransportState } from "agents/mcp"; import { jsonRpcErrorBody } from "@executor-js/host-mcp"; import { RequestWebOrigin } from "@executor-js/api/server"; @@ -99,6 +99,9 @@ const SESSION_TIMEOUT_MS = 5 * 60 * 1000; const TRANSPORT_STATE_KEY = "transport"; const SESSION_META_KEY = "session-meta"; const LAST_ACTIVITY_KEY = "last-activity-ms"; +const MCP_EVENT_STORE_KEY_PREFIX = "__mcp_event__:"; +const MCP_STREAM_REQUEST_IDS_KEY_PREFIX = "__mcp_stream_reqs__:"; +const STORAGE_DELETE_CHUNK_SIZE = 128; const approvalResponseKey = (executionId: string) => `approval-response:${executionId}`; // --------------------------------------------------------------------------- @@ -340,6 +343,17 @@ export abstract class McpSessionDOBase< }; } + private async deleteStoragePrefix(prefix: string): Promise { + for (;;) { + const rows = await this.ctx.storage.list({ + prefix, + limit: STORAGE_DELETE_CHUNK_SIZE, + }); + if (rows.size === 0) return; + await this.ctx.storage.delete([...rows.keys()]); + } + } + private clearSessionState(): Effect.Effect { return Effect.promise(async () => { this.sessionMeta = null; @@ -354,6 +368,10 @@ export abstract class McpSessionDOBase< this.ctx.storage.delete(SESSION_META_KEY).catch(() => false), // oxlint-disable-next-line executor/no-promise-catch -- boundary: Durable Object storage cleanup is best-effort after session invalidation this.ctx.storage.delete(LAST_ACTIVITY_KEY).catch(() => false), + // oxlint-disable-next-line executor/no-promise-catch -- boundary: agents DurableObjectEventStore cleanup is best-effort after session invalidation + this.deleteStoragePrefix(MCP_EVENT_STORE_KEY_PREFIX).catch(() => undefined), + // oxlint-disable-next-line executor/no-promise-catch -- boundary: agents stream request-id mapping cleanup is best-effort after session invalidation + this.deleteStoragePrefix(MCP_STREAM_REQUEST_IDS_KEY_PREFIX).catch(() => undefined), // oxlint-disable-next-line executor/no-promise-catch -- boundary: Durable Object alarm cleanup is best-effort after session invalidation this.ctx.storage.deleteAlarm().catch(() => undefined), ]); @@ -378,6 +396,7 @@ export abstract class McpSessionDOBase< const transport = yield* makeMcpWorkerTransport({ sessionIdGenerator: () => self.sessionId, storage: self.makeStorage(), + eventStore: new DurableObjectEventStore(self.ctx.storage), enableJsonResponse: options.enableJsonResponse, }); self.transportJsonResponseMode = options.enableJsonResponse ?? false; diff --git a/packages/hosts/cloudflare/src/mcp/worker-transport.ts b/packages/hosts/cloudflare/src/mcp/worker-transport.ts index 7bb7713bfd..b3c1bee4a1 100644 --- a/packages/hosts/cloudflare/src/mcp/worker-transport.ts +++ b/packages/hosts/cloudflare/src/mcp/worker-transport.ts @@ -17,36 +17,6 @@ export type McpWorkerTransport = Readonly<{ close: () => Effect.Effect; }>; -type HandleRequestResult = { - readonly response: Response; - readonly replacedStandaloneSse: boolean; -}; - -const closeExistingStandaloneSse = (transport: WorkerTransport): boolean => { - const streamId = - typeof Reflect.get(transport, "standaloneSseStreamId") === "string" - ? Reflect.get(transport, "standaloneSseStreamId") - : "_GET_stream"; - const streamMapping = Reflect.get(transport, "streamMapping"); - if (!(streamMapping instanceof Map)) return false; - - const stream = streamMapping.get(streamId); - if (!stream) return false; - - if ( - typeof stream === "object" && - stream !== null && - typeof Reflect.get(stream, "cleanup") === "function" - ) { - Reflect.get(stream, "cleanup")(); - } - streamMapping.delete(streamId); - return true; -}; - -const isStandaloneSseGet = (request: Request): boolean => - request.method === "GET" && (request.headers.get("accept") ?? "").includes("text/event-stream"); - export const makeMcpWorkerTransport = ( options: WorkerTransportOptions, ): Effect.Effect => @@ -60,41 +30,13 @@ export const makeMcpWorkerTransport = ( catch: (cause) => new McpWorkerTransportError({ cause }), }).pipe(Effect.withSpan(`mcp.worker_transport.${name}`)); - const handleWithStandaloneSseReplacement = async ( - request: Request, - ): Promise => { - if (!isStandaloneSseGet(request)) { - return { - response: await transport.handleRequest(request), - replacedStandaloneSse: false, - }; - } - - const initial = await transport.handleRequest(request); - if (initial.status !== 409) { - return { response: initial, replacedStandaloneSse: false }; - } - - const replacedStandaloneSse = closeExistingStandaloneSse(transport); - return { - response: replacedStandaloneSse ? await transport.handleRequest(request) : initial, - replacedStandaloneSse, - }; - }; - return { transport, connect: (server: McpServer) => use("connect", () => server.connect(transport)), handleRequest: (request: Request) => - Effect.gen(function* () { - const result = yield* use("handle_request", () => - requestIdQueue.run(request, () => handleWithStandaloneSseReplacement(request)), - ); - yield* Effect.annotateCurrentSpan({ - "mcp.transport.replaced_standalone_sse": result.replacedStandaloneSse, - }); - return result.response; - }), + use("handle_request", () => + requestIdQueue.run(request, () => transport.handleRequest(request)), + ), close: () => Effect.ignore( Effect.tryPromise({ From 31621819cca567342a73e985dbb5982f6ebe0db2 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Fri, 19 Jun 2026 00:12:00 +0530 Subject: [PATCH 41/57] chore(cloudflare): improve MCP session observability (#39) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Add structured MCP session request logs and span attributes around the Cloudflare Durable Object request path. This makes long-lived stream behavior visible by method, headers, session state, response status, and elapsed wall time. ## Changes - Emit one structured `mcp_session_request` log per Cloudflare MCP session request. - Add request/transport/event-store attributes to MCP session spans. - Convert MCP session-mode logging to structured JSON. ## Tests - `bun run --cwd packages/hosts/cloudflare typecheck` - `bun run --cwd packages/hosts/cloudflare test` - `bun run --cwd packages/hosts/mcp typecheck` - `bun run --cwd packages/hosts/mcp test` - `bun run typecheck` - `bun run test` - `bunx oxfmt --check ` - `bunx oxlint -c .oxlintrc.jsonc --deny-warnings` ### [Stack](https://github.com/kitlangton/stack) 1. #38 2. **#39** πŸ‘ˆ current 3. #40 --- .../src/mcp/session-durable-object.ts | 78 +++++++++++++++---- packages/hosts/mcp/src/tool-server.ts | 4 +- 2 files changed, 63 insertions(+), 19 deletions(-) diff --git a/packages/hosts/cloudflare/src/mcp/session-durable-object.ts b/packages/hosts/cloudflare/src/mcp/session-durable-object.ts index 357311b35a..5fe507803f 100644 --- a/packages/hosts/cloudflare/src/mcp/session-durable-object.ts +++ b/packages/hosts/cloudflare/src/mcp/session-durable-object.ts @@ -104,6 +104,9 @@ const MCP_STREAM_REQUEST_IDS_KEY_PREFIX = "__mcp_stream_reqs__:"; const STORAGE_DELETE_CHUNK_SIZE = 128; const approvalResponseKey = (executionId: string) => `approval-response:${executionId}`; +const isStandaloneSseGet = (request: Request): boolean => + request.method === "GET" && (request.headers.get("accept") ?? "").includes("text/event-stream"); + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -354,6 +357,45 @@ export abstract class McpSessionDOBase< } } + private requestAttrs(request: Request): Record { + return { + "mcp.request.method": request.method, + "mcp.request.session_id_present": !!request.headers.get("mcp-session-id"), + "mcp.request.accept": request.headers.get("accept") ?? "", + "mcp.request.content_type": request.headers.get("content-type") ?? "", + "mcp.request.content_length": request.headers.get("content-length") ?? "", + "mcp.request.last_event_id_present": !!request.headers.get("last-event-id"), + "mcp.request.standalone_sse_get": isStandaloneSseGet(request), + }; + } + + private logRequestCompleted(input: { + readonly request: Request; + readonly response: Response; + readonly startedAt: number; + }): void { + console.info( + JSON.stringify({ + event: "mcp_session_request", + sessionId: this.sessionId, + method: input.request.method, + accept: input.request.headers.get("accept") ?? "", + contentType: input.request.headers.get("content-type") ?? "", + contentLength: input.request.headers.get("content-length") ?? "", + sessionIdPresent: !!input.request.headers.get("mcp-session-id"), + lastEventIdPresent: !!input.request.headers.get("last-event-id"), + standaloneSseGet: isStandaloneSseGet(input.request), + responseStatus: input.response.status, + responseContentType: input.response.headers.get("content-type") ?? "", + eventStoreEnabled: true, + enableJsonResponse: this.transportJsonResponseMode ?? false, + initialized: this.initialized, + hasTransport: !!this.transport, + responseInitMs: Date.now() - input.startedAt, + }), + ); + } + private clearSessionState(): Effect.Effect { return Effect.promise(async () => { this.sessionMeta = null; @@ -526,10 +568,7 @@ export abstract class McpSessionDOBase< return "restored" as const; }).pipe( Effect.withSpan("McpSessionDO.restoreRuntime", { - attributes: { - "mcp.request.method": request.method, - "mcp.request.session_id_present": !!request.headers.get("mcp-session-id"), - }, + attributes: self.requestAttrs(request), }), // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: cold DO restore is re-entered from Promise-only Durable Object method Effect.orDie, @@ -677,6 +716,7 @@ export abstract class McpSessionDOBase< async handleRequest(request: Request): Promise { const methodEnteredAt = Date.now(); + const startedAt = methodEnteredAt; // Wrap the dispatch in an Effect span so every DO request β€” not just // the rare new-session `init()` β€” shows up in Axiom. Basic attributes // only (method, session-id presence, response status); rich client @@ -700,10 +740,20 @@ export abstract class McpSessionDOBase< return yield* self.dispatchRequest(request).pipe( Effect.tap((response) => - Effect.annotateCurrentSpan({ - "mcp.response.status_code": response.status, - "mcp.response.content_type": response.headers.get("content-type") ?? "", - "mcp.transport.enable_json_response": self.transportJsonResponseMode ?? false, + Effect.gen(function* () { + yield* Effect.annotateCurrentSpan({ + "mcp.response.status_code": response.status, + "mcp.response.content_type": response.headers.get("content-type") ?? "", + "mcp.transport.enable_json_response": self.transportJsonResponseMode ?? false, + "mcp.transport.event_store_enabled": true, + }); + yield* Effect.sync(() => + self.logRequestCompleted({ + request, + response, + startedAt, + }), + ); }), ), Effect.ensuring( @@ -719,10 +769,7 @@ export abstract class McpSessionDOBase< // it ends so the flushed span names the frame. Effect.tapCause((cause) => self.recordCauseOnSpan(cause)), Effect.withSpan("McpSessionDO.handleRequest", { - attributes: { - "mcp.request.method": request.method, - "mcp.request.session_id_present": !!request.headers.get("mcp-session-id"), - }, + attributes: self.requestAttrs(request), }), (eff) => this.withTelemetry(eff, incoming), (eff) => self.withSpanFlush(eff), @@ -880,17 +927,14 @@ export abstract class McpSessionDOBase< ); const response = yield* transport.handleRequest(request).pipe( Effect.withSpan("McpSessionDO.transport.handleRequest", { - attributes: { - "mcp.request.method": request.method, - "mcp.request.content_type": request.headers.get("content-type") ?? "", - "mcp.request.content_length": request.headers.get("content-length") ?? "", - }, + attributes: self.requestAttrs(request), }), ); yield* Effect.annotateCurrentSpan({ "mcp.response.status_code": response.status, "mcp.response.content_type": response.headers.get("content-type") ?? "", "mcp.transport.enable_json_response": self.transportJsonResponseMode ?? false, + "mcp.transport.event_store_enabled": true, }); if (request.method === "DELETE") { yield* Effect.promise(() => self.cleanup()).pipe(Effect.withSpan("mcp.session.cleanup")); diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index 6f61b8684e..b196d7818a 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -629,9 +629,9 @@ export const createExecutorMcpServer = ( ); yield* Effect.sync(() => { - console.error( - "[executor] MCP session mode", + console.info( JSON.stringify({ + event: "mcp_session_mode", ...capabilitySnapshot(server), elicitationMode: elicitationMode.mode, resumeEnabled: elicitationMode.mode !== "native", From a3a62fc8cada43d9316dfb7cd12bca9499b55d88 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Fri, 19 Jun 2026 00:12:18 +0530 Subject: [PATCH 42/57] refactor(mcp): use streamable http for remote clients (#40) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Remove the legacy configurable SSE remote-client mode and use Streamable HTTP for remote MCP clients. This keeps the remote MCP path aligned with the transport used by the Cloudflare Durable Object session work. ## Changes - Remove remote MCP transport selection from public API payloads and stored config. - Drop `SSEClientTransport` from remote client connection setup. - Update e2e payloads and expectations to the flattened remote config shape. ## Tests - `bun run --cwd packages/plugins/mcp typecheck` - `bun run --cwd packages/plugins/mcp test` - `bun run --cwd e2e typecheck` - `bun run typecheck` - `bun run test` - `bunx oxfmt --check ` - `bunx oxlint -c .oxlintrc.jsonc --deny-warnings` ### [Stack](https://github.com/kitlangton/stack) 1. #38 2. #39 3. **#40** πŸ‘ˆ current --- e2e/cloud/sources-api.test.ts | 3 --- e2e/cloud/sources-refresh.test.ts | 1 - packages/plugins/mcp/src/api/group.ts | 1 - packages/plugins/mcp/src/api/handlers.ts | 2 -- packages/plugins/mcp/src/sdk/connection.ts | 22 ++-------------------- packages/plugins/mcp/src/sdk/index.ts | 1 - packages/plugins/mcp/src/sdk/plugin.ts | 9 ++------- packages/plugins/mcp/src/sdk/types.ts | 14 +++----------- 8 files changed, 7 insertions(+), 46 deletions(-) diff --git a/e2e/cloud/sources-api.test.ts b/e2e/cloud/sources-api.test.ts index e174428b90..517eccdf7e 100644 --- a/e2e/cloud/sources-api.test.ts +++ b/e2e/cloud/sources-api.test.ts @@ -219,7 +219,6 @@ scenario( transport: "remote", name: "Broken MCP", endpoint: "http://127.0.0.1:1/mcp", - remoteTransport: "auto", slug, }, }); @@ -231,7 +230,6 @@ scenario( config: { transport: "remote", endpoint: "http://127.0.0.1:1/mcp", - remoteTransport: "auto", }, }); }), @@ -260,7 +258,6 @@ scenario( transport: "remote", name: "E2E MCP", endpoint: server.endpoint, - remoteTransport: "streamable-http", slug, }, }); diff --git a/e2e/cloud/sources-refresh.test.ts b/e2e/cloud/sources-refresh.test.ts index fe2b70afd8..f97705be1a 100644 --- a/e2e/cloud/sources-refresh.test.ts +++ b/e2e/cloud/sources-refresh.test.ts @@ -93,7 +93,6 @@ scenario( transport: "remote", name: "Refresh MCP", endpoint: server.endpoint, - remoteTransport: "streamable-http", slug, }, }); diff --git a/packages/plugins/mcp/src/api/group.ts b/packages/plugins/mcp/src/api/group.ts index 19b3e141b4..0530835555 100644 --- a/packages/plugins/mcp/src/api/group.ts +++ b/packages/plugins/mcp/src/api/group.ts @@ -32,7 +32,6 @@ const AddRemoteServerPayload = Schema.Struct({ transport: Schema.optional(Schema.Literal("remote")), name: Schema.String, endpoint: Schema.String, - remoteTransport: Schema.optional(Schema.Literals(["streamable-http", "sse", "auto"])), slug: Schema.optional(Schema.String), queryParams: Schema.optional(StringMap), headers: Schema.optional(StringMap), diff --git a/packages/plugins/mcp/src/api/handlers.ts b/packages/plugins/mcp/src/api/handlers.ts index 5c2bc239cd..898f5cbbaa 100644 --- a/packages/plugins/mcp/src/api/handlers.ts +++ b/packages/plugins/mcp/src/api/handlers.ts @@ -54,7 +54,6 @@ const toServerInput = ( transport?: "remote"; name: string; endpoint: string; - remoteTransport?: "streamable-http" | "sse" | "auto"; queryParams?: Record; headers?: Record; slug?: string; @@ -68,7 +67,6 @@ const toServerInput = ( transport: "remote", name: p.name, endpoint: p.endpoint, - remoteTransport: p.remoteTransport, queryParams: p.queryParams, headers: p.headers, slug: p.slug, diff --git a/packages/plugins/mcp/src/sdk/connection.ts b/packages/plugins/mcp/src/sdk/connection.ts index 208bc2f306..8585be4ff5 100644 --- a/packages/plugins/mcp/src/sdk/connection.ts +++ b/packages/plugins/mcp/src/sdk/connection.ts @@ -1,6 +1,5 @@ import type { OAuthClientProvider } from "@modelcontextprotocol/sdk/client/auth.js"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; -import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { CfWorkerJsonSchemaValidator } from "@modelcontextprotocol/sdk/validation/cfworker"; import { Effect } from "effect"; @@ -33,9 +32,8 @@ export type McpConnector = Effect.Effect; export type RemoteConnectorInput = Omit< McpRemoteIntegrationConfig, - "authenticationTemplate" | "remoteTransport" | "headers" | "queryParams" + "authenticationTemplate" | "headers" | "queryParams" > & { - readonly remoteTransport?: McpRemoteIntegrationConfig["remoteTransport"]; readonly headers?: Record; readonly queryParams?: Record; readonly authProvider?: OAuthClientProvider; @@ -144,12 +142,11 @@ export const createMcpConnector = (input: ConnectorInput): McpConnector => { // Remote transport const headers = input.headers ?? {}; - const remoteTransport = input.remoteTransport ?? "auto"; const requestInit = Object.keys(headers).length > 0 ? { headers } : undefined; const endpoint = buildEndpointUrl(input.endpoint, input.queryParams ?? {}); - const connectStreamableHttp = connectClient({ + return connectClient({ transport: "streamable-http", createTransport: () => new StreamableHTTPClientTransport(endpoint, { @@ -157,19 +154,4 @@ export const createMcpConnector = (input: ConnectorInput): McpConnector => { authProvider: input.authProvider, }), }); - - const connectSse = connectClient({ - transport: "sse", - createTransport: () => - new SSEClientTransport(endpoint, { - requestInit, - authProvider: input.authProvider, - }), - }); - - if (remoteTransport === "streamable-http") return connectStreamableHttp; - if (remoteTransport === "sse") return connectSse; - - // auto β€” try streamable-http first, fall back to SSE - return connectStreamableHttp.pipe(Effect.catch(() => connectSse)); }; diff --git a/packages/plugins/mcp/src/sdk/index.ts b/packages/plugins/mcp/src/sdk/index.ts index cf0991cd41..ef4a82f287 100644 --- a/packages/plugins/mcp/src/sdk/index.ts +++ b/packages/plugins/mcp/src/sdk/index.ts @@ -18,7 +18,6 @@ export { McpIntegrationConfig, McpRemoteIntegrationConfig, McpStdioIntegrationConfig, - McpRemoteTransport, McpTransport, McpToolAnnotations, McpToolBinding, diff --git a/packages/plugins/mcp/src/sdk/plugin.ts b/packages/plugins/mcp/src/sdk/plugin.ts index 41afb42dd6..6646ada705 100644 --- a/packages/plugins/mcp/src/sdk/plugin.ts +++ b/packages/plugins/mcp/src/sdk/plugin.ts @@ -46,7 +46,6 @@ import { probeMcpEndpointShape, type McpShapeProbeResult } from "./probe-shape"; import { McpAuthMethodInput, McpAuthShorthand, - McpRemoteTransport, type McpAuthMethod, type McpToolAnnotations, expandMcpAuthMethodInputs, @@ -146,7 +145,6 @@ const McpRemoteServerInputSchema = Schema.Struct({ transport: Schema.optional(Schema.Literal("remote")), name: Schema.String, endpoint: Schema.String, - remoteTransport: Schema.optional(McpRemoteTransport), headers: Schema.optional(Schema.Record(Schema.String, Schema.String)), queryParams: Schema.optional(Schema.Record(Schema.String, Schema.String)), slug: Schema.optional(Schema.String), @@ -270,7 +268,6 @@ const toIntegrationConfig = (input: McpServerInput): McpIntegrationConfigType => return { transport: "remote", endpoint: input.endpoint, - remoteTransport: input.remoteTransport ?? "auto", queryParams: input.queryParams, headers: input.headers, authenticationTemplate: input.authenticationTemplate @@ -480,7 +477,6 @@ const buildConnectorInput = ( return Effect.succeed({ transport: "remote" as const, endpoint: config.endpoint, - remoteTransport: config.remoteTransport ?? "auto", queryParams: Object.keys(queryParams).length > 0 ? queryParams : undefined, headers: Object.keys(headers).length > 0 ? headers : undefined, authProvider, @@ -914,7 +910,7 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { const parsed = parseMcpIntegrationConfig(credential.config); if (!parsed) { return yield* new McpConnectionError({ - transport: "auto", + transport: "streamable-http", message: `MCP integration "${toolRow.integration}" has no usable config`, }); } @@ -927,8 +923,7 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { }); } - const transport: string = - parsed.transport === "stdio" ? "stdio" : (parsed.remoteTransport ?? "auto"); + const transport: string = parsed.transport === "stdio" ? "stdio" : "streamable-http"; // An apikey method with unresolved inputs fails the invocation // explicitly instead of dialing unauthenticated. diff --git a/packages/plugins/mcp/src/sdk/types.ts b/packages/plugins/mcp/src/sdk/types.ts index 9ce33e0abb..ae2275ec70 100644 --- a/packages/plugins/mcp/src/sdk/types.ts +++ b/packages/plugins/mcp/src/sdk/types.ts @@ -1,4 +1,4 @@ -import { Effect, Option, Schema } from "effect"; +import { Option, Schema } from "effect"; import { ApiKeyAuthMethod, ApiKeyAuthTemplate, @@ -21,14 +21,11 @@ import { // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- -// Transport / remote transport +// Transport // --------------------------------------------------------------------------- -export const McpRemoteTransport = Schema.Literals(["streamable-http", "sse", "auto"]); -export type McpRemoteTransport = typeof McpRemoteTransport.Type; - /** All transport types (used in the connector layer) */ -export const McpTransport = Schema.Literals(["streamable-http", "sse", "stdio", "auto"]); +export const McpTransport = Schema.Literals(["streamable-http", "stdio"]); export type McpTransport = typeof McpTransport.Type; // --------------------------------------------------------------------------- @@ -159,11 +156,6 @@ export const McpRemoteIntegrationConfig = Schema.Struct({ transport: Schema.Literal("remote"), /** The MCP server endpoint URL */ endpoint: Schema.String, - /** Transport preference for this remote server */ - remoteTransport: McpRemoteTransport.pipe( - Schema.optionalKey, - Schema.withConstructorDefault(Effect.succeed("auto" as const)), - ), /** Static query params appended to the endpoint URL (non-credential) */ queryParams: Schema.optional(StringMap), /** Static headers sent on every request (non-credential) */ From ef2cbfecefe216f674052a6983b6a227e7c2e1a2 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Fri, 19 Jun 2026 00:13:45 +0530 Subject: [PATCH 43/57] perf(sdk): cache tool schema views (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Caches fully assembled `ToolSchemaView` results in the configured `KeyValueStore`, using a content-addressed key so repeated schema views can skip preview assembly work after the raw schema data is unchanged. ## Changes - Adds a schema-view cache entry schema and deterministic key builder that includes tool metadata, input/output schemas, definitions, and TypeScript preview cache/compiler versions. - Reads and writes the schema-view cache from `executor.tools.schema` for static and persisted tools. - Adds regression coverage for cache-key invalidation and executor cache writes. ## Tests - `bun run --cwd packages/core/sdk test src/tool-schema-view-cache.test.ts src/executor.test.ts` - `bun run --cwd packages/core/sdk typecheck` ### [Stack](https://github.com/kitlangton/stack) 1. **#36** πŸ‘ˆ current 2. #37 --- packages/core/sdk/src/cache-key.ts | 13 +++ packages/core/sdk/src/executor.test.ts | 42 ++++++---- packages/core/sdk/src/executor.ts | 81 +++++++++++++++++-- .../sdk/src/tool-schema-view-cache.test.ts | 74 +++++++++++++++++ .../core/sdk/src/tool-schema-view-cache.ts | 48 +++++++++++ .../sdk/src/tool-typescript-preview-cache.ts | 30 +++---- 6 files changed, 247 insertions(+), 41 deletions(-) create mode 100644 packages/core/sdk/src/cache-key.ts create mode 100644 packages/core/sdk/src/tool-schema-view-cache.test.ts create mode 100644 packages/core/sdk/src/tool-schema-view-cache.ts diff --git a/packages/core/sdk/src/cache-key.ts b/packages/core/sdk/src/cache-key.ts new file mode 100644 index 0000000000..e303c78079 --- /dev/null +++ b/packages/core/sdk/src/cache-key.ts @@ -0,0 +1,13 @@ +const normalizeForCacheKey = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(normalizeForCacheKey); + if (value === null || typeof value !== "object") return value; + + const out: Record = {}; + for (const key of Object.keys(value as Record).sort()) { + out[key] = normalizeForCacheKey((value as Record)[key]); + } + return out; +}; + +export const cacheKeyPayload = (value: unknown): string => + JSON.stringify(normalizeForCacheKey(value)); diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 4198314a5b..8feb74d61a 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -18,6 +18,7 @@ import type { CredentialProvider } from "./provider"; import { IntegrationDetectionResult } from "./types"; import { makeTestExecutor } from "./testing"; import { serveOAuthTestServer } from "./testing/oauth-test-server"; +import { toolSchemaViewCacheKey } from "./tool-schema-view-cache"; import { toolTypeScriptPreviewCacheKey } from "./tool-typescript-preview-cache"; // removed: v1 secret browser-handoff, source.configure, case-insensitive tool-id @@ -45,6 +46,21 @@ const CONN = ConnectionName.make("main"); const addr = (tool: string): ToolAddress => ToolAddress.make(`tools.${INTEG}.org.${CONN}.${tool}`); +const demoDefinitions = { + Pet: { anyOf: [{ $ref: "#/$defs/Dog" }, { $ref: "#/$defs/Cat" }] }, + Dog: { + type: "object", + properties: { collar: { $ref: "#/$defs/Collar" } }, + }, + Cat: { type: "object", properties: { lives: { type: "number" } } }, + Collar: { type: "object", properties: { id: { type: "string" } } }, + Owner: { type: "object", properties: { pet: { $ref: "#/$defs/Pet" } } }, + // Regression fixture: unused provider definitions can be malformed. + // Tool previews must compile only the definitions reachable from the + // requested tool's schema roots. + Unused: { type: "object", properties: { broken: { $ref: "#/$defs/Missing" } } }, +}; + // --------------------------------------------------------------------------- // A plugin that registers an integration, produces per-connection tools via // resolveTools (with shared $defs), and supports ctx.transaction rollback. @@ -76,20 +92,7 @@ const demoPlugin = definePlugin(() => ({ }, { name: ToolName.make("run"), description: "run" }, ], - definitions: { - Pet: { anyOf: [{ $ref: "#/$defs/Dog" }, { $ref: "#/$defs/Cat" }] }, - Dog: { - type: "object", - properties: { collar: { $ref: "#/$defs/Collar" } }, - }, - Cat: { type: "object", properties: { lives: { type: "number" } } }, - Collar: { type: "object", properties: { id: { type: "string" } } }, - Owner: { type: "object", properties: { pet: { $ref: "#/$defs/Pet" } } }, - // Regression fixture: unused provider definitions can be malformed. - // Tool previews must compile only the definitions reachable from the - // requested tool's schema roots. - Unused: { type: "object", properties: { broken: { $ref: "#/$defs/Missing" } } }, - }, + definitions: demoDefinitions, }), invokeTool: ({ toolRow }) => Effect.succeed({ ran: toolRow.name }), extension: (ctx) => ({ @@ -435,6 +438,17 @@ describe("createExecutor", () => { }); const cached = yield* keyValueStore.get(cacheKey); expect(cached).toContain("preview"); + + const schemaViewCacheKey = yield* toolSchemaViewCacheKey({ + address: String(addr("inspect")), + name: schema?.name, + description: schema?.description, + inputSchema: schema?.inputSchema, + outputSchema: schema?.outputSchema, + definitions: demoDefinitions, + }); + const cachedSchemaView = yield* keyValueStore.get(schemaViewCacheKey); + expect(cachedSchemaView).toContain("view"); }), ); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index fc23b9991d..1dff079780 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -152,6 +152,11 @@ import { ToolTypeScriptPreviewCacheEntry, toolTypeScriptPreviewCacheKey, } from "./tool-typescript-preview-cache"; +import { + TOOL_SCHEMA_VIEW_CACHE_VERSION, + ToolSchemaViewCacheEntry, + toolSchemaViewCacheKey, +} from "./tool-schema-view-cache"; import { refreshAccessToken, exchangeClientCredentials, @@ -1612,6 +1617,10 @@ export const createExecutor = + Effect.gen(function* () { + if (toolSchemaViewStore === undefined) return null; + + const entry = yield* toolSchemaViewStore + .get(key) + .pipe(Effect.catch(() => Effect.succeed(Option.none()))); + if (Option.isNone(entry)) return null; + + return entry.value.view; + }), + }); const transaction = (effect: Effect.Effect) => fuma.transaction(effect); // Populated once, never mutated after startup. @@ -2797,12 +2821,23 @@ export const createExecutor = (); for (const def of definitionRows) defs.set(def.name, decodeJsonColumn(def.schema)); + const definitions = Object.fromEntries(defs); + const key = yield* toolSchemaViewCacheKey({ + address: String(address), + name: String(tool.name), + description: tool.description, + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + definitions, + }); + const cached = yield* Cache.get(toolSchemaViewCache, key); + if (cached !== null) return cached; const referenced = collectReferencedDefinitions( [tool.inputSchema, tool.outputSchema], @@ -2850,8 +2908,7 @@ export const createExecutor = 0 ? (referenced as Record) : undefined, - inputTypeScript: Option.getOrUndefined(view)?.inputTypeScript, - outputTypeScript: Option.getOrUndefined(view)?.outputTypeScript, - typeScriptDefinitions: Option.getOrUndefined(view)?.typeScriptDefinitions, + inputTypeScript: Option.getOrUndefined(preview)?.inputTypeScript, + outputTypeScript: Option.getOrUndefined(preview)?.outputTypeScript, + typeScriptDefinitions: Option.getOrUndefined(preview)?.typeScriptDefinitions, }); + yield* Cache.set(toolSchemaViewCache, key, view); + if (toolSchemaViewStore !== undefined) { + yield* toolSchemaViewStore + .set(key, { + version: TOOL_SCHEMA_VIEW_CACHE_VERSION, + typeScriptPreviewCacheVersion: TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, + typeScriptPreviewCompilerVersion: TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, + view, + }) + .pipe(Effect.ignore); + } + return view; }); // ------------------------------------------------------------------ diff --git a/packages/core/sdk/src/tool-schema-view-cache.test.ts b/packages/core/sdk/src/tool-schema-view-cache.test.ts new file mode 100644 index 0000000000..9dc1a05b95 --- /dev/null +++ b/packages/core/sdk/src/tool-schema-view-cache.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { toolSchemaViewCacheKey } from "./tool-schema-view-cache"; + +describe("toolSchemaViewCacheKey", () => { + it.effect("is stable for object key order and changes when schema inputs change", () => + Effect.gen(function* () { + const base = { + address: "tools.demo.org.main.inspect", + name: "inspect", + description: "Inspect a thing", + inputSchema: { + type: "object", + properties: { + pet: { $ref: "#/$defs/Pet" }, + owner: { type: "string" }, + }, + }, + outputSchema: { $ref: "#/$defs/Owner" }, + definitions: { + Pet: { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" }, + }, + }, + Owner: { type: "object", properties: { pet: { $ref: "#/$defs/Pet" } } }, + }, + }; + + const reordered = { + ...base, + inputSchema: { + properties: { + owner: { type: "string" }, + pet: { $ref: "#/$defs/Pet" }, + }, + type: "object", + }, + definitions: { + Owner: { properties: { pet: { $ref: "#/$defs/Pet" } }, type: "object" }, + Pet: { + properties: { + age: { type: "number" }, + name: { type: "string" }, + }, + type: "object", + }, + }, + }; + + const sameKey = yield* toolSchemaViewCacheKey(base); + expect(yield* toolSchemaViewCacheKey(reordered)).toBe(sameKey); + expect(yield* toolSchemaViewCacheKey({ ...base, description: "Changed" })).not.toBe(sameKey); + expect( + yield* toolSchemaViewCacheKey({ + ...base, + definitions: { + ...base.definitions, + Pet: { type: "object", properties: { name: { type: "number" } } }, + }, + }), + ).not.toBe(sameKey); + expect( + yield* toolSchemaViewCacheKey({ + ...base, + inputSchema: { type: "object", properties: { pet: { type: "string" } } }, + }), + ).not.toBe(sameKey); + }), + ); +}); diff --git a/packages/core/sdk/src/tool-schema-view-cache.ts b/packages/core/sdk/src/tool-schema-view-cache.ts new file mode 100644 index 0000000000..d5ca7a4def --- /dev/null +++ b/packages/core/sdk/src/tool-schema-view-cache.ts @@ -0,0 +1,48 @@ +import { Effect, Schema } from "effect"; + +import { sha256Hex } from "./blob"; +import { cacheKeyPayload } from "./cache-key"; +import { + TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, + TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, +} from "./tool-typescript-preview-cache"; +import { ToolSchemaView } from "./types"; + +export const TOOL_SCHEMA_VIEW_CACHE_PREFIX = "tool-schema-view/"; +export const TOOL_SCHEMA_VIEW_CACHE_VERSION = "v1"; + +export const ToolSchemaViewCacheEntry = Schema.Struct({ + version: Schema.Literal(TOOL_SCHEMA_VIEW_CACHE_VERSION), + typeScriptPreviewCacheVersion: Schema.Literal(TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION), + typeScriptPreviewCompilerVersion: Schema.Literal(TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION), + view: ToolSchemaView, +}); + +interface ToolSchemaViewCacheKeyInput { + readonly address: string; + readonly name?: string; + readonly description?: string; + readonly inputSchema?: unknown; + readonly outputSchema?: unknown; + readonly definitions: Readonly>; +} + +const cachePayload = (input: ToolSchemaViewCacheKeyInput): string => + cacheKeyPayload({ + version: TOOL_SCHEMA_VIEW_CACHE_VERSION, + typeScriptPreviewCacheVersion: TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, + typeScriptPreviewCompilerVersion: TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, + address: input.address, + name: input.name, + description: input.description, + inputSchema: input.inputSchema, + outputSchema: input.outputSchema, + definitions: input.definitions, + }); + +export const toolSchemaViewCacheKey = (input: ToolSchemaViewCacheKeyInput): Effect.Effect => + sha256Hex(cachePayload(input)).pipe( + Effect.map( + (hash) => `${TOOL_SCHEMA_VIEW_CACHE_PREFIX}${TOOL_SCHEMA_VIEW_CACHE_VERSION}/${hash}`, + ), + ); diff --git a/packages/core/sdk/src/tool-typescript-preview-cache.ts b/packages/core/sdk/src/tool-typescript-preview-cache.ts index 8b6356becb..b93a359eb9 100644 --- a/packages/core/sdk/src/tool-typescript-preview-cache.ts +++ b/packages/core/sdk/src/tool-typescript-preview-cache.ts @@ -1,6 +1,7 @@ import { Effect, Schema } from "effect"; import { sha256Hex } from "./blob"; +import { cacheKeyPayload } from "./cache-key"; import type { TypeScriptRenderOptions } from "./schema-types"; export const TOOL_TYPESCRIPT_PREVIEW_CACHE_PREFIX = "tool-typescript-preview/"; @@ -26,28 +27,15 @@ interface ToolTypeScriptPreviewCacheKeyInput { readonly options?: TypeScriptRenderOptions; } -const normalizeForHash = (value: unknown): unknown => { - if (Array.isArray(value)) return value.map(normalizeForHash); - if (value === null || typeof value !== "object") return value; - - const out: Record = {}; - for (const key of Object.keys(value as Record).sort()) { - out[key] = normalizeForHash((value as Record)[key]); - } - return out; -}; - const cachePayload = (input: ToolTypeScriptPreviewCacheKeyInput): string => - JSON.stringify( - normalizeForHash({ - version: TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, - compilerVersion: TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, - inputSchema: input.inputSchema, - outputSchema: input.outputSchema, - definitions: input.definitions, - options: input.options ?? {}, - }), - ); + cacheKeyPayload({ + version: TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, + compilerVersion: TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, + inputSchema: input.inputSchema, + outputSchema: input.outputSchema, + definitions: input.definitions, + options: input.options ?? {}, + }); export const toolTypeScriptPreviewCacheKey = ( input: ToolTypeScriptPreviewCacheKeyInput, From 63a22f382308363c8af66718bb401a95d73299d7 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Fri, 19 Jun 2026 14:14:32 +0530 Subject: [PATCH 44/57] feat(host-cloudflare): serve MCP with Agents (#41) ## Summary Migrates the host-cloudflare `/mcp` route to the Agents `McpAgent.serve()` Streamable HTTP bridge while keeping the existing Executor MCP server construction and approval hooks. ## Changes - Adds a shared `McpAgentSessionDOBase` for Cloudflare-hosted Executor MCP sessions. - Routes host-cloudflare `/mcp` at the Worker edge so authenticated session props can flow through `ExecutionContext` into the hibernatable Durable Object. - Removes the host-cloudflare app-local `McpSessionStore` mount and its forwarded-request session-store wiring. - Updates workerd e2e coverage to accept Streamable HTTP SSE-framed JSON-RPC responses and cover initialized GET streams. ## Tests - `bun run --cwd packages/hosts/cloudflare typecheck` - `bun run --cwd apps/host-cloudflare typecheck` - `bunx oxlint packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts apps/host-cloudflare/src/mcp/agent-handler.ts apps/host-cloudflare/src/mcp/session-durable-object.ts apps/host-cloudflare/src/mcp/index.ts apps/host-cloudflare/src/worker.ts apps/host-cloudflare/src/app.ts apps/host-cloudflare/src/worker.e2e.node.test.ts --deny-warnings` - `bun run --cwd apps/host-cloudflare test src/worker.e2e.node.test.ts` - `bun run --cwd apps/cloud typecheck` - `bun run --cwd apps/host-cloudflare build` --- apps/host-cloudflare/src/app.ts | 14 +- apps/host-cloudflare/src/mcp/agent-handler.ts | 153 ++++++ apps/host-cloudflare/src/mcp/index.ts | 51 -- .../src/mcp/session-durable-object.ts | 14 +- apps/host-cloudflare/src/mcp/session-store.ts | 43 -- .../src/worker.e2e.node.test.ts | 73 ++- apps/host-cloudflare/src/worker.ts | 28 +- packages/hosts/cloudflare/package.json | 4 + .../src/mcp/agent-session-durable-object.ts | 489 ++++++++++++++++++ 9 files changed, 745 insertions(+), 124 deletions(-) create mode 100644 apps/host-cloudflare/src/mcp/agent-handler.ts delete mode 100644 apps/host-cloudflare/src/mcp/session-store.ts create mode 100644 packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts diff --git a/apps/host-cloudflare/src/app.ts b/apps/host-cloudflare/src/app.ts index b6056505c5..bc1b814f6b 100644 --- a/apps/host-cloudflare/src/app.ts +++ b/apps/host-cloudflare/src/app.ts @@ -15,7 +15,7 @@ import { } from "./execution"; import { ErrorCaptureLive } from "./observability"; import { cloudflareAccountMiddleware } from "./account/account-provider"; -import { makeCloudflareMcpSeams } from "./mcp"; +import { makeCloudflareMcpAgentHandler } from "./mcp/agent-handler"; import { preloadQuickJs } from "./quickjs"; // =========================================================================== @@ -60,10 +60,9 @@ export const makeCloudflareApp = async (env: CloudflareEnv) => { env.CACHE === undefined ? identityLayer : Layer.mergeAll(identityLayer, layerCloudflareKeyValueStore(env.CACHE)); - // MCP runs through the `MCP_SESSION` Durable Object (cross-isolate sessions); - // each session DO opens its own D1 handle, so it takes `env`, not `dbHandle`. - // The `/mcp` GATE auth runs in the Worker, so it shares the alias lookup. - const mcp = makeCloudflareMcpSeams(config, env, aliasLookup); + // `/mcp` is mounted in worker.ts through the Agents Streamable HTTP bridge + // because it needs the Cloudflare ExecutionContext. + const mcpAgentHandler = makeCloudflareMcpAgentHandler(config, aliasLookup); const { appLayer, toWebHandler } = ExecutorApp.make({ plugins, @@ -80,13 +79,10 @@ export const makeCloudflareApp = async (env: CloudflareEnv) => { // auth context; `me` reflects the Access principal. Members/keys are // Access-managed, so the rest of the surface is stubbed. account: cloudflareAccountMiddleware(config, aliasLookup), - // The MCP serving envelope: Access-JWT auth + the shared in-process session - // store over the QuickJS engine. - mcp: { auth: mcp.auth, sessions: mcp.sessions, reporter: mcp.reporter }, }, config: { mountPrefix: "/api", failure: textFailureStrategy }, boot: bootLayer, }); - return { appLayer, toWebHandler }; + return { appLayer, toWebHandler, mcpAgentHandler }; }; diff --git a/apps/host-cloudflare/src/mcp/agent-handler.ts b/apps/host-cloudflare/src/mcp/agent-handler.ts new file mode 100644 index 0000000000..93d306f587 --- /dev/null +++ b/apps/host-cloudflare/src/mcp/agent-handler.ts @@ -0,0 +1,153 @@ +import { Effect, Predicate } from "effect"; + +import { + McpAuthProvider, + jsonRpcErrorBody, + type AuthOutcome, + type Principal, +} from "@executor-js/host-mcp"; +import { + currentPropagationHeaders, + readElicitationMode, + withVerifiedIdentityHeaders, +} from "@executor-js/cloudflare/mcp/do-headers"; +import type { McpSessionProps } from "@executor-js/cloudflare/mcp/agent-durable-object"; + +import type { ServiceTokenAliasLookup } from "../auth/service-token-alias"; +import type { CloudflareConfig, CloudflareEnv } from "../config"; +import { cloudflareAccessMcpAuth } from "./auth"; +import { McpSessionDO } from "./session-durable-object"; + +interface McpAgentSessionStub { + readonly validateMcpSessionOwner: (identity: { + readonly accountId: string; + readonly organizationId: string; + }) => Promise<"ok" | "not_found" | "forbidden">; + readonly _cf_scheduleDestroy: () => Promise; +} + +const corsPreflightResponse = (): Response => + new Response(null, { + status: 204, + headers: { + "access-control-allow-origin": "*", + "access-control-allow-methods": "GET, POST, DELETE, OPTIONS", + "access-control-allow-headers": + "content-type, authorization, mcp-session-id, accept, mcp-protocol-version", + "access-control-expose-headers": "mcp-session-id, WWW-Authenticate", + }, + }); + +const jsonRpcResponse = ( + status: number, + code: number, + message: string, + challenge?: string, +): Response => + challenge === undefined + ? jsonRpcErrorBody(status, code, message) + : jsonRpcErrorBody(status, code, message, { challenge }); + +const renderAuthError = ( + auth: McpAuthProvider["Service"], + request: Request, + outcome: Exclude, +): Response => { + if (Predicate.isTagged(outcome, "Unauthorized")) { + return jsonRpcResponse( + 401, + -32001, + "Unauthorized", + outcome.challenge ?? `Bearer resource_metadata="${auth.resourceMetadataUrl(request)}"`, + ); + } + if (Predicate.isTagged(outcome, "Forbidden")) { + return jsonRpcResponse(403, outcome.code ?? -32001, outcome.message); + } + return jsonRpcResponse(503, -32001, outcome.message); +}; + +const sessionStub = (env: CloudflareEnv, sessionId: string): McpAgentSessionStub => + // oxlint-disable-next-line executor/no-double-cast -- boundary: Workers types expose only DurableObjectStub, but RPC methods are generated from the bound DO class. + env.MCP_SESSION.get( + env.MCP_SESSION.idFromName(`streamable-http:${sessionId}`), + ) as unknown as McpAgentSessionStub; + +const authenticate = ( + request: Request, + config: CloudflareConfig, + aliasLookup?: ServiceTokenAliasLookup, +) => + Effect.gen(function* () { + const auth = yield* McpAuthProvider; + const outcome = yield* auth.authenticate(request); + return { auth, outcome }; + }).pipe(Effect.provide(cloudflareAccessMcpAuth(config, aliasLookup))); + +const propsForPrincipal = ( + request: Request, + principal: Principal, +): Effect.Effect => + Effect.gen(function* () { + const propagation = yield* currentPropagationHeaders(request); + return { + session: { + organizationId: principal.organizationId, + userId: principal.accountId, + elicitationMode: readElicitationMode(request), + webOrigin: new URL(request.url).origin, + ...(principal.actor ? { actor: principal.actor } : {}), + }, + propagation, + }; + }); + +export const makeCloudflareMcpAgentHandler = ( + config: CloudflareConfig, + aliasLookup?: ServiceTokenAliasLookup, +) => { + const serve = McpSessionDO.serve("/mcp", { + binding: "MCP_SESSION", + transport: "streamable-http", + }); + + return async (request: Request, env: CloudflareEnv, ctx: ExecutionContext): Promise => { + if (request.method === "OPTIONS") return corsPreflightResponse(); + const sessionId = request.headers.get("mcp-session-id"); + + const { auth, outcome } = await Effect.runPromise(authenticate(request, config, aliasLookup)); + if (!Predicate.isTagged(outcome, "Authenticated")) { + if (Predicate.isTagged(outcome, "Forbidden") && sessionId) { + await Effect.runPromise( + Effect.ignore(Effect.tryPromise(() => sessionStub(env, sessionId)._cf_scheduleDestroy())), + ); + } + return renderAuthError(auth, request, outcome); + } + + if (!sessionId && request.method === "DELETE") { + return new Response(null, { status: 204, headers: { "access-control-allow-origin": "*" } }); + } + + if (sessionId) { + const owner = await sessionStub(env, sessionId).validateMcpSessionOwner({ + accountId: outcome.principal.accountId, + organizationId: outcome.principal.organizationId, + }); + if (owner === "not_found") { + return jsonRpcResponse(404, -32001, "Session not found"); + } + if (owner === "forbidden") { + return jsonRpcResponse(403, -32003, "MCP session does not belong to the current bearer"); + } + } + + const props = await Effect.runPromise(propsForPrincipal(request, outcome.principal)); + (ctx as ExecutionContext & { props?: McpSessionProps }).props = props; + const forwarded = withVerifiedIdentityHeaders(request, { + accountId: outcome.principal.accountId, + organizationId: outcome.principal.organizationId, + }); + return serve.fetch(forwarded, env, ctx); + }; +}; diff --git a/apps/host-cloudflare/src/mcp/index.ts b/apps/host-cloudflare/src/mcp/index.ts index 748cbbd760..9f4b41ee14 100644 --- a/apps/host-cloudflare/src/mcp/index.ts +++ b/apps/host-cloudflare/src/mcp/index.ts @@ -1,53 +1,2 @@ -import type { Layer } from "effect"; - -import type { McpAuthProvider, McpErrorReporter, McpSessionStore } from "@executor-js/host-mcp"; - -import type { ServiceTokenAliasLookup } from "../auth/service-token-alias"; -import type { CloudflareConfig, CloudflareEnv } from "../config"; -import { cloudflareAccessMcpAuth } from "./auth"; -import { cloudflareMcpReporter, makeCloudflareMcpSessionStore } from "./session-store"; - export { cloudflareAccessMcpAuth } from "./auth"; -export { cloudflareMcpReporter, makeCloudflareMcpSessionStore } from "./session-store"; export { McpSessionDO } from "./session-durable-object"; - -// --------------------------------------------------------------------------- -// The Cloudflare MCP serving seams, fed to `ExecutorApp.make`'s `mcp` group. -// -// `ExecutorApp.make` mounts the shared, provider-neutral MCP serving envelope -// (@executor-js/host-mcp) at the top-level `/mcp`, outside the API's execution -// middleware. The Cloudflare host provides the two envelope seams plus the -// error-reporter override: -// - McpAuthProvider -> `cloudflareAccessMcpAuth`: validate the Access JWT -// (same identity as the API gate); no MCP OAuth. -// - McpSessionStore -> the shared Durable-Object dispatcher over the host's -// `MCP_SESSION` namespace (cross-isolate, same as cloud). -// - McpErrorReporter -> `cloudflareMcpReporter`: route 500 defects through the -// host's console capture. -// --------------------------------------------------------------------------- - -export interface CloudflareMcpSeams { - /** Validate the Access JWT to an MCP `AuthOutcome`; declares no discovery routes. */ - readonly auth: Layer.Layer; - /** The Durable-Object session store seam (dispatch + lifetime). */ - readonly sessions: Layer.Layer; - /** Route 500 defects through the host's console `ErrorCapture`. */ - readonly reporter: Layer.Layer; -} - -/** - * Build the Cloudflare MCP serving seams over the host's `MCP_SESSION` Durable - * Object namespace. No per-session DB handle is threaded here β€” each session DO - * opens its own D1 handle in its own isolate. The `/mcp` GATE auth runs in the - * Worker (not the DO), so it takes the app-level service-token alias lookup so a - * service-token MCP client resolves its aliased identity just like the API gate. - */ -export const makeCloudflareMcpSeams = ( - config: CloudflareConfig, - env: CloudflareEnv, - aliasLookup?: ServiceTokenAliasLookup, -): CloudflareMcpSeams => ({ - auth: cloudflareAccessMcpAuth(config, aliasLookup), - sessions: makeCloudflareMcpSessionStore(env), - reporter: cloudflareMcpReporter, -}); diff --git a/apps/host-cloudflare/src/mcp/session-durable-object.ts b/apps/host-cloudflare/src/mcp/session-durable-object.ts index 2640aba23c..65e9246b81 100644 --- a/apps/host-cloudflare/src/mcp/session-durable-object.ts +++ b/apps/host-cloudflare/src/mcp/session-durable-object.ts @@ -3,11 +3,11 @@ import { Effect } from "effect"; import { createExecutorMcpServer } from "@executor-js/host-mcp/tool-server"; import type { ExecutorDbHandle } from "@executor-js/api/server"; import { - McpSessionDOBase, + McpAgentSessionDOBase, type BuiltMcpServer, type McpSessionInit, type SessionMeta, -} from "@executor-js/cloudflare/mcp/durable-object"; +} from "@executor-js/cloudflare/mcp/agent-durable-object"; import { loadConfig, type CloudflareConfig, type CloudflareEnv } from "../config"; import { createD1ExecutorDb } from "../db/d1"; @@ -34,14 +34,14 @@ import { preloadQuickJs } from "../quickjs"; // own lifecycle (the binding is the connection), so `end` is `close` β€” a no-op. type CfSessionDbHandle = ExecutorDbHandle & { readonly end: () => Promise }; -export class McpSessionDO extends McpSessionDOBase { +export class McpSessionDO extends McpAgentSessionDOBase { private readonly cfEnv: CloudflareEnv; private readonly cfConfig: CloudflareConfig; - // `ctx`'s type is taken from the base constructor so it tracks whichever - // `@cloudflare/workers-types` the shared package resolves (avoids a - // cross-version `DurableObjectState` mismatch at the `super` call). - constructor(ctx: ConstructorParameters[0], env: CloudflareEnv) { + constructor( + ctx: ConstructorParameters>[0], + env: CloudflareEnv, + ) { super(ctx, env); this.cfEnv = env; this.cfConfig = loadConfig(env); diff --git a/apps/host-cloudflare/src/mcp/session-store.ts b/apps/host-cloudflare/src/mcp/session-store.ts deleted file mode 100644 index cb4a8b1146..0000000000 --- a/apps/host-cloudflare/src/mcp/session-store.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Layer } from "effect"; - -import { makeConsoleMcpErrorReporter } from "@executor-js/api/server"; -import type { McpErrorReporter, McpSessionStore } from "@executor-js/host-mcp"; -import { - makeDurableObjectMcpSessionStore, - type McpSessionDOStub, -} from "@executor-js/cloudflare/mcp/session-store"; - -import type { CloudflareEnv } from "../config"; -import { ErrorCaptureLive } from "../observability"; - -// --------------------------------------------------------------------------- -// Cloudflare McpSessionStore wiring β€” the SAME shared Durable-Object dispatcher -// as cloud (@executor-js/cloudflare), over host-cloudflare's `MCP_SESSION` -// namespace. The dispatch/identity/trace/peek logic all lives in the shared -// package; the host supplies ONLY its DO stub accessors (the session id IS the -// DO id, so every follow-up request routes back to the same isolate). -// -// This replaces the in-process store: an in-memory session map is invisible to -// the next Worker isolate, so `tools/list` after `initialize` failed in -// production ("Not connected"). The DO holds the session in one addressable -// isolate, fixing that across the board. -// --------------------------------------------------------------------------- - -// The DO RPC stub structurally satisfies `McpSessionDOStub` (init/handleRequest/ -// clearSession), but `@cloudflare/workers-types` types it as a generic -// `DurableObjectStub`. Narrow at this one boundary via an `unknown` hop β€” a -// single cast, so no double-cast through the worker-types stub type. -const toSessionStub = (stub: unknown): McpSessionDOStub => stub as McpSessionDOStub; - -/** Build the DO-backed MCP session store over the host's `MCP_SESSION` namespace. */ -export const makeCloudflareMcpSessionStore = (env: CloudflareEnv): Layer.Layer => - makeDurableObjectMcpSessionStore({ - getStub: (sessionId) => - toSessionStub(env.MCP_SESSION.get(env.MCP_SESSION.idFromString(sessionId))), - newStub: () => toSessionStub(env.MCP_SESSION.get(env.MCP_SESSION.newUniqueId())), - // host-cf has no Sentry; a 500-defect surfaces through the reporter seam below. - }); - -/** Route 500-defects through the host's console `ErrorCapture`. */ -export const cloudflareMcpReporter: Layer.Layer = - makeConsoleMcpErrorReporter(ErrorCaptureLive); diff --git a/apps/host-cloudflare/src/worker.e2e.node.test.ts b/apps/host-cloudflare/src/worker.e2e.node.test.ts index 63287f4614..150a9ab2b8 100644 --- a/apps/host-cloudflare/src/worker.e2e.node.test.ts +++ b/apps/host-cloudflare/src/worker.e2e.node.test.ts @@ -4,6 +4,7 @@ import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { afterAll, beforeAll, describe, expect, it } from "@effect/vitest"; +import { Schema } from "effect"; import { unstable_dev, type Unstable_DevWorker } from "wrangler"; // --------------------------------------------------------------------------- @@ -29,6 +30,28 @@ const SPEC = JSON.stringify({ }, }); +const decodeUnknownJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); + +interface JsonReadableResponse { + readonly headers: { get(name: string): string | null }; + json(): Promise; + text(): Promise; +} + +const readMcpJson = async (response: JsonReadableResponse): Promise => { + const contentType = response.headers.get("content-type") ?? ""; + if (contentType.includes("application/json")) { + return (await response.json()) as A; + } + const text = await response.text(); + const data = text + .split("\n") + .find((line) => line.startsWith("data: ")) + ?.slice("data: ".length); + expect(data).toBeTruthy(); + return decodeUnknownJson(data!) as A; +}; + describe("cloudflare host e2e (workerd/miniflare)", () => { let worker: Unstable_DevWorker; @@ -200,13 +223,55 @@ describe("cloudflare host e2e (workerd/miniflare)", () => { method: "tools/list", }); expect(list.status).toBe(200); - const listed = (await list.json()) as { + const listed = await readMcpJson<{ result?: { tools?: ReadonlyArray<{ name: string }> }; - }; + }>(list); const toolNames = listed.result?.tools?.map((t) => t.name) ?? []; expect(toolNames).toContain("execute"); }, 60_000); + it("serves streamable HTTP GET only for initialized sessions", async () => { + const missing = await worker.fetch("/mcp", { + method: "GET", + headers: { accept: "text/event-stream" }, + }); + expect(missing.status).toBe(400); + await missing.text(); + + const init = await worker.fetch("/mcp", { + method: "POST", + headers: { + "content-type": "application/json", + accept: "application/json, text/event-stream", + }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "test", version: "1" }, + }, + }), + }); + expect(init.status).toBe(200); + const sessionId = init.headers.get("mcp-session-id"); + expect(sessionId).toBeTruthy(); + await init.text(); + + const stream = await worker.fetch("/mcp", { + method: "GET", + headers: { + accept: "text/event-stream", + "mcp-session-id": sessionId!, + }, + }); + expect(stream.status).toBe(200); + expect(stream.headers.get("content-type")).toContain("text/event-stream"); + await stream.body?.cancel(); + }, 60_000); + it("invokes the execute tool over MCP (initialize β†’ tools/call β†’ QuickJS)", async () => { const accept = "application/json, text/event-stream"; const rpc = (sessionId: string | null, body: unknown) => @@ -246,9 +311,9 @@ describe("cloudflare host e2e (workerd/miniflare)", () => { params: { name: "execute", arguments: { code: "export default 6 * 7" } }, }); expect(call.status).toBe(200); - const result = (await call.json()) as { + const result = await readMcpJson<{ result?: { structuredContent?: { result?: number } }; - }; + }>(call); expect(result.result?.structuredContent?.result).toBe(42); }, 60_000); }); diff --git a/apps/host-cloudflare/src/worker.ts b/apps/host-cloudflare/src/worker.ts index 228b60a551..2602e9948a 100644 --- a/apps/host-cloudflare/src/worker.ts +++ b/apps/host-cloudflare/src/worker.ts @@ -6,25 +6,33 @@ import type { CloudflareEnv } from "./config"; export { McpSessionDO } from "./mcp"; // --------------------------------------------------------------------------- -// The Worker fetch entry. `ExecutorApp.make`'s `toWebHandler()` produces a -// `(Request) => Promise` β€” exactly a Worker handler β€” so the entry is -// thin: build the app ONCE per isolate (memoized; the build runs the D1 schema -// bring-up), then forward every request to its handler. `env` (the D1 binding + -// Access vars) arrives with the request and is captured at build time. +// The Worker fetch entry. Most requests go to `ExecutorApp.make`'s Effect web +// handler. `/mcp` stays at this edge boundary because `McpAgent.serve()` needs +// the Cloudflare `ExecutionContext` to pass authenticated session props into the +// hibernatable Durable Object bridge. // --------------------------------------------------------------------------- -let handlerPromise: Promise<(request: Request) => Promise> | null = null; +let handlerPromise: Promise<{ + readonly app: (request: Request) => Promise; + readonly mcp: (request: Request, env: CloudflareEnv, ctx: ExecutionContext) => Promise; +}> | null = null; -const resolveHandler = (env: CloudflareEnv): Promise<(request: Request) => Promise> => { +const resolveHandler = (env: CloudflareEnv) => { if (!handlerPromise) { - handlerPromise = makeCloudflareApp(env).then(({ toWebHandler }) => toWebHandler().handler); + handlerPromise = makeCloudflareApp(env).then(({ toWebHandler, mcpAgentHandler }) => ({ + app: toWebHandler().handler, + mcp: mcpAgentHandler, + })); } return handlerPromise; }; export default { - fetch: async (request: Request, env: CloudflareEnv): Promise => { + fetch: async (request: Request, env: CloudflareEnv, ctx: ExecutionContext): Promise => { const serve = await resolveHandler(env); - return serve(request); + if (new URL(request.url).pathname === "/mcp") { + return serve.mcp(request, env, ctx); + } + return serve.app(request); }, }; diff --git a/packages/hosts/cloudflare/package.json b/packages/hosts/cloudflare/package.json index 94ec974294..2f8b4e21d7 100644 --- a/packages/hosts/cloudflare/package.json +++ b/packages/hosts/cloudflare/package.json @@ -32,6 +32,10 @@ "types": "./src/mcp/session-durable-object.ts", "default": "./src/mcp/session-durable-object.ts" }, + "./mcp/agent-durable-object": { + "types": "./src/mcp/agent-session-durable-object.ts", + "default": "./src/mcp/agent-session-durable-object.ts" + }, "./mcp/request-id-queue": { "types": "./src/mcp/request-id-queue.ts", "default": "./src/mcp/request-id-queue.ts" diff --git a/packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts b/packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts new file mode 100644 index 0000000000..651f816e15 --- /dev/null +++ b/packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts @@ -0,0 +1,489 @@ +import { Cause, Deferred, Effect } from "effect"; +import type * as Tracer from "effect/Tracer"; +import { McpAgent } from "agents/mcp"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; + +import { RequestWebOrigin } from "@executor-js/api/server"; +import type { ExecutionActor } from "@executor-js/sdk/core"; +import { + formatPausedExecution, + type ExecutionEngine, + type ResumeResponse, +} from "@executor-js/execution"; + +import type { IncomingPropagationHeaders, McpElicitationMode } from "./do-headers"; + +export type IncomingTraceHeaders = IncomingPropagationHeaders; + +export interface McpSessionInit { + readonly organizationId: string; + readonly userId: string; + readonly elicitationMode: McpElicitationMode; + readonly webOrigin?: string; + readonly actor?: ExecutionActor; +} + +export interface McpSessionProps extends Record { + readonly session: McpSessionInit; + readonly propagation?: IncomingTraceHeaders; +} + +export type McpApprovalOwner = { + readonly accountId: string; + readonly organizationId: string; +}; + +type McpSessionApprovalErrorResult = + | { readonly status: "not_found" } + | { readonly status: "forbidden" }; + +export type McpSessionApprovalResult = + | { + readonly status: "ok"; + readonly text: string; + readonly structured: Record; + } + | McpSessionApprovalErrorResult; + +export type McpSessionResumeApprovalResult = + | { + readonly status: "ok"; + readonly executionStatus: "completed" | "paused"; + readonly text: string; + readonly structured: Record; + readonly isError?: boolean; + } + | McpSessionApprovalErrorResult; + +export interface SessionDbHandle { + readonly end: () => Promise | void; +} + +export interface SessionMeta { + readonly organizationId: string; + readonly organizationName: string; + readonly userId: string; + readonly elicitationMode?: "browser" | "model" | "native"; + readonly webOrigin?: string; + readonly actor?: ExecutionActor; +} + +export interface BuiltMcpServer { + readonly mcpServer: McpServer; + readonly engine: ExecutionEngine; +} + +export interface BrowserApprovalStore { + readonly takeResponse: (executionId: string) => Effect.Effect; + readonly waitForResponse: (executionId: string) => Effect.Effect; +} + +const SESSION_META_KEY = "session-meta"; +const LAST_ACTIVITY_KEY = "last-activity-ms"; +const SESSION_TIMEOUT_MS = 5 * 60 * 1000; +const approvalResponseKey = (executionId: string) => `approval-response:${executionId}`; + +const resumeApprovalResult = ( + executionId: string, + response: ResumeResponse, +): Extract => { + const textByAction = { + accept: "I've approved it", + decline: "I've denied it", + cancel: "I've canceled it", + } satisfies Record; + const statusByAction = { + accept: "approved", + decline: "denied", + cancel: "canceled", + } satisfies Record; + + return { + status: "ok", + executionStatus: "completed", + text: textByAction[response.action], + structured: { status: statusByAction[response.action], executionId }, + isError: false, + }; +}; + +const isSessionProps = (props: unknown): props is McpSessionProps => + typeof props === "object" && + props !== null && + "session" in props && + typeof (props as { readonly session?: unknown }).session === "object" && + (props as { readonly session?: unknown }).session !== null; + +export abstract class McpAgentSessionDOBase< + Env extends Cloudflare.Env = Cloudflare.Env, + TDbHandle extends SessionDbHandle = SessionDbHandle, +> extends McpAgent { + server!: McpServer; + private engine: ExecutionEngine | null = null; + private dbHandle: TDbHandle | null = null; + private sessionMeta: SessionMeta | null = null; + private initialized = false; + private lastActivityMs = 0; + private approvalResponses = new Map(); + private approvalWaiters = new Map>(); + + protected abstract openSessionDb(): TDbHandle | Promise; + + protected abstract resolveSessionMeta(token: McpSessionInit): Effect.Effect; + + protected abstract buildMcpServer( + sessionMeta: SessionMeta, + dbHandle: TDbHandle, + ): Effect.Effect; + + protected withTelemetry( + effect: Effect.Effect, + _incoming?: IncomingTraceHeaders, + ): Effect.Effect { + return effect; + } + + protected captureCause(cause: Cause.Cause): void {} + + protected flushTelemetry(): Promise { + return Promise.resolve(); + } + + protected get sessionId(): string { + return this.getSessionId(); + } + + protected currentParentSpan(): Tracer.AnySpan | undefined { + return undefined; + } + + protected readonly browserApprovalStore: BrowserApprovalStore = { + takeResponse: (executionId) => this.takeApprovalResponse(executionId), + waitForResponse: (executionId) => this.waitForApprovalResponse(executionId), + }; + + private openSessionDbHandle(): Effect.Effect { + return Effect.promise(() => Promise.resolve(this.openSessionDb())); + } + + private loadSessionMeta(): Effect.Effect { + return Effect.promise(async () => { + if (this.sessionMeta) return this.sessionMeta; + const stored = await this.ctx.storage.get(SESSION_META_KEY); + this.sessionMeta = stored ?? null; + return this.sessionMeta; + }).pipe(Effect.withSpan("mcp.session.load_meta")); + } + + private async saveSessionMeta(sessionMeta: SessionMeta): Promise { + this.sessionMeta = sessionMeta; + await this.ctx.storage.put(SESSION_META_KEY, sessionMeta); + } + + private async markActivity(now = Date.now()): Promise { + this.lastActivityMs = now; + await Promise.all([ + this.ctx.storage.put(LAST_ACTIVITY_KEY, now), + this.ctx.storage.setAlarm(now + SESSION_TIMEOUT_MS), + ]); + } + + private async loadLastActivity(): Promise { + if (this.lastActivityMs > 0) return this.lastActivityMs; + const stored = await this.ctx.storage.get(LAST_ACTIVITY_KEY); + this.lastActivityMs = stored ?? 0; + return this.lastActivityMs; + } + + private resolveAndStoreSessionMeta(token: McpSessionInit) { + const self = this; + return Effect.gen(function* () { + const resolved = yield* self.resolveSessionMeta(token); + const sessionMeta: SessionMeta = { + ...resolved, + ...(token.webOrigin ? { webOrigin: token.webOrigin } : {}), + ...(token.actor ? { actor: token.actor } : {}), + }; + yield* Effect.promise(() => self.saveSessionMeta(sessionMeta)).pipe( + Effect.withSpan("mcp.session.save_meta"), + ); + return sessionMeta; + }).pipe(Effect.withSpan("mcp.session.resolve_and_store_meta")); + } + + private recordCauseOnSpan(cause: Cause.Cause): Effect.Effect { + const errors = Cause.prettyErrors(cause); + if (errors.length === 0) return Effect.void; + const first = errors[0]; + return Effect.annotateCurrentSpan({ + "exception.type": first?.name ?? "Error", + "exception.message": first?.message ?? "unknown", + "exception.stacktrace": Cause.pretty(cause), + }); + } + + private withSpanFlush(effect: Effect.Effect): Effect.Effect { + const self = this; + return effect.pipe(Effect.ensuring(Effect.promise(() => self.flushTelemetry()))); + } + + private buildRuntime(sessionMeta: SessionMeta, dbHandle: TDbHandle) { + const built = this.buildMcpServer(sessionMeta, dbHandle); + return sessionMeta.webOrigin + ? built.pipe(Effect.provideService(RequestWebOrigin, { origin: sessionMeta.webOrigin })) + : built; + } + + private closeRuntime(): Effect.Effect { + const self = this; + return Effect.gen(function* () { + if (self.server) { + const server = self.server; + yield* Effect.promise(() => server.close()).pipe(Effect.ignore); + } + self.engine = null; + if (self.dbHandle) { + const dbHandle = self.dbHandle; + self.dbHandle = null; + yield* Effect.promise(() => Promise.resolve(dbHandle.end())).pipe(Effect.ignore); + } + self.initialized = false; + }); + } + + private ensureRuntimeForApproval(): Effect.Effect { + const self = this; + return Effect.gen(function* () { + if (self.initialized && self.engine) return true; + + const sessionMeta = yield* self.loadSessionMeta(); + if (!sessionMeta) return false; + + yield* self.closeRuntime(); + const dbHandle = yield* self.openSessionDbHandle(); + const { mcpServer, engine } = yield* self.buildRuntime(sessionMeta, dbHandle); + self.dbHandle = dbHandle; + self.server = mcpServer; + self.engine = engine; + self.initialized = true; + yield* Effect.promise(() => self.markActivity()).pipe( + Effect.withSpan("McpSessionDO.markActivity"), + ); + return true; + }).pipe(Effect.withSpan("McpSessionDO.ensure_runtime_for_approval")); + } + + async init(): Promise { + if (this.initialized) return; + const props = isSessionProps(this.props) ? this.props : null; + if (!props) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: McpAgent.init is a Promise-only framework hook and props are required before any Effect runtime exists. + throw new Error("MCP session props are required"); + } + const self = this; + const program = Effect.gen(function* () { + const sessionMeta = yield* self.resolveAndStoreSessionMeta(props.session); + const dbHandle = yield* self.openSessionDbHandle(); + const { mcpServer, engine } = yield* self.buildRuntime(sessionMeta, dbHandle); + self.dbHandle = dbHandle; + self.server = mcpServer; + self.engine = engine; + self.initialized = true; + yield* Effect.promise(() => self.markActivity()).pipe( + Effect.withSpan("McpSessionDO.markActivity"), + ); + }).pipe( + Effect.tapCause((cause) => + Effect.gen(function* () { + console.error("[mcp-session] init failed:", Cause.pretty(cause)); + self.captureCause(cause); + yield* self.recordCauseOnSpan(cause); + }), + ), + Effect.catchCause((cause) => + Effect.gen(function* () { + yield* Effect.promise(() => self.cleanup()); + return yield* Effect.failCause(cause); + }), + ), + Effect.withSpan("McpSessionDO.init", { + attributes: { + "mcp.auth.organization_id": props?.session.organizationId ?? "", + }, + }), + ); + const traced = this.withTelemetry(program, props?.propagation); + return Effect.runPromise( + traced.pipe( + // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: Durable Object init method can only reject its Promise + Effect.orDie, + (effect) => self.withSpanFlush(effect), + ), + ); + } + + async validateMcpSessionOwner( + identity: McpApprovalOwner, + ): Promise<"ok" | "not_found" | "forbidden"> { + const self = this; + return Effect.runPromise( + Effect.gen(function* () { + const sessionMeta = yield* self.loadSessionMeta(); + if (!sessionMeta) return "not_found" as const; + yield* Effect.promise(() => self.markActivity()).pipe( + Effect.withSpan("McpSessionDO.markActivity"), + ); + return identity.accountId === sessionMeta.userId && + identity.organizationId === sessionMeta.organizationId + ? ("ok" as const) + : ("forbidden" as const); + }).pipe( + Effect.withSpan("McpSessionDO.validateMcpSessionOwner"), + // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: DO RPC exposes Promise results + Effect.orDie, + ), + ); + } + + async getPausedExecutionForApproval( + executionId: string, + identity: McpApprovalOwner, + incoming?: IncomingTraceHeaders, + ): Promise { + const self = this; + return Effect.runPromise( + Effect.gen(function* () { + const owner = yield* self.validateApprovalIdentity(identity); + if (owner !== "ok") return { status: owner } as const; + + const restored = yield* self.ensureRuntimeForApproval(); + if (!restored || !self.engine) return { status: "not_found" } as const; + + const paused = yield* self.engine.getPausedExecution(executionId); + if (!paused) return { status: "not_found" } as const; + + const formatted = formatPausedExecution(paused); + return { + status: "ok" as const, + text: formatted.text, + structured: formatted.structured, + }; + }).pipe( + Effect.withSpan("McpSessionDO.getPausedExecutionForApproval", { + attributes: { "mcp.execution.id": executionId }, + }), + (eff) => this.withTelemetry(eff, incoming), + // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: DO RPC exposes Promise results + Effect.orDie, + (eff) => self.withSpanFlush(eff), + ), + ); + } + + async resumeExecutionForApproval( + executionId: string, + identity: McpApprovalOwner, + response: ResumeResponse, + incoming?: IncomingTraceHeaders, + ): Promise { + const self = this; + return Effect.runPromise( + Effect.gen(function* () { + const owner = yield* self.validateApprovalIdentity(identity); + if (owner !== "ok") return { status: owner } as const; + + const restored = yield* self.ensureRuntimeForApproval(); + if (!restored || !self.engine) return { status: "not_found" } as const; + + const paused = yield* self.engine.getPausedExecution(executionId); + if (!paused) return { status: "not_found" } as const; + + self.approvalResponses.set(executionId, response); + yield* Effect.promise(() => + self.ctx.storage.put(approvalResponseKey(executionId), response), + ); + const waiter = self.approvalWaiters.get(executionId); + if (waiter) yield* Deferred.succeed(waiter, response); + return resumeApprovalResult(executionId, response); + }).pipe( + Effect.withSpan("McpSessionDO.resumeExecutionForApproval", { + attributes: { "mcp.execution.id": executionId }, + }), + (eff) => this.withTelemetry(eff, incoming), + // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: DO RPC exposes Promise results + Effect.orDie, + (eff) => self.withSpanFlush(eff), + ), + ); + } + + override async destroy(): Promise { + await this.cleanup(); + await super.destroy(); + } + + override async alarm(): Promise { + const lastActivityMs = await this.loadLastActivity(); + if (lastActivityMs > 0 && Date.now() - lastActivityMs >= SESSION_TIMEOUT_MS) { + await this.destroy(); + return; + } + await super.alarm(); + } + + private validateApprovalIdentity( + identity: McpApprovalOwner, + ): Effect.Effect<"ok" | "not_found" | "forbidden"> { + const self = this; + return Effect.gen(function* () { + const sessionMeta = yield* self.loadSessionMeta(); + if (!sessionMeta) return "not_found" as const; + return identity.accountId === sessionMeta.userId && + identity.organizationId === sessionMeta.organizationId + ? ("ok" as const) + : ("forbidden" as const); + }).pipe(Effect.withSpan("mcp.session.validate_approval_identity")); + } + + private takeApprovalResponse(executionId: string): Effect.Effect { + const self = this; + return Effect.promise(async () => { + const memoryResponse = self.approvalResponses.get(executionId); + if (memoryResponse) { + self.approvalResponses.delete(executionId); + await self.ctx.storage.delete(approvalResponseKey(executionId)); + return memoryResponse; + } + const stored = await self.ctx.storage.get(approvalResponseKey(executionId)); + if (!stored) return null; + await self.ctx.storage.delete(approvalResponseKey(executionId)); + return stored; + }); + } + + private waitForApprovalResponse(executionId: string): Effect.Effect { + const self = this; + return Effect.gen(function* () { + const existing = yield* self.takeApprovalResponse(executionId); + if (existing) return existing; + + const waiter = + self.approvalWaiters.get(executionId) ?? (yield* Deferred.make()); + self.approvalWaiters.set(executionId, waiter); + yield* Deferred.await(waiter).pipe( + Effect.ensuring( + Effect.sync(() => { + if (self.approvalWaiters.get(executionId) === waiter) { + self.approvalWaiters.delete(executionId); + } + }), + ), + ); + return yield* self.takeApprovalResponse(executionId); + }); + } + + private async cleanup(): Promise { + await Effect.runPromise(this.closeRuntime()); + } +} From 234cb8e8ed95db60d538ea00d40aa358d3013112 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Fri, 19 Jun 2026 19:33:23 +0530 Subject: [PATCH 45/57] feat(plugin-semantic-search): add staged reindex pipeline (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds a staged, fingerprint-aware semantic-search reindex pipeline so hosts can drive durable indexing work without rebuilding unchanged tool documents. ## Changes - Adds staged index collections, job planning, and snapshot/fingerprint helpers for semantic-search indexing. - Updates reindex status/API surfaces and React atoms/page wiring for the staged pipeline. - Lets schema consumers opt out of TypeScript preview generation when they only need raw schema content for fingerprinting. ## Tests - `bun run --cwd packages/plugins/semantic-search test src/sdk/fingerprint.test.ts src/sdk/indexer.test.ts` - `bun run --cwd packages/core/sdk test src/tool-schema-view-cache.test.ts src/executor.test.ts` - `bun run --cwd packages/plugins/semantic-search typecheck` - `bun run --cwd packages/core/sdk typecheck` ### [Stack](https://github.com/kitlangton/stack) 1. #36 2. **#37** πŸ‘ˆ current 3. #42 4. #43 --- packages/core/sdk/src/executor.ts | 55 +- packages/core/sdk/src/index.ts | 1 + packages/core/sdk/src/promise.ts | 1 + packages/core/sdk/src/shared.ts | 1 + .../sdk/src/tool-schema-view-cache.test.ts | 3 + .../core/sdk/src/tool-schema-view-cache.ts | 2 + packages/core/sdk/src/types.ts | 12 + .../plugins/semantic-search/src/api/group.ts | 12 +- .../semantic-search/src/api/handlers.ts | 3 +- .../semantic-search/src/react/SearchPage.tsx | 2 +- .../semantic-search/src/react/atoms.ts | 2 +- .../semantic-search/src/sdk/collections.ts | 82 + .../semantic-search/src/sdk/documents.ts | 155 +- .../src/sdk/embedder-openai.ts | 6 +- .../semantic-search/src/sdk/embedder.ts | 6 +- .../src/sdk/fingerprint.test.ts | 63 +- .../semantic-search/src/sdk/fingerprint.ts | 45 +- .../plugins/semantic-search/src/sdk/index.ts | 29 +- .../semantic-search/src/sdk/indexer.test.ts | 609 ------- .../semantic-search/src/sdk/indexer.ts | 408 ----- .../plugins/semantic-search/src/sdk/plugin.ts | 235 ++- .../src/sdk/store-cloudflare.ts | 34 +- .../src/sdk/tool-search-index.test.ts | 553 ++++++ .../src/sdk/tool-search-index.ts | 1477 +++++++++++++++++ 24 files changed, 2574 insertions(+), 1222 deletions(-) delete mode 100644 packages/plugins/semantic-search/src/sdk/indexer.test.ts delete mode 100644 packages/plugins/semantic-search/src/sdk/indexer.ts create mode 100644 packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts create mode 100644 packages/plugins/semantic-search/src/sdk/tool-search-index.ts diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 1dff079780..6506a10b3a 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -142,7 +142,7 @@ import { ORG_SUBJECT, type ExecutorOwnerPolicyContext, } from "./owner-policy"; -import { ToolSchemaView, type IntegrationDetectionResult } from "./types"; +import { ToolSchemaView, type IntegrationDetectionResult, type ToolSchemaOptions } from "./types"; import { type Tool, type ToolAnnotations, type ToolDef, type ToolListFilter } from "./tool"; import { buildToolTypeScriptPreview, type ToolTypeScriptPreview } from "./schema-types"; import { collectReferencedDefinitions } from "./schema-refs"; @@ -312,7 +312,10 @@ export type Executor = { readonly tools: { readonly list: (filter?: ToolListFilter) => Effect.Effect; - readonly schema: (address: ToolAddress) => Effect.Effect; + readonly schema: ( + address: ToolAddress, + options?: ToolSchemaOptions, + ) => Effect.Effect; }; readonly providers: { @@ -2816,8 +2819,14 @@ export const createExecutor = => Effect.gen(function* () { + // The JSON-schema β†’ TypeScript codegen is the CPU-heavy part of this + // surface; callers that only need the schema's content (a reindex + // computing a content fingerprint) opt out and skip it. Raw schema roots + // + referenced `$defs` are returned either way. + const includeTypeScript = options?.includeTypeScript ?? true; const staticEntry = staticTools.get(String(address)); if (staticEntry) { const tool = staticToolToTool(staticEntry); @@ -2828,24 +2837,29 @@ export const createExecutor = (Object.entries(referenced)); - const preview = yield* buildCachedToolTypeScriptPreview({ - inputSchema: tool.inputSchema, - outputSchema: tool.outputSchema, - defs: referencedDefs, - }).pipe(Effect.option); + const preview = includeTypeScript + ? Option.getOrUndefined( + yield* buildCachedToolTypeScriptPreview({ + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + defs: referencedDefs, + }).pipe(Effect.option), + ) + : undefined; const view = ToolSchemaView.make({ address, @@ -2918,9 +2937,9 @@ export const createExecutor = 0 ? (referenced as Record) : undefined, - inputTypeScript: Option.getOrUndefined(preview)?.inputTypeScript, - outputTypeScript: Option.getOrUndefined(preview)?.outputTypeScript, - typeScriptDefinitions: Option.getOrUndefined(preview)?.typeScriptDefinitions, + inputTypeScript: preview?.inputTypeScript, + outputTypeScript: preview?.outputTypeScript, + typeScriptDefinitions: preview?.typeScriptDefinitions, }); yield* Cache.set(toolSchemaViewCache, key, view); if (toolSchemaViewStore !== undefined) { diff --git a/packages/core/sdk/src/index.ts b/packages/core/sdk/src/index.ts index 3a1c5f4478..362feda40a 100644 --- a/packages/core/sdk/src/index.ts +++ b/packages/core/sdk/src/index.ts @@ -99,6 +99,7 @@ export type { CredentialProvider, ProviderEntry } from "./provider"; // Public projections / detection. export { ToolSchemaView, IntegrationDetectionResult } from "./types"; +export type { ToolSchemaOptions } from "./types"; // Core schema. export { diff --git a/packages/core/sdk/src/promise.ts b/packages/core/sdk/src/promise.ts index a486c8e4b8..28ea7c4a0e 100644 --- a/packages/core/sdk/src/promise.ts +++ b/packages/core/sdk/src/promise.ts @@ -44,6 +44,7 @@ export type { UpdateToolPolicyInput, } from "./policies"; export { ToolSchemaView, IntegrationDetectionResult } from "./types"; +export type { ToolSchemaOptions } from "./types"; export type { Tool, ToolDef, ToolListFilter, ToolAnnotations } from "./tool"; export type { AnyPlugin, PluginExtensions } from "./plugin"; export type { diff --git a/packages/core/sdk/src/shared.ts b/packages/core/sdk/src/shared.ts index 30724bfbde..cdd07df0c5 100644 --- a/packages/core/sdk/src/shared.ts +++ b/packages/core/sdk/src/shared.ts @@ -97,6 +97,7 @@ export type { ToolPolicyAction } from "./core-schema"; // Schema-side views + onboarding autodetect. export { ToolSchemaView, IntegrationDetectionResult } from "./types"; +export type { ToolSchemaOptions } from "./types"; // OAuth wire contracts (data + tagged errors; the flow impl is server-only). export { diff --git a/packages/core/sdk/src/tool-schema-view-cache.test.ts b/packages/core/sdk/src/tool-schema-view-cache.test.ts index 9dc1a05b95..425b987b1c 100644 --- a/packages/core/sdk/src/tool-schema-view-cache.test.ts +++ b/packages/core/sdk/src/tool-schema-view-cache.test.ts @@ -69,6 +69,9 @@ describe("toolSchemaViewCacheKey", () => { inputSchema: { type: "object", properties: { pet: { type: "string" } } }, }), ).not.toBe(sameKey); + expect(yield* toolSchemaViewCacheKey({ ...base, includeTypeScript: false })).not.toBe( + sameKey, + ); }), ); }); diff --git a/packages/core/sdk/src/tool-schema-view-cache.ts b/packages/core/sdk/src/tool-schema-view-cache.ts index d5ca7a4def..91dc7830ec 100644 --- a/packages/core/sdk/src/tool-schema-view-cache.ts +++ b/packages/core/sdk/src/tool-schema-view-cache.ts @@ -25,6 +25,7 @@ interface ToolSchemaViewCacheKeyInput { readonly inputSchema?: unknown; readonly outputSchema?: unknown; readonly definitions: Readonly>; + readonly includeTypeScript?: boolean; } const cachePayload = (input: ToolSchemaViewCacheKeyInput): string => @@ -38,6 +39,7 @@ const cachePayload = (input: ToolSchemaViewCacheKeyInput): string => inputSchema: input.inputSchema, outputSchema: input.outputSchema, definitions: input.definitions, + includeTypeScript: input.includeTypeScript ?? true, }); export const toolSchemaViewCacheKey = (input: ToolSchemaViewCacheKeyInput): Effect.Effect => diff --git a/packages/core/sdk/src/types.ts b/packages/core/sdk/src/types.ts index 852d409dbd..3369ff8b4e 100644 --- a/packages/core/sdk/src/types.ts +++ b/packages/core/sdk/src/types.ts @@ -28,6 +28,18 @@ export const ToolSchemaView = Schema.Struct({ }); export type ToolSchemaView = typeof ToolSchemaView.Type; +/** Options for `executor.tools.schema`. */ +export interface ToolSchemaOptions { + /** Generate the TypeScript preview strings (`inputTypeScript` / + * `outputTypeScript` / `typeScriptDefinitions`). Defaults to `true`. Pass + * `false` to skip the CPU-heavy JSON-schema β†’ TypeScript codegen and return + * only the raw JSON schema roots plus referenced `$defs` β€” the cheap tier for + * callers that need the schema's content but not its rendered types (e.g. a + * reindex computing a content fingerprint). Mirrors `tools.list`'s + * `includeAnnotations` opt-out of an expensive per-tool derivation. */ + readonly includeTypeScript?: boolean; +} + // --------------------------------------------------------------------------- // Integration detection β€” optional capability on `PluginSpec.detect`. When a // user pastes a URL in the onboarding UI, `executor.integrations.detect(url)` diff --git a/packages/plugins/semantic-search/src/api/group.ts b/packages/plugins/semantic-search/src/api/group.ts index 9271ae3fe9..21b47ffa8a 100644 --- a/packages/plugins/semantic-search/src/api/group.ts +++ b/packages/plugins/semantic-search/src/api/group.ts @@ -4,14 +4,14 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; import { InternalError } from "@executor-js/api"; // --------------------------------------------------------------------------- -// HTTP surface for the Semantic search plugin: reindex + operator search/status. +// HTTP surface for the Semantic search plugin: index + operator search/status. // // Routes are flat and plugin-id-prefixed (`/semantic-search/...`), matching the // execution-history/graphql convention β€” the per-request executor is already // owner-scoped at the host edge, so there is no `:scopeId` segment. // -// - reindex (POST) reconciles the whole tool catalog for the scoped tenant -// against Vectorize + the lexical store (incremental fingerprint diff). +// - reindex (POST) indexes the whole tool catalog for the scoped tenant +// through the same index refresh/chunk/embed pipeline. // - search (GET) runs a live `tools.search` through the SAME provider the // engine uses, so the operator console sees what the agent would. // - status (GET) reports index population (vector fingerprints + lexical docs). @@ -20,12 +20,12 @@ import { InternalError } from "@executor-js/api"; // `capture` downgrades it to `InternalError`. // --------------------------------------------------------------------------- -/** Result of a reindex reconcile: counts for each category of tool processed. */ +/** Result of an index run: counts for each category of tool processed. */ export const ReindexResponse = Schema.Struct({ namespace: Schema.String, total: Schema.Number, - reembedded: Schema.Number, - unchanged: Schema.Number, + indexed: Schema.Number, + skipped: Schema.Number, removed: Schema.Number, }); diff --git a/packages/plugins/semantic-search/src/api/handlers.ts b/packages/plugins/semantic-search/src/api/handlers.ts index 688298287c..10a8dd1c94 100644 --- a/packages/plugins/semantic-search/src/api/handlers.ts +++ b/packages/plugins/semantic-search/src/api/handlers.ts @@ -15,7 +15,8 @@ import { SemanticSearchGroup } from "./group"; // `Layer.succeed(SemanticSearchExtensionService, executor.semanticSearch)`. // The handler also yields the per-request `ExecutorService` (the scoped // executor) and hands it to `reindex`, since the catalog lives on the executor, -// not the plugin ctx. +// not the plugin ctx. `reindex` is a index run, not a legacy page +// reconciler. // --------------------------------------------------------------------------- export class SemanticSearchExtensionService extends Context.Service< diff --git a/packages/plugins/semantic-search/src/react/SearchPage.tsx b/packages/plugins/semantic-search/src/react/SearchPage.tsx index 9dc567d7e5..cfab0fa6c2 100644 --- a/packages/plugins/semantic-search/src/react/SearchPage.tsx +++ b/packages/plugins/semantic-search/src/react/SearchPage.tsx @@ -68,7 +68,7 @@ export function SearchPage() { const exit = await doReindex({ reactivityKeys: [] }); setReindexing(false); if (Exit.isFailure(exit)) { - setNotice("Reindex failed β€” a full-catalog reconcile can exceed the Worker CPU limit."); + setNotice("Reindex failed β€” a full-catalog index run can exceed the Worker CPU limit."); return; } setNotice("Reindex complete."); diff --git a/packages/plugins/semantic-search/src/react/atoms.ts b/packages/plugins/semantic-search/src/react/atoms.ts index 6843a6ca69..80888f1dd7 100644 --- a/packages/plugins/semantic-search/src/react/atoms.ts +++ b/packages/plugins/semantic-search/src/react/atoms.ts @@ -24,5 +24,5 @@ export const statusAtom = SemanticSearchClient.query("semanticSearch", "status", reactivityKeys: [], }); -// Explicit reindex (reconcile the whole catalog into the vector + lexical index). +// Explicit index run (index fingerprint/materialize/embed pipeline). export const reindexMutation = SemanticSearchClient.mutation("semanticSearch", "reindex"); diff --git a/packages/plugins/semantic-search/src/sdk/collections.ts b/packages/plugins/semantic-search/src/sdk/collections.ts index 3ed562c185..5c38b750a1 100644 --- a/packages/plugins/semantic-search/src/sdk/collections.ts +++ b/packages/plugins/semantic-search/src/sdk/collections.ts @@ -27,3 +27,85 @@ export type FingerprintRow = typeof FingerprintRow.Type; export const toolFingerprints = definePluginStorageCollection("toolFingerprints", FingerprintRow, { indexes: ["path"], }); + +export const IndexRunStatus = Schema.Literals(["running", "completed", "failed"]); +export type IndexRunStatus = typeof IndexRunStatus.Type; + +export const IndexRun = Schema.Struct({ + runId: Schema.String, + namespace: Schema.String, + status: IndexRunStatus, + partitionCount: Schema.Number, + total: Schema.Number, + createdAt: Schema.String, + updatedAt: Schema.String, + error: Schema.optional(Schema.String), +}); +export type IndexRun = typeof IndexRun.Type; + +export const indexRuns = definePluginStorageCollection("indexRuns", IndexRun, { + indexes: ["runId", "namespace", "status"], +}); + +export const IndexJobStatus = Schema.Literals([ + "pendingScan", + "skipped", + "pendingChunk", + "pendingEmbedding", + "indexed", + "failed", +]); +export type IndexJobStatus = typeof IndexJobStatus.Type; + +export const IndexJob = Schema.Struct({ + runId: Schema.String, + namespace: Schema.String, + partition: Schema.Number, + ordinal: Schema.Number, + address: Schema.String, + path: Schema.String, + name: Schema.String, + integration: Schema.String, + description: Schema.String, + status: IndexJobStatus, + fingerprint: Schema.optional(Schema.String), + oldChunkIds: Schema.Array(Schema.String), + chunkIds: Schema.Array(Schema.String), + lexicalTextKey: Schema.optional(Schema.String), + createdAt: Schema.String, + updatedAt: Schema.String, + error: Schema.optional(Schema.String), +}); +export type IndexJob = typeof IndexJob.Type; + +export const indexJobs = definePluginStorageCollection("indexJobs", IndexJob, { + indexes: ["runId", "namespace", "partition", "status", "path", "ordinal"], +}); + +export const IndexChunkStatus = Schema.Literals(["pendingEmbedding", "indexed", "failed"]); +export type IndexChunkStatus = typeof IndexChunkStatus.Type; + +export const IndexChunk = Schema.Struct({ + runId: Schema.String, + namespace: Schema.String, + partition: Schema.Number, + path: Schema.String, + chunkId: Schema.String, + facet: Schema.String, + chunkIndex: Schema.Number, + embeddingTextKey: Schema.String, + embeddingTextBytes: Schema.Number, + embeddingTextTokens: Schema.Number, + name: Schema.String, + integration: Schema.String, + description: Schema.String, + status: IndexChunkStatus, + createdAt: Schema.String, + updatedAt: Schema.String, + error: Schema.optional(Schema.String), +}); +export type IndexChunk = typeof IndexChunk.Type; + +export const indexChunks = definePluginStorageCollection("indexChunks", IndexChunk, { + indexes: ["runId", "namespace", "partition", "status", "path"], +}); diff --git a/packages/plugins/semantic-search/src/sdk/documents.ts b/packages/plugins/semantic-search/src/sdk/documents.ts index 9019e8cd12..26aca43ccd 100644 --- a/packages/plugins/semantic-search/src/sdk/documents.ts +++ b/packages/plugins/semantic-search/src/sdk/documents.ts @@ -3,9 +3,17 @@ import { Effect } from "effect"; import type { ToolDocumentInput } from "./chunker"; import { SemanticSearchError } from "./errors"; +import { cyrb53, type FingerprintInput } from "./fingerprint"; const ADDRESS_PREFIX = "tools."; +export interface IndexableToolDescriptor { + readonly address: Tool["address"] | string; + readonly name: Tool["name"] | string; + readonly integration: Tool["integration"] | string; + readonly description?: string; +} + // --------------------------------------------------------------------------- // HTML-stripping helper // --------------------------------------------------------------------------- @@ -70,6 +78,29 @@ export const buildLexicalText = (doc: ToolDocumentInput): string => { export const addressToPath = (address: string): string => address.startsWith(ADDRESS_PREFIX) ? address.slice(ADDRESS_PREFIX.length) : address; +export interface ListToolDescriptorsOptions { + readonly maxTools?: number; +} + +const selectToolDescriptors = ( + tools: readonly Tool[], + options?: ListToolDescriptorsOptions, +): readonly Tool[] => { + const sorted = [...tools].sort((a, b) => String(a.address).localeCompare(String(b.address))); + const maxTools = options?.maxTools; + if (maxTools === undefined) return sorted; + const limit = Math.max(0, Math.floor(maxTools)); + if (limit >= sorted.length) return sorted; + return [...sorted] + .sort((a, b) => { + const left = cyrb53(addressToPath(String(a.address))); + const right = cyrb53(addressToPath(String(b.address))); + return left === right ? String(a.address).localeCompare(String(b.address)) : left - right; + }) + .slice(0, limit) + .sort((a, b) => String(a.address).localeCompare(String(b.address))); +}; + /** List the live tool descriptors, stably sorted by address. * * Cheap β€” descriptors only, NO per-tool schema fetch β€” so it is safe to call @@ -78,21 +109,58 @@ export const addressToPath = (address: string): string => * even though `tools.list` has no native pagination. */ export const listToolDescriptors = ( executor: Executor, + options?: ListToolDescriptorsOptions, ): Effect.Effect => executor.tools.list({ includeAnnotations: false }).pipe( Effect.mapError( (cause) => new SemanticSearchError({ message: "Failed to list tools for indexing.", cause }), ), - Effect.map((tools) => - [...tools].sort((a, b) => String(a.address).localeCompare(String(b.address))), - ), + Effect.map((tools) => selectToolDescriptors(tools, options)), ); +/** Collect `ToolDocumentInput` for a single tool descriptor. + * + * Attempts to fetch its schema via `tools.schema(address)` to populate the + * TypeScript facets; on failure it degrades gracefully to an identity-only + * document. + */ +export const collectDocForTool = ( + executor: Executor, + tool: IndexableToolDescriptor, +): Effect.Effect => { + const path = addressToPath(String(tool.address)); + const base: ToolDocumentInput = { + path, + name: String(tool.name), + integration: String(tool.integration), + description: stripHtml(String(tool.description ?? "")), + }; + return executor.tools.schema(tool.address as Tool["address"]).pipe( + Effect.map((view): ToolDocumentInput => { + const doc: ToolDocumentInput = + view === null + ? base + : { + ...base, + ...(view.inputTypeScript !== undefined + ? { inputTypeScript: view.inputTypeScript } + : {}), + ...(view.outputTypeScript !== undefined + ? { outputTypeScript: view.outputTypeScript } + : {}), + ...(view.typeScriptDefinitions !== undefined + ? { typeScriptDefinitions: view.typeScriptDefinitions } + : {}), + }; + return { ...doc, lexicalText: buildLexicalText(doc) }; + }), + // Degrade: schema fetch failed β€” use identity-only document. + Effect.catch(() => Effect.succeed({ ...base, lexicalText: buildLexicalText(base) })), + ); +}; + /** Collect `ToolDocumentInput` for a specific set of tool descriptors. * - * For each tool we attempt to fetch its schema via `tools.schema(address)` to - * populate the TypeScript facets; on any per-tool failure we degrade gracefully - * to an identity-only document (never failing the whole batch for one tool). * This per-tool schema β†’ TypeScript codegen is the CPU-heavy part of indexing, * so callers bound the input (one page) to stay within a single invocation's * CPU budget. @@ -100,41 +168,60 @@ export const listToolDescriptors = ( * Bounded concurrency (16) keeps the walk fast while avoiding unbounded fan-out. */ export const collectDocsForTools = ( executor: Executor, - tools: readonly Tool[], + tools: readonly IndexableToolDescriptor[], ): Effect.Effect => + Effect.forEach(tools, (tool) => collectDocForTool(executor, tool), { concurrency: 16 }); + +/** A tool paired with the raw-schema fingerprint input the diff hashes. */ +export interface ToolFingerprintInput { + readonly tool: IndexableToolDescriptor; + readonly input: FingerprintInput; +} + +/** Collect the cheap fingerprint inputs for a set of tools β€” the raw JSON schema + * (roots + referenced `$defs`), NO TypeScript codegen. + * + * This is the cheap tier the incremental reindex diffs on: it fetches each + * tool's schema with `includeTypeScript: false`, so the CPU-heavy + * `tools.schema` codegen is skipped entirely. Only tools whose fingerprint + * changed go on to pay the codegen (via `collectDocsForTools`). On any per-tool + * schema-fetch failure we degrade to an identity-only fingerprint (path + name + * + description), which simply re-classifies the tool as "changed" and lets the + * full pass retry it β€” never failing the whole batch for one tool. + * + * Bounded concurrency (16) keeps the walk fast while avoiding unbounded fan-out. */ +export const collectFingerprintInputs = ( + executor: Executor, + tools: readonly IndexableToolDescriptor[], +): Effect.Effect => Effect.forEach( tools, (tool) => { - const path = addressToPath(String(tool.address)); - const base: ToolDocumentInput = { - path, + const base: FingerprintInput = { + path: addressToPath(String(tool.address)), name: String(tool.name), - integration: String(tool.integration), description: stripHtml(String(tool.description ?? "")), }; - // Attempt schema fetch; on any failure degrade to identity-only. - return executor.tools.schema(tool.address).pipe( - Effect.map((view): ToolDocumentInput => { - const doc: ToolDocumentInput = - view === null - ? base - : { - ...base, - ...(view.inputTypeScript !== undefined - ? { inputTypeScript: view.inputTypeScript } - : {}), - ...(view.outputTypeScript !== undefined - ? { outputTypeScript: view.outputTypeScript } - : {}), - ...(view.typeScriptDefinitions !== undefined - ? { typeScriptDefinitions: view.typeScriptDefinitions } - : {}), - }; - return { ...doc, lexicalText: buildLexicalText(doc) }; - }), - // Degrade: schema fetch failed β€” use identity-only document. - Effect.catch(() => Effect.succeed({ ...base, lexicalText: buildLexicalText(base) })), - ); + return executor.tools + .schema(tool.address as Tool["address"], { includeTypeScript: false }) + .pipe( + Effect.map((view): ToolFingerprintInput => { + if (view === null) return { tool, input: base }; + return { + tool, + input: { + ...base, + ...(view.inputSchema !== undefined ? { inputSchema: view.inputSchema } : {}), + ...(view.outputSchema !== undefined ? { outputSchema: view.outputSchema } : {}), + ...(view.schemaDefinitions !== undefined + ? { schemaDefinitions: view.schemaDefinitions } + : {}), + }, + }; + }), + // Degrade: schema fetch failed β€” fall back to an identity-only fingerprint. + Effect.catch(() => Effect.succeed({ tool, input: base })), + ); }, { concurrency: 16 }, ); diff --git a/packages/plugins/semantic-search/src/sdk/embedder-openai.ts b/packages/plugins/semantic-search/src/sdk/embedder-openai.ts index 7472061171..e65b80548f 100644 --- a/packages/plugins/semantic-search/src/sdk/embedder-openai.ts +++ b/packages/plugins/semantic-search/src/sdk/embedder-openai.ts @@ -28,7 +28,7 @@ interface OpenAiEmbeddingsResponse { readonly data?: readonly { readonly embedding?: readonly number[] }[]; } -const chunk = (items: readonly A[], size: number): readonly (readonly A[])[] => { +const chunk = (items: readonly A[], size: number): A[][] => { const safe = Math.max(1, Math.floor(size)); const out: A[][] = []; for (let i = 0; i < items.length; i += safe) out.push(items.slice(i, i + safe)); @@ -42,7 +42,7 @@ export const makeOpenAiCompatibleEmbedder = ( const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE; const embedBatch = ( - texts: readonly string[], + texts: string[], ): Effect.Effect => { if (texts.length === 0) return Effect.succeed([]); return Effect.gen(function* () { @@ -57,7 +57,7 @@ export const makeOpenAiCompatibleEmbedder = ( // `dimensions` is honored by OpenAI + most compatible servers; ignored otherwise. body: JSON.stringify({ model: options.model, - input: [...texts], + input: texts, dimensions: options.dimensions, }), }), diff --git a/packages/plugins/semantic-search/src/sdk/embedder.ts b/packages/plugins/semantic-search/src/sdk/embedder.ts index fb22ec6db5..bd2fef0f68 100644 --- a/packages/plugins/semantic-search/src/sdk/embedder.ts +++ b/packages/plugins/semantic-search/src/sdk/embedder.ts @@ -35,7 +35,7 @@ const DEFAULT_BATCH_SIZE = 32; const DOCUMENT_TASK_TYPE = "RETRIEVAL_DOCUMENT"; const QUERY_TASK_TYPE = "RETRIEVAL_QUERY"; -const chunk = (items: readonly A[], size: number): readonly (readonly A[])[] => { +const chunk = (items: readonly A[], size: number): A[][] => { const safe = Math.max(1, Math.floor(size)); const out: A[][] = []; for (let i = 0; i < items.length; i += safe) { @@ -51,7 +51,7 @@ export const makeGeminiEmbedder = (options: GeminiEmbedderOptions): ToolEmbedder const google = createGoogleGenerativeAI({ apiKey: options.apiKey }); const embedTexts = ( - texts: readonly string[], + texts: string[], taskType: typeof DOCUMENT_TASK_TYPE | typeof QUERY_TASK_TYPE, ): Effect.Effect => Effect.tryPromise({ @@ -69,7 +69,7 @@ export const makeGeminiEmbedder = (options: GeminiEmbedderOptions): ToolEmbedder } const { embeddings } = await embedMany({ model: textModel, - values: [...texts], + values: texts, maxParallelCalls: 5, providerOptions, }); diff --git a/packages/plugins/semantic-search/src/sdk/fingerprint.test.ts b/packages/plugins/semantic-search/src/sdk/fingerprint.test.ts index 85bb03166c..057b91cf48 100644 --- a/packages/plugins/semantic-search/src/sdk/fingerprint.test.ts +++ b/packages/plugins/semantic-search/src/sdk/fingerprint.test.ts @@ -6,8 +6,14 @@ const base = { path: "github.repos.get", name: "repos.get", description: "Get a GitHub repository by owner and name.", - inputTypeScript: "{ owner: string; repo: string }", - outputTypeScript: "{ id: number; full_name: string }", + inputSchema: { + type: "object", + properties: { owner: { type: "string" }, repo: { type: "string" } }, + }, + outputSchema: { + type: "object", + properties: { id: { type: "number" }, full_name: { type: "string" } }, + }, } as const; describe("fingerprintTool", () => { @@ -23,24 +29,61 @@ describe("fingerprintTool", () => { expect(changed).not.toBe(original); }); - it("changes when inputTypeScript changes", () => { + it("changes when inputSchema changes", () => { const original = fingerprintTool(base); const changed = fingerprintTool({ ...base, - inputTypeScript: "{ owner: string; repo: string; ref?: string }", + inputSchema: { + type: "object", + properties: { + owner: { type: "string" }, + repo: { type: "string" }, + ref: { type: "string" }, + }, + }, }); expect(changed).not.toBe(original); }); - it("changes when outputTypeScript changes", () => { + it("changes when outputSchema changes", () => { const original = fingerprintTool(base); const changed = fingerprintTool({ ...base, - outputTypeScript: "{ id: number; full_name: string; private: boolean }", + outputSchema: { + type: "object", + properties: { + id: { type: "number" }, + full_name: { type: "string" }, + private: { type: "boolean" }, + }, + }, }); expect(changed).not.toBe(original); }); + it("changes when a referenced $def changes (def-only change)", () => { + const withDef = fingerprintTool({ + ...base, + schemaDefinitions: { Repo: { type: "object", properties: { id: { type: "number" } } } }, + }); + const changedDef = fingerprintTool({ + ...base, + schemaDefinitions: { Repo: { type: "object", properties: { id: { type: "string" } } } }, + }); + expect(changedDef).not.toBe(withDef); + }); + + it("is canonical β€” schema key order does not change the hash", () => { + const reordered = fingerprintTool({ + ...base, + inputSchema: { + properties: { repo: { type: "string" }, owner: { type: "string" } }, + type: "object", + }, + }); + expect(reordered).toBe(fingerprintTool(base)); + }); + it("changes when path changes", () => { const original = fingerprintTool(base); const changed = fingerprintTool({ ...base, path: "github.repos.list" }); @@ -59,8 +102,8 @@ describe("fingerprintTool", () => { path: "github.issues.create", name: "issues.create", description: "Create an issue in a repository.", - inputTypeScript: "{ title: string; body?: string }", - outputTypeScript: "{ id: number; number: number }", + inputSchema: { type: "object", properties: { title: { type: "string" } } }, + outputSchema: { type: "object", properties: { id: { type: "number" } } }, }); expect(tool1).not.toBe(tool2); }); @@ -73,13 +116,11 @@ describe("fingerprintTool", () => { expect(a).not.toBe(b); }); - it("treats absent optional fields the same as empty strings", () => { + it("treats absent optional fields the same as empty content", () => { const withEmpty = fingerprintTool({ path: "x.y", name: "y", description: "", - inputTypeScript: "", - outputTypeScript: "", }); const withAbsent = fingerprintTool({ path: "x.y", diff --git a/packages/plugins/semantic-search/src/sdk/fingerprint.ts b/packages/plugins/semantic-search/src/sdk/fingerprint.ts index 9ffb59e4b2..d66b05b454 100644 --- a/packages/plugins/semantic-search/src/sdk/fingerprint.ts +++ b/packages/plugins/semantic-search/src/sdk/fingerprint.ts @@ -1,21 +1,47 @@ // --------------------------------------------------------------------------- -// Deterministic content hash over the five fields that define a tool's -// semantic identity. Used by the incremental reindex to detect changes and -// skip unchanged tools. +// Deterministic content hash over the fields that define a tool's semantic +// identity. Used by the incremental reindex to detect changes and skip +// unchanged tools. +// +// The hash keys off the RAW JSON schema (input/output roots + the referenced +// `$defs`), NOT the generated TypeScript. The TS is a deterministic function of +// the raw schema, so hashing the raw schema is change-equivalent β€” but the raw +// schema is read cheaply from the tool row, whereas the TS costs the per-tool +// codegen. That lets the reindex decide "unchanged" without codegen, and pay it +// only for tools that actually changed. Referenced `$defs` are included so a +// change to a shared definition a tool `$ref`s still flips the fingerprint. // --------------------------------------------------------------------------- export interface FingerprintInput { readonly path: string; readonly name: string; readonly description?: string; - readonly inputTypeScript?: string; - readonly outputTypeScript?: string; + /** Raw input JSON schema root (the `tool` row column, not the codegen). */ + readonly inputSchema?: unknown; + /** Raw output JSON schema root. */ + readonly outputSchema?: unknown; + /** Referenced `$defs` the input/output schemas resolve against. */ + readonly schemaDefinitions?: Record; } +// Canonical JSON: object keys sorted, `undefined` dropped, arrays preserved. +// JSON.stringify is key-insertion-ordered, so two schemas that differ only in +// key order would hash differently; canonicalizing first avoids those false +// "changed" verdicts (and the needless re-embed they would trigger). +const canonicalize = (value: unknown): string => { + if (value === undefined) return ""; + if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null"; + if (Array.isArray(value)) return `[${value.map(canonicalize).join(",")}]`; + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalize(v)}`).join(",")}}`; +}; + // cyrb53 β€” a fast, stable 53-bit hash with good avalanche properties. // Produces a consistent unsigned integer across all JS environments. // Source: https://github.com/bryc/code/blob/master/jshash/experimental/cyrb53.js -function cyrb53(str: string, seed = 0): number { +export function cyrb53(str: string, seed = 0): number { let h1 = 0xdeadbeef ^ seed; let h2 = 0x41c6ce57 ^ seed; for (let i = 0; i < str.length; i++) { @@ -34,15 +60,16 @@ function cyrb53(str: string, seed = 0): number { // {name:"a", description:"b"} hash to the same value. const FIELD_SEPARATOR = String.fromCharCode(0); -/** Stable, deterministic fingerprint over all five semantic fields of a tool. +/** Stable, deterministic fingerprint over a tool's identity + raw schema. * Changes when ANY field changes and is identical for identical input. */ export const fingerprintTool = (input: FingerprintInput): string => { const content = [ input.path, input.name, input.description ?? "", - input.inputTypeScript ?? "", - input.outputTypeScript ?? "", + canonicalize(input.inputSchema), + canonicalize(input.outputSchema), + canonicalize(input.schemaDefinitions), ].join(FIELD_SEPARATOR); return cyrb53(content).toString(36); }; diff --git a/packages/plugins/semantic-search/src/sdk/index.ts b/packages/plugins/semantic-search/src/sdk/index.ts index 92e72c44ef..29aef276b0 100644 --- a/packages/plugins/semantic-search/src/sdk/index.ts +++ b/packages/plugins/semantic-search/src/sdk/index.ts @@ -14,14 +14,18 @@ export { export { type VectorStore, type VectorMatch, type VectorMatches, type VectorInput } from "./store"; export { makeVectorizeStore, type VectorizeIndex, MAX_TOP_K } from "./store-cloudflare"; export { - reconcileToolCatalog, - reconcileToolCatalogPage, + ToolSearchIndex, + create, + scan, + chunk, + embed, + commit, + status, + complete, + run, + make as makeToolSearchIndex, sweepRemoved, - DEFAULT_REINDEX_PAGE_SIZE, - type ReconcileResult, - type ReconcilePageResult, - type SweepResult, -} from "./indexer"; +} from "./tool-search-index"; export { SemanticSearchError } from "./errors"; // Chunker @@ -43,7 +47,16 @@ export { makeHybridToolDiscoveryProvider, type HybridOptions } from "./hybrid"; export { fingerprintTool, type FingerprintInput } from "./fingerprint"; // Collections (plugin storage) -export { toolFingerprints, FingerprintRow } from "./collections"; +export { + toolFingerprints, + indexRuns, + indexJobs, + indexChunks, + FingerprintRow, + IndexRun, + IndexJob, + IndexChunk, +} from "./collections"; // Store limit decorator export { withCloudflareLimits } from "./store-cloudflare-limits"; diff --git a/packages/plugins/semantic-search/src/sdk/indexer.test.ts b/packages/plugins/semantic-search/src/sdk/indexer.test.ts deleted file mode 100644 index ad2005ffdc..0000000000 --- a/packages/plugins/semantic-search/src/sdk/indexer.test.ts +++ /dev/null @@ -1,609 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { Effect } from "effect"; -import type { - Executor, - Owner, - PluginStorageCollectionFacade, - PluginStorageEntry, -} from "@executor-js/sdk/core"; - -import { makeFacetChunker } from "./chunker"; -import type { FingerprintRow } from "./collections"; -import { toolFingerprints } from "./collections"; -import type { ToolEmbedder } from "./embedder"; -import { reconcileToolCatalog, reconcileToolCatalogPage, sweepRemoved } from "./indexer"; -import type { VectorStore, VectorInput } from "./store"; -import type { FtsDocumentInput, FtsLexicalStore } from "./store-fts"; - -// --------------------------------------------------------------------------- -// Test helpers -// --------------------------------------------------------------------------- - -/** Build a minimal fake Executor that exposes a fixed tool list + schema map. */ -const makeExecutor = ( - tools: ReadonlyArray<{ - address: string; - name: string; - integration: string; - description: string; - inputTypeScript?: string; - outputTypeScript?: string; - }>, - schemas: Map = new Map(), -): Executor => { - const toolObjects = tools.map((t) => ({ - address: t.address as never, - name: t.name as never, - integration: t.integration as never, - description: t.description, - owner: "org" as never, - connection: "default" as never, - pluginId: "test", - })); - // oxlint-disable-next-line executor/no-double-cast -- test stub: only the exercised methods are implemented - return { - tools: { - list: () => Effect.succeed(toolObjects as never), - schema: (address: never) => { - const addrStr = String(address); - const key = addrStr.startsWith("tools.") ? addrStr.slice(6) : addrStr; - const view = schemas.get(key) ?? null; - if (view === null) return Effect.succeed(null); - return Effect.succeed({ - address: address as never, - ...view, - }); - }, - }, - } as unknown as Executor; -}; - -/** Build a fake VectorStore that records upserts and deletes. */ -const makeStore = () => { - const upserted: VectorInput[] = []; - const deleted: string[] = []; - const store: VectorStore = { - maxTopK: 200, - query: () => Effect.succeed([]), - upsert: (vectors) => - Effect.sync(() => { - upserted.push(...vectors); - }), - deleteByIds: (ids) => - Effect.sync(() => { - deleted.push(...ids); - }), - }; - return { store, upserted, deleted }; -}; - -/** Build a fake FtsLexicalStore that records upserts/deletes. */ -const makeFakeLexical = () => { - const upserted: FtsDocumentInput[] = []; - const deleted: string[] = []; - const store: FtsLexicalStore = { - upsert: (docs) => - Effect.sync(() => { - upserted.push(...docs); - }), - deleteByIds: (ids) => - Effect.sync(() => { - deleted.push(...ids); - }), - search: () => Effect.succeed([]), - count: () => Effect.succeed(upserted.length), - }; - return { store, upserted, deleted }; -}; - -/** Build a fake PluginStorageCollectionFacade backed by an in-test Map. */ -const makeFingerprints = ( - initial: Map = new Map(), -): PluginStorageCollectionFacade => { - const storage = new Map(initial); - let idCounter = 0; - - const makeEntry = (key: string, data: FingerprintRow): PluginStorageEntry => ({ - id: String(idCounter++), - owner: "org" as Owner, - pluginId: "vectorize-search", - collection: "toolFingerprints", - key, - data, - createdAt: new Date(0), - updatedAt: new Date(0), - }); - - // oxlint-disable-next-line executor/no-double-cast -- test stub: only the exercised methods are implemented - return { - get: ({ key }: { key: string }) => - Effect.succeed(storage.has(key) ? makeEntry(key, storage.get(key)!) : null), - getForOwner: ({ key }: { key: string }) => - Effect.succeed(storage.has(key) ? makeEntry(key, storage.get(key)!) : null), - list: () => Effect.succeed([...storage.entries()].map(([key, data]) => makeEntry(key, data))), - put: ({ key, data }: { key: string; owner: string; data: unknown }) => { - storage.set(key, data as FingerprintRow); - return Effect.succeed(makeEntry(key, data as FingerprintRow)); - }, - query: (input?: { where?: unknown }) => { - let entries = [...storage.entries()].map(([key, data]) => makeEntry(key, data)); - if (input?.where) { - // Simple path-eq filter for tests. - const where = input.where as Record; - if (typeof where["path"] === "string") { - entries = entries.filter((e) => e.data.path === where["path"]); - } - } - return Effect.succeed(entries); - }, - count: () => Effect.succeed(storage.size), - queryKeyset: () => Effect.succeed({ entries: [], nextCursor: null }), - aggregate: { - count: () => Effect.succeed(storage.size), - groupCount: () => Effect.succeed([]), - timeBuckets: () => Effect.succeed([]), - stats: () => Effect.succeed({ count: 0, min: null, max: null, percentiles: [] }), - }, - remove: ({ key }: { key: string }) => - Effect.sync(() => { - storage.delete(key); - }), - } as unknown as PluginStorageCollectionFacade; -}; - -const owner: Owner = "org" as Owner; -const namespace = "test-ns"; -const chunker = makeFacetChunker(); -const fakeEmbedder: ToolEmbedder = { - model: "test", - dimensions: 3, - embedDocuments: (texts) => Effect.succeed(texts.map(() => [0.1, 0.2, 0.3])), - embedQuery: () => Effect.succeed([1, 0, 0]), -}; - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -describe("reconcileToolCatalog", () => { - it.effect("first run: embeds and upserts all tools, writes fingerprint rows", () => - Effect.gen(function* () { - const { store, upserted, deleted } = makeStore(); - const fingerprints = makeFingerprints(); - - const result = yield* reconcileToolCatalog({ - namespace, - executor: makeExecutor([ - { - address: "tools.github.repos.get", - name: "repos.get", - integration: "github", - description: "Get a repository", - }, - { - address: "tools.github.repos.list", - name: "repos.list", - integration: "github", - description: "List repositories", - }, - ]), - embedder: fakeEmbedder, - store, - chunker, - fingerprints, - owner, - }); - - expect(result.total).toBe(2); - expect(result.reembedded).toBe(2); - expect(result.unchanged).toBe(0); - expect(result.removed).toBe(0); - // Each tool produces at least 1 identity chunk. - expect(upserted.length).toBeGreaterThanOrEqual(2); - expect(deleted).toHaveLength(0); - // All upserted records belong to our namespace. - for (const v of upserted) { - expect(v.namespace).toBe(namespace); - } - }), - ); - - it.effect("reads the catalog once for the whole in-process reconcile (no N+1 tools.list)", () => - Effect.gen(function* () { - let listCalls = 0; - const inner = makeExecutor([ - { address: "tools.a.one", name: "one", integration: "a", description: "1" }, - { address: "tools.a.two", name: "two", integration: "a", description: "2" }, - { address: "tools.a.three", name: "three", integration: "a", description: "3" }, - ]); - // oxlint-disable-next-line executor/no-double-cast -- test stub: counting wrapper over the fake executor - const executor = { - tools: { - list: (...args: never[]) => { - listCalls++; - return inner.tools.list(...args); - }, - schema: inner.tools.schema, - }, - } as unknown as Executor; - - // pageSize 1 over 3 tools would re-list per page (3) + sweep (1) = 4 the old - // way; a single shared snapshot must collapse that to exactly one read. - yield* reconcileToolCatalog({ - namespace, - executor, - embedder: fakeEmbedder, - store: makeStore().store, - chunker, - fingerprints: makeFingerprints(), - owner, - pageSize: 1, - }); - - expect(listCalls).toBe(1); - }), - ); - - it.effect("second run with no changes: reembeds nothing, unchanged equals total", () => - Effect.gen(function* () { - const executor = makeExecutor([ - { - address: "tools.github.repos.get", - name: "repos.get", - integration: "github", - description: "Get a repository", - }, - ]); - - // First run to populate fingerprints. - const fingerprints = makeFingerprints(); - const { store: store1 } = makeStore(); - yield* reconcileToolCatalog({ - namespace, - executor, - embedder: fakeEmbedder, - store: store1, - chunker, - fingerprints, - owner, - }); - - // Second run β€” same executor, same data. - const { store: store2, upserted: upserted2 } = makeStore(); - const result2 = yield* reconcileToolCatalog({ - namespace, - executor, - embedder: fakeEmbedder, - store: store2, - chunker, - fingerprints, - owner, - }); - - expect(result2.total).toBe(1); - expect(result2.reembedded).toBe(0); - expect(result2.unchanged).toBe(1); - expect(upserted2).toHaveLength(0); - }), - ); - - it.effect( - "changing one tool's description re-embeds only that tool AND deletes old chunk ids", - () => - Effect.gen(function* () { - const executor1 = makeExecutor([ - { - address: "tools.github.repos.get", - name: "repos.get", - integration: "github", - description: "Get a repository", - }, - { - address: "tools.github.repos.list", - name: "repos.list", - integration: "github", - description: "List repositories", - }, - ]); - - // First run. - const fingerprints = makeFingerprints(); - const { store: store1, upserted: upserted1 } = makeStore(); - yield* reconcileToolCatalog({ - namespace, - executor: executor1, - embedder: fakeEmbedder, - store: store1, - chunker, - fingerprints, - owner, - }); - - // Capture old chunk ids for repos.get. - const oldGetChunkIds = upserted1 - .filter((v) => v.metadata?.["path"] === "github.repos.get") - .map((v) => v.id); - expect(oldGetChunkIds.length).toBeGreaterThan(0); - - // Second run β€” repos.get description changed; repos.list is unchanged. - const executor2 = makeExecutor([ - { - address: "tools.github.repos.get", - name: "repos.get", - integration: "github", - description: "Fetch a repository β€” updated description", - }, - { - address: "tools.github.repos.list", - name: "repos.list", - integration: "github", - description: "List repositories", - }, - ]); - const { store: store2, upserted: upserted2, deleted: deleted2 } = makeStore(); - const result2 = yield* reconcileToolCatalog({ - namespace, - executor: executor2, - embedder: fakeEmbedder, - store: store2, - chunker, - fingerprints, - owner, - }); - - expect(result2.reembedded).toBe(1); - expect(result2.unchanged).toBe(1); - // Old chunk ids for repos.get must have been deleted. - for (const id of oldGetChunkIds) { - expect(deleted2).toContain(id); - } - // Only repos.get chunks were re-upserted. - for (const v of upserted2) { - expect(v.metadata?.["path"]).toBe("github.repos.get"); - } - }), - ); - - it.effect("a tool removed from the live list is swept: vector chunks + fingerprint deleted", () => - Effect.gen(function* () { - const executor1 = makeExecutor([ - { - address: "tools.github.repos.get", - name: "repos.get", - integration: "github", - description: "Get a repository", - }, - { - address: "tools.github.repos.delete", - name: "repos.delete", - integration: "github", - description: "Delete a repository", - }, - ]); - - // First run β€” both tools indexed. - const fingerprints = makeFingerprints(); - const { store: store1 } = makeStore(); - yield* reconcileToolCatalog({ - namespace, - executor: executor1, - embedder: fakeEmbedder, - store: store1, - chunker, - fingerprints, - owner, - }); - - // Second run β€” repos.delete is no longer in the catalog. - const executor2 = makeExecutor([ - { - address: "tools.github.repos.get", - name: "repos.get", - integration: "github", - description: "Get a repository", - }, - ]); - const { store: store2, deleted: deleted2 } = makeStore(); - const result2 = yield* reconcileToolCatalog({ - namespace, - executor: executor2, - embedder: fakeEmbedder, - store: store2, - chunker, - fingerprints, - owner, - }); - - expect(result2.total).toBe(1); - expect(result2.unchanged).toBe(1); - expect(result2.removed).toBe(1); - // repos.delete's vector chunks are swept (repos.get is unchanged, so it - // contributes no deletions); its fingerprint row is removed. - expect(deleted2.length).toBeGreaterThan(0); - const remaining = yield* fingerprints.list(); - expect(remaining.map((e) => e.key)).toEqual(["github.repos.get"]); - }), - ); - - it.effect( - "populates the lexical store with every live tool, keyed by namespace-prefixed id", - () => - Effect.gen(function* () { - const fingerprints = makeFingerprints(); - const tools = [ - { - address: "tools.github.repos.get", - name: "repos.get", - integration: "github", - description: "Get a repository", - }, - { - address: "tools.calendar.events.create", - name: "events.create", - integration: "calendar", - description: "Create a calendar event", - }, - ]; - - const lexical = makeFakeLexical(); - yield* reconcileToolCatalog({ - namespace, - executor: makeExecutor(tools), - embedder: fakeEmbedder, - store: makeStore().store, - chunker, - fingerprints, - owner, - lexicalStore: lexical.store, - }); - - // One lexical doc per tool, id namespace-prefixed so paths can't collide - // across namespaces in a shared store. - expect(lexical.upserted.map((d) => d.id).sort()).toEqual([ - `${namespace}:calendar.events.create`, - `${namespace}:github.repos.get`, - ]); - const repo = lexical.upserted.find((d) => d.id === `${namespace}:github.repos.get`); - expect(repo?.path).toBe("github.repos.get"); - expect(repo?.integration).toBe("github"); - expect(repo?.namespace).toBe(namespace); - expect((repo?.lexicalText ?? "").length).toBeGreaterThan(0); - - // A SECOND reindex with identical tools re-embeds nothing (fingerprints - // match) yet still fully populates the lexical store β€” so attaching a - // lexical store to an already-indexed deployment is not silently empty. - const lexical2 = makeFakeLexical(); - const result2 = yield* reconcileToolCatalog({ - namespace, - executor: makeExecutor(tools), - embedder: fakeEmbedder, - store: makeStore().store, - chunker, - fingerprints, - owner, - lexicalStore: lexical2.store, - }); - expect(result2.reembedded).toBe(0); - expect(result2.unchanged).toBe(2); - expect(lexical2.upserted).toHaveLength(2); - }), - ); -}); - -describe("reconcileToolCatalogPage", () => { - it.effect("pages through the catalog one slice at a time, advancing the cursor", () => - Effect.gen(function* () { - const executor = makeExecutor([ - { address: "tools.a.one", name: "one", integration: "a", description: "1" }, - { address: "tools.a.two", name: "two", integration: "a", description: "2" }, - { address: "tools.a.three", name: "three", integration: "a", description: "3" }, - ]); - const fingerprints = makeFingerprints(); - const { store } = makeStore(); - const base = { - namespace, - executor, - embedder: fakeEmbedder, - store, - chunker, - fingerprints, - owner, - pageSize: 1, - }; - - const p0 = yield* reconcileToolCatalogPage({ ...base, cursor: 0 }); - expect(p0.total).toBe(3); - expect(p0.processed).toBe(1); - expect(p0.reembedded).toBe(1); - expect(p0.nextCursor).toBe(1); - - const p1 = yield* reconcileToolCatalogPage({ ...base, cursor: 1 }); - expect(p1.processed).toBe(1); - expect(p1.nextCursor).toBe(2); - - const p2 = yield* reconcileToolCatalogPage({ ...base, cursor: 2 }); - expect(p2.processed).toBe(1); - expect(p2.nextCursor).toBe(null); - - // Every tool now has a fingerprint row (full coverage across pages). - const rows = yield* fingerprints.list(); - expect(rows.length).toBe(3); - }), - ); - - it.effect("a cursor past the end is an empty, terminal page", () => - Effect.gen(function* () { - const executor = makeExecutor([ - { address: "tools.a.one", name: "one", integration: "a", description: "1" }, - ]); - const page = yield* reconcileToolCatalogPage({ - namespace, - executor, - embedder: fakeEmbedder, - store: makeStore().store, - chunker, - fingerprints: makeFingerprints(), - owner, - cursor: 5, - }); - expect(page.total).toBe(1); - expect(page.processed).toBe(0); - expect(page.nextCursor).toBe(null); - }), - ); -}); - -describe("sweepRemoved", () => { - it.effect("deletes vector chunks + fingerprint for tools no longer in the catalog", () => - Effect.gen(function* () { - const fingerprints = makeFingerprints( - new Map([ - [ - "a.gone", - { path: "a.gone", integration: "a", fingerprint: "x", chunkIds: ["a.gone#0"] }, - ], - [ - "a.live", - { path: "a.live", integration: "a", fingerprint: "y", chunkIds: ["a.live#0"] }, - ], - ]), - ); - const { store, deleted } = makeStore(); - const executor = makeExecutor([ - { address: "tools.a.live", name: "live", integration: "a", description: "" }, - ]); - - const result = yield* sweepRemoved({ namespace, executor, store, fingerprints, owner }); - - expect(result.removed).toBe(1); - expect(deleted).toContain("a.gone#0"); - expect(deleted).not.toContain("a.live#0"); - const rows = yield* fingerprints.list(); - expect(rows.map((e) => e.key)).toEqual(["a.live"]); - }), - ); - - it.effect("an empty live catalog is a no-op (never wipes the index)", () => - Effect.gen(function* () { - const fingerprints = makeFingerprints( - new Map([ - ["a.x", { path: "a.x", integration: "a", fingerprint: "x", chunkIds: ["a.x#0"] }], - ]), - ); - const { store, deleted } = makeStore(); - - const result = yield* sweepRemoved({ - namespace, - executor: makeExecutor([]), - store, - fingerprints, - owner, - }); - - expect(result.removed).toBe(0); - expect(deleted).toHaveLength(0); - const rows = yield* fingerprints.list(); - expect(rows.length).toBe(1); - }), - ); -}); diff --git a/packages/plugins/semantic-search/src/sdk/indexer.ts b/packages/plugins/semantic-search/src/sdk/indexer.ts deleted file mode 100644 index c4885ceb61..0000000000 --- a/packages/plugins/semantic-search/src/sdk/indexer.ts +++ /dev/null @@ -1,408 +0,0 @@ -import type { Executor, Tool } from "@executor-js/sdk/core"; -import type { Owner, PluginStorageCollectionFacade } from "@executor-js/sdk/core"; -import { Effect } from "effect"; - -import type { Chunker } from "./chunker"; -import type { FingerprintRow } from "./collections"; -import { toolFingerprints } from "./collections"; -import { - addressToPath, - buildLexicalText, - collectDocsForTools, - listToolDescriptors, -} from "./documents"; -import type { ToolEmbedder } from "./embedder"; -import { SemanticSearchError } from "./errors"; -import { fingerprintTool } from "./fingerprint"; -import type { VectorStore, VectorInput } from "./store"; -import type { FtsDocumentInput, FtsLexicalStore } from "./store-fts"; - -// --------------------------------------------------------------------------- -// Results -// --------------------------------------------------------------------------- - -export interface ReconcileResult { - /** The namespace this reconcile ran against. */ - readonly namespace: string; - /** Total tools in the live catalog. */ - readonly total: number; - /** Number of tools whose embeddings were refreshed (new + changed). */ - readonly reembedded: number; - /** Number of tools whose stored fingerprint matched β€” skipped re-embedding. */ - readonly unchanged: number; - /** Number of tools removed from the index because they left the catalog. */ - readonly removed: number; -} - -export interface ReconcilePageResult { - readonly namespace: string; - /** Total tools in the live catalog (size of the sorted descriptor list). */ - readonly total: number; - /** Tools handled in this page. */ - readonly processed: number; - readonly reembedded: number; - readonly unchanged: number; - /** Index of the next page, or `null` when the catalog is exhausted. */ - readonly nextCursor: number | null; -} - -export interface SweepResult { - readonly namespace: string; - /** Tools deleted from the index (vector chunks + lexical doc + fingerprint). */ - readonly removed: number; -} - -/** Default tools-per-page. Bounds the CPU-heavy schemaβ†’TS codegen + embedding so - * one page fits within a single Worker invocation's CPU budget. */ -export const DEFAULT_REINDEX_PAGE_SIZE = 200; - -interface ReconcileStores { - readonly namespace: string; - readonly executor: Executor; - readonly store: VectorStore; - readonly fingerprints: PluginStorageCollectionFacade; - readonly owner: Owner; - /** Optional FTS5 lexical store kept in lockstep with the vector index. */ - readonly lexicalStore?: FtsLexicalStore; -} - -interface ReconcileDeps extends ReconcileStores { - readonly embedder: ToolEmbedder; - readonly chunker: Chunker; -} - -// --------------------------------------------------------------------------- -// reconcileToolCatalogPage β€” reconcile ONE bounded slice of the catalog -// --------------------------------------------------------------------------- - -/** Reconcile a single page of the live tool catalog into the vector + lexical - * index, returning the cursor for the next page. - * - * `tools.list` has no native pagination, so a page re-lists the (stably sorted) - * catalog β€” cheap, descriptors only β€” and slices `[cursor, cursor + pageSize)`. - * Only the slice pays the CPU-heavy `tools.schema` codegen + embedding, so each - * page stays within one invocation's CPU budget. This is the unit a durable - * driver (e.g. a Cloudflare Workflow step) calls in a loop; removal of tools - * that left the catalog is handled separately by `sweepRemoved`. - * - * Pass a pre-listed `descriptors` snapshot to reuse one catalog read across an - * in-process page loop (see `reconcileToolCatalog`); omit it and the page reads - * the catalog itself (the durable-step path, where each step is its own read). - * - * NOTE: the schema β†’ TS codegen runs for EVERY tool in the slice, including - * fingerprint-unchanged ones, because the fingerprint is derived from the - * schema β€” a schema-only change can't be detected without fetching it. So - * per-page CPU is dominated by codegen and is similar on re-runs; the - * incremental win is skipping the EMBEDDING of unchanged tools, not their - * schema fetch. */ -export const reconcileToolCatalogPage = ( - input: ReconcileDeps & { - readonly cursor: number; - readonly pageSize?: number; - readonly descriptors?: readonly Tool[]; - }, -): Effect.Effect => - Effect.gen(function* () { - const { namespace, executor, embedder, store, chunker, fingerprints, owner } = input; - const pageSize = input.pageSize ?? DEFAULT_REINDEX_PAGE_SIZE; - const cursor = Math.max(0, input.cursor); - - const all = input.descriptors ?? (yield* listToolDescriptors(executor)); - const total = all.length; - const slice = all.slice(cursor, cursor + pageSize); - const nextCursor = cursor + slice.length < total ? cursor + slice.length : null; - - if (slice.length === 0) { - return { namespace, total, processed: 0, reembedded: 0, unchanged: 0, nextCursor }; - } - - // Collect docs (schema β†’ TS) for THIS slice only. - const docs = yield* collectDocsForTools(executor, slice); - - // Lexical store carries no embedding cost, so rebuild it in full for the - // slice every page (delete-then-insert keyed by namespace-prefixed id). - if (input.lexicalStore) { - const lexicalDocs: readonly FtsDocumentInput[] = docs.map((doc) => ({ - id: `${namespace}:${doc.path}`, - namespace, - path: doc.path, - name: doc.name, - description: doc.description, - integration: doc.integration, - lexicalText: buildLexicalText(doc), - })); - yield* input.lexicalStore.upsert(lexicalDocs); - } - - // Load stored fingerprints for just this slice's paths (not the whole index). - const slicePaths = docs.map((doc) => doc.path); - const storedByPath = yield* loadFingerprintsForPaths(fingerprints, owner, slicePaths); - - // Diff: classify each tool in the slice as new/changed (re-embed) or unchanged. - const toEmbed: Array<{ - doc: (typeof docs)[number]; - fingerprint: string; - oldChunkIds: readonly string[] | null; - }> = []; - let unchanged = 0; - for (const doc of docs) { - const fingerprint = fingerprintTool(doc); - const stored = storedByPath.get(doc.path); - if (stored !== undefined && stored.fingerprint === fingerprint) { - unchanged++; - } else { - toEmbed.push({ doc, fingerprint, oldChunkIds: stored?.chunkIds ?? null }); - } - } - - if (toEmbed.length === 0) { - return { namespace, total, processed: slice.length, reembedded: 0, unchanged, nextCursor }; - } - - yield* embedAndUpsert({ namespace, embedder, store, chunker, fingerprints, owner }, toEmbed); - - return { - namespace, - total, - processed: slice.length, - reembedded: toEmbed.length, - unchanged, - nextCursor, - }; - }); - -// --------------------------------------------------------------------------- -// sweepRemoved β€” delete tools that left the catalog -// --------------------------------------------------------------------------- - -/** Delete index entries for tools no longer present in the live catalog. - * - * Uses the cheap `listToolDescriptors` (paths only β€” no schema codegen) for - * liveness and diffs it against all stored fingerprints, so it fits one - * invocation regardless of catalog size. For each removed tool it deletes the - * vector chunks, the lexical document, and the fingerprint row. Guards against - * an empty live list (a transient `tools.list` failure must not wipe the index). - * Accepts a pre-listed `descriptors` snapshot to share one catalog read with a - * preceding in-process page loop. */ -export const sweepRemoved = ( - input: ReconcileStores & { readonly descriptors?: readonly Tool[] }, -): Effect.Effect => - Effect.gen(function* () { - const { namespace, executor, store, fingerprints, owner } = input; - - const live = input.descriptors ?? (yield* listToolDescriptors(executor)); - // Safety: never treat an empty live catalog as "everything removed". - if (live.length === 0) { - return { namespace, removed: 0 }; - } - const livePaths = new Set(live.map((tool) => addressToPath(String(tool.address)))); - - const storedEntries = yield* fingerprints.list().pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ - message: "Failed to load stored fingerprints for sweep.", - cause, - }), - ), - ); - const removed = storedEntries.filter((entry) => !livePaths.has(entry.key)); - if (removed.length === 0) { - return { namespace, removed: 0 }; - } - - yield* Effect.forEach( - removed, - (entry) => - Effect.gen(function* () { - const { key: path, data } = entry; - if (data.chunkIds.length > 0) { - yield* store.deleteByIds([...data.chunkIds]); - } - if (input.lexicalStore) { - yield* input.lexicalStore.deleteByIds([`${namespace}:${path}`]); - } - yield* fingerprints.remove({ owner, key: path }).pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ - message: `Failed to delete fingerprint row for removed tool "${path}".`, - cause, - }), - ), - ); - }), - { concurrency: 8, discard: true }, - ); - - return { namespace, removed: removed.length }; - }); - -// --------------------------------------------------------------------------- -// reconcileToolCatalog β€” full reconcile (loop pages + sweep) -// --------------------------------------------------------------------------- - -/** Reconcile the entire live tool catalog into the vector + lexical index. - * - * Drives `reconcileToolCatalogPage` over every page in-process, then runs - * `sweepRemoved`. This is the convenience entry for hosts that can complete the - * whole reconcile in one execution (local / self-host, or small catalogs). At - * large catalog sizes on a CPU-bounded host (a Worker), drive the page + - * sweep primitives from a durable scheduler instead so each step gets its own - * CPU budget. */ -export const reconcileToolCatalog = ( - input: ReconcileDeps & { readonly pageSize?: number }, -): Effect.Effect => - Effect.gen(function* () { - // One catalog snapshot for the whole in-process run: avoids N+1 `tools.list` - // round-trips across pages + sweep, and gives a consistent `total` and diff - // even if the catalog changes mid-run. (The durable-step path re-lists per - // step by design, since each step is a separate invocation.) - const descriptors = yield* listToolDescriptors(input.executor); - const total = descriptors.length; - let cursor = 0; - let reembedded = 0; - let unchanged = 0; - - for (;;) { - const page = yield* reconcileToolCatalogPage({ ...input, cursor, descriptors }); - reembedded += page.reembedded; - unchanged += page.unchanged; - if (page.nextCursor === null) break; - cursor = page.nextCursor; - } - - const swept = yield* sweepRemoved({ ...input, descriptors }); - - return { namespace: input.namespace, total, reembedded, unchanged, removed: swept.removed }; - }); - -// --------------------------------------------------------------------------- -// Internal helpers -// --------------------------------------------------------------------------- - -/** Load stored fingerprint rows for a specific set of paths (one page), keyed by - * path. Avoids loading the whole fingerprint store per page. */ -const loadFingerprintsForPaths = ( - fingerprints: PluginStorageCollectionFacade, - owner: Owner, - paths: readonly string[], -): Effect.Effect, SemanticSearchError> => - Effect.forEach( - paths, - (path) => - fingerprints - .getForOwner({ owner, key: path }) - .pipe(Effect.map((entry) => [path, entry?.data ?? null] as const)), - { concurrency: 16 }, - ).pipe( - Effect.map( - (pairs) => - new Map(pairs.flatMap(([path, data]) => (data === null ? [] : [[path, data] as const]))), - ), - Effect.mapError( - (cause) => new SemanticSearchError({ message: "Failed to load stored fingerprints.", cause }), - ), - ); - -/** Embed the new/changed tools of a page, upsert their vectors (deleting any old - * chunks first), and persist their fingerprint rows. */ -const embedAndUpsert = ( - deps: { - readonly namespace: string; - readonly embedder: ToolEmbedder; - readonly store: VectorStore; - readonly chunker: Chunker; - readonly fingerprints: PluginStorageCollectionFacade; - readonly owner: Owner; - }, - toEmbed: ReadonlyArray<{ - readonly doc: { - readonly path: string; - readonly name: string; - readonly description: string; - readonly integration: string; - }; - readonly fingerprint: string; - readonly oldChunkIds: readonly string[] | null; - }>, -): Effect.Effect => - Effect.gen(function* () { - const { namespace, embedder, store, chunker, fingerprints, owner } = deps; - - const chunkedGroups = toEmbed.map((item) => ({ - ...item, - chunks: chunker.chunk(namespace, item.doc), - })); - - const allTexts: string[] = []; - for (const group of chunkedGroups) { - for (const chunk of group.chunks) { - allTexts.push(chunk.embeddingText); - } - } - - const allVectors = yield* embedder.embedDocuments(allTexts); - - let vectorOffset = 0; - const records: VectorInput[] = []; - const fingerprintUpdates: Array<{ path: string; row: FingerprintRow }> = []; - - for (const group of chunkedGroups) { - const { doc, fingerprint, oldChunkIds, chunks } = group; - const newChunkIds: string[] = []; - - for (const chunkItem of chunks) { - const vec = allVectors[vectorOffset++]; - if (vec === undefined) { - return yield* new SemanticSearchError({ - message: `reconcileToolCatalogPage: embedding vector missing at offset ${vectorOffset - 1}`, - }); - } - records.push({ - id: chunkItem.id, - values: [...vec], - namespace, - metadata: { - path: doc.path, - name: doc.name, - description: doc.description, - integration: doc.integration, - facet: chunkItem.facet, - chunkIndex: chunkItem.chunkIndex, - }, - }); - newChunkIds.push(chunkItem.id); - } - - fingerprintUpdates.push({ - path: doc.path, - row: { path: doc.path, integration: doc.integration, fingerprint, chunkIds: newChunkIds }, - }); - - // Delete OLD chunk ids for changed tools before upserting new ones. - if (oldChunkIds !== null && oldChunkIds.length > 0) { - yield* store.deleteByIds([...oldChunkIds]); - } - } - - // `store.upsert` chunks internally (≀50/batch) so the Vectorize binding never - // exceeds Cloudflare's per-upsert caps regardless of page size. - yield* store.upsert(records); - - yield* Effect.forEach( - fingerprintUpdates, - ({ path, row }) => - fingerprints.put({ owner, key: path, data: row }).pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ - message: `Failed to persist fingerprint row for tool "${path}".`, - cause, - }), - ), - Effect.asVoid, - ), - { concurrency: 8, discard: true }, - ); - }); diff --git a/packages/plugins/semantic-search/src/sdk/plugin.ts b/packages/plugins/semantic-search/src/sdk/plugin.ts index ef4314ba9a..cdd3b2dc2c 100644 --- a/packages/plugins/semantic-search/src/sdk/plugin.ts +++ b/packages/plugins/semantic-search/src/sdk/plugin.ts @@ -7,18 +7,16 @@ import { import { Effect } from "effect"; import { type Chunker, makeFacetChunker } from "./chunker"; -import { toolFingerprints } from "./collections"; +import { indexChunks, indexJobs, indexRuns, toolFingerprints } from "./collections"; import { makeGeminiEmbedder, type ToolEmbedder } from "./embedder"; import { SemanticSearchError } from "./errors"; import { makeHybridToolDiscoveryProvider } from "./hybrid"; import { - reconcileToolCatalog, - reconcileToolCatalogPage, + make as makeToolSearchIndex, + run as runToolSearchIndex, sweepRemoved, - type ReconcilePageResult, - type ReconcileResult, - type SweepResult, -} from "./indexer"; + type ToolSearchIndex, +} from "./tool-search-index"; import { makeVectorToolDiscoveryProvider } from "./provider"; import type { VectorStore } from "./store"; import { type FtsLexicalStore, makeFtsLexicalProvider } from "./store-fts"; @@ -40,6 +38,13 @@ export interface SemanticSearchPluginOptions { readonly model?: string; /** Embedding dimensionality β€” MUST equal the vector index's dimensions. */ readonly dimensions?: number; + /** Gemini embedding batch size (texts per request; the Google provider allows + * up to 2048 values per call, but Workers memory is the practical ceiling). + * Larger batches + * mean fewer, fatter requests β€” lower peak RPM for the same tokens β€” which is + * the key lever when a reindex fans out across many concurrent workers and is + * request-rate-bound. Defaults to the embedder's own default. */ + readonly embedderBatchSize?: number; /** Inject a custom embedder (tests). Overrides `model`/`geminiApiKey`/`dimensions`. */ readonly embedder?: ToolEmbedder; /** Chunker used when indexing the tool catalog. Defaults to `makeFacetChunker()`. @@ -70,6 +75,7 @@ const notConfigured = (): Effect.Effect => /** Default page size for the operator `search` surface. */ const DEFAULT_SEARCH_LIMIT = 20; +const DEFAULT_IN_PROCESS_PARTITIONS = 1; /** A live `tools.search` result page from the operator search surface. */ export interface SemanticSearchResultPage { @@ -98,110 +104,148 @@ const makeSemanticSearchExtension = (deps: { readonly embedder: ToolEmbedder | undefined; readonly store: VectorStore | undefined; readonly chunker: Chunker; - readonly fingerprints: Parameters[0]["fingerprints"] | undefined; - readonly owner: Parameters[0]["owner"] | undefined; + readonly fingerprints: Parameters[0]["fingerprints"] | undefined; + readonly indexRuns: Parameters[0]["runs"] | undefined; + readonly indexJobs: Parameters[0]["jobs"] | undefined; + readonly indexChunks: Parameters[0]["chunks"] | undefined; + readonly blobs: Parameters[0]["blobs"] | undefined; + readonly owner: Parameters[0]["owner"] | undefined; readonly lexicalStore: FtsLexicalStore | undefined; readonly provider: ToolDiscoveryProvider | undefined; -}) => ({ - reindex: (executor: Executor): Effect.Effect => - deps.embedder && deps.store && deps.fingerprints && deps.owner - ? reconcileToolCatalog({ - namespace: deps.namespace, - executor, - embedder: deps.embedder, - store: deps.store, - chunker: deps.chunker, - fingerprints: deps.fingerprints, - owner: deps.owner, - lexicalStore: deps.lexicalStore, - }) - : notConfigured(), - - /** Reconcile ONE bounded page of the catalog. The unit a durable driver (a - * Cloudflare Workflow step) calls in a loop so each step gets its own CPU - * budget. Pair with `sweep` after the last page. */ - reindexPage: ( - executor: Executor, - input: { readonly cursor: number; readonly pageSize?: number }, - ): Effect.Effect => - deps.embedder && deps.store && deps.fingerprints && deps.owner - ? reconcileToolCatalogPage({ +}) => { + const unconfiguredIndex: ToolSearchIndex.Service = { + create: () => notConfigured(), + scan: () => notConfigured(), + chunk: () => notConfigured(), + embed: () => notConfigured(), + commit: () => notConfigured(), + fail: () => notConfigured(), + reconcile: () => notConfigured(), + status: () => notConfigured(), + complete: () => notConfigured(), + }; + const index = (executor: Executor): ToolSearchIndex.Service => + deps.embedder && + deps.store && + deps.fingerprints && + deps.indexRuns && + deps.indexJobs && + deps.indexChunks && + deps.blobs && + deps.owner + ? makeToolSearchIndex({ namespace: deps.namespace, executor, embedder: deps.embedder, store: deps.store, chunker: deps.chunker, + runs: deps.indexRuns, + jobs: deps.indexJobs, + chunks: deps.indexChunks, fingerprints: deps.fingerprints, + blobs: deps.blobs, owner: deps.owner, lexicalStore: deps.lexicalStore, - cursor: input.cursor, - pageSize: input.pageSize, }) - : notConfigured(), + : unconfiguredIndex; - /** Delete index entries for tools that left the catalog. Run as the terminal - * step after a paged reindex (or standalone). Needs no embedder. */ - sweep: (executor: Executor): Effect.Effect => - deps.store && deps.fingerprints && deps.owner - ? sweepRemoved({ - namespace: deps.namespace, - executor, - store: deps.store, - fingerprints: deps.fingerprints, - owner: deps.owner, - lexicalStore: deps.lexicalStore, - }) - : notConfigured(), + return { + index, + reindex: (executor: Executor): Effect.Effect => + deps.embedder && + deps.store && + deps.fingerprints && + deps.indexRuns && + deps.indexJobs && + deps.indexChunks && + deps.blobs && + deps.owner + ? runToolSearchIndex({ + namespace: deps.namespace, + executor, + embedder: deps.embedder, + store: deps.store, + chunker: deps.chunker, + runs: deps.indexRuns, + jobs: deps.indexJobs, + chunks: deps.indexChunks, + fingerprints: deps.fingerprints, + blobs: deps.blobs, + owner: deps.owner, + lexicalStore: deps.lexicalStore, + runId: `manual-${Date.now()}`, + partitionCount: DEFAULT_IN_PROCESS_PARTITIONS, + }) + : notConfigured(), - /** Run a live `tools.search` through the same provider the engine uses, so the - * operator console sees exactly what the agent would. Inert until configured. */ - search: ( - executor: Executor, - input: { readonly query: string; readonly namespace?: string; readonly limit?: number }, - ): Effect.Effect => { - const namespace = input.namespace ?? deps.namespace; - return deps.provider - ? deps.provider - .searchTools({ + /** Delete index entries for tools that left the catalog. Needs no embedder. */ + sweep: ( + executor: Executor, + ): Effect.Effect< + { readonly namespace: string; readonly removed: number }, + SemanticSearchError + > => + deps.store && deps.fingerprints && deps.owner + ? sweepRemoved({ + namespace: deps.namespace, executor, - query: input.query, - namespace, - limit: input.limit ?? DEFAULT_SEARCH_LIMIT, - offset: 0, + store: deps.store, + fingerprints: deps.fingerprints, + owner: deps.owner, + lexicalStore: deps.lexicalStore, }) - .pipe( - Effect.map((page) => ({ namespace, query: input.query, items: page.items })), - Effect.mapError( - (cause) => - new SemanticSearchError({ message: "Semantic search query failed.", cause }), - ), - ) - : notConfigured(); - }, + : notConfigured(), - /** Index status for the operator console: vector (fingerprint) + lexical counts. */ - status: (): Effect.Effect => - Effect.gen(function* () { - const indexed = deps.fingerprints - ? yield* deps.fingerprints - .count() + /** Run a live `tools.search` through the same provider the engine uses, so the + * operator console sees exactly what the agent would. Inert until configured. */ + search: ( + executor: Executor, + input: { readonly query: string; readonly namespace?: string; readonly limit?: number }, + ): Effect.Effect => { + const namespace = input.namespace ?? deps.namespace; + return deps.provider + ? deps.provider + .searchTools({ + executor, + query: input.query, + namespace, + limit: input.limit ?? DEFAULT_SEARCH_LIMIT, + offset: 0, + }) .pipe( + Effect.map((page) => ({ namespace, query: input.query, items: page.items })), Effect.mapError( (cause) => - new SemanticSearchError({ message: "Failed to count indexed tools.", cause }), + new SemanticSearchError({ message: "Semantic search query failed.", cause }), ), ) - : 0; - // A transient lexical-count failure must not mask the (already-fetched) - // vector count: degrade `lexical` to null and still return `indexed`. - const lexical = deps.lexicalStore - ? yield* deps.lexicalStore - .count(deps.namespace) - .pipe(Effect.catch(() => Effect.succeed(null))) - : null; - return { namespace: deps.namespace, indexed, lexical }; - }), -}); + : notConfigured(); + }, + + /** Index status for the operator console: vector (fingerprint) + lexical counts. */ + status: (): Effect.Effect => + Effect.gen(function* () { + const indexed = deps.fingerprints + ? yield* deps.fingerprints + .count() + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Failed to count indexed tools.", cause }), + ), + ) + : 0; + // A transient lexical-count failure must not mask the (already-fetched) + // vector count: degrade `lexical` to null and still return `indexed`. + const lexical = deps.lexicalStore + ? yield* deps.lexicalStore + .count(deps.namespace) + .pipe(Effect.catch(() => Effect.succeed(null))) + : null; + return { namespace: deps.namespace, indexed, lexical }; + }), + }; +}; /** The `executor.semanticSearch` surface, derived from its factory. */ export type SemanticSearchExtension = ReturnType; @@ -227,6 +271,7 @@ export const semanticSearchPlugin = definePlugin((options?: SemanticSearchPlugin apiKey: options.geminiApiKey, model: options.model, dimensions: options.dimensions, + batchSize: options.embedderBatchSize, }) : undefined); // Inert without both a vector store and an embedder β€” the engine then @@ -257,9 +302,13 @@ export const semanticSearchPlugin = definePlugin((options?: SemanticSearchPlugin return { id: "semanticSearch" as const, packageName: "@executor-js/plugin-semantic-search", - pluginStorage: { toolFingerprints }, + pluginStorage: { toolFingerprints, indexRuns, indexJobs, indexChunks }, storage: (deps) => ({ fingerprints: deps.pluginStorage.collection(toolFingerprints), + indexRuns: deps.pluginStorage.collection(indexRuns), + indexJobs: deps.pluginStorage.collection(indexJobs), + indexChunks: deps.pluginStorage.collection(indexChunks), + indexBlobs: deps.blobs, // The tool catalog is an org-level artifact, so fingerprints are ALWAYS // org-scoped. Scoping by the triggering principal (user vs cron) would // split the fingerprint store into disjoint partitions, so each reindex @@ -273,6 +322,10 @@ export const semanticSearchPlugin = definePlugin((options?: SemanticSearchPlugin store, chunker, fingerprints: ctx.storage.fingerprints, + indexRuns: ctx.storage.indexRuns, + indexJobs: ctx.storage.indexJobs, + indexChunks: ctx.storage.indexChunks, + blobs: ctx.storage.indexBlobs, owner: ctx.storage.owner, lexicalStore, provider, diff --git a/packages/plugins/semantic-search/src/sdk/store-cloudflare.ts b/packages/plugins/semantic-search/src/sdk/store-cloudflare.ts index 9ce11e203f..75691dc65a 100644 --- a/packages/plugins/semantic-search/src/sdk/store-cloudflare.ts +++ b/packages/plugins/semantic-search/src/sdk/store-cloudflare.ts @@ -35,15 +35,6 @@ export const MAX_METADATA_TOP_K = 20; * under both limits and upsert the batches sequentially. */ const UPSERT_BATCH_SIZE = 50; -const chunk = (items: readonly A[], size: number): readonly (readonly A[])[] => { - const safe = Math.max(1, Math.floor(size)); - const out: A[][] = []; - for (let i = 0; i < items.length; i += safe) { - out.push(items.slice(i, i + safe)); - } - return out; -}; - /** Effect-wrapped `VectorStore` backed by a Cloudflare Vectorize binding. * `maxTopK` reflects the `returnMetadata:"all"` cap (20). */ export const makeVectorizeStore = (index: VectorizeIndex): VectorStore => ({ @@ -51,7 +42,7 @@ export const makeVectorizeStore = (index: VectorizeIndex): VectorStore => ({ query: ({ vector, namespace, topK }) => Effect.tryPromise({ try: () => - index.query([...vector], { + index.query(vector, { topK: Math.min(Math.max(1, topK), MAX_TOP_K), namespace, returnMetadata: "all", @@ -61,21 +52,26 @@ export const makeVectorizeStore = (index: VectorizeIndex): VectorStore => ({ upsert: (vectors) => vectors.length === 0 ? Effect.void - : Effect.forEach( - chunk([...vectors], UPSERT_BATCH_SIZE), - (batch) => - Effect.tryPromise({ + : Effect.gen(function* () { + let batchIndex = 0; + for (let start = 0; start < vectors.length; start += UPSERT_BATCH_SIZE) { + const batch = vectors.slice(start, start + UPSERT_BATCH_SIZE); + yield* Effect.tryPromise({ try: () => index.upsert(batch), catch: (cause) => - new SemanticSearchError({ message: "Vectorize upsert failed.", cause }), - }), - { concurrency: 1, discard: true }, - ), + new SemanticSearchError({ + message: `Vectorize upsert failed for batch ${batchIndex} (${batch.length} vectors).`, + cause, + }), + }); + batchIndex++; + } + }), deleteByIds: (ids) => ids.length === 0 ? Effect.void : Effect.tryPromise({ - try: () => index.deleteByIds([...ids]), + try: () => index.deleteByIds(ids), catch: (cause) => new SemanticSearchError({ message: "Vectorize delete failed.", cause }), }).pipe(Effect.asVoid), }); diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts new file mode 100644 index 0000000000..cdb230886f --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -0,0 +1,553 @@ +import { describe, expect, it } from "@effect/vitest"; +import type { + Executor, + Owner, + PluginStorageCollectionFacade, + PluginStorageEntry, +} from "@executor-js/sdk/core"; +import { makeInMemoryBlobStore, pluginBlobStore } from "@executor-js/sdk/core"; +import { Effect } from "effect"; + +import { makeFacetChunker } from "./chunker"; +import { + type FingerprintRow, + type IndexChunk, + type IndexJob, + type IndexRun, + indexChunks, + indexJobs, + indexRuns, + toolFingerprints, +} from "./collections"; +import type { ToolEmbedder } from "./embedder"; +import { chunk, commit, create, embed, fail, reconcile, scan, status } from "./tool-search-index"; +import type { VectorInput, VectorStore } from "./store"; + +const owner: Owner = "org" as Owner; +const namespace = "test-ns"; + +const makeBlobs = () => + pluginBlobStore(makeInMemoryBlobStore(), { org: "test-org", user: null }, "semanticSearch"); + +const makeExecutor = (counters: { raw: number; codegen: number }): Executor => { + const executor: Pick = { + tools: { + list: () => + Effect.succeed([ + { + address: "tools.github.repos.get" as never, + name: "repos.get" as never, + integration: "github" as never, + description: "Get a repository", + owner, + connection: "default" as never, + pluginId: "test", + }, + ]), + schema: (address, options) => { + const includeTypeScript = options?.includeTypeScript ?? true; + if (includeTypeScript) counters.codegen++; + else counters.raw++; + return Effect.succeed({ + address, + inputSchema: { type: "object", properties: { owner: { type: "string" } } }, + inputTypeScript: includeTypeScript ? "{ owner: string }" : undefined, + }); + }, + }, + }; + return executor as Executor; +}; + +type TestCollection = PluginStorageCollectionFacade & { + readonly data: Map; +}; + +const makeCollection = (collection: string): TestCollection => { + const data = new Map(); + let id = 0; + const entry = (key: string, value: T): PluginStorageEntry => ({ + id: String(id++), + owner, + pluginId: "semanticSearch", + collection, + key, + data: value, + createdAt: new Date(0), + updatedAt: new Date(0), + }); + const matches = (value: T, where: Record | undefined): boolean => { + if (!where) return true; + return Object.entries(where).every( + ([key, expected]) => (value as Record)[key] === expected, + ); + }; + const facade = { + data, + get: ({ key }: { key: string }) => + Effect.succeed(data.has(key) ? entry(key, data.get(key)!) : null), + getForOwner: ({ key }: { key: string }) => + Effect.succeed(data.has(key) ? entry(key, data.get(key)!) : null), + list: () => Effect.succeed([...data.entries()].map(([key, value]) => entry(key, value))), + put: ({ key, data: value }: { key: string; data: T }) => + Effect.sync(() => { + data.set(key, value); + return entry(key, value); + }), + query: (input?: { + where?: Record; + orderBy?: readonly { readonly field: string; readonly direction?: "asc" | "desc" }[]; + limit?: number; + }) => + Effect.succeed( + [...data.entries()] + .filter(([, value]) => matches(value, input?.where)) + .sort(([leftKey, left], [rightKey, right]) => { + for (const order of input?.orderBy ?? []) { + const direction = order.direction === "desc" ? -1 : 1; + const a = (left as Record)[order.field]; + const b = (right as Record)[order.field]; + if (typeof a === "number" && typeof b === "number" && a !== b) + return (a - b) * direction; + if (String(a) !== String(b)) return String(a).localeCompare(String(b)) * direction; + } + return leftKey.localeCompare(rightKey); + }) + .slice(0, input?.limit) + .map(([key, value]) => entry(key, value)), + ), + count: (input?: { where?: Record }) => + Effect.succeed([...data.values()].filter((value) => matches(value, input?.where)).length), + queryKeyset: () => Effect.succeed({ entries: [], nextCursor: null }), + aggregate: { + count: () => Effect.succeed(data.size), + groupCount: () => Effect.succeed([]), + timeBuckets: () => Effect.succeed([]), + stats: () => Effect.succeed({ count: 0, min: null, max: null, percentiles: [] }), + }, + remove: ({ key }: { key: string }) => Effect.sync(() => data.delete(key)), + }; + // oxlint-disable-next-line executor/no-double-cast -- test fixture implements the storage facade methods exercised by these indexer tests + return facade as unknown as TestCollection; +}; + +describe("ToolSearchIndex", () => { + it.effect("scans, chunks, embeds, and commits changed tools", () => + Effect.gen(function* () { + const counters = { raw: 0, codegen: 0 }; + const executor = makeExecutor(counters); + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + const upserted: VectorInput[] = []; + const store: VectorStore = { + maxTopK: 100, + query: () => Effect.succeed([]), + upsert: (vectors) => Effect.sync(() => void upserted.push(...vectors)), + deleteByIds: () => Effect.void, + }; + const embedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => Effect.succeed(texts.map(() => [0.1, 0.2, 0.3])), + embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), + }; + + const base = { namespace, executor, runs, jobs, chunks, fingerprints, blobs, owner }; + yield* create({ ...base, runId: "run-1", partitionCount: 1 }); + const scanned = yield* scan({ + ...base, + runId: "run-1", + partition: 0, + limit: 10, + }); + const chunked = yield* chunk({ + ...base, + embedder, + store, + chunker: makeFacetChunker(), + runId: "run-1", + paths: scanned.paths, + limit: 10, + }); + const embedded = yield* embed({ + ...base, + embedder, + store, + chunker: makeFacetChunker(), + runId: "run-1", + chunkRefs: chunked.chunkRefs, + }); + for (const path of embedded.paths) { + yield* commit({ + ...base, + embedder, + store, + chunker: makeFacetChunker(), + runId: "run-1", + path, + }); + } + + expect(scanned).toMatchObject({ processed: 1, changed: 1, skipped: 0 }); + expect(chunked.processed).toBe(1); + expect(chunked.chunks).toBeGreaterThan(0); + expect(embedded).toMatchObject({ processed: chunked.chunks, chunks: chunked.chunks }); + expect(counters).toEqual({ raw: 1, codegen: 1 }); + expect(upserted).toHaveLength(chunked.chunks); + expect([...fingerprints.data.values()]).toHaveLength(1); + expect([...jobs.data.values()][0]?.status).toBe("indexed"); + }), + ); + + it.effect("marks pending path work failed and reports progress timestamps", () => + Effect.gen(function* () { + const counters = { raw: 0, codegen: 0 }; + const executor = makeExecutor(counters); + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + const store: VectorStore = { + maxTopK: 100, + query: () => Effect.succeed([]), + upsert: () => Effect.void, + deleteByIds: () => Effect.void, + }; + const embedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => Effect.succeed(texts.map(() => [0.1, 0.2, 0.3])), + embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), + }; + + const base = { namespace, executor, runs, jobs, chunks, fingerprints, blobs, owner }; + yield* create({ ...base, runId: "run-fail", partitionCount: 1 }); + const scanned = yield* scan({ + ...base, + runId: "run-fail", + partition: 0, + limit: 10, + }); + yield* chunk({ + ...base, + embedder, + store, + chunker: makeFacetChunker(), + runId: "run-fail", + paths: scanned.paths, + limit: 10, + }); + + const result = yield* fail({ + ...base, + runId: "run-fail", + paths: scanned.paths, + error: "queue exhausted", + }); + const current = yield* status({ ...base, runId: "run-fail" }); + + expect(result.jobs).toBe(1); + expect(result.chunks).toBeGreaterThan(0); + expect(result.runFailed).toBe(false); + expect([...jobs.data.values()][0]?.status).toBe("failed"); + expect([...chunks.data.values()].every((chunk) => chunk.status === "failed")).toBe(true); + expect(current.failed).toBe(1); + expect(current.lastProgressAt).toBeDefined(); + }), + ); + + it.effect("reconciles pending scan partitions, chunk paths, embedding chunks, and commits", () => + Effect.gen(function* () { + const counters = { raw: 0, codegen: 0 }; + const executor = makeExecutor(counters); + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + const base = { namespace, executor, runs, jobs, chunks, fingerprints, blobs, owner }; + + yield* create({ ...base, runId: "run-reconcile", partitionCount: 1 }); + const beforeScan = yield* reconcile({ ...base, runId: "run-reconcile" }); + const scanned = yield* scan({ + ...base, + runId: "run-reconcile", + partition: 0, + limit: 10, + }); + const afterScan = yield* reconcile({ ...base, runId: "run-reconcile" }); + + expect(beforeScan.scanPartitions).toEqual([0]); + expect(afterScan.pendingChunkPaths).toEqual(scanned.paths); + expect(afterScan.scanPartitions).toEqual([]); + }), + ); + + it.effect("limits seeded tools when maxTools is provided", () => + Effect.gen(function* () { + const executor: Pick = { + tools: { + list: () => + Effect.succeed([ + { + address: "tools.github.repos.get" as never, + name: "repos.get" as never, + integration: "github" as never, + description: "Get a repository", + owner, + connection: "default" as never, + pluginId: "test", + }, + { + address: "tools.stripe.customers.list" as never, + name: "customers.list" as never, + integration: "stripe" as never, + description: "List customers", + owner, + connection: "default" as never, + pluginId: "test", + }, + { + address: "tools.slack.chat.postMessage" as never, + name: "chat.postMessage" as never, + integration: "slack" as never, + description: "Post a message", + owner, + connection: "default" as never, + pluginId: "test", + }, + ]), + schema: () => Effect.succeed(null), + }, + }; + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + const base = { + namespace, + executor: executor as Executor, + runs, + jobs, + chunks, + fingerprints, + blobs, + owner, + }; + + const started = yield* create({ + ...base, + runId: "run-limited", + partitionCount: 1, + maxTools: 2, + }); + const scanned = yield* scan({ + ...base, + runId: "run-limited", + partition: 0, + limit: 10, + maxTools: 2, + }); + + expect(started.total).toBe(2); + expect(scanned.processed).toBe(2); + expect(jobs.data.size).toBe(2); + }), + ); + + it.effect("embeds only the chunks that fit the page budget and commits the job later", () => + Effect.gen(function* () { + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + const mutableEmbeddedGroups: string[][] = []; + const upserted: VectorInput[] = []; + const store: VectorStore = { + maxTopK: 100, + query: () => Effect.succeed([]), + upsert: (vectors) => Effect.sync(() => void upserted.push(...vectors)), + deleteByIds: () => Effect.void, + }; + const embedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => + Effect.sync(() => { + mutableEmbeddedGroups.push([...texts]); + return texts.map(() => [0.1, 0.2, 0.3]); + }), + embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), + }; + + const createdAt = new Date(0).toISOString(); + const job: IndexJob = { + runId: "run-budget", + namespace, + partition: 0, + ordinal: 0, + address: "tools.github.repos.get", + path: "github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + status: "pendingEmbedding", + fingerprint: "fp-budget", + oldChunkIds: [], + chunkIds: ["chunk-0", "chunk-1", "chunk-2"], + lexicalTextKey: "test/lexical.txt", + createdAt, + updatedAt: createdAt, + }; + yield* blobs.put("test/lexical.txt", "github github.repos.get repos.get", { owner }); + yield* jobs.put({ owner, key: `${job.runId}:${job.path}`, data: job }); + yield* Effect.forEach( + job.chunkIds, + (chunkId, chunkIndex) => + Effect.gen(function* () { + const embeddingText = `chunk text ${chunkIndex}`; + const chunk: IndexChunk = { + runId: job.runId, + namespace, + partition: 0, + path: job.path, + chunkId, + facet: "description", + chunkIndex, + embeddingTextKey: `test/${chunkId}.txt`, + embeddingTextBytes: embeddingText.length, + embeddingTextTokens: 3, + name: job.name, + integration: job.integration, + description: job.description, + status: "pendingEmbedding", + createdAt, + updatedAt: createdAt, + }; + yield* blobs.put(chunk.embeddingTextKey, embeddingText, { owner }); + yield* chunks.put({ + owner, + key: `${chunk.runId}:${chunk.path}:${chunk.chunkId}`, + data: chunk, + }); + }), + { discard: true }, + ); + + const base = { + namespace, + executor: makeExecutor({ raw: 0, codegen: 0 }), + runs, + jobs, + chunks, + fingerprints, + blobs, + owner, + embedder, + store, + chunker: makeFacetChunker(), + }; + + const first = yield* embed({ + ...base, + runId: job.runId, + chunkRefs: job.chunkIds.map((chunkId) => ({ path: job.path, chunkId })), + maxChunks: 2, + }); + const afterFirstJob = jobs.data.get(`${job.runId}:${job.path}`); + const firstChunkStatuses = [...chunks.data.values()].map((chunk) => chunk.status); + + const second = yield* embed({ + ...base, + runId: job.runId, + chunkRefs: job.chunkIds.map((chunkId) => ({ path: job.path, chunkId })), + maxChunks: 10, + }); + const committed = yield* commit({ ...base, runId: job.runId, path: job.path }); + const afterCommitJob = jobs.data.get(`${job.runId}:${job.path}`); + + expect(first).toMatchObject({ processed: 2, chunks: 2 }); + expect(afterFirstJob?.status).toBe("pendingEmbedding"); + expect(firstChunkStatuses.filter((status) => status === "indexed")).toHaveLength(2); + expect(firstChunkStatuses.filter((status) => status === "pendingEmbedding")).toHaveLength(1); + expect(second).toMatchObject({ processed: 1, chunks: 1 }); + expect(committed.committed).toBe(true); + expect(afterCommitJob?.status).toBe("indexed"); + expect(mutableEmbeddedGroups.map((group) => group.length)).toEqual([2, 1]); + expect(upserted).toHaveLength(3); + expect([...fingerprints.data.values()]).toHaveLength(1); + }), + ); + + it.effect("commits zero-chunk embed jobs so fingerprints can warm future runs", () => + Effect.gen(function* () { + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + const store: VectorStore = { + maxTopK: 100, + query: () => Effect.succeed([]), + upsert: () => Effect.die("zero-chunk jobs must not upsert vectors"), + deleteByIds: () => Effect.void, + }; + const embedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: () => Effect.die("zero-chunk jobs must not embed"), + embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), + }; + + const createdAt = new Date(0).toISOString(); + const job: IndexJob = { + runId: "run-zero", + namespace, + partition: 0, + ordinal: 0, + address: "tools.github.repos.empty", + path: "github.repos.empty", + name: "repos.empty", + integration: "github", + description: "Empty tool", + status: "pendingEmbedding", + fingerprint: "fp-zero", + oldChunkIds: [], + chunkIds: [], + createdAt, + updatedAt: createdAt, + }; + yield* jobs.put({ owner, key: `${job.runId}:${job.path}`, data: job }); + + const result = yield* commit({ + namespace, + executor: makeExecutor({ raw: 0, codegen: 0 }), + runs, + jobs, + chunks, + fingerprints, + blobs, + owner, + embedder, + store, + chunker: makeFacetChunker(), + runId: job.runId, + path: job.path, + }); + + expect(result).toMatchObject({ committed: true }); + expect(jobs.data.get(`${job.runId}:${job.path}`)?.status).toBe("indexed"); + expect(fingerprints.data.get(job.path)).toMatchObject({ + path: job.path, + fingerprint: "fp-zero", + chunkIds: [], + }); + }), + ); +}); diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts new file mode 100644 index 0000000000..47776a21ae --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts @@ -0,0 +1,1477 @@ +import type { + Executor, + Owner, + PluginBlobStore, + PluginStorageCollectionFacade, +} from "@executor-js/sdk/core"; +import { sha256Hex } from "@executor-js/sdk/core"; +import { Context, Effect, Predicate } from "effect"; + +import type { Chunker, ToolChunk } from "./chunker"; +import { + type FingerprintRow, + type IndexChunk, + type IndexJob, + type IndexRun, + indexChunks, + indexJobs, + indexRuns, + toolFingerprints, +} from "./collections"; +import { + addressToPath, + buildLexicalText, + collectDocForTool, + collectFingerprintInputs, + type IndexableToolDescriptor, + listToolDescriptors, +} from "./documents"; +import type { ToolEmbedder } from "./embedder"; +import { SemanticSearchError } from "./errors"; +import { cyrb53, fingerprintTool } from "./fingerprint"; +import type { VectorInput, VectorStore } from "./store"; +import type { FtsDocumentInput, FtsLexicalStore } from "./store-fts"; + +export interface IndexCollections { + readonly runs: PluginStorageCollectionFacade; + readonly jobs: PluginStorageCollectionFacade; + readonly chunks: PluginStorageCollectionFacade; + readonly fingerprints: PluginStorageCollectionFacade; + readonly blobs: PluginBlobStore; + readonly owner: Owner; +} + +interface IndexStores extends IndexCollections { + readonly namespace: string; + readonly executor: Executor; + readonly lexicalStore?: FtsLexicalStore; +} + +interface IndexDeps extends IndexStores { + readonly embedder: ToolEmbedder; + readonly store: VectorStore; + readonly chunker: Chunker; +} + +export declare namespace ToolSearchIndex { + export interface CreateInput { + readonly runId: string; + readonly partitionCount: number; + readonly maxTools?: number; + } + + export interface CreateResult { + readonly runId: string; + readonly namespace: string; + readonly total: number; + readonly partitionCount: number; + } + + export interface ScanInput { + readonly runId: string; + readonly partition: number; + readonly limit?: number; + readonly maxTools?: number; + } + + export interface ScanResult { + readonly runId: string; + readonly partition: number; + readonly processed: number; + readonly changed: number; + readonly skipped: number; + readonly paths: readonly string[]; + readonly hasMore: boolean; + } + + export interface ChunkRef { + readonly path: string; + readonly chunkId: string; + } + + export interface ChunkInput { + readonly runId: string; + readonly paths: readonly string[]; + readonly limit?: number; + readonly concurrency?: number; + } + + export interface ChunkResult { + readonly runId: string; + readonly processed: number; + readonly chunks: number; + readonly paths: readonly string[]; + readonly chunkRefs: readonly ChunkRef[]; + readonly commitPaths: readonly string[]; + } + + export interface EmbedInput { + readonly runId: string; + readonly chunkRefs: readonly ChunkRef[]; + readonly maxChunks?: number; + readonly maxEstimatedInputTokens?: number; + readonly maxEstimatedResponseBytes?: number; + readonly maxEstimatedTokensPerText?: number; + } + + export interface EmbedResult { + readonly runId: string; + readonly processed: number; + readonly chunks: number; + readonly paths: readonly string[]; + readonly chunkRefs: readonly ChunkRef[]; + } + + export interface CommitInput { + readonly runId: string; + readonly path: string; + } + + export interface CommitResult { + readonly runId: string; + readonly path: string; + readonly committed: boolean; + } + + export interface CompleteInput { + readonly runId: string; + } + + export interface CompleteResult { + readonly runId: string; + readonly removed: number; + } + + export interface FailInput { + readonly runId: string; + readonly partition?: number; + readonly paths?: readonly string[]; + readonly chunkRefs?: readonly ChunkRef[]; + readonly error: string; + } + + export interface FailResult { + readonly runId: string; + readonly jobs: number; + readonly chunks: number; + readonly runFailed: boolean; + } + + export interface ReconcileInput { + readonly runId: string; + } + + export interface ReconcileResult { + readonly runId: string; + readonly scanPartitions: readonly number[]; + readonly pendingChunkPaths: readonly string[]; + readonly pendingEmbeddingChunks: readonly ChunkRef[]; + readonly pendingCommitPaths: readonly string[]; + } + + export interface StatusInput { + readonly runId: string; + } + + export interface Status { + readonly runId: string; + readonly namespace: string; + readonly status: IndexRun["status"]; + readonly total: number; + readonly pendingScan: number; + readonly skipped: number; + readonly pendingChunk: number; + readonly pendingEmbedding: number; + readonly indexed: number; + readonly failed: number; + readonly updatedAt?: string; + readonly lastProgressAt?: string; + } + + export interface Result { + readonly namespace: string; + readonly total: number; + readonly indexed: number; + readonly skipped: number; + readonly removed: number; + } + + export interface Service { + readonly create: (input: CreateInput) => Effect.Effect; + readonly scan: (input: ScanInput) => Effect.Effect; + readonly chunk: (input: ChunkInput) => Effect.Effect; + readonly embed: (input: EmbedInput) => Effect.Effect; + readonly commit: (input: CommitInput) => Effect.Effect; + readonly fail: (input: FailInput) => Effect.Effect; + readonly reconcile: ( + input: ReconcileInput, + ) => Effect.Effect; + readonly status: (input: StatusInput) => Effect.Effect; + readonly complete: (input: CompleteInput) => Effect.Effect; + } +} + +export class ToolSearchIndex extends Context.Service()( + "@executor-js/semantic-search/ToolSearchIndex", +) {} + +const DEFAULT_PAGE_LIMIT = 25; +const DEFAULT_CHUNK_CONCURRENCY = 1; +const DEFAULT_EMBED_MAX_CHUNKS = 128; +const DEFAULT_EMBED_MAX_ESTIMATED_INPUT_TOKENS = 64_000; +const DEFAULT_EMBED_MAX_ESTIMATED_RESPONSE_BYTES = 8 * 1024 * 1024; +const DEFAULT_EMBED_MAX_ESTIMATED_TOKENS_PER_TEXT = 2_048; +const DEFAULT_EMBED_COMMIT_GROUP_SIZE = 32; +const ESTIMATED_CHARS_PER_TOKEN = 4; +const ESTIMATED_RESPONSE_BYTES_PER_DIMENSION = 16; +const ESTIMATED_RESPONSE_BYTES_PER_VECTOR_OVERHEAD = 512; +const INDEX_STORAGE_CONCURRENCY = 2; + +const nowIso = (): string => new Date().toISOString(); + +const partitionForPath = (path: string, partitionCount: number): number => + Math.abs(cyrb53(path)) % Math.max(1, Math.floor(partitionCount)); + +const jobKey = (runId: string, path: string): string => `${runId}:${path}`; + +const chunkKey = (runId: string, path: string, chunkId: string): string => + `${runId}:${path}:${chunkId}`; + +const jobToDescriptor = (job: IndexJob): IndexableToolDescriptor => ({ + address: job.address, + name: job.name, + integration: job.integration, + description: job.description, +}); + +const queryChunksForJob = ( + deps: IndexCollections, + job: IndexJob, +): Effect.Effect< + readonly { readonly key: string; readonly data: IndexChunk }[], + SemanticSearchError +> => + deps.chunks + .query({ + where: { runId: job.runId, path: job.path }, + orderBy: [{ field: "path", direction: "asc" }], + }) + .pipe( + Effect.map((entries) => [...entries].sort((a, b) => a.data.chunkIndex - b.data.chunkIndex)), + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to query index chunks.", cause }), + ), + ); + +const getChunksByRefs = ( + deps: IndexCollections, + input: { + readonly runId: string; + readonly chunkRefs: readonly ToolSearchIndex.ChunkRef[]; + }, +): Effect.Effect< + readonly { readonly key: string; readonly data: IndexChunk }[], + SemanticSearchError +> => + Effect.forEach( + input.chunkRefs, + (ref) => + deps.chunks + .getForOwner({ owner: deps.owner, key: chunkKey(input.runId, ref.path, ref.chunkId) }) + .pipe(Effect.map((entry) => (entry?.data.status === "pendingEmbedding" ? entry : null))), + { concurrency: INDEX_STORAGE_CONCURRENCY }, + ).pipe( + Effect.map((entries) => entries.flatMap((entry) => (entry === null ? [] : [entry]))), + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to load index chunks.", cause }), + ), + ); + +interface EmbedBudget { + readonly maxChunks: number; + readonly maxEstimatedInputTokens: number; + readonly maxEstimatedResponseBytes: number; + readonly maxEstimatedTokensPerText: number; +} + +const resolveEmbedBudget = (input: ToolSearchIndex.EmbedInput): EmbedBudget => ({ + maxChunks: Math.max(1, Math.floor(input.maxChunks ?? DEFAULT_EMBED_MAX_CHUNKS)), + maxEstimatedInputTokens: Math.max( + 1, + Math.floor(input.maxEstimatedInputTokens ?? DEFAULT_EMBED_MAX_ESTIMATED_INPUT_TOKENS), + ), + maxEstimatedResponseBytes: Math.max( + 1, + Math.floor(input.maxEstimatedResponseBytes ?? DEFAULT_EMBED_MAX_ESTIMATED_RESPONSE_BYTES), + ), + maxEstimatedTokensPerText: Math.max( + 1, + Math.floor(input.maxEstimatedTokensPerText ?? DEFAULT_EMBED_MAX_ESTIMATED_TOKENS_PER_TEXT), + ), +}); + +const estimateTokens = (text: string): number => + Math.max(1, Math.ceil(text.length / ESTIMATED_CHARS_PER_TOKEN)); + +const utf8Bytes = (text: string): number => new TextEncoder().encode(text).byteLength; + +const payloadKey = (kind: "embedding-text" | "lexical-text", digest: string): string => + `semantic-search/index/${kind}/${digest}.txt`; + +const putPayloadText = ( + deps: IndexCollections, + kind: "embedding-text" | "lexical-text", + text: string, +): Effect.Effect => + Effect.gen(function* () { + const digest = yield* sha256Hex(text); + const key = payloadKey(kind, digest); + yield* deps.blobs.put(key, text, { owner: deps.owner }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to persist index ${kind} payload "${key}".`, + cause, + }), + ), + ); + return key; + }); + +const getPayloadText = ( + deps: IndexCollections, + key: string, +): Effect.Effect => + deps.blobs.get(key).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: `Failed to load index payload "${key}".`, cause }), + ), + Effect.flatMap((text) => + text === null + ? Effect.fail(new SemanticSearchError({ message: `Index payload "${key}" is missing.` })) + : Effect.succeed(text), + ), + ); + +const estimateEmbeddingResponseBytes = (dimensions: number): number => + Math.max( + 1, + Math.floor( + dimensions * ESTIMATED_RESPONSE_BYTES_PER_DIMENSION + + ESTIMATED_RESPONSE_BYTES_PER_VECTOR_OVERHEAD, + ), + ); + +const wouldExceedEmbedBudget = ( + current: { + readonly chunks: number; + readonly inputTokens: number; + readonly responseBytes: number; + }, + next: { readonly inputTokens: number; readonly responseBytes: number }, + budget: EmbedBudget, +): boolean => + current.chunks + 1 > budget.maxChunks || + current.inputTokens + next.inputTokens > budget.maxEstimatedInputTokens || + current.responseBytes + next.responseBytes > budget.maxEstimatedResponseBytes; + +const loadFingerprints = ( + deps: IndexCollections, + paths: readonly string[], +): Effect.Effect, SemanticSearchError> => + Effect.forEach( + paths, + (path) => + deps.fingerprints + .getForOwner({ owner: deps.owner, key: path }) + .pipe(Effect.map((entry) => [path, entry?.data ?? null] as const)), + { concurrency: INDEX_STORAGE_CONCURRENCY }, + ).pipe( + Effect.map( + (pairs) => + new Map(pairs.flatMap(([path, data]) => (data === null ? [] : [[path, data] as const]))), + ), + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to load index fingerprint rows.", + cause, + }), + ), + ); + +const putJob = (deps: IndexCollections, job: IndexJob): Effect.Effect => + deps.jobs.put({ owner: deps.owner, key: jobKey(job.runId, job.path), data: job }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to persist index job "${job.path}".`, + cause, + }), + ), + Effect.asVoid, + ); + +const removeChunksForJob = ( + deps: IndexCollections, + job: IndexJob, +): Effect.Effect => + queryChunksForJob(deps, job).pipe( + Effect.flatMap((entries) => + Effect.forEach( + entries, + (entry) => + deps.chunks.remove({ owner: deps.owner, key: entry.key }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to remove index chunk "${entry.key}".`, + cause, + }), + ), + ), + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ), + ), + ); + +const getJobsByPaths = ( + deps: IndexCollections, + input: { + readonly runId: string; + readonly paths: readonly string[]; + readonly status: IndexJob["status"]; + readonly limit?: number; + }, +): Effect.Effect< + readonly { readonly key: string; readonly data: IndexJob }[], + SemanticSearchError +> => + Effect.forEach( + input.paths, + (path) => + deps.jobs + .getForOwner({ owner: deps.owner, key: jobKey(input.runId, path) }) + .pipe(Effect.map((entry) => (entry?.data.status === input.status ? entry : null))), + { concurrency: INDEX_STORAGE_CONCURRENCY }, + ).pipe( + Effect.map((entries) => + entries + .flatMap((entry) => (entry === null ? [] : [entry])) + .slice(0, input.limit ?? input.paths.length), + ), + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to load index jobs.", cause }), + ), + ); + +const queryJobsByStatus = ( + deps: IndexCollections, + input: { + readonly runId: string; + readonly status: IndexJob["status"]; + }, +): Effect.Effect< + readonly { readonly key: string; readonly data: IndexJob }[], + SemanticSearchError +> => + deps.jobs + .query({ + where: { runId: input.runId, status: input.status }, + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to load ${input.status} index jobs.`, + cause, + }), + ), + ); + +const loadLatestJobUpdatedAt = ( + collection: PluginStorageCollectionFacade, + runId: string, +): Effect.Effect => + collection + .query({ + where: { runId }, + }) + .pipe( + Effect.map((entries) => + entries + .map((entry) => entry.data.updatedAt) + .sort() + .at(-1), + ), + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to load latest index job progress timestamp.", + cause, + }), + ), + ); + +const loadLatestChunkUpdatedAt = ( + collection: PluginStorageCollectionFacade, + runId: string, +): Effect.Effect => + collection + .query({ + where: { runId }, + }) + .pipe( + Effect.map((entries) => + entries + .map((entry) => entry.data.updatedAt) + .sort() + .at(-1), + ), + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to load latest index chunk progress timestamp.", + cause, + }), + ), + ); + +export const fail = ( + input: IndexCollections & ToolSearchIndex.FailInput, +): Effect.Effect => + Effect.gen(function* () { + const updatedAt = nowIso(); + const paths = new Set(input.paths ?? []); + const chunkRefs = input.chunkRefs ?? []; + let jobs = 0; + let chunks = 0; + + if (paths.size > 0) { + yield* Effect.forEach( + [...paths], + (path) => + Effect.gen(function* () { + const entry = yield* input.jobs + .getForOwner({ owner: input.owner, key: jobKey(input.runId, path) }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to load index job "${path}" for failure marking.`, + cause, + }), + ), + ); + if ( + entry !== null && + (entry.data.status === "pendingChunk" || entry.data.status === "pendingEmbedding") + ) { + yield* putJob(input, { + ...entry.data, + status: "failed", + error: input.error, + updatedAt, + }); + jobs++; + } + + const chunkEntries = yield* input.chunks + .query({ + where: { runId: input.runId, path, status: "pendingEmbedding" }, + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to load pending chunks for "${path}" failure marking.`, + cause, + }), + ), + ); + yield* Effect.forEach( + chunkEntries, + (chunk) => + input.chunks + .put({ + owner: input.owner, + key: chunk.key, + data: { ...chunk.data, status: "failed", error: input.error, updatedAt }, + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to mark chunk "${chunk.data.chunkId}" failed.`, + cause, + }), + ), + ), + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ); + chunks += chunkEntries.length; + }), + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ); + } + + if (chunkRefs.length > 0) { + yield* Effect.forEach( + chunkRefs, + (ref) => + Effect.gen(function* () { + const entry = yield* input.chunks + .getForOwner({ + owner: input.owner, + key: chunkKey(input.runId, ref.path, ref.chunkId), + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to load chunk "${ref.chunkId}" for failure marking.`, + cause, + }), + ), + ); + if (entry !== null && entry.data.status === "pendingEmbedding") { + yield* input.chunks + .put({ + owner: input.owner, + key: entry.key, + data: { ...entry.data, status: "failed", error: input.error, updatedAt }, + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to mark chunk "${ref.chunkId}" failed.`, + cause, + }), + ), + ); + chunks++; + } + }), + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ); + } + + const run = yield* input.runs.getForOwner({ owner: input.owner, key: input.runId }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to load index run for failure marking.", + cause, + }), + ), + ); + const runFailed = paths.size === 0 && chunkRefs.length === 0; + if (runFailed && run !== null) { + yield* input.runs + .put({ + owner: input.owner, + key: input.runId, + data: { + ...run.data, + status: "failed", + error: input.error, + updatedAt, + }, + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Failed to mark index run failed.", cause }), + ), + ); + } + + return { + runId: input.runId, + jobs, + chunks, + runFailed, + }; + }); + +export const reconcile = ( + input: IndexCollections & ToolSearchIndex.ReconcileInput, +): Effect.Effect => + Effect.gen(function* () { + const run = yield* input.runs.getForOwner({ owner: input.owner, key: input.runId }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to load index run for reconciliation.", + cause, + }), + ), + ); + if (run === null) { + return yield* new SemanticSearchError({ + message: `Index run "${input.runId}" does not exist.`, + }); + } + + const [pendingChunkJobs, pendingEmbeddingJobs, pendingEmbeddingChunks] = yield* Effect.all( + [ + queryJobsByStatus(input, { runId: input.runId, status: "pendingChunk" }), + queryJobsByStatus(input, { runId: input.runId, status: "pendingEmbedding" }), + input.chunks + .query({ + where: { runId: input.runId, status: "pendingEmbedding" }, + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to load pending embedding chunks.", + cause, + }), + ), + ), + ], + { concurrency: INDEX_STORAGE_CONCURRENCY }, + ); + const observed = yield* input.jobs.count({ where: { runId: input.runId } }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to count index jobs for reconciliation.", + cause, + }), + ), + ); + const scanPartitions = + observed < run.data.total + ? Array.from({ length: run.data.partitionCount }, (_, partition) => partition) + : []; + const pendingEmbeddingPaths = new Set(pendingEmbeddingChunks.map((entry) => entry.data.path)); + const pendingCommitPaths = pendingEmbeddingJobs + .map((entry) => entry.data.path) + .filter((path) => !pendingEmbeddingPaths.has(path)); + + return { + runId: input.runId, + scanPartitions, + pendingChunkPaths: [...new Set(pendingChunkJobs.map((entry) => entry.data.path))], + pendingEmbeddingChunks: pendingEmbeddingChunks.map((entry) => ({ + path: entry.data.path, + chunkId: entry.data.chunkId, + })), + pendingCommitPaths: [...new Set(pendingCommitPaths)], + }; + }); + +export const create = ( + input: IndexStores & ToolSearchIndex.CreateInput, +): Effect.Effect => + Effect.gen(function* () { + const partitionCount = Math.max(1, Math.floor(input.partitionCount)); + const descriptors = yield* listToolDescriptors(input.executor, { maxTools: input.maxTools }); + const createdAt = nowIso(); + + yield* input.runs + .put({ + owner: input.owner, + key: input.runId, + data: { + runId: input.runId, + namespace: input.namespace, + status: "running", + partitionCount, + total: descriptors.length, + createdAt, + updatedAt: createdAt, + }, + }) + .pipe( + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to create index run.", cause }), + ), + ); + + return { + runId: input.runId, + namespace: input.namespace, + total: descriptors.length, + partitionCount, + }; + }); + +export const scan = ( + input: IndexStores & ToolSearchIndex.ScanInput, +): Effect.Effect => + Effect.gen(function* () { + const run = yield* input.runs + .getForOwner({ owner: input.owner, key: input.runId }) + .pipe( + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to load index run.", cause }), + ), + ); + if (run === null) { + return yield* new SemanticSearchError({ + message: `Index run "${input.runId}" does not exist.`, + }); + } + + const descriptors = yield* listToolDescriptors(input.executor, { maxTools: input.maxTools }); + const partitionDescriptors = descriptors + .map((tool, ordinal) => ({ tool, ordinal, path: addressToPath(String(tool.address)) })) + .filter(({ path }) => partitionForPath(path, run.data.partitionCount) === input.partition); + const existing = yield* input.jobs + .count({ where: { runId: input.runId, partition: input.partition } }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Failed to count scanned index jobs.", cause }), + ), + ); + const selected = partitionDescriptors.slice( + existing, + existing + (input.limit ?? DEFAULT_PAGE_LIMIT), + ); + if (selected.length === 0) { + return { + runId: input.runId, + partition: input.partition, + processed: 0, + changed: 0, + skipped: 0, + paths: [], + hasMore: false, + }; + } + + const fingerprintInputs = yield* collectFingerprintInputs( + input.executor, + selected.map(({ tool }) => tool), + ); + const stored = yield* loadFingerprints( + input, + fingerprintInputs.map(({ input: fp }) => fp.path), + ); + + let changed = 0; + let skipped = 0; + const changedPaths: string[] = []; + const selectedByPath = new Map(selected.map((entry) => [entry.path, entry])); + const updatedAt = nowIso(); + + yield* Effect.forEach( + fingerprintInputs, + ({ input: fp }) => { + const selectedEntry = selectedByPath.get(fp.path); + if (selectedEntry === undefined) return Effect.void; + const fingerprint = fingerprintTool(fp); + const storedRow = stored.get(fp.path); + const { tool, ordinal, path } = selectedEntry; + const next: IndexJob = + storedRow !== undefined && storedRow.fingerprint === fingerprint + ? { + runId: input.runId, + namespace: input.namespace, + partition: input.partition, + ordinal, + address: String(tool.address), + path, + name: String(tool.name), + integration: String(tool.integration), + description: String(tool.description ?? ""), + status: "skipped", + fingerprint, + oldChunkIds: storedRow.chunkIds, + chunkIds: storedRow.chunkIds, + updatedAt, + createdAt: updatedAt, + } + : { + runId: input.runId, + namespace: input.namespace, + partition: input.partition, + ordinal, + address: String(tool.address), + path, + name: String(tool.name), + integration: String(tool.integration), + description: String(tool.description ?? ""), + status: "pendingChunk", + fingerprint, + oldChunkIds: storedRow?.chunkIds ?? [], + chunkIds: [], + updatedAt, + createdAt: updatedAt, + }; + if (next.status === "skipped") skipped++; + else { + changed++; + changedPaths.push(next.path); + } + return putJob(input, next); + }, + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ); + + return { + runId: input.runId, + partition: input.partition, + processed: selected.length, + changed, + skipped, + paths: changedPaths, + hasMore: existing + selected.length < partitionDescriptors.length, + }; + }); + +export const chunk = ( + input: IndexDeps & ToolSearchIndex.ChunkInput, +): Effect.Effect => + Effect.gen(function* () { + const jobs = yield* getJobsByPaths(input, { + runId: input.runId, + status: "pendingChunk", + paths: input.paths, + limit: input.limit, + }); + if (jobs.length === 0) { + return { + runId: input.runId, + processed: 0, + chunks: 0, + paths: [], + chunkRefs: [], + commitPaths: [], + }; + } + + const updatedAt = nowIso(); + const chunkConcurrency = Math.max( + 1, + Math.floor(input.concurrency ?? DEFAULT_CHUNK_CONCURRENCY), + ); + const paths: string[] = []; + const chunkRefs: ToolSearchIndex.ChunkRef[] = []; + const commitPaths: string[] = []; + + const chunkCounts = yield* Effect.forEach( + jobs, + (entry) => + Effect.gen(function* () { + const job = entry.data; + const doc = yield* collectDocForTool(input.executor, jobToDescriptor(job)); + const chunks = input.chunker.chunk(input.namespace, doc); + const lexicalText = doc.lexicalText ?? buildLexicalText(doc); + const lexicalTextKey = yield* putPayloadText(input, "lexical-text", lexicalText); + yield* removeChunksForJob(input, job); + yield* Effect.forEach(chunks, (chunk) => putChunk(input, job, chunk, updatedAt), { + concurrency: INDEX_STORAGE_CONCURRENCY, + discard: true, + }); + for (const chunk of chunks) { + chunkRefs.push({ path: job.path, chunkId: chunk.id }); + } + yield* putJob(input, { + ...job, + status: "pendingEmbedding", + chunkIds: chunks.map((chunk) => chunk.id), + lexicalTextKey, + updatedAt, + }); + paths.push(job.path); + if (chunks.length === 0) { + commitPaths.push(job.path); + } + return chunks.length; + }), + { concurrency: chunkConcurrency }, + ); + const chunkCount = chunkCounts.reduce((sum, count) => sum + count, 0); + + return { + runId: input.runId, + processed: jobs.length, + chunks: chunkCount, + paths, + chunkRefs, + commitPaths, + }; + }); + +const putChunk = ( + deps: IndexCollections, + job: IndexJob, + chunk: ToolChunk, + timestamp: string, +): Effect.Effect => + Effect.gen(function* () { + const embeddingTextKey = yield* putPayloadText(deps, "embedding-text", chunk.embeddingText); + yield* deps.chunks.put({ + owner: deps.owner, + key: chunkKey(job.runId, job.path, chunk.id), + data: { + runId: job.runId, + namespace: job.namespace, + partition: job.partition, + path: job.path, + chunkId: chunk.id, + facet: chunk.facet, + chunkIndex: chunk.chunkIndex, + embeddingTextKey, + embeddingTextBytes: utf8Bytes(chunk.embeddingText), + embeddingTextTokens: estimateTokens(chunk.embeddingText), + name: chunk.name, + integration: chunk.integration, + description: job.description, + status: "pendingEmbedding", + createdAt: timestamp, + updatedAt: timestamp, + }, + }); + }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: `Failed to persist chunk "${chunk.id}".`, cause }), + ), + Effect.asVoid, + ); + +export const embed = ( + input: IndexDeps & ToolSearchIndex.EmbedInput, +): Effect.Effect => + Effect.gen(function* () { + const budget = resolveEmbedBudget(input); + const pendingChunks = yield* getChunksByRefs(input, { + runId: input.runId, + chunkRefs: input.chunkRefs, + }); + const selectedChunks: { readonly key: string; readonly data: IndexChunk }[] = []; + const vectorResponseBytes = estimateEmbeddingResponseBytes(input.embedder.dimensions); + let selectedInputTokens = 0; + let selectedResponseBytes = 0; + + for (const entry of pendingChunks) { + const inputTokens = entry.data.embeddingTextTokens; + if (inputTokens > budget.maxEstimatedTokensPerText) { + return yield* new SemanticSearchError({ + message: `Index chunk "${entry.data.chunkId}" is estimated at ${inputTokens} tokens, above the per-text embedding budget of ${budget.maxEstimatedTokensPerText}. Lower the chunker facet budget or raise maxEstimatedTokensPerText.`, + }); + } + + const next = { inputTokens, responseBytes: vectorResponseBytes }; + const current = { + chunks: selectedChunks.length, + inputTokens: selectedInputTokens, + responseBytes: selectedResponseBytes, + }; + if (selectedChunks.length > 0 && wouldExceedEmbedBudget(current, next, budget)) { + break; + } + + selectedChunks.push(entry); + selectedInputTokens += inputTokens; + selectedResponseBytes += vectorResponseBytes; + } + + if (selectedChunks.length === 0) { + return { runId: input.runId, processed: 0, chunks: 0, paths: [], chunkRefs: [] }; + } + + const updatedAt = nowIso(); + const affectedPaths = new Set(); + const affectedChunks: ToolSearchIndex.ChunkRef[] = []; + let indexedChunks = 0; + + for (let start = 0; start < selectedChunks.length; start += DEFAULT_EMBED_COMMIT_GROUP_SIZE) { + const group = selectedChunks.slice(start, start + DEFAULT_EMBED_COMMIT_GROUP_SIZE); + const texts = yield* Effect.forEach( + group, + (entry) => getPayloadText(input, entry.data.embeddingTextKey), + { concurrency: INDEX_STORAGE_CONCURRENCY }, + ); + const vectors = yield* input.embedder.embedDocuments(texts); + const records: VectorInput[] = []; + + for (let i = 0; i < group.length; i++) { + const chunk = group[i]?.data; + const vec = vectors[i]; + if (chunk === undefined || vec === undefined) { + return yield* new SemanticSearchError({ + message: `ToolSearchIndex.embed: embedding vector missing at group offset ${i}; expected ${group.length} vectors and received ${vectors.length}.`, + }); + } + records.push({ + id: chunk.chunkId, + values: vec, + namespace: input.namespace, + metadata: { + path: chunk.path, + name: chunk.name, + description: chunk.description, + integration: chunk.integration, + facet: chunk.facet, + chunkIndex: chunk.chunkIndex, + }, + }); + } + + yield* input.store.upsert(records); + yield* Effect.forEach( + group, + (entry) => + input.chunks + .put({ + owner: input.owner, + key: entry.key, + data: { ...entry.data, status: "indexed", updatedAt }, + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to mark chunk "${entry.data.chunkId}" indexed.`, + cause, + }), + ), + ), + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ); + + indexedChunks += records.length; + for (const entry of group) { + affectedPaths.add(entry.data.path); + affectedChunks.push({ path: entry.data.path, chunkId: entry.data.chunkId }); + } + } + + return { + runId: input.runId, + processed: affectedChunks.length, + chunks: indexedChunks, + paths: [...affectedPaths], + chunkRefs: affectedChunks, + }; + }); + +export const commit = ( + input: IndexDeps & ToolSearchIndex.CommitInput, +): Effect.Effect => + Effect.gen(function* () { + const entry = yield* input.jobs + .getForOwner({ owner: input.owner, key: jobKey(input.runId, input.path) }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to load index job "${input.path}" for commit.`, + cause, + }), + ), + ); + if (entry === null || entry.data.status !== "pendingEmbedding") { + return { runId: input.runId, path: input.path, committed: false }; + } + + const committed = yield* finalizeCompletedEmbedJobs(input, [entry], undefined, nowIso()); + return { runId: input.runId, path: input.path, committed: committed > 0 }; + }); + +const finalizeCompletedEmbedJobs = ( + input: IndexDeps, + jobs: readonly { readonly key: string; readonly data: IndexJob }[], + preloadedChunks: + | ReadonlyMap + | undefined, + updatedAt: string, +): Effect.Effect => + Effect.gen(function* () { + let finalized = 0; + for (const { data: job } of jobs) { + const chunkEntries = preloadedChunks?.get(job.path) ?? (yield* queryChunksForJob(input, job)); + if (chunkEntries.some((entry) => entry.data.status !== "indexed")) { + continue; + } + + const fingerprint = job.fingerprint; + if (fingerprint === undefined) { + return yield* new SemanticSearchError({ + message: `Index job "${job.path}" reached embedding without a fingerprint.`, + }); + } + + yield* input.fingerprints + .put({ + owner: input.owner, + key: job.path, + data: { + path: job.path, + integration: job.integration, + fingerprint, + chunkIds: job.chunkIds, + }, + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to persist fingerprint row for "${job.path}".`, + cause, + }), + ), + ); + + if (job.oldChunkIds.length > 0) { + yield* input.store.deleteByIds(job.oldChunkIds); + } + + if (input.lexicalStore) { + const lexicalText = + job.lexicalTextKey === undefined + ? `${job.integration} Β· ${job.path} Β· ${job.name}` + : yield* getPayloadText(input, job.lexicalTextKey); + const lexicalDoc: FtsDocumentInput = { + id: `${input.namespace}:${job.path}`, + namespace: input.namespace, + path: job.path, + name: job.name, + description: job.description, + integration: job.integration, + lexicalText, + }; + yield* input.lexicalStore.upsert([lexicalDoc]); + } + + yield* putJob(input, { ...job, status: "indexed", updatedAt }); + finalized++; + } + return finalized; + }); + +export const status = ( + input: IndexCollections & { readonly namespace: string; readonly runId: string }, +): Effect.Effect => + Effect.gen(function* () { + const run = yield* input.runs + .getForOwner({ owner: input.owner, key: input.runId }) + .pipe( + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to load index run.", cause }), + ), + ); + const count = (status: IndexJob["status"]) => + input.jobs + .count({ where: { runId: input.runId, status } }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: `Failed to count ${status} jobs.`, cause }), + ), + ); + const counts = yield* Effect.all( + { + storedPendingScan: count("pendingScan"), + skipped: count("skipped"), + pendingChunk: count("pendingChunk"), + pendingEmbedding: count("pendingEmbedding"), + indexed: count("indexed"), + failed: count("failed"), + latestJobUpdatedAt: loadLatestJobUpdatedAt(input.jobs, input.runId), + latestChunkUpdatedAt: loadLatestChunkUpdatedAt(input.chunks, input.runId), + }, + { concurrency: INDEX_STORAGE_CONCURRENCY }, + ); + const observed = + counts.storedPendingScan + + counts.skipped + + counts.pendingChunk + + counts.pendingEmbedding + + counts.indexed + + counts.failed; + const total = run?.data.total ?? observed; + const pendingScan = Math.max(0, total - observed + counts.storedPendingScan); + const lastProgressAt = [ + run?.data.updatedAt, + counts.latestJobUpdatedAt, + counts.latestChunkUpdatedAt, + ] + .filter(Predicate.isNotUndefined) + .sort() + .at(-1); + return { + runId: input.runId, + namespace: run?.data.namespace ?? input.namespace, + status: run?.data.status ?? "running", + total, + pendingScan, + skipped: counts.skipped, + pendingChunk: counts.pendingChunk, + pendingEmbedding: counts.pendingEmbedding, + indexed: counts.indexed, + failed: counts.failed, + updatedAt: run?.data.updatedAt, + lastProgressAt, + }; + }); + +export const sweepRemoved = (input: { + readonly namespace: string; + readonly executor: Executor; + readonly store: VectorStore; + readonly fingerprints: PluginStorageCollectionFacade; + readonly owner: Owner; + readonly lexicalStore?: FtsLexicalStore; +}): Effect.Effect<{ readonly namespace: string; readonly removed: number }, SemanticSearchError> => + Effect.gen(function* () { + const live = yield* listToolDescriptors(input.executor); + if (live.length === 0) return { namespace: input.namespace, removed: 0 }; + + const livePaths = new Set(live.map((tool) => addressToPath(String(tool.address)))); + const stored = yield* input.fingerprints.list().pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to load fingerprint rows for removed-tool sweep.", + cause, + }), + ), + ); + const removed = stored.filter((entry) => !livePaths.has(entry.key)); + + yield* Effect.forEach( + removed, + (entry) => + Effect.gen(function* () { + if (entry.data.chunkIds.length > 0) { + yield* input.store.deleteByIds(entry.data.chunkIds); + } + if (input.lexicalStore) { + yield* input.lexicalStore.deleteByIds([`${input.namespace}:${entry.key}`]); + } + yield* input.fingerprints.remove({ owner: input.owner, key: entry.key }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to delete fingerprint row for removed tool "${entry.key}".`, + cause, + }), + ), + ); + }), + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ); + + return { namespace: input.namespace, removed: removed.length }; + }); + +export const complete = ( + input: IndexDeps & ToolSearchIndex.CompleteInput, +): Effect.Effect => + Effect.gen(function* () { + const result = yield* sweepRemoved({ + namespace: input.namespace, + executor: input.executor, + store: input.store, + fingerprints: input.fingerprints, + owner: input.owner, + lexicalStore: input.lexicalStore, + }); + const existing = yield* input.runs.getForOwner({ owner: input.owner, key: input.runId }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to load index run for completion.", + cause, + }), + ), + ); + const updatedAt = nowIso(); + if (existing !== null) { + yield* input.runs + .put({ + owner: input.owner, + key: input.runId, + data: { ...existing.data, status: "completed", updatedAt }, + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Failed to mark index run completed.", cause }), + ), + ); + } + return { runId: input.runId, removed: result.removed }; + }); + +export const run = ( + input: IndexDeps & { + readonly runId: string; + readonly partitionCount: number; + readonly pageLimit?: number; + readonly maxTools?: number; + }, +): Effect.Effect => + Effect.gen(function* () { + const started = yield* create(input); + for (let partition = 0; partition < started.partitionCount; partition++) { + for (;;) { + const page = yield* scan({ + ...input, + partition, + limit: input.pageLimit, + }); + if (page.processed === 0) break; + if (page.paths.length > 0) { + const chunked = yield* chunk({ + ...input, + paths: page.paths, + limit: input.pageLimit, + }); + let pendingChunks = [...chunked.chunkRefs]; + for (const path of chunked.commitPaths) { + yield* commit({ ...input, runId: input.runId, path }); + } + while (pendingChunks.length > 0) { + const embedded = yield* embed({ + ...input, + chunkRefs: pendingChunks, + maxChunks: input.pageLimit, + }); + if (embedded.processed === 0 || embedded.chunks === 0) break; + const embeddedKeys = new Set( + embedded.chunkRefs.map((ref) => `${ref.path}:${ref.chunkId}`), + ); + pendingChunks = pendingChunks.filter( + (ref) => !embeddedKeys.has(`${ref.path}:${ref.chunkId}`), + ); + for (const path of embedded.paths) { + yield* commit({ ...input, runId: input.runId, path }); + } + } + } + if (!page.hasMore) break; + } + } + const completed = yield* complete(input); + const current = yield* status(input); + return { + namespace: input.namespace, + total: current.total, + indexed: current.indexed, + skipped: current.skipped, + removed: completed.removed, + }; + }); + +export const make = (input: IndexDeps): ToolSearchIndex.Service => ({ + create: (options) => create({ ...input, ...options }), + scan: (options) => scan({ ...input, ...options }), + chunk: (options) => chunk({ ...input, ...options }), + embed: (options) => embed({ ...input, ...options }), + commit: (options) => commit({ ...input, ...options }), + fail: (options) => fail({ ...input, ...options }), + reconcile: (options) => reconcile({ ...input, ...options }), + status: (options) => status({ ...input, ...options }), + complete: (options) => complete({ ...input, ...options }), +}); From b418fd9858d9d79f93b4da02cc9334a90a9ae792 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Fri, 19 Jun 2026 19:34:10 +0530 Subject: [PATCH 46/57] feat(sdk): add tool schema manifest (#42) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds a core SDK tool schema manifest that stores cheap, stable schema fingerprints during connection refresh and exposes them through `executor.tools.manifest()`. ## Changes - Adds `tool_schema_manifest` to the core schema and public SDK type exports. - Writes manifest rows alongside tool/definition rows during connection refresh and removes stale rows when connections or integrations are removed. - Adds `tools.manifest()` with policy filtering and static tool support so indexers can diff without loading full schema views. ## Tests - `bun run test -- src/executor.test.ts` in `packages/core/sdk` - `bun run typecheck` in `packages/core/sdk` - `bunx oxlint -c .oxlintrc.jsonc packages/core/sdk/src/types.ts packages/core/sdk/src/core-schema.ts packages/core/sdk/src/executor.ts packages/core/sdk/src/executor.test.ts packages/core/sdk/src/index.ts packages/core/sdk/src/shared.ts packages/core/sdk/src/promise.ts --deny-warnings` ### [Stack](https://github.com/kitlangton/stack) 1. #37 2. **#42** πŸ‘ˆ current 3. #43 --- packages/core/sdk/src/core-schema.ts | 26 ++ packages/core/sdk/src/executor.test.ts | 134 +++++++++ packages/core/sdk/src/executor.ts | 270 +++++++++++++++++- packages/core/sdk/src/index.ts | 3 +- packages/core/sdk/src/promise.ts | 2 +- packages/core/sdk/src/shared.ts | 2 +- packages/core/sdk/src/types.ts | 26 ++ .../src/sdk/tool-search-index.test.ts | 103 ++++--- 8 files changed, 519 insertions(+), 47 deletions(-) diff --git a/packages/core/sdk/src/core-schema.ts b/packages/core/sdk/src/core-schema.ts index 1e8ff2d604..64345da833 100644 --- a/packages/core/sdk/src/core-schema.ts +++ b/packages/core/sdk/src/core-schema.ts @@ -243,6 +243,31 @@ export const coreTables = defineTables({ ["tenant", "owner", "subject", "integration", "connection", "name"], ), + // Compact source-refresh-written manifest for tool schema invalidation. This + // lets indexing and other change-detection paths compare hashes without + // loading every tool's raw schema and definition graph. + tool_schema_manifest: ownedExecutorTable( + "tool_schema_manifest", + { + integration: keyColumn("integration"), + connection: keyColumn("connection"), + plugin_id: textColumn("plugin_id"), + name: keyColumn("name"), + path: keyColumn("path"), + description: textColumn("description"), + descriptor_hash: textColumn("descriptor_hash"), + input_schema_hash: textColumn("input_schema_hash"), + output_schema_hash: textColumn("output_schema_hash"), + definition_set_hash: textColumn("definition_set_hash"), + index_fingerprint: textColumn("index_fingerprint"), + fingerprint_version: textColumn("fingerprint_version"), + source_revision: nullableTextColumn("source_revision"), + created_at: dateColumn("created_at"), + updated_at: dateColumn("updated_at"), + }, + ["tenant", "owner", "subject", "integration", "connection", "name"], + ), + // User-authored tool policies (approve / require_approval / block). tool_policy: ownedExecutorTable( "tool_policy", @@ -308,6 +333,7 @@ export const TOOL_INVOCATION_COLUMNS = [ "updated_at", ] as const satisfies readonly (keyof ToolRow)[]; export type DefinitionRow = FumaRow; +export type ToolSchemaManifestRow = FumaRow; export type ToolPolicyRow = FumaRow; export type PluginStorageRow = FumaRow; export type BlobRow = FumaRow; diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 8feb74d61a..182bcf65fa 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -452,6 +452,140 @@ describe("createExecutor", () => { }), ); + it.effect("tools.manifest returns refresh-written schema fingerprints", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [demoPlugin] as const, + }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { + provider: ProviderKey.make("memory"), + id: ProviderItemId.make("v"), + }, + }); + + const manifests = yield* executor.tools.manifest({ + integration: INTEG, + includeBlocked: true, + }); + expect(manifests.map((manifest) => manifest.path).sort()).toEqual([ + `${INTEG}.org.${CONN}.inspect`, + `${INTEG}.org.${CONN}.run`, + ]); + const inspect = manifests.find((manifest) => manifest.name === "inspect"); + expect(inspect).toMatchObject({ + address: addr("inspect"), + integration: String(INTEG), + connection: String(CONN), + pluginId: "demo", + fingerprintVersion: "tool-schema-manifest/v1", + }); + expect(inspect?.inputSchemaHash).toHaveLength(64); + expect(inspect?.outputSchemaHash).toHaveLength(64); + expect(inspect?.definitionSetHash).toHaveLength(64); + expect(inspect?.indexFingerprint).toHaveLength(64); + + const schema = yield* executor.tools.schema(addr("inspect"), { includeTypeScript: false }); + expect(schema?.schemaDefinitions).not.toHaveProperty("Unused"); + }), + ); + + it.effect("tools.manifest includes static tools", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + coreTools: { webBaseUrl: "http://localhost:3000" }, + }); + const manifests = yield* executor.tools.manifest({ + integration: IntegrationSlug.make("executor"), + includeBlocked: true, + }); + expect(manifests.map((manifest) => manifest.address)).toContain( + ToolAddress.make("executor.coreTools.integrations.list"), + ); + const listed = manifests.find( + (manifest) => manifest.address === ToolAddress.make("executor.coreTools.integrations.list"), + ); + expect(listed).toMatchObject({ + path: "executor.coreTools.integrations.list", + integration: "executor", + connection: "coreTools", + fingerprintVersion: "tool-schema-manifest/v1", + }); + }), + ); + + it.effect("connections.refresh replaces stale tool manifest rows", () => + Effect.gen(function* () { + let includeInspect = true; + const dynamicPlugin = definePlugin(() => ({ + id: "dynamicManifest" as const, + credentialProviders: [memoryProvider()], + storage: () => ({}), + resolveTools: () => + Effect.succeed({ + tools: [ + ...(includeInspect + ? [ + { + name: ToolName.make("inspect"), + description: "inspect", + inputSchema: { type: "object" }, + }, + ] + : []), + { name: ToolName.make("run"), description: "run" }, + ], + definitions: {}, + }), + invokeTool: ({ toolRow }) => Effect.succeed({ ran: toolRow.name }), + extension: (ctx) => ({ + seed: () => + ctx.core.integrations.register({ + slug: INTEG, + description: "Dynamic", + config: {}, + }), + }), + }))(); + + const executor = yield* makeTestExecutor({ + plugins: [dynamicPlugin] as const, + }); + yield* executor.dynamicManifest.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { + provider: ProviderKey.make("memory"), + id: ProviderItemId.make("v"), + }, + }); + + expect((yield* executor.tools.manifest({ integration: INTEG })).map((m) => m.name)).toEqual([ + "inspect", + "run", + ]); + + includeInspect = false; + yield* executor.connections.refresh({ + owner: "org", + integration: INTEG, + name: CONN, + }); + + expect((yield* executor.tools.manifest({ integration: INTEG })).map((m) => m.name)).toEqual([ + "run", + ]); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 6506a10b3a..f01626a888 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -30,7 +30,14 @@ import { type FumaTables, type StorageFailure, } from "./fuma-runtime"; -import { makeFumaBlobStore, pluginBlobStore, type BlobStore, type OwnerPartitions } from "./blob"; +import { + makeFumaBlobStore, + pluginBlobStore, + sha256Hex, + type BlobStore, + type OwnerPartitions, +} from "./blob"; +import { cacheKeyPayload } from "./cache-key"; import { coreToolsPlugin } from "./core-tools"; import type { Connection, @@ -47,6 +54,7 @@ import { type CoreSchema, type IntegrationRow, type OAuthClientRow, + type ToolSchemaManifestRow, type ToolInvocationRow, type ToolRow, type ToolPolicyRow, @@ -142,7 +150,12 @@ import { ORG_SUBJECT, type ExecutorOwnerPolicyContext, } from "./owner-policy"; -import { ToolSchemaView, type IntegrationDetectionResult, type ToolSchemaOptions } from "./types"; +import { + ToolSchemaManifest, + ToolSchemaView, + type IntegrationDetectionResult, + type ToolSchemaOptions, +} from "./types"; import { type Tool, type ToolAnnotations, type ToolDef, type ToolListFilter } from "./tool"; import { buildToolTypeScriptPreview, type ToolTypeScriptPreview } from "./schema-types"; import { collectReferencedDefinitions } from "./schema-refs"; @@ -312,6 +325,9 @@ export type Executor = { readonly tools: { readonly list: (filter?: ToolListFilter) => Effect.Effect; + readonly manifest: ( + filter?: ToolListFilter, + ) => Effect.Effect; readonly schema: ( address: ToolAddress, options?: ToolSchemaOptions, @@ -580,6 +596,139 @@ const rowToTool = ( }; }; +const TOOL_SCHEMA_MANIFEST_FINGERPRINT_VERSION = "tool-schema-manifest/v1"; + +const toolManifestPath = (address: string): string => + address.startsWith(`${ADDRESS_PREFIX}.`) ? address.slice(ADDRESS_PREFIX.length + 1) : address; + +const hashManifestValue = (value: unknown): Effect.Effect => + sha256Hex(cacheKeyPayload(value)); + +interface ToolManifestInput { + readonly address?: ToolAddress; + readonly owner: Owner; + readonly integration: IntegrationSlug; + readonly connection: ConnectionName; + readonly pluginId: string; + readonly name: ToolName; + readonly description: string; + readonly inputSchema?: unknown; + readonly outputSchema?: unknown; + readonly definitions: ReadonlyMap; + readonly sourceRevision?: string; +} + +const buildToolSchemaManifest = (input: ToolManifestInput): Effect.Effect => + Effect.gen(function* () { + const address = + input.address ?? toolAddress(input.owner, input.integration, input.connection, input.name); + const path = toolManifestPath(String(address)); + const referencedDefinitions = collectReferencedDefinitions( + [input.inputSchema, input.outputSchema], + input.definitions, + ); + const [descriptorHash, inputSchemaHash, outputSchemaHash, definitionSetHash, indexFingerprint] = + yield* Effect.all( + [ + hashManifestValue({ + path, + name: String(input.name), + description: input.description, + integration: String(input.integration), + connection: String(input.connection), + pluginId: input.pluginId, + }), + hashManifestValue(input.inputSchema), + hashManifestValue(input.outputSchema), + hashManifestValue(referencedDefinitions), + hashManifestValue({ + path, + name: String(input.name), + description: input.description, + inputSchema: input.inputSchema, + outputSchema: input.outputSchema, + schemaDefinitions: referencedDefinitions, + }), + ], + { concurrency: 5 }, + ); + + return ToolSchemaManifest.make({ + address, + path, + owner: input.owner, + integration: String(input.integration), + connection: String(input.connection), + pluginId: input.pluginId, + name: String(input.name), + description: input.description, + descriptorHash, + inputSchemaHash, + outputSchemaHash, + definitionSetHash, + indexFingerprint, + fingerprintVersion: TOOL_SCHEMA_MANIFEST_FINGERPRINT_VERSION, + sourceRevision: input.sourceRevision, + }); + }); + +const rowToToolSchemaManifest = (row: ToolSchemaManifestRow): ToolSchemaManifest => { + const owner = row.owner as Owner; + const integration = IntegrationSlug.make(row.integration); + const connection = ConnectionName.make(row.connection); + const name = ToolName.make(row.name); + return ToolSchemaManifest.make({ + address: toolAddress(owner, integration, connection, name), + path: row.path, + owner, + integration: row.integration, + connection: row.connection, + pluginId: row.plugin_id, + name: row.name, + description: row.description, + descriptorHash: row.descriptor_hash, + inputSchemaHash: row.input_schema_hash, + outputSchemaHash: row.output_schema_hash, + definitionSetHash: row.definition_set_hash, + indexFingerprint: row.index_fingerprint, + fingerprintVersion: row.fingerprint_version, + sourceRevision: row.source_revision ?? undefined, + }); +}; + +const manifestToPolicyTool = (manifest: ToolSchemaManifest): Tool => ({ + address: manifest.address, + owner: manifest.owner as Owner, + integration: IntegrationSlug.make(manifest.integration), + connection: ConnectionName.make(manifest.connection), + name: ToolName.make(manifest.name), + pluginId: manifest.pluginId, + description: manifest.description, +}); + +const connectionToolSourceRevision = ( + integrationRow: IntegrationRow, + connectionRow: ConnectionRow | null, +): Effect.Effect => + hashManifestValue({ + integration: integrationRow.slug, + pluginId: integrationRow.plugin_id, + config: decodeJsonColumn(integrationRow.config), + connection: + connectionRow === null + ? null + : { + integration: connectionRow.integration, + name: connectionRow.name, + template: connectionRow.template, + provider: connectionRow.provider, + itemIds: decodeJsonColumn(connectionRow.item_ids), + oauthClient: connectionRow.oauth_client, + oauthClientOwner: connectionRow.oauth_client_owner, + oauthScope: connectionRow.oauth_scope, + }, + }); + // --------------------------------------------------------------------------- // Condition builders // --------------------------------------------------------------------------- @@ -1736,6 +1885,30 @@ export const createExecutor = => { + const tool = staticToolToTool(entry); + return buildToolSchemaManifest({ + address: tool.address, + owner: tool.owner, + integration: tool.integration, + connection: tool.connection, + pluginId: tool.pluginId, + name: tool.name, + description: tool.description, + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + definitions: new Map(), + }); + }; + + const staticToolManifestCache = yield* Cache.make({ + capacity: 2_048, + timeToLive: Duration.minutes(10), + lookup: buildStaticToolSchemaManifest, + }); + const registerCredentialProvider = ( provider: CredentialProvider, sourceLabel: string, @@ -2214,6 +2387,7 @@ export const createExecutor = b("integration", "=", String(slug)); yield* core.deleteMany("tool", { where }); yield* core.deleteMany("definition", { where }); + yield* core.deleteMany("tool_schema_manifest", { where }); yield* core.deleteMany("connection", { where }); yield* core.deleteMany("integration", { where: (b: AnyCb) => b("slug", "=", String(slug)), @@ -2279,6 +2453,7 @@ export const createExecutor = + buildToolSchemaManifest({ + owner, + integration: ref.integration, + connection: ref.name, + pluginId: integrationRow.plugin_id, + name: tool.name, + description: tool.description ?? "", + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + definitions: definitionMap, + sourceRevision, + }).pipe( + Effect.map((manifest) => ({ + tenant: keys.tenant, + owner: keys.owner, + subject: keys.subject, + integration: String(ref.integration), + connection: String(ref.name), + plugin_id: integrationRow.plugin_id, + name: String(tool.name), + path: manifest.path, + description: manifest.description, + descriptor_hash: manifest.descriptorHash, + input_schema_hash: manifest.inputSchemaHash, + output_schema_hash: manifest.outputSchemaHash, + definition_set_hash: manifest.definitionSetHash, + index_fingerprint: manifest.indexFingerprint, + fingerprint_version: manifest.fingerprintVersion, + source_revision: manifest.sourceRevision ?? null, + created_at: now, + updated_at: now, + })), + ), + { concurrency: 16 }, + ); yield* transaction( Effect.gen(function* () { yield* core.deleteMany("tool", { where }); yield* core.deleteMany("definition", { where }); + yield* core.deleteMany("tool_schema_manifest", { where }); yield* core.createMany("tool", toolRows); yield* core.createMany("definition", definitionRows); + yield* core.createMany("tool_schema_manifest", manifestRows); }), ); @@ -2715,6 +2933,7 @@ export const createExecutor = b.and( @@ -2817,6 +3036,52 @@ export const createExecutor = => + Effect.gen(function* () { + const rows = yield* core.findMany("tool_schema_manifest", { + where: (b: AnyCb) => + b.and( + filter?.integration === undefined + ? true + : b("integration", "=", String(filter.integration)), + filter?.owner === undefined ? true : b("owner", "=", filter.owner), + filter?.connection === undefined + ? true + : b("connection", "=", String(filter.connection)), + ), + }); + const manifests = rows.map(rowToToolSchemaManifest); + + const staticManifests = yield* Effect.forEach( + [...staticTools.values()], + (entry) => { + return Cache.get(staticToolManifestCache, entry); + }, + { concurrency: 16 }, + ); + + const includeBlocked = filter?.includeBlocked ?? false; + const policyRows = yield* core.findMany("tool_policy", {}); + const visible: ToolSchemaManifest[] = []; + for (const manifest of [...manifests, ...staticManifests]) { + const tool = manifestToPolicyTool(manifest); + if (!matchesToolFilter(tool, filter)) continue; + if (!includeBlocked) { + const effective = resolveEffectivePolicy( + normalizedPolicyId(tool), + policyRows, + ownerRankForRow, + ); + if (effective.action === "block") continue; + } + visible.push(manifest); + } + + return visible.sort((a, b) => String(a.address).localeCompare(String(b.address))); + }); + const toolSchema = ( address: ToolAddress, options?: ToolSchemaOptions, @@ -3607,6 +3872,7 @@ export const createExecutor = pluginBlobStore(makeInMemoryBlobStore(), { org: "test-org", user: null }, "semanticSearch"); +const manifestForTool = (tool: Tool): ToolSchemaManifest => ({ + address: tool.address, + path: String(tool.address).replace(/^tools\./, ""), + owner, + integration: String(tool.integration), + connection: String(tool.connection), + pluginId: tool.pluginId, + name: String(tool.name), + description: tool.description ?? "", + descriptorHash: `descriptor:${String(tool.address)}`, + inputSchemaHash: `input:${String(tool.address)}`, + outputSchemaHash: `output:${String(tool.address)}`, + definitionSetHash: `definitions:${String(tool.address)}`, + indexFingerprint: `fingerprint:${String(tool.address)}`, + fingerprintVersion: "tool-schema-manifest/v1", +}); + const makeExecutor = (counters: { raw: number; codegen: number }): Executor => { + const tool: Tool = { + address: "tools.github.repos.get" as never, + name: "repos.get" as never, + integration: "github" as never, + description: "Get a repository", + owner, + connection: "default" as never, + pluginId: "test", + }; const executor: Pick = { tools: { - list: () => - Effect.succeed([ - { - address: "tools.github.repos.get" as never, - name: "repos.get" as never, - integration: "github" as never, - description: "Get a repository", - owner, - connection: "default" as never, - pluginId: "test", - }, - ]), + list: () => Effect.succeed([tool]), + manifest: () => Effect.succeed([manifestForTool(tool)]), schema: (address, options) => { const includeTypeScript = options?.includeTypeScript ?? true; if (includeTypeScript) counters.codegen++; @@ -289,38 +307,39 @@ describe("ToolSearchIndex", () => { it.effect("limits seeded tools when maxTools is provided", () => Effect.gen(function* () { + const tools: readonly Tool[] = [ + { + address: "tools.github.repos.get" as never, + name: "repos.get" as never, + integration: "github" as never, + description: "Get a repository", + owner, + connection: "default" as never, + pluginId: "test", + }, + { + address: "tools.stripe.customers.list" as never, + name: "customers.list" as never, + integration: "stripe" as never, + description: "List customers", + owner, + connection: "default" as never, + pluginId: "test", + }, + { + address: "tools.slack.chat.postMessage" as never, + name: "chat.postMessage" as never, + integration: "slack" as never, + description: "Post a message", + owner, + connection: "default" as never, + pluginId: "test", + }, + ]; const executor: Pick = { tools: { - list: () => - Effect.succeed([ - { - address: "tools.github.repos.get" as never, - name: "repos.get" as never, - integration: "github" as never, - description: "Get a repository", - owner, - connection: "default" as never, - pluginId: "test", - }, - { - address: "tools.stripe.customers.list" as never, - name: "customers.list" as never, - integration: "stripe" as never, - description: "List customers", - owner, - connection: "default" as never, - pluginId: "test", - }, - { - address: "tools.slack.chat.postMessage" as never, - name: "chat.postMessage" as never, - integration: "slack" as never, - description: "Post a message", - owner, - connection: "default" as never, - pluginId: "test", - }, - ]), + list: () => Effect.succeed(tools), + manifest: () => Effect.succeed(tools.map((tool) => manifestForTool(tool))), schema: () => Effect.succeed(null), }, }; From b9448beff72fb27bb761bf75b82df5b91bbe9107 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Fri, 19 Jun 2026 19:34:44 +0530 Subject: [PATCH 47/57] perf(semantic-search): scan from tool manifests (#43) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Rewires semantic-search indexing to use the core tool schema manifest as its scan/diff source instead of loading each tool schema during scan. ## Changes - Adds manifest listing/slicing for index scan inputs. - Makes `ToolSearchIndex.create`, `scan`, and removed-tool sweep consume manifest paths and fingerprints. - Keeps TypeScript schema/codegen work in the changed-tool chunking path only, and updates tests to assert scan does not perform raw schema reads. ## Tests - `bun run test -- src/sdk/tool-search-index.test.ts` in `packages/plugins/semantic-search` - `bun run typecheck` in `packages/plugins/semantic-search` - `bunx oxlint -c .oxlintrc.jsonc packages/plugins/semantic-search/src/sdk/documents.ts packages/plugins/semantic-search/src/sdk/tool-search-index.ts packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts --deny-warnings` ### [Stack](https://github.com/kitlangton/stack) 1. #37 2. #42 3. **#43** πŸ‘ˆ current --- apps/host-selfhost/src/multi-user.test.ts | 2 +- .../semantic-search/src/sdk/documents.ts | 112 ++++++++---------- .../src/sdk/tool-search-index.test.ts | 9 +- .../src/sdk/tool-search-index.ts | 68 +++++------ 4 files changed, 88 insertions(+), 103 deletions(-) diff --git a/apps/host-selfhost/src/multi-user.test.ts b/apps/host-selfhost/src/multi-user.test.ts index b705356088..262451aaee 100644 --- a/apps/host-selfhost/src/multi-user.test.ts +++ b/apps/host-selfhost/src/multi-user.test.ts @@ -182,7 +182,7 @@ test("multiple accounts share one org but isolate per-user connections", async ( true, ); expect(bobConns.some((a) => a.includes(connectionName("alice-private")))).toBe(false); -}); +}, 20_000); test("each account can execute code in its own scoped sandbox", async () => { const carol = await signUp("carol@multi.test"); diff --git a/packages/plugins/semantic-search/src/sdk/documents.ts b/packages/plugins/semantic-search/src/sdk/documents.ts index 26aca43ccd..f8d50ab698 100644 --- a/packages/plugins/semantic-search/src/sdk/documents.ts +++ b/packages/plugins/semantic-search/src/sdk/documents.ts @@ -1,9 +1,9 @@ -import type { Executor, Tool } from "@executor-js/sdk/core"; +import type { Executor, Tool, ToolSchemaManifest } from "@executor-js/sdk/core"; import { Effect } from "effect"; import type { ToolDocumentInput } from "./chunker"; import { SemanticSearchError } from "./errors"; -import { cyrb53, type FingerprintInput } from "./fingerprint"; +import { cyrb53 } from "./fingerprint"; const ADDRESS_PREFIX = "tools."; @@ -86,12 +86,15 @@ const selectToolDescriptors = ( tools: readonly Tool[], options?: ListToolDescriptorsOptions, ): readonly Tool[] => { - const sorted = [...tools].sort((a, b) => String(a.address).localeCompare(String(b.address))); const maxTools = options?.maxTools; - if (maxTools === undefined) return sorted; + if (maxTools === undefined) { + return [...tools].sort((a, b) => String(a.address).localeCompare(String(b.address))); + } const limit = Math.max(0, Math.floor(maxTools)); - if (limit >= sorted.length) return sorted; - return [...sorted] + if (limit >= tools.length) { + return [...tools].sort((a, b) => String(a.address).localeCompare(String(b.address))); + } + return [...tools] .sort((a, b) => { const left = cyrb53(addressToPath(String(a.address))); const right = cyrb53(addressToPath(String(b.address))); @@ -101,6 +104,28 @@ const selectToolDescriptors = ( .sort((a, b) => String(a.address).localeCompare(String(b.address))); }; +const selectToolManifests = ( + manifests: readonly ToolSchemaManifest[], + options?: ListToolDescriptorsOptions, +): readonly ToolSchemaManifest[] => { + const maxTools = options?.maxTools; + if (maxTools === undefined) { + return [...manifests].sort((a, b) => a.path.localeCompare(b.path)); + } + const limit = Math.max(0, Math.floor(maxTools)); + if (limit >= manifests.length) { + return [...manifests].sort((a, b) => a.path.localeCompare(b.path)); + } + return [...manifests] + .sort((a, b) => { + const left = cyrb53(a.path); + const right = cyrb53(b.path); + return left === right ? a.path.localeCompare(b.path) : left - right; + }) + .slice(0, limit) + .sort((a, b) => a.path.localeCompare(b.path)); +}; + /** List the live tool descriptors, stably sorted by address. * * Cheap β€” descriptors only, NO per-tool schema fetch β€” so it is safe to call @@ -118,6 +143,23 @@ export const listToolDescriptors = ( Effect.map((tools) => selectToolDescriptors(tools, options)), ); +/** List live tool schema manifests, stably sorted by indexed path. + * + * This is the manifest tier for indexing: it carries precomputed raw-schema + * fingerprints from source refresh, so scan/diff can avoid per-tool + * `tools.schema(includeTypeScript: false)` reads entirely. */ +export const listToolManifests = ( + executor: Executor, + options?: ListToolDescriptorsOptions, +): Effect.Effect => + executor.tools.manifest().pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Failed to list tool manifests for indexing.", cause }), + ), + Effect.map((manifests) => selectToolManifests(manifests, options)), + ); + /** Collect `ToolDocumentInput` for a single tool descriptor. * * Attempts to fetch its schema via `tools.schema(address)` to populate the @@ -172,63 +214,9 @@ export const collectDocsForTools = ( ): Effect.Effect => Effect.forEach(tools, (tool) => collectDocForTool(executor, tool), { concurrency: 16 }); -/** A tool paired with the raw-schema fingerprint input the diff hashes. */ -export interface ToolFingerprintInput { - readonly tool: IndexableToolDescriptor; - readonly input: FingerprintInput; -} - -/** Collect the cheap fingerprint inputs for a set of tools β€” the raw JSON schema - * (roots + referenced `$defs`), NO TypeScript codegen. - * - * This is the cheap tier the incremental reindex diffs on: it fetches each - * tool's schema with `includeTypeScript: false`, so the CPU-heavy - * `tools.schema` codegen is skipped entirely. Only tools whose fingerprint - * changed go on to pay the codegen (via `collectDocsForTools`). On any per-tool - * schema-fetch failure we degrade to an identity-only fingerprint (path + name - * + description), which simply re-classifies the tool as "changed" and lets the - * full pass retry it β€” never failing the whole batch for one tool. - * - * Bounded concurrency (16) keeps the walk fast while avoiding unbounded fan-out. */ -export const collectFingerprintInputs = ( - executor: Executor, - tools: readonly IndexableToolDescriptor[], -): Effect.Effect => - Effect.forEach( - tools, - (tool) => { - const base: FingerprintInput = { - path: addressToPath(String(tool.address)), - name: String(tool.name), - description: stripHtml(String(tool.description ?? "")), - }; - return executor.tools - .schema(tool.address as Tool["address"], { includeTypeScript: false }) - .pipe( - Effect.map((view): ToolFingerprintInput => { - if (view === null) return { tool, input: base }; - return { - tool, - input: { - ...base, - ...(view.inputSchema !== undefined ? { inputSchema: view.inputSchema } : {}), - ...(view.outputSchema !== undefined ? { outputSchema: view.outputSchema } : {}), - ...(view.schemaDefinitions !== undefined - ? { schemaDefinitions: view.schemaDefinitions } - : {}), - }, - }; - }), - // Degrade: schema fetch failed β€” fall back to an identity-only fingerprint. - Effect.catch(() => Effect.succeed({ tool, input: base })), - ); - }, - { concurrency: 16 }, - ); - /** Collect the full `ToolDocumentInput` set from the live catalog (list + schema). - * Convenience for non-paged callers; paged reindex uses `listToolDescriptors` - * to slice, then `collectDocsForTools` on each slice. */ + * Convenience for non-paged callers; the queued indexer uses manifests for + * scan/diff and only calls `collectDocForTool` for changed jobs. */ export const collectToolDocumentInputs = ( executor: Executor, ): Effect.Effect => diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts index 40ec39f1c2..c133200c25 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -31,7 +31,10 @@ const namespace = "test-ns"; const makeBlobs = () => pluginBlobStore(makeInMemoryBlobStore(), { org: "test-org", user: null }, "semanticSearch"); -const manifestForTool = (tool: Tool): ToolSchemaManifest => ({ +const manifestForTool = ( + tool: Tool, + fingerprint = `fingerprint:${String(tool.address)}`, +): ToolSchemaManifest => ({ address: tool.address, path: String(tool.address).replace(/^tools\./, ""), owner, @@ -44,7 +47,7 @@ const manifestForTool = (tool: Tool): ToolSchemaManifest => ({ inputSchemaHash: `input:${String(tool.address)}`, outputSchemaHash: `output:${String(tool.address)}`, definitionSetHash: `definitions:${String(tool.address)}`, - indexFingerprint: `fingerprint:${String(tool.address)}`, + indexFingerprint: fingerprint, fingerprintVersion: "tool-schema-manifest/v1", }); @@ -213,7 +216,7 @@ describe("ToolSearchIndex", () => { expect(chunked.processed).toBe(1); expect(chunked.chunks).toBeGreaterThan(0); expect(embedded).toMatchObject({ processed: chunked.chunks, chunks: chunked.chunks }); - expect(counters).toEqual({ raw: 1, codegen: 1 }); + expect(counters).toEqual({ raw: 0, codegen: 1 }); expect(upserted).toHaveLength(chunked.chunks); expect([...fingerprints.data.values()]).toHaveLength(1); expect([...jobs.data.values()][0]?.status).toBe("indexed"); diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts index 47776a21ae..00e92e9d83 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts @@ -22,13 +22,13 @@ import { addressToPath, buildLexicalText, collectDocForTool, - collectFingerprintInputs, type IndexableToolDescriptor, listToolDescriptors, + listToolManifests, } from "./documents"; import type { ToolEmbedder } from "./embedder"; import { SemanticSearchError } from "./errors"; -import { cyrb53, fingerprintTool } from "./fingerprint"; +import { cyrb53 } from "./fingerprint"; import type { VectorInput, VectorStore } from "./store"; import type { FtsDocumentInput, FtsLexicalStore } from "./store-fts"; @@ -770,7 +770,7 @@ export const create = ( ): Effect.Effect => Effect.gen(function* () { const partitionCount = Math.max(1, Math.floor(input.partitionCount)); - const descriptors = yield* listToolDescriptors(input.executor, { maxTools: input.maxTools }); + const manifests = yield* listToolManifests(input.executor, { maxTools: input.maxTools }); const createdAt = nowIso(); yield* input.runs @@ -782,7 +782,7 @@ export const create = ( namespace: input.namespace, status: "running", partitionCount, - total: descriptors.length, + total: manifests.length, createdAt, updatedAt: createdAt, }, @@ -796,7 +796,7 @@ export const create = ( return { runId: input.runId, namespace: input.namespace, - total: descriptors.length, + total: manifests.length, partitionCount, }; }); @@ -818,10 +818,13 @@ export const scan = ( }); } - const descriptors = yield* listToolDescriptors(input.executor, { maxTools: input.maxTools }); - const partitionDescriptors = descriptors - .map((tool, ordinal) => ({ tool, ordinal, path: addressToPath(String(tool.address)) })) - .filter(({ path }) => partitionForPath(path, run.data.partitionCount) === input.partition); + const manifests = yield* listToolManifests(input.executor, { maxTools: input.maxTools }); + const partitionManifests = manifests + .map((manifest, ordinal) => ({ manifest, ordinal })) + .filter( + ({ manifest }) => + partitionForPath(manifest.path, run.data.partitionCount) === input.partition, + ); const existing = yield* input.jobs .count({ where: { runId: input.runId, partition: input.partition } }) .pipe( @@ -830,7 +833,7 @@ export const scan = ( new SemanticSearchError({ message: "Failed to count scanned index jobs.", cause }), ), ); - const selected = partitionDescriptors.slice( + const selected = partitionManifests.slice( existing, existing + (input.limit ?? DEFAULT_PAGE_LIMIT), ); @@ -846,43 +849,34 @@ export const scan = ( }; } - const fingerprintInputs = yield* collectFingerprintInputs( - input.executor, - selected.map(({ tool }) => tool), - ); const stored = yield* loadFingerprints( input, - fingerprintInputs.map(({ input: fp }) => fp.path), + selected.map(({ manifest }) => manifest.path), ); let changed = 0; let skipped = 0; const changedPaths: string[] = []; - const selectedByPath = new Map(selected.map((entry) => [entry.path, entry])); const updatedAt = nowIso(); yield* Effect.forEach( - fingerprintInputs, - ({ input: fp }) => { - const selectedEntry = selectedByPath.get(fp.path); - if (selectedEntry === undefined) return Effect.void; - const fingerprint = fingerprintTool(fp); - const storedRow = stored.get(fp.path); - const { tool, ordinal, path } = selectedEntry; + selected, + ({ manifest, ordinal }) => { + const storedRow = stored.get(manifest.path); const next: IndexJob = - storedRow !== undefined && storedRow.fingerprint === fingerprint + storedRow !== undefined && storedRow.fingerprint === manifest.indexFingerprint ? { runId: input.runId, namespace: input.namespace, partition: input.partition, ordinal, - address: String(tool.address), - path, - name: String(tool.name), - integration: String(tool.integration), - description: String(tool.description ?? ""), + address: String(manifest.address), + path: manifest.path, + name: manifest.name, + integration: manifest.integration, + description: manifest.description, status: "skipped", - fingerprint, + fingerprint: manifest.indexFingerprint, oldChunkIds: storedRow.chunkIds, chunkIds: storedRow.chunkIds, updatedAt, @@ -893,13 +887,13 @@ export const scan = ( namespace: input.namespace, partition: input.partition, ordinal, - address: String(tool.address), - path, - name: String(tool.name), - integration: String(tool.integration), - description: String(tool.description ?? ""), + address: String(manifest.address), + path: manifest.path, + name: manifest.name, + integration: manifest.integration, + description: manifest.description, status: "pendingChunk", - fingerprint, + fingerprint: manifest.indexFingerprint, oldChunkIds: storedRow?.chunkIds ?? [], chunkIds: [], updatedAt, @@ -922,7 +916,7 @@ export const scan = ( changed, skipped, paths: changedPaths, - hasMore: existing + selected.length < partitionDescriptors.length, + hasMore: existing + selected.length < partitionManifests.length, }; }); From 84ce77fae15213c566a007733cbdc713dcd772ee Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Fri, 19 Jun 2026 21:02:24 +0530 Subject: [PATCH 48/57] fix(semantic-search): reconstruct queued index work Recover downstream queue messages from persisted index state when chunk or embed queue turns are retried after D1 writes have already advanced job or chunk status. --- .../src/sdk/tool-search-index.test.ts | 161 ++++++++++++++++++ .../src/sdk/tool-search-index.ts | 63 +++++-- 2 files changed, 212 insertions(+), 12 deletions(-) diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts index c133200c25..c9805b7bdf 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -308,6 +308,62 @@ describe("ToolSearchIndex", () => { }), ); + it.effect("reconstructs embed messages when a chunk message retries after persisting chunks", () => + Effect.gen(function* () { + const counters = { raw: 0, codegen: 0 }; + const executor = makeExecutor(counters); + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + const store: VectorStore = { + maxTopK: 100, + query: () => Effect.succeed([]), + upsert: () => Effect.void, + deleteByIds: () => Effect.void, + }; + const embedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => Effect.succeed(texts.map(() => [0.1, 0.2, 0.3])), + embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), + }; + const base = { namespace, executor, runs, jobs, chunks, fingerprints, blobs, owner }; + + yield* create({ ...base, runId: "run-chunk-retry", partitionCount: 1 }); + const scanned = yield* scan({ + ...base, + runId: "run-chunk-retry", + partition: 0, + limit: 10, + }); + const first = yield* chunk({ + ...base, + embedder, + store, + chunker: makeFacetChunker(), + runId: "run-chunk-retry", + paths: scanned.paths, + limit: 10, + }); + const retry = yield* chunk({ + ...base, + embedder, + store, + chunker: makeFacetChunker(), + runId: "run-chunk-retry", + paths: scanned.paths, + limit: 10, + }); + + expect(first.chunkRefs.length).toBeGreaterThan(0); + expect(retry.chunkRefs).toEqual(first.chunkRefs); + expect(retry.commitPaths).toEqual([]); + expect(counters.codegen).toBe(1); + }), + ); + it.effect("limits seeded tools when maxTools is provided", () => Effect.gen(function* () { const tools: readonly Tool[] = [ @@ -507,6 +563,111 @@ describe("ToolSearchIndex", () => { }), ); + it.effect("reconstructs commit messages when an embed message retries after indexing chunks", () => + Effect.gen(function* () { + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + let embedCalls = 0; + const store: VectorStore = { + maxTopK: 100, + query: () => Effect.succeed([]), + upsert: () => Effect.void, + deleteByIds: () => Effect.void, + }; + const embedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => + Effect.sync(() => { + embedCalls++; + return texts.map(() => [0.1, 0.2, 0.3]); + }), + embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), + }; + + const createdAt = new Date(0).toISOString(); + const lexicalTextKey = "test/lexical-retry.txt"; + const job: IndexJob = { + runId: "run-embed-retry", + namespace, + partition: 0, + ordinal: 0, + address: "tools.github.repos.get", + path: "github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + status: "pendingEmbedding", + fingerprint: "fp-embed-retry", + oldChunkIds: [], + chunkIds: ["chunk-0"], + lexicalTextKey, + createdAt, + updatedAt: createdAt, + }; + const chunkRow: IndexChunk = { + runId: job.runId, + namespace, + partition: 0, + path: job.path, + chunkId: "chunk-0", + facet: "description", + chunkIndex: 0, + embeddingTextKey: "test/chunk-retry.txt", + embeddingTextBytes: "chunk retry text".length, + embeddingTextTokens: 4, + name: job.name, + integration: job.integration, + description: job.description, + status: "pendingEmbedding", + createdAt, + updatedAt: createdAt, + }; + yield* blobs.put(lexicalTextKey, "github github.repos.get repos.get", { owner }); + yield* blobs.put(chunkRow.embeddingTextKey, "chunk retry text", { owner }); + yield* jobs.put({ owner, key: `${job.runId}:${job.path}`, data: job }); + yield* chunks.put({ + owner, + key: `${chunkRow.runId}:${chunkRow.path}:${chunkRow.chunkId}`, + data: chunkRow, + }); + + const base = { + namespace, + executor: makeExecutor({ raw: 0, codegen: 0 }), + runs, + jobs, + chunks, + fingerprints, + blobs, + owner, + embedder, + store, + chunker: makeFacetChunker(), + }; + const first = yield* embed({ + ...base, + runId: job.runId, + chunkRefs: [{ path: job.path, chunkId: "chunk-0" }], + }); + const retry = yield* embed({ + ...base, + runId: job.runId, + chunkRefs: [{ path: job.path, chunkId: "chunk-0" }], + }); + const committed = yield* commit({ ...base, runId: job.runId, path: job.path }); + + expect(first).toMatchObject({ processed: 1, chunks: 1, paths: [job.path] }); + expect(retry).toMatchObject({ processed: 0, chunks: 0, paths: [job.path] }); + expect(committed.committed).toBe(true); + expect(embedCalls).toBe(1); + expect(jobs.data.get(`${job.runId}:${job.path}`)?.status).toBe("indexed"); + }), + ); + it.effect("commits zero-chunk embed jobs so fingerprints can warm future runs", () => Effect.gen(function* () { const runs = makeCollection(indexRuns.name); diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts index 00e92e9d83..3d4c312f84 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts @@ -278,7 +278,7 @@ const getChunksByRefs = ( (ref) => deps.chunks .getForOwner({ owner: deps.owner, key: chunkKey(input.runId, ref.path, ref.chunkId) }) - .pipe(Effect.map((entry) => (entry?.data.status === "pendingEmbedding" ? entry : null))), + .pipe(Effect.map((entry) => entry ?? null)), { concurrency: INDEX_STORAGE_CONCURRENCY }, ).pipe( Effect.map((entries) => entries.flatMap((entry) => (entry === null ? [] : [entry]))), @@ -931,13 +931,38 @@ export const chunk = ( limit: input.limit, }); if (jobs.length === 0) { + const pendingEmbeddingJobs = yield* getJobsByPaths(input, { + runId: input.runId, + status: "pendingEmbedding", + paths: input.paths, + limit: input.limit, + }); + const existing = yield* Effect.forEach( + pendingEmbeddingJobs, + (entry) => + queryChunksForJob(input, entry.data).pipe( + Effect.map((chunkEntries) => ({ job: entry.data, chunkEntries })), + ), + { concurrency: INDEX_STORAGE_CONCURRENCY }, + ); + const chunkRefs = existing.flatMap(({ chunkEntries }) => + chunkEntries + .filter((entry) => entry.data.status === "pendingEmbedding") + .map((entry) => ({ path: entry.data.path, chunkId: entry.data.chunkId })), + ); + const commitPaths = existing + .filter(({ chunkEntries }) => + chunkEntries.every((entry) => entry.data.status === "indexed"), + ) + .map(({ job }) => job.path); + return { runId: input.runId, - processed: 0, - chunks: 0, - paths: [], - chunkRefs: [], - commitPaths: [], + processed: pendingEmbeddingJobs.length, + chunks: chunkRefs.length, + paths: pendingEmbeddingJobs.map((entry) => entry.data.path), + chunkRefs, + commitPaths, }; } @@ -1037,16 +1062,18 @@ export const embed = ( ): Effect.Effect => Effect.gen(function* () { const budget = resolveEmbedBudget(input); - const pendingChunks = yield* getChunksByRefs(input, { + const chunkEntries = yield* getChunksByRefs(input, { runId: input.runId, chunkRefs: input.chunkRefs, }); + const alreadyIndexed = chunkEntries.filter((entry) => entry.data.status === "indexed"); const selectedChunks: { readonly key: string; readonly data: IndexChunk }[] = []; const vectorResponseBytes = estimateEmbeddingResponseBytes(input.embedder.dimensions); let selectedInputTokens = 0; let selectedResponseBytes = 0; - for (const entry of pendingChunks) { + for (const entry of chunkEntries) { + if (entry.data.status !== "pendingEmbedding") continue; const inputTokens = entry.data.embeddingTextTokens; if (inputTokens > budget.maxEstimatedTokensPerText) { return yield* new SemanticSearchError({ @@ -1070,12 +1097,24 @@ export const embed = ( } if (selectedChunks.length === 0) { - return { runId: input.runId, processed: 0, chunks: 0, paths: [], chunkRefs: [] }; + return { + runId: input.runId, + processed: 0, + chunks: 0, + paths: [...new Set(alreadyIndexed.map((entry) => entry.data.path))], + chunkRefs: alreadyIndexed.map((entry) => ({ + path: entry.data.path, + chunkId: entry.data.chunkId, + })), + }; } const updatedAt = nowIso(); - const affectedPaths = new Set(); - const affectedChunks: ToolSearchIndex.ChunkRef[] = []; + const affectedPaths = new Set(alreadyIndexed.map((entry) => entry.data.path)); + const affectedChunks: ToolSearchIndex.ChunkRef[] = alreadyIndexed.map((entry) => ({ + path: entry.data.path, + chunkId: entry.data.chunkId, + })); let indexedChunks = 0; for (let start = 0; start < selectedChunks.length; start += DEFAULT_EMBED_COMMIT_GROUP_SIZE) { @@ -1142,7 +1181,7 @@ export const embed = ( return { runId: input.runId, - processed: affectedChunks.length, + processed: indexedChunks, chunks: indexedChunks, paths: [...affectedPaths], chunkRefs: affectedChunks, From fed76059c999196545e9dbc9c9163538d9fc5238 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sat, 20 Jun 2026 02:41:28 +0530 Subject: [PATCH 49/57] perf(semantic-search): reduce queued index work (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Improves the semantic-search indexing path by making derived schema caches durable through Effect `KeyValueStore`, reducing queued index materialization work, and bounding Cloudflare Vectorize metadata before upsert. ## Changes - Replace large per-isolate schema `Effect.Cache` layers with direct typed `KeyValueStore` cache reads/writes and an in-memory default when hosts do not provide cache storage. - Make queued indexing use raw schema reads plus compact schema facet text instead of TypeScript preview generation. - Cache materialized index documents by tool fingerprint and batch commit work across paths. - Cap Vectorize metadata descriptions and reject oversized metadata before calling Vectorize. ## Tests - `bun run --cwd /Users/aryasaatvik/Developer/executor-worktrees/selfhost/packages/core/sdk test -- src/executor.test.ts src/tool-schema-view-cache.test.ts src/tool-typescript-preview-cache.test.ts` - `bun run --cwd /Users/aryasaatvik/Developer/executor-worktrees/selfhost/packages/plugins/semantic-search test -- src/sdk/tool-search-index.test.ts src/sdk/store-cloudflare-limits.test.ts src/sdk/chunker.test.ts` - `bun run --cwd /Users/aryasaatvik/Developer/executor-worktrees/selfhost/packages/plugins/semantic-search typecheck` - `bun run --cwd /Users/aryasaatvik/Developer/executor-worktrees/selfhost/packages/core/sdk typecheck` - `bun run --cwd /Users/aryasaatvik/Developer/executor-worktrees/selfhost/packages/core/api typecheck` - `bun run typecheck` in `/Users/aryasaatvik/Developer/AryaLabsHQ/executor` ## Known tradeoffs - Host queue batching is committed in the dogfood host checkout separately and is not part of this package PR. - This was not deployed and no indexing run was triggered. ### [Stack](https://github.com/kitlangton/stack) 1. **#44** πŸ‘ˆ current 2. #45 --- .../core/api/src/server/scoped-executor.ts | 6 +- packages/core/sdk/src/executor.test.ts | 24 +- packages/core/sdk/src/executor.ts | 225 ++++++----- packages/core/sdk/src/test-config.ts | 4 +- .../core/sdk/src/tool-schema-view-cache.ts | 28 ++ .../semantic-search/src/sdk/chunker.test.ts | 10 +- .../semantic-search/src/sdk/chunker.ts | 34 +- .../semantic-search/src/sdk/collections.ts | 1 + .../semantic-search/src/sdk/documents.ts | 107 +++-- .../plugins/semantic-search/src/sdk/eval.ts | 56 +-- .../plugins/semantic-search/src/sdk/plugin.ts | 3 +- .../src/sdk/store-cloudflare-limits.test.ts | 33 ++ .../src/sdk/store-cloudflare-limits.ts | 19 + .../src/sdk/tool-search-index.test.ts | 374 ++++++++++-------- .../src/sdk/tool-search-index.ts | 139 +++++-- 15 files changed, 690 insertions(+), 373 deletions(-) diff --git a/packages/core/api/src/server/scoped-executor.ts b/packages/core/api/src/server/scoped-executor.ts index f99f919299..761d8313e8 100644 --- a/packages/core/api/src/server/scoped-executor.ts +++ b/packages/core/api/src/server/scoped-executor.ts @@ -182,7 +182,7 @@ export const makeScopedExecutor = < const { db, blobs } = yield* DbProvider; const { plugins: pluginsFactory } = yield* PluginsProvider; const config = yield* HostConfig; - const keyValueStore = yield* Effect.serviceOption(KeyValueStore.KeyValueStore); + const cache = yield* Effect.serviceOption(KeyValueStore.KeyValueStore); // Explicit config wins; otherwise fall back to the request origin if a host // provided one (HTTP middleware / MCP session DO). Stays `undefined` for // non-request callers β€” `coreTools.webBaseUrl` is optional and only the @@ -222,9 +222,9 @@ export const makeScopedExecutor = < subject: Subject.make(accountId), db, blobs, - ...Option.match(keyValueStore, { + ...Option.match(cache, { onNone: () => ({}), - onSome: (store) => ({ keyValueStore: store }), + onSome: (store) => ({ cache: store }), }), plugins, httpClientLayer, diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 182bcf65fa..9030a08351 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -18,7 +18,7 @@ import type { CredentialProvider } from "./provider"; import { IntegrationDetectionResult } from "./types"; import { makeTestExecutor } from "./testing"; import { serveOAuthTestServer } from "./testing/oauth-test-server"; -import { toolSchemaViewCacheKey } from "./tool-schema-view-cache"; +import { toolSchemaViewManifestCacheKey } from "./tool-schema-view-cache"; import { toolTypeScriptPreviewCacheKey } from "./tool-typescript-preview-cache"; // removed: v1 secret browser-handoff, source.configure, case-insensitive tool-id @@ -392,7 +392,7 @@ describe("createExecutor", () => { it.effect("tools.schema returns roots with shared reachable definitions", () => Effect.gen(function* () { const cacheRows = new Map(); - const keyValueStore = KeyValueStore.makeStringOnly({ + const cache = KeyValueStore.makeStringOnly({ get: (key) => Effect.sync(() => cacheRows.get(key)), set: (key, value) => Effect.sync(() => { @@ -409,7 +409,7 @@ describe("createExecutor", () => { }); const executor = yield* makeTestExecutor({ plugins: [demoPlugin] as const, - keyValueStore, + cache, }); yield* executor.demo.seed(); yield* executor.connections.create({ @@ -436,18 +436,20 @@ describe("createExecutor", () => { outputSchema: schema?.outputSchema, definitions: defs, }); - const cached = yield* keyValueStore.get(cacheKey); + const cached = yield* cache.get(cacheKey); expect(cached).toContain("preview"); - const schemaViewCacheKey = yield* toolSchemaViewCacheKey({ + const manifest = (yield* executor.tools.manifest({ integration: INTEG })).find( + (entry) => entry.address === addr("inspect"), + ); + expect(manifest).toBeDefined(); + if (manifest === undefined) return; + const schemaViewCacheKey = yield* toolSchemaViewManifestCacheKey({ address: String(addr("inspect")), - name: schema?.name, - description: schema?.description, - inputSchema: schema?.inputSchema, - outputSchema: schema?.outputSchema, - definitions: demoDefinitions, + indexFingerprint: manifest.indexFingerprint, + fingerprintVersion: manifest.fingerprintVersion, }); - const cachedSchemaView = yield* keyValueStore.get(schemaViewCacheKey); + const cachedSchemaView = yield* cache.get(schemaViewCacheKey); expect(cachedSchemaView).toContain("view"); }), ); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index f01626a888..8e96cf6138 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -169,6 +169,7 @@ import { TOOL_SCHEMA_VIEW_CACHE_VERSION, ToolSchemaViewCacheEntry, toolSchemaViewCacheKey, + toolSchemaViewManifestCacheKey, } from "./tool-schema-view-cache"; import { refreshAccessToken, @@ -387,12 +388,11 @@ export interface ExecutorConfig isStorageFailure(cause) ? cause : new StorageError({ message, cause }); +const MEMORY_CACHE_CAPACITY = 2_048; +const MEMORY_CACHE_TTL_MS = 10 * 60 * 1000; + +const makeMemoryCacheStore = (): KeyValueStore.KeyValueStore => { + const rows = new Map(); + const evictExpired = (now: number): void => { + for (const [key, entry] of rows) { + if (entry.expiresAt <= now) rows.delete(key); + } + }; + const evictCapacity = (): void => { + while (rows.size > MEMORY_CACHE_CAPACITY) { + const oldest = rows.keys().next().value; + if (oldest === undefined) break; + rows.delete(oldest); + } + }; + return KeyValueStore.makeStringOnly({ + get: (key) => + Effect.sync(() => { + const now = Date.now(); + const entry = rows.get(key); + if (entry === undefined) return undefined; + if (entry.expiresAt <= now) { + rows.delete(key); + return undefined; + } + rows.delete(key); + rows.set(key, entry); + return entry.value; + }), + set: (key, value) => + Effect.sync(() => { + const now = Date.now(); + evictExpired(now); + rows.set(key, { value, expiresAt: now + MEMORY_CACHE_TTL_MS }); + evictCapacity(); + }), + remove: (key) => + Effect.sync(() => { + rows.delete(key); + }), + clear: Effect.sync(() => { + rows.clear(); + }), + size: Effect.sync(() => { + evictExpired(Date.now()); + return rows.size; + }), + }); +}; + const pluginStorageFailure = (pluginId: string, hook: string, cause: unknown): StorageFailure => storageFailureFromUnknown(`${hook} failed for plugin ${pluginId}`, cause); @@ -1765,44 +1817,12 @@ export const createExecutor = - Effect.gen(function* () { - if (typeScriptPreviewStore === undefined) return null; - - const entry = yield* typeScriptPreviewStore - .get(key) - .pipe(Effect.catch(() => Effect.succeed(Option.none()))); - if (Option.isNone(entry)) return null; - - return entry.value.preview; - }), - }); - const toolSchemaViewCache = yield* Cache.make({ - capacity: 2_048, - timeToLive: Duration.minutes(10), - lookup: (key: string) => - Effect.gen(function* () { - if (toolSchemaViewStore === undefined) return null; - - const entry = yield* toolSchemaViewStore - .get(key) - .pipe(Effect.catch(() => Effect.succeed(Option.none()))); - if (Option.isNone(entry)) return null; - - return entry.value.view; - }), - }); + const cacheStore = config.cache ?? makeMemoryCacheStore(); + const typeScriptPreviewStore = KeyValueStore.toSchemaStore( + cacheStore, + ToolTypeScriptPreviewCacheEntry, + ); + const toolSchemaViewStore = KeyValueStore.toSchemaStore(cacheStore, ToolSchemaViewCacheEntry); const transaction = (effect: Effect.Effect) => fuma.transaction(effect); // Populated once, never mutated after startup. @@ -1836,8 +1856,10 @@ export const createExecutor = Effect.succeed(Option.none()))); + if (Option.isSome(cached)) return cached.value.preview; const preview = yield* Effect.tryPromise({ try: () => @@ -1849,16 +1871,13 @@ export const createExecutor = storageFailureFromUnknown("Failed to build tool TypeScript preview", cause), }); - yield* Cache.set(typeScriptPreviewCache, key, preview); - if (typeScriptPreviewStore !== undefined) { - yield* typeScriptPreviewStore - .set(key, { - version: TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, - compilerVersion: TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, - preview, - }) - .pipe(Effect.ignore); - } + yield* typeScriptPreviewStore + .set(key, { + version: TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, + compilerVersion: TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, + preview, + }) + .pipe(Effect.ignore); return preview; }); @@ -3104,8 +3123,10 @@ export const createExecutor = Effect.succeed(Option.none()))); + if (Option.isSome(cached)) return cached.value.view; const preview = includeTypeScript ? Option.getOrUndefined( @@ -3126,22 +3147,45 @@ export const createExecutor = + b.and( + byOwner(parsed.owner)(b), + b("integration", "=", String(parsed.integration)), + b("connection", "=", String(parsed.connection)), + b("name", "=", String(parsed.tool)), + ), + }); + const manifest = manifestRow === null ? null : rowToToolSchemaManifest(manifestRow); + const manifestCacheKey = + manifest === null + ? null + : yield* toolSchemaViewManifestCacheKey({ + address: String(address), + indexFingerprint: manifest.indexFingerprint, + fingerprintVersion: manifest.fingerprintVersion, + includeTypeScript, + }); + if (manifestCacheKey !== null) { + const cached = yield* toolSchemaViewStore + .get(manifestCacheKey) + .pipe(Effect.catch(() => Effect.succeed(Option.none()))); + if (Option.isSome(cached)) return cached.value.view; + } + const row = yield* core.findFirst("tool", { where: (b: AnyCb) => b.and( @@ -3165,17 +3209,23 @@ export const createExecutor = (); for (const def of definitionRows) defs.set(def.name, decodeJsonColumn(def.schema)); const definitions = Object.fromEntries(defs); - const key = yield* toolSchemaViewCacheKey({ - address: String(address), - name: String(tool.name), - description: tool.description, - inputSchema: tool.inputSchema, - outputSchema: tool.outputSchema, - definitions, - includeTypeScript, - }); - const cached = yield* Cache.get(toolSchemaViewCache, key); - if (cached !== null) return cached; + const key = + manifestCacheKey ?? + (yield* toolSchemaViewCacheKey({ + address: String(address), + name: String(tool.name), + description: tool.description, + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + definitions, + includeTypeScript, + })); + if (manifestCacheKey === null) { + const cached = yield* toolSchemaViewStore + .get(key) + .pipe(Effect.catch(() => Effect.succeed(Option.none()))); + if (Option.isSome(cached)) return cached.value.view; + } const referenced = collectReferencedDefinitions( [tool.inputSchema, tool.outputSchema], @@ -3206,17 +3256,14 @@ export const createExecutor = ["coreTools"]; - readonly keyValueStore?: KeyValueStore.KeyValueStore; + readonly cache?: KeyValueStore.KeyValueStore; /** Override the OAuth callback URL. Pass `null` to construct an executor with * no OAuth callback (exercises the fail-loud redirect path). */ readonly redirectUri?: string | null; @@ -162,7 +162,7 @@ export const makeTestConfig = cacheKeyPayload({ version: TOOL_SCHEMA_VIEW_CACHE_VERSION, @@ -42,9 +49,30 @@ const cachePayload = (input: ToolSchemaViewCacheKeyInput): string => includeTypeScript: input.includeTypeScript ?? true, }); +const manifestCachePayload = (input: ToolSchemaViewManifestCacheKeyInput): string => + cacheKeyPayload({ + version: TOOL_SCHEMA_VIEW_CACHE_VERSION, + typeScriptPreviewCacheVersion: TOOL_TYPESCRIPT_PREVIEW_CACHE_VERSION, + typeScriptPreviewCompilerVersion: TOOL_TYPESCRIPT_PREVIEW_COMPILER_VERSION, + address: input.address, + indexFingerprint: input.indexFingerprint, + fingerprintVersion: input.fingerprintVersion, + includeTypeScript: input.includeTypeScript ?? true, + }); + export const toolSchemaViewCacheKey = (input: ToolSchemaViewCacheKeyInput): Effect.Effect => sha256Hex(cachePayload(input)).pipe( Effect.map( (hash) => `${TOOL_SCHEMA_VIEW_CACHE_PREFIX}${TOOL_SCHEMA_VIEW_CACHE_VERSION}/${hash}`, ), ); + +export const toolSchemaViewManifestCacheKey = ( + input: ToolSchemaViewManifestCacheKeyInput, +): Effect.Effect => + sha256Hex(manifestCachePayload(input)).pipe( + Effect.map( + (hash) => + `${TOOL_SCHEMA_VIEW_CACHE_PREFIX}${TOOL_SCHEMA_VIEW_CACHE_VERSION}/manifest/${hash}`, + ), + ); diff --git a/packages/plugins/semantic-search/src/sdk/chunker.test.ts b/packages/plugins/semantic-search/src/sdk/chunker.test.ts index 09627b6c04..cecc72ac88 100644 --- a/packages/plugins/semantic-search/src/sdk/chunker.test.ts +++ b/packages/plugins/semantic-search/src/sdk/chunker.test.ts @@ -29,7 +29,7 @@ When creating a subscription, you must specify at least one item referencing a r Subscriptions support a rich set of lifecycle features: free trial periods via trial_period_days or trial_end, proration when upgrading or downgrading mid-cycle, metered billing via usage records, and multiple tax rates. You can also attach metadata, set a custom billing anchor, or pass coupon and promotion codes at creation time. After creation the subscription moves through statuses: incomplete, trialing, active, past_due, unpaid, and canceled. Webhooks such as customer.subscription.created and invoice.payment_succeeded are the recommended integration path for reacting to these transitions reliably.`, - inputTypeScript: `{ + inputSchemaText: `{ customer: string; items: Array<{ price: string; @@ -42,7 +42,7 @@ After creation the subscription moves through statuses: incomplete, trialing, ac promotion_code?: string; metadata?: Record; }`, - outputTypeScript: `{ + outputSchemaText: `{ id: string; object: "subscription"; customer: string; @@ -113,7 +113,7 @@ describe("makeFacetChunker", () => { } }); - it("tool with inputTypeScript and outputTypeScript β†’ has input and output facets", () => { + it("tool with inputSchemaText and outputSchemaText β†’ has input and output facets", () => { const chunker = makeFacetChunker(); const chunks = chunker.chunk("ns", stripeSubscriptionsDoc); @@ -132,11 +132,11 @@ describe("makeFacetChunker", () => { expect(outputChunks[0]!.embeddingText).toContain("status:"); }); - it("trivial outputTypeScript β†’ no output facet", () => { + it("trivial outputSchemaText β†’ no output facet", () => { const chunker = makeFacetChunker(); const doc: ToolDocumentInput = { ...stripeSubscriptionsDoc, - outputTypeScript: "{}", + outputSchemaText: "{}", }; const chunks = chunker.chunk("ns", doc); expect(chunks.filter((c) => c.facet === "output")).toHaveLength(0); diff --git a/packages/plugins/semantic-search/src/sdk/chunker.ts b/packages/plugins/semantic-search/src/sdk/chunker.ts index c99ea42dc4..6b8a4bc92d 100644 --- a/packages/plugins/semantic-search/src/sdk/chunker.ts +++ b/packages/plugins/semantic-search/src/sdk/chunker.ts @@ -1,3 +1,5 @@ +import { Schema } from "effect"; + // --------------------------------------------------------------------------- // Facet-aware chunker (P1). A tool becomes one OR MORE chunks, each targeting // a distinct semantic facet: identity (header + description lead), description @@ -34,23 +36,23 @@ const cyrb53 = (str: string): string => { // --------------------------------------------------------------------------- /** The input representation of a tool used by the chunker β€” richer than the - * core Tool contract, because it includes the schema strings produced by - * tools.schema(). */ -export interface ToolDocumentInput { - readonly path: string; - readonly name: string; - readonly integration: string; - readonly description: string; - readonly inputTypeScript?: string; - readonly outputTypeScript?: string; - readonly typeScriptDefinitions?: Record; + * core Tool contract, because it includes compact schema facet text. */ +export const ToolDocumentInput = Schema.Struct({ + path: Schema.String, + name: Schema.String, + integration: Schema.String, + description: Schema.String, + inputSchemaText: Schema.optional(Schema.String), + outputSchemaText: Schema.optional(Schema.String), + schemaDefinitionText: Schema.optional(Schema.String), /** * Broad single-string representation of the tool for FTS indexing. * Populated by the projector (`buildLexicalText` in documents.ts); absent * when the input comes from a source that does not populate it. */ - readonly lexicalText?: string; -} + lexicalText: Schema.optional(Schema.String), +}); +export type ToolDocumentInput = typeof ToolDocumentInput.Type; /** Which semantic facet a chunk represents. */ export type ChunkFacet = "identity" | "description" | "input" | "output"; @@ -259,8 +261,8 @@ export const makeFacetChunker = (options?: FacetChunkerOptions): Chunker => { // ----------------------------------------------------------------------- // Input facet // ----------------------------------------------------------------------- - if (!isTrivialSchema(doc.inputTypeScript)) { - const text = `${integration} ${path} input\n${doc.inputTypeScript}`; + if (!isTrivialSchema(doc.inputSchemaText)) { + const text = `${integration} ${path} input\n${doc.inputSchemaText}`; // If the schema exceeds the budget, keep the structural head (entire // text is preferable; hard-slicing is safer than silently losing it). const embeddingText = text.length <= budget ? text : text.slice(0, budget); @@ -279,8 +281,8 @@ export const makeFacetChunker = (options?: FacetChunkerOptions): Chunker => { // ----------------------------------------------------------------------- // Output facet // ----------------------------------------------------------------------- - if (!isTrivialSchema(doc.outputTypeScript)) { - const text = `${integration} ${path} output\n${doc.outputTypeScript}`; + if (!isTrivialSchema(doc.outputSchemaText)) { + const text = `${integration} ${path} output\n${doc.outputSchemaText}`; const embeddingText = text.length <= budget ? text : text.slice(0, budget); chunks.push({ id: makeId(namespace, path, "output", globalIndex), diff --git a/packages/plugins/semantic-search/src/sdk/collections.ts b/packages/plugins/semantic-search/src/sdk/collections.ts index 5c38b750a1..25e26d4fb2 100644 --- a/packages/plugins/semantic-search/src/sdk/collections.ts +++ b/packages/plugins/semantic-search/src/sdk/collections.ts @@ -69,6 +69,7 @@ export const IndexJob = Schema.Struct({ description: Schema.String, status: IndexJobStatus, fingerprint: Schema.optional(Schema.String), + oldFingerprint: Schema.optional(Schema.String), oldChunkIds: Schema.Array(Schema.String), chunkIds: Schema.Array(Schema.String), lexicalTextKey: Schema.optional(Schema.String), diff --git a/packages/plugins/semantic-search/src/sdk/documents.ts b/packages/plugins/semantic-search/src/sdk/documents.ts index f8d50ab698..5250b10c84 100644 --- a/packages/plugins/semantic-search/src/sdk/documents.ts +++ b/packages/plugins/semantic-search/src/sdk/documents.ts @@ -56,16 +56,14 @@ export const stripHtml = (s: string): string => export const buildLexicalText = (doc: ToolDocumentInput): string => { const parts: string[] = [doc.integration, doc.path, doc.name, stripHtml(doc.description)]; - if (doc.inputTypeScript !== undefined) { - parts.push(doc.inputTypeScript); + if (doc.inputSchemaText !== undefined) { + parts.push(doc.inputSchemaText); } - if (doc.outputTypeScript !== undefined) { - parts.push(doc.outputTypeScript); + if (doc.outputSchemaText !== undefined) { + parts.push(doc.outputSchemaText); } - if (doc.typeScriptDefinitions !== undefined) { - for (const def of Object.values(doc.typeScriptDefinitions)) { - parts.push(def); - } + if (doc.schemaDefinitionText !== undefined) { + parts.push(doc.schemaDefinitionText); } return parts.join(" Β· "); @@ -160,11 +158,70 @@ export const listToolManifests = ( Effect.map((manifests) => selectToolManifests(manifests, options)), ); +const isPlainRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +const collectSchemaTerms = (value: unknown, terms: Set, depth = 0): void => { + if (depth > 12 || terms.size >= 500) return; + if (Array.isArray(value)) { + for (const item of value) collectSchemaTerms(item, terms, depth + 1); + return; + } + if (!isPlainRecord(value)) return; + + for (const [key, entry] of Object.entries(value)) { + if (terms.size >= 500) return; + if (key === "properties" && isPlainRecord(entry)) { + for (const property of Object.keys(entry)) terms.add(property); + collectSchemaTerms(entry, terms, depth + 1); + continue; + } + if (key === "required" && Array.isArray(entry)) { + for (const required of entry) { + if (typeof required === "string") terms.add(`required ${required}`); + } + continue; + } + if ((key === "type" || key === "format" || key === "title") && typeof entry === "string") { + terms.add(`${key} ${entry}`); + continue; + } + if (key === "enum" && Array.isArray(entry)) { + for (const item of entry) { + if (typeof item === "string" && item.length <= 80) terms.add(item); + } + continue; + } + if (key === "description" && typeof entry === "string") { + const description = stripHtml(entry); + if (description.length > 0) terms.add(description.slice(0, 240)); + continue; + } + collectSchemaTerms(entry, terms, depth + 1); + } +}; + +const schemaFacetText = (label: string, schema: unknown): string | undefined => { + const terms = new Set(); + collectSchemaTerms(schema, terms); + if (terms.size === 0) return undefined; + return `${label} ${Array.from(terms).join(" ")}`; +}; + +const schemaDefinitionsFacetText = ( + definitions: Readonly> | undefined, +): string | undefined => { + if (definitions === undefined || Object.keys(definitions).length === 0) return undefined; + const terms = new Set(Object.keys(definitions)); + for (const definition of Object.values(definitions)) collectSchemaTerms(definition, terms); + return terms.size === 0 ? undefined : `definitions ${Array.from(terms).join(" ")}`; +}; + /** Collect `ToolDocumentInput` for a single tool descriptor. * - * Attempts to fetch its schema via `tools.schema(address)` to populate the - * TypeScript facets; on failure it degrades gracefully to an identity-only - * document. + * Attempts to fetch raw schema via `tools.schema(address, { includeTypeScript: + * false })` to populate compact schema facets; on failure it degrades + * gracefully to an identity-only document. */ export const collectDocForTool = ( executor: Executor, @@ -177,23 +234,19 @@ export const collectDocForTool = ( integration: String(tool.integration), description: stripHtml(String(tool.description ?? "")), }; - return executor.tools.schema(tool.address as Tool["address"]).pipe( + return executor.tools.schema(tool.address as Tool["address"], { includeTypeScript: false }).pipe( Effect.map((view): ToolDocumentInput => { - const doc: ToolDocumentInput = - view === null - ? base - : { - ...base, - ...(view.inputTypeScript !== undefined - ? { inputTypeScript: view.inputTypeScript } - : {}), - ...(view.outputTypeScript !== undefined - ? { outputTypeScript: view.outputTypeScript } - : {}), - ...(view.typeScriptDefinitions !== undefined - ? { typeScriptDefinitions: view.typeScriptDefinitions } - : {}), - }; + if (view === null) return { ...base, lexicalText: buildLexicalText(base) }; + + const inputSchemaText = schemaFacetText("input", view.inputSchema); + const outputSchemaText = schemaFacetText("output", view.outputSchema); + const schemaDefinitionText = schemaDefinitionsFacetText(view.schemaDefinitions); + const doc: ToolDocumentInput = { + ...base, + ...(inputSchemaText !== undefined ? { inputSchemaText } : {}), + ...(outputSchemaText !== undefined ? { outputSchemaText } : {}), + ...(schemaDefinitionText !== undefined ? { schemaDefinitionText } : {}), + }; return { ...doc, lexicalText: buildLexicalText(doc) }; }), // Degrade: schema fetch failed β€” use identity-only document. diff --git a/packages/plugins/semantic-search/src/sdk/eval.ts b/packages/plugins/semantic-search/src/sdk/eval.ts index c57d770d21..d29ccb9d88 100644 --- a/packages/plugins/semantic-search/src/sdk/eval.ts +++ b/packages/plugins/semantic-search/src/sdk/eval.ts @@ -72,7 +72,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "google", description: "Creates an event on the specified calendar. The event can include a title, description, start/end time, attendees, location, and recurrence rules.", - inputTypeScript: `{ + inputSchemaText: `{ calendarId: string; // 'primary' or a calendar id summary: string; // event title description?: string; @@ -82,7 +82,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ location?: string; recurrence?: string[]; // RRULE strings }`, - outputTypeScript: `{ + outputSchemaText: `{ id: string; htmlLink: string; status: 'confirmed' | 'tentative' | 'cancelled'; @@ -97,12 +97,12 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "google", description: "Creates an event based on a simple text string such as 'Appointment at Somewhere on June 3rd 10am-10:25am'. Google parses the natural-language text to populate the event fields.", - inputTypeScript: `{ + inputSchemaText: `{ calendarId: string; text: string; // natural-language event description sendUpdates?: 'all' | 'externalOnly' | 'none'; }`, - outputTypeScript: `{ + outputSchemaText: `{ id: string; htmlLink: string; summary: string; @@ -116,12 +116,12 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "google", description: "Deletes an event from a calendar. Once deleted the event is removed from the calendar view. For recurring events you can target a single instance or the entire series.", - inputTypeScript: `{ + inputSchemaText: `{ calendarId: string; eventId: string; sendUpdates?: 'all' | 'externalOnly' | 'none'; }`, - outputTypeScript: "void", + outputSchemaText: "void", }, { path: "tools.google.org.x.gmail.users.messages.send", @@ -129,12 +129,12 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "google", description: "Sends the specified message to the recipients in the To, Cc, and Bcc headers. The message body is supplied as a base64url-encoded RFC 2822 email string.", - inputTypeScript: `{ + inputSchemaText: `{ userId: string; // 'me' for the authenticated user raw: string; // base64url-encoded RFC 2822 message threadId?: string; // supply to reply in an existing thread }`, - outputTypeScript: `{ + outputSchemaText: `{ id: string; threadId: string; labelIds: string[]; @@ -146,7 +146,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "github", description: "Create an issue in a repository. Any user with pull access to a repository can create an issue. Issues can include a title, body, labels, assignees, and milestone.", - inputTypeScript: `{ + inputSchemaText: `{ owner: string; repo: string; title: string; @@ -155,7 +155,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ assignees?: string[]; milestone?: number; }`, - outputTypeScript: `{ + outputSchemaText: `{ id: number; number: number; html_url: string; @@ -169,7 +169,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "github", description: "Create a pull request to propose and collaborate on changes to a repository. The pull request includes a title, body, head branch, and base branch to merge into.", - inputTypeScript: `{ + inputSchemaText: `{ owner: string; repo: string; title: string; @@ -178,7 +178,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ body?: string; draft?: boolean; }`, - outputTypeScript: `{ + outputSchemaText: `{ id: number; number: number; html_url: string; @@ -193,8 +193,8 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "github", description: "Gets a repository's metadata, including its description, default branch, visibility, star count, and clone URL.", - inputTypeScript: `{ owner: string; repo: string }`, - outputTypeScript: `{ + inputSchemaText: `{ owner: string; repo: string }`, + outputSchemaText: `{ id: number; name: string; full_name: string; @@ -210,7 +210,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "stripe", description: "To charge a credit or debit card, you create a Charge object. Stripe attempts to immediately collect funds from the customer's payment source. Pass amount in the smallest currency unit (e.g. cents for USD).", - inputTypeScript: `{ + inputSchemaText: `{ amount: number; // in smallest currency unit (e.g. cents) currency: string; // ISO 4217 code, e.g. 'usd' source?: string; // card token or payment method id @@ -219,7 +219,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ metadata?: Record; capture?: boolean; // default true; false for auth-only }`, - outputTypeScript: `{ + outputSchemaText: `{ id: string; amount: number; currency: string; @@ -234,14 +234,14 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "stripe", description: "Refund a charge that has previously been created but not yet refunded. Funds will be refunded to the credit or debit card that was originally charged. You can refund the full amount or a partial amount.", - inputTypeScript: `{ + inputSchemaText: `{ charge?: string; // charge id to refund payment_intent?: string; // or payment_intent id amount?: number; // omit to refund the full charge reason?: 'duplicate' | 'fraudulent' | 'requested_by_customer'; metadata?: Record; }`, - outputTypeScript: `{ + outputSchemaText: `{ id: string; amount: number; currency: string; @@ -255,7 +255,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "stripe", description: "Creates a new customer object you can later charge or attach payment methods to. Customer objects allow you to perform recurring charges, track multiple payments, and store payment details.", - inputTypeScript: `{ + inputSchemaText: `{ email?: string; name?: string; phone?: string; @@ -263,7 +263,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ payment_method?: string; metadata?: Record; }`, - outputTypeScript: `{ + outputSchemaText: `{ id: string; email: string | null; name: string | null; @@ -277,14 +277,14 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "slack", description: "Sends a message to a channel. The channel can be a public channel, private channel, or direct message. Supports rich formatting via Block Kit or simple markdown text.", - inputTypeScript: `{ + inputSchemaText: `{ channel: string; // channel ID or name text?: string; // plain text fallback blocks?: unknown[]; // Block Kit blocks thread_ts?: string; // reply in a thread mrkdwn?: boolean; }`, - outputTypeScript: `{ + outputSchemaText: `{ ok: boolean; channel: string; ts: string; // message timestamp (also the message id) @@ -297,7 +297,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ integration: "linear", description: "Create a new issue in a Linear team, with title, description, assignee, and priority. Issues track work items and bugs within a team's backlog.", - inputTypeScript: `{ + inputSchemaText: `{ teamId: string; title: string; description?: string; // markdown @@ -306,7 +306,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ labelIds?: string[]; stateId?: string; }`, - outputTypeScript: `{ + outputSchemaText: `{ id: string; identifier: string; // e.g. 'ENG-123' title: string; @@ -320,7 +320,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ name: "subscriptions.create", integration: "stripe", description: SUBSCRIPTION_DESC, - inputTypeScript: `{ + inputSchemaText: `{ customer: string; items: Array<{ price: string; quantity?: number }>; trial_period_days?: number; @@ -332,7 +332,7 @@ const SAMPLE: readonly ToolDocumentInput[] = [ billing_cycle_anchor?: number | 'now'; metadata?: Record; }`, - outputTypeScript: `{ + outputSchemaText: `{ id: string; status: 'active' | 'trialing' | 'past_due' | 'canceled' | 'unpaid' | 'incomplete'; current_period_start: number; @@ -429,8 +429,8 @@ const loadRealCatalog = (): readonly ToolDocumentInput[] => { name: row.name, integration: row.integration, description: stripHtml((colToString(row.description as string | Buffer | null) ?? "").trim()), - inputTypeScript: inputTs ? inputTs.slice(0, 1500) : undefined, - outputTypeScript: outputTs ? outputTs.slice(0, 1500) : undefined, + inputSchemaText: inputTs ? inputTs.slice(0, 1500) : undefined, + outputSchemaText: outputTs ? outputTs.slice(0, 1500) : undefined, }; tools.push({ ...doc, lexicalText: buildLexicalText(doc) }); diff --git a/packages/plugins/semantic-search/src/sdk/plugin.ts b/packages/plugins/semantic-search/src/sdk/plugin.ts index cdd3b2dc2c..3afa876b5b 100644 --- a/packages/plugins/semantic-search/src/sdk/plugin.ts +++ b/packages/plugins/semantic-search/src/sdk/plugin.ts @@ -185,12 +185,13 @@ const makeSemanticSearchExtension = (deps: { { readonly namespace: string; readonly removed: number }, SemanticSearchError > => - deps.store && deps.fingerprints && deps.owner + deps.store && deps.fingerprints && deps.blobs && deps.owner ? sweepRemoved({ namespace: deps.namespace, executor, store: deps.store, fingerprints: deps.fingerprints, + blobs: deps.blobs, owner: deps.owner, lexicalStore: deps.lexicalStore, }) diff --git a/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.test.ts b/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.test.ts index 8cc8334b97..dae7238a04 100644 --- a/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.test.ts +++ b/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.test.ts @@ -114,6 +114,39 @@ describe("withCloudflareLimits", () => { ); }); + describe("upsert β€” metadata byte-length guard", () => { + it.effect("rejects metadata over 10 KiB", () => + Effect.gen(function* () { + const { store, state } = makeFakeStore(); + const wrapped = withCloudflareLimits(store); + const metadata = { description: "x".repeat(10 * 1024) }; + expect(Buffer.byteLength(JSON.stringify(metadata), "utf8")).toBeGreaterThan(10 * 1024); + + const exit = yield* wrapped + .upsert([{ id: "valid-id", values: [1, 0, 0], metadata }]) + .pipe(Effect.exit); + + expect(Exit.isFailure(exit)).toBe(true); + expect(state.upsertCalls).toHaveLength(0); + }), + ); + + it.effect("passes through metadata within the 10 KiB limit", () => + Effect.gen(function* () { + const { store, state } = makeFakeStore(); + const wrapped = withCloudflareLimits(store); + const metadata = { description: "x".repeat(1_024) }; + + const exit = yield* wrapped + .upsert([{ id: "valid-id", values: [1, 0, 0], metadata }]) + .pipe(Effect.exit); + + expect(Exit.isSuccess(exit)).toBe(true); + expect(state.upsertCalls).toHaveLength(1); + }), + ); + }); + describe("query β€” topK cap guard", () => { it.effect("rejects topK = 21 (one over the metadata-all cap)", () => Effect.gen(function* () { diff --git a/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.ts b/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.ts index 8126e87468..784088e468 100644 --- a/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.ts +++ b/packages/plugins/semantic-search/src/sdk/store-cloudflare-limits.ts @@ -12,6 +12,9 @@ import type { VectorStore, VectorInput } from "./store"; // β€’ id: max 64 bytes (UTF-8 encoded). The raw `${namespace}#${path}` // format used before the facet chunker could exceed this for long // OpenAPI paths β€” a failure found by deploying, not testing. +// β€’ metadata: max 10 KiB per vector. Tool descriptions from provider specs +// can exceed this by themselves, so callers must store bounded +// metadata and keep full text in the blob/document stores. // β€’ topK: max 20 when `returnMetadata` is "all". `makeVectorizeStore` always // passes `returnMetadata: "all"`, so the query path is capped at 20. // We reject rather than clamp so the caller is forced to stay within @@ -19,6 +22,7 @@ import type { VectorStore, VectorInput } from "./store"; // --------------------------------------------------------------------------- const MAX_ID_BYTES = 64; +const MAX_METADATA_BYTES = 10 * 1024; const MAX_QUERY_TOP_K = 20; // UTF-8 byte length via the Web-standard TextEncoder β€” available in the @@ -26,6 +30,11 @@ const MAX_QUERY_TOP_K = 20; // `Buffer` would require. const utf8ByteLength = (value: string): number => new TextEncoder().encode(value).length; +const metadataByteLength = (value: Record | undefined): number => { + if (value === undefined) return 0; + return utf8ByteLength(JSON.stringify(value)); +}; + /** Wraps `inner` with Cloudflare Vectorize runtime-limit validation. Any call * that would violate a limit fails immediately with `SemanticSearchError` * before the inner store is touched. All other calls pass through unchanged. @@ -42,6 +51,16 @@ export const withCloudflareLimits = (inner: VectorStore): VectorStore => ({ }), ); } + const oversizedMetadata = vectors.find( + (v) => metadataByteLength(v.metadata) > MAX_METADATA_BYTES, + ); + if (oversizedMetadata !== undefined) { + return Effect.fail( + new SemanticSearchError({ + message: `Vector metadata exceeds the Cloudflare Vectorize 10 KiB limit (${metadataByteLength(oversizedMetadata.metadata)} bytes): "${oversizedMetadata.id}".`, + }), + ); + } return inner.upsert(vectors); }, diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts index c9805b7bdf..ed0a86c941 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -51,12 +51,15 @@ const manifestForTool = ( fingerprintVersion: "tool-schema-manifest/v1", }); -const makeExecutor = (counters: { raw: number; codegen: number }): Executor => { +const makeExecutor = ( + counters: { raw: number; codegen: number }, + options?: { readonly description?: string }, +): Executor => { const tool: Tool = { address: "tools.github.repos.get" as never, name: "repos.get" as never, integration: "github" as never, - description: "Get a repository", + description: options?.description ?? "Get a repository", owner, connection: "default" as never, pluginId: "test", @@ -156,18 +159,42 @@ describe("ToolSearchIndex", () => { it.effect("scans, chunks, embeds, and commits changed tools", () => Effect.gen(function* () { const counters = { raw: 0, codegen: 0 }; - const executor = makeExecutor(counters); + const executor = makeExecutor(counters, { description: "Get a repository ".repeat(700) }); const runs = makeCollection(indexRuns.name); const jobs = makeCollection(indexJobs.name); const chunks = makeCollection(indexChunks.name); const fingerprints = makeCollection(toolFingerprints.name); const blobs = makeBlobs(); + const path = "github.repos.get"; + const oldFingerprint = "fingerprint:old"; + const nextFingerprint = "fingerprint:tools.github.repos.get"; + yield* fingerprints.put({ + owner, + key: path, + data: { + path, + integration: "github", + fingerprint: oldFingerprint, + chunkIds: ["old-chunk"], + }, + }); + yield* blobs.put( + `semantic-search/index/document/v1/${oldFingerprint}.json`, + JSON.stringify({ + path, + name: "repos.get", + integration: "github", + description: "old", + }), + { owner }, + ); const upserted: VectorInput[] = []; + const deleted: string[][] = []; const store: VectorStore = { maxTopK: 100, query: () => Effect.succeed([]), upsert: (vectors) => Effect.sync(() => void upserted.push(...vectors)), - deleteByIds: () => Effect.void, + deleteByIds: (ids) => Effect.sync(() => void deleted.push([...ids])), }; const embedder: ToolEmbedder = { model: "test", @@ -201,14 +228,14 @@ describe("ToolSearchIndex", () => { runId: "run-1", chunkRefs: chunked.chunkRefs, }); - for (const path of embedded.paths) { + if (embedded.paths.length > 0) { yield* commit({ ...base, embedder, store, chunker: makeFacetChunker(), runId: "run-1", - path, + paths: embedded.paths, }); } @@ -216,9 +243,22 @@ describe("ToolSearchIndex", () => { expect(chunked.processed).toBe(1); expect(chunked.chunks).toBeGreaterThan(0); expect(embedded).toMatchObject({ processed: chunked.chunks, chunks: chunked.chunks }); - expect(counters).toEqual({ raw: 0, codegen: 1 }); + expect(counters).toEqual({ raw: 1, codegen: 0 }); expect(upserted).toHaveLength(chunked.chunks); + for (const vector of upserted) { + expect( + Buffer.byteLength(String(vector.metadata?.description ?? ""), "utf8"), + ).toBeLessThanOrEqual(2_048); + } expect([...fingerprints.data.values()]).toHaveLength(1); + expect(fingerprints.data.get(path)?.fingerprint).toBe(nextFingerprint); + expect(yield* blobs.has(`semantic-search/index/document/v1/${oldFingerprint}.json`)).toBe( + false, + ); + expect(yield* blobs.has(`semantic-search/index/document/v1/${nextFingerprint}.json`)).toBe( + true, + ); + expect(deleted).toEqual([["old-chunk"]]); expect([...jobs.data.values()][0]?.status).toBe("indexed"); }), ); @@ -308,60 +348,62 @@ describe("ToolSearchIndex", () => { }), ); - it.effect("reconstructs embed messages when a chunk message retries after persisting chunks", () => - Effect.gen(function* () { - const counters = { raw: 0, codegen: 0 }; - const executor = makeExecutor(counters); - const runs = makeCollection(indexRuns.name); - const jobs = makeCollection(indexJobs.name); - const chunks = makeCollection(indexChunks.name); - const fingerprints = makeCollection(toolFingerprints.name); - const blobs = makeBlobs(); - const store: VectorStore = { - maxTopK: 100, - query: () => Effect.succeed([]), - upsert: () => Effect.void, - deleteByIds: () => Effect.void, - }; - const embedder: ToolEmbedder = { - model: "test", - dimensions: 3, - embedDocuments: (texts) => Effect.succeed(texts.map(() => [0.1, 0.2, 0.3])), - embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), - }; - const base = { namespace, executor, runs, jobs, chunks, fingerprints, blobs, owner }; - - yield* create({ ...base, runId: "run-chunk-retry", partitionCount: 1 }); - const scanned = yield* scan({ - ...base, - runId: "run-chunk-retry", - partition: 0, - limit: 10, - }); - const first = yield* chunk({ - ...base, - embedder, - store, - chunker: makeFacetChunker(), - runId: "run-chunk-retry", - paths: scanned.paths, - limit: 10, - }); - const retry = yield* chunk({ - ...base, - embedder, - store, - chunker: makeFacetChunker(), - runId: "run-chunk-retry", - paths: scanned.paths, - limit: 10, - }); + it.effect( + "reconstructs embed messages when a chunk message retries after persisting chunks", + () => + Effect.gen(function* () { + const counters = { raw: 0, codegen: 0 }; + const executor = makeExecutor(counters); + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + const store: VectorStore = { + maxTopK: 100, + query: () => Effect.succeed([]), + upsert: () => Effect.void, + deleteByIds: () => Effect.void, + }; + const embedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => Effect.succeed(texts.map(() => [0.1, 0.2, 0.3])), + embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), + }; + const base = { namespace, executor, runs, jobs, chunks, fingerprints, blobs, owner }; + + yield* create({ ...base, runId: "run-chunk-retry", partitionCount: 1 }); + const scanned = yield* scan({ + ...base, + runId: "run-chunk-retry", + partition: 0, + limit: 10, + }); + const first = yield* chunk({ + ...base, + embedder, + store, + chunker: makeFacetChunker(), + runId: "run-chunk-retry", + paths: scanned.paths, + limit: 10, + }); + const retry = yield* chunk({ + ...base, + embedder, + store, + chunker: makeFacetChunker(), + runId: "run-chunk-retry", + paths: scanned.paths, + limit: 10, + }); - expect(first.chunkRefs.length).toBeGreaterThan(0); - expect(retry.chunkRefs).toEqual(first.chunkRefs); - expect(retry.commitPaths).toEqual([]); - expect(counters.codegen).toBe(1); - }), + expect(first.chunkRefs.length).toBeGreaterThan(0); + expect(retry.chunkRefs).toEqual(first.chunkRefs); + expect(retry.commitPaths).toEqual([]); + expect(counters).toEqual({ raw: 1, codegen: 0 }); + }), ); it.effect("limits seeded tools when maxTools is provided", () => @@ -547,7 +589,7 @@ describe("ToolSearchIndex", () => { chunkRefs: job.chunkIds.map((chunkId) => ({ path: job.path, chunkId })), maxChunks: 10, }); - const committed = yield* commit({ ...base, runId: job.runId, path: job.path }); + const committed = yield* commit({ ...base, runId: job.runId, paths: [job.path] }); const afterCommitJob = jobs.data.get(`${job.runId}:${job.path}`); expect(first).toMatchObject({ processed: 2, chunks: 2 }); @@ -555,7 +597,7 @@ describe("ToolSearchIndex", () => { expect(firstChunkStatuses.filter((status) => status === "indexed")).toHaveLength(2); expect(firstChunkStatuses.filter((status) => status === "pendingEmbedding")).toHaveLength(1); expect(second).toMatchObject({ processed: 1, chunks: 1 }); - expect(committed.committed).toBe(true); + expect(committed.committed).toBe(1); expect(afterCommitJob?.status).toBe("indexed"); expect(mutableEmbeddedGroups.map((group) => group.length)).toEqual([2, 1]); expect(upserted).toHaveLength(3); @@ -563,109 +605,111 @@ describe("ToolSearchIndex", () => { }), ); - it.effect("reconstructs commit messages when an embed message retries after indexing chunks", () => - Effect.gen(function* () { - const runs = makeCollection(indexRuns.name); - const jobs = makeCollection(indexJobs.name); - const chunks = makeCollection(indexChunks.name); - const fingerprints = makeCollection(toolFingerprints.name); - const blobs = makeBlobs(); - let embedCalls = 0; - const store: VectorStore = { - maxTopK: 100, - query: () => Effect.succeed([]), - upsert: () => Effect.void, - deleteByIds: () => Effect.void, - }; - const embedder: ToolEmbedder = { - model: "test", - dimensions: 3, - embedDocuments: (texts) => - Effect.sync(() => { - embedCalls++; - return texts.map(() => [0.1, 0.2, 0.3]); - }), - embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), - }; - - const createdAt = new Date(0).toISOString(); - const lexicalTextKey = "test/lexical-retry.txt"; - const job: IndexJob = { - runId: "run-embed-retry", - namespace, - partition: 0, - ordinal: 0, - address: "tools.github.repos.get", - path: "github.repos.get", - name: "repos.get", - integration: "github", - description: "Get a repository", - status: "pendingEmbedding", - fingerprint: "fp-embed-retry", - oldChunkIds: [], - chunkIds: ["chunk-0"], - lexicalTextKey, - createdAt, - updatedAt: createdAt, - }; - const chunkRow: IndexChunk = { - runId: job.runId, - namespace, - partition: 0, - path: job.path, - chunkId: "chunk-0", - facet: "description", - chunkIndex: 0, - embeddingTextKey: "test/chunk-retry.txt", - embeddingTextBytes: "chunk retry text".length, - embeddingTextTokens: 4, - name: job.name, - integration: job.integration, - description: job.description, - status: "pendingEmbedding", - createdAt, - updatedAt: createdAt, - }; - yield* blobs.put(lexicalTextKey, "github github.repos.get repos.get", { owner }); - yield* blobs.put(chunkRow.embeddingTextKey, "chunk retry text", { owner }); - yield* jobs.put({ owner, key: `${job.runId}:${job.path}`, data: job }); - yield* chunks.put({ - owner, - key: `${chunkRow.runId}:${chunkRow.path}:${chunkRow.chunkId}`, - data: chunkRow, - }); + it.effect( + "reconstructs commit messages when an embed message retries after indexing chunks", + () => + Effect.gen(function* () { + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + let embedCalls = 0; + const store: VectorStore = { + maxTopK: 100, + query: () => Effect.succeed([]), + upsert: () => Effect.void, + deleteByIds: () => Effect.void, + }; + const embedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => + Effect.sync(() => { + embedCalls++; + return texts.map(() => [0.1, 0.2, 0.3]); + }), + embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), + }; + + const createdAt = new Date(0).toISOString(); + const lexicalTextKey = "test/lexical-retry.txt"; + const job: IndexJob = { + runId: "run-embed-retry", + namespace, + partition: 0, + ordinal: 0, + address: "tools.github.repos.get", + path: "github.repos.get", + name: "repos.get", + integration: "github", + description: "Get a repository", + status: "pendingEmbedding", + fingerprint: "fp-embed-retry", + oldChunkIds: [], + chunkIds: ["chunk-0"], + lexicalTextKey, + createdAt, + updatedAt: createdAt, + }; + const chunkRow: IndexChunk = { + runId: job.runId, + namespace, + partition: 0, + path: job.path, + chunkId: "chunk-0", + facet: "description", + chunkIndex: 0, + embeddingTextKey: "test/chunk-retry.txt", + embeddingTextBytes: "chunk retry text".length, + embeddingTextTokens: 4, + name: job.name, + integration: job.integration, + description: job.description, + status: "pendingEmbedding", + createdAt, + updatedAt: createdAt, + }; + yield* blobs.put(lexicalTextKey, "github github.repos.get repos.get", { owner }); + yield* blobs.put(chunkRow.embeddingTextKey, "chunk retry text", { owner }); + yield* jobs.put({ owner, key: `${job.runId}:${job.path}`, data: job }); + yield* chunks.put({ + owner, + key: `${chunkRow.runId}:${chunkRow.path}:${chunkRow.chunkId}`, + data: chunkRow, + }); - const base = { - namespace, - executor: makeExecutor({ raw: 0, codegen: 0 }), - runs, - jobs, - chunks, - fingerprints, - blobs, - owner, - embedder, - store, - chunker: makeFacetChunker(), - }; - const first = yield* embed({ - ...base, - runId: job.runId, - chunkRefs: [{ path: job.path, chunkId: "chunk-0" }], - }); - const retry = yield* embed({ - ...base, - runId: job.runId, - chunkRefs: [{ path: job.path, chunkId: "chunk-0" }], - }); - const committed = yield* commit({ ...base, runId: job.runId, path: job.path }); + const base = { + namespace, + executor: makeExecutor({ raw: 0, codegen: 0 }), + runs, + jobs, + chunks, + fingerprints, + blobs, + owner, + embedder, + store, + chunker: makeFacetChunker(), + }; + const first = yield* embed({ + ...base, + runId: job.runId, + chunkRefs: [{ path: job.path, chunkId: "chunk-0" }], + }); + const retry = yield* embed({ + ...base, + runId: job.runId, + chunkRefs: [{ path: job.path, chunkId: "chunk-0" }], + }); + const committed = yield* commit({ ...base, runId: job.runId, paths: [job.path] }); - expect(first).toMatchObject({ processed: 1, chunks: 1, paths: [job.path] }); - expect(retry).toMatchObject({ processed: 0, chunks: 0, paths: [job.path] }); - expect(committed.committed).toBe(true); - expect(embedCalls).toBe(1); - expect(jobs.data.get(`${job.runId}:${job.path}`)?.status).toBe("indexed"); - }), + expect(first).toMatchObject({ processed: 1, chunks: 1, paths: [job.path] }); + expect(retry).toMatchObject({ processed: 0, chunks: 0, paths: [job.path] }); + expect(committed.committed).toBe(1); + expect(embedCalls).toBe(1); + expect(jobs.data.get(`${job.runId}:${job.path}`)?.status).toBe("indexed"); + }), ); it.effect("commits zero-chunk embed jobs so fingerprints can warm future runs", () => @@ -721,10 +765,10 @@ describe("ToolSearchIndex", () => { store, chunker: makeFacetChunker(), runId: job.runId, - path: job.path, + paths: [job.path], }); - expect(result).toMatchObject({ committed: true }); + expect(result).toMatchObject({ committed: 1 }); expect(jobs.data.get(`${job.runId}:${job.path}`)?.status).toBe("indexed"); expect(fingerprints.data.get(job.path)).toMatchObject({ path: job.path, diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts index 3d4c312f84..d2f068c555 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts @@ -5,9 +5,9 @@ import type { PluginStorageCollectionFacade, } from "@executor-js/sdk/core"; import { sha256Hex } from "@executor-js/sdk/core"; -import { Context, Effect, Predicate } from "effect"; +import { Context, Effect, Predicate, Schema } from "effect"; -import type { Chunker, ToolChunk } from "./chunker"; +import { type Chunker, type ToolChunk, ToolDocumentInput } from "./chunker"; import { type FingerprintRow, type IndexChunk, @@ -124,13 +124,14 @@ export declare namespace ToolSearchIndex { export interface CommitInput { readonly runId: string; - readonly path: string; + readonly paths: readonly string[]; } export interface CommitResult { readonly runId: string; - readonly path: string; - readonly committed: boolean; + readonly processed: number; + readonly committed: number; + readonly paths: readonly string[]; } export interface CompleteInput { @@ -226,9 +227,25 @@ const ESTIMATED_CHARS_PER_TOKEN = 4; const ESTIMATED_RESPONSE_BYTES_PER_DIMENSION = 16; const ESTIMATED_RESPONSE_BYTES_PER_VECTOR_OVERHEAD = 512; const INDEX_STORAGE_CONCURRENCY = 2; +const VECTOR_METADATA_DESCRIPTION_BYTES = 2_048; const nowIso = (): string => new Date().toISOString(); +const truncateUtf8 = (value: string, maxBytes: number): string => { + if (maxBytes <= 0) return ""; + const encoder = new TextEncoder(); + const scratch = new Uint8Array(4); + let bytes = 0; + const chars: string[] = []; + for (const char of value) { + const { written } = encoder.encodeInto(char, scratch); + if (bytes + written > maxBytes) break; + chars.push(char); + bytes += written; + } + return chars.length === value.length ? value : chars.join(""); +}; + const partitionForPath = (path: string, partitionCount: number): number => Math.abs(cyrb53(path)) % Math.max(1, Math.floor(partitionCount)); @@ -318,6 +335,11 @@ const utf8Bytes = (text: string): number => new TextEncoder().encode(text).byteL const payloadKey = (kind: "embedding-text" | "lexical-text", digest: string): string => `semantic-search/index/${kind}/${digest}.txt`; +const indexDocumentKey = (fingerprint: string): string => + `semantic-search/index/document/v1/${fingerprint}.json`; + +const decodeToolDocument = Schema.decodeUnknownEffect(Schema.fromJsonString(ToolDocumentInput)); + const putPayloadText = ( deps: IndexCollections, kind: "embedding-text" | "lexical-text", @@ -354,6 +376,54 @@ const getPayloadText = ( ), ); +const getCachedToolDocument = ( + deps: IndexCollections, + fingerprint: string, +): Effect.Effect => { + const key = indexDocumentKey(fingerprint); + return deps.blobs.get(key).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: `Failed to load index document "${key}".`, cause }), + ), + Effect.flatMap((text) => { + if (text === null) return Effect.sync((): ToolDocumentInput | undefined => undefined); + return decodeToolDocument(text).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to decode index document "${key}".`, + cause, + }), + ), + ); + }), + ); +}; + +const putCachedToolDocument = ( + deps: IndexCollections, + fingerprint: string, + doc: ToolDocumentInput, +): Effect.Effect => + deps.blobs.put(indexDocumentKey(fingerprint), JSON.stringify(doc), { owner: deps.owner }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: `Failed to persist index document for fingerprint "${fingerprint}".`, + cause, + }), + ), + ); + +const deleteCachedToolDocument = ( + deps: Pick, + fingerprint: string | undefined, +): Effect.Effect => + fingerprint === undefined + ? Effect.void + : deps.blobs.delete(indexDocumentKey(fingerprint), { owner: deps.owner }).pipe(Effect.ignore); + const estimateEmbeddingResponseBytes = (dimensions: number): number => Math.max( 1, @@ -877,6 +947,7 @@ export const scan = ( description: manifest.description, status: "skipped", fingerprint: manifest.indexFingerprint, + oldFingerprint: storedRow.fingerprint, oldChunkIds: storedRow.chunkIds, chunkIds: storedRow.chunkIds, updatedAt, @@ -894,6 +965,7 @@ export const scan = ( description: manifest.description, status: "pendingChunk", fingerprint: manifest.indexFingerprint, + oldFingerprint: storedRow?.fingerprint, oldChunkIds: storedRow?.chunkIds ?? [], chunkIds: [], updatedAt, @@ -980,7 +1052,17 @@ export const chunk = ( (entry) => Effect.gen(function* () { const job = entry.data; - const doc = yield* collectDocForTool(input.executor, jobToDescriptor(job)); + const doc = yield* Effect.gen(function* () { + const fingerprint = job.fingerprint; + if (fingerprint === undefined) { + return yield* collectDocForTool(input.executor, jobToDescriptor(job)); + } + const cached = yield* getCachedToolDocument(input, fingerprint); + if (cached !== undefined) return cached; + const next = yield* collectDocForTool(input.executor, jobToDescriptor(job)); + yield* putCachedToolDocument(input, fingerprint, next); + return next; + }); const chunks = input.chunker.chunk(input.namespace, doc); const lexicalText = doc.lexicalText ?? buildLexicalText(doc); const lexicalTextKey = yield* putPayloadText(input, "lexical-text", lexicalText); @@ -1142,7 +1224,7 @@ export const embed = ( metadata: { path: chunk.path, name: chunk.name, - description: chunk.description, + description: truncateUtf8(chunk.description, VECTOR_METADATA_DESCRIPTION_BYTES), integration: chunk.integration, facet: chunk.facet, chunkIndex: chunk.chunkIndex, @@ -1192,23 +1274,22 @@ export const commit = ( input: IndexDeps & ToolSearchIndex.CommitInput, ): Effect.Effect => Effect.gen(function* () { - const entry = yield* input.jobs - .getForOwner({ owner: input.owner, key: jobKey(input.runId, input.path) }) - .pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ - message: `Failed to load index job "${input.path}" for commit.`, - cause, - }), - ), - ); - if (entry === null || entry.data.status !== "pendingEmbedding") { - return { runId: input.runId, path: input.path, committed: false }; + const entries = yield* getJobsByPaths(input, { + runId: input.runId, + status: "pendingEmbedding", + paths: input.paths, + }); + if (entries.length === 0) { + return { runId: input.runId, processed: 0, committed: 0, paths: [] }; } - const committed = yield* finalizeCompletedEmbedJobs(input, [entry], undefined, nowIso()); - return { runId: input.runId, path: input.path, committed: committed > 0 }; + const committed = yield* finalizeCompletedEmbedJobs(input, entries, undefined, nowIso()); + return { + runId: input.runId, + processed: entries.length, + committed, + paths: entries.map((entry) => entry.data.path), + }; }); const finalizeCompletedEmbedJobs = ( @@ -1258,6 +1339,9 @@ const finalizeCompletedEmbedJobs = ( if (job.oldChunkIds.length > 0) { yield* input.store.deleteByIds(job.oldChunkIds); } + if (job.oldFingerprint !== undefined && job.oldFingerprint !== fingerprint) { + yield* deleteCachedToolDocument(input, job.oldFingerprint); + } if (input.lexicalStore) { const lexicalText = @@ -1353,6 +1437,7 @@ export const sweepRemoved = (input: { readonly executor: Executor; readonly store: VectorStore; readonly fingerprints: PluginStorageCollectionFacade; + readonly blobs: PluginBlobStore; readonly owner: Owner; readonly lexicalStore?: FtsLexicalStore; }): Effect.Effect<{ readonly namespace: string; readonly removed: number }, SemanticSearchError> => @@ -1382,6 +1467,7 @@ export const sweepRemoved = (input: { if (input.lexicalStore) { yield* input.lexicalStore.deleteByIds([`${input.namespace}:${entry.key}`]); } + yield* deleteCachedToolDocument(input, entry.data.fingerprint); yield* input.fingerprints.remove({ owner: input.owner, key: entry.key }).pipe( Effect.mapError( (cause) => @@ -1407,6 +1493,7 @@ export const complete = ( executor: input.executor, store: input.store, fingerprints: input.fingerprints, + blobs: input.blobs, owner: input.owner, lexicalStore: input.lexicalStore, }); @@ -1462,8 +1549,8 @@ export const run = ( limit: input.pageLimit, }); let pendingChunks = [...chunked.chunkRefs]; - for (const path of chunked.commitPaths) { - yield* commit({ ...input, runId: input.runId, path }); + if (chunked.commitPaths.length > 0) { + yield* commit({ ...input, runId: input.runId, paths: chunked.commitPaths }); } while (pendingChunks.length > 0) { const embedded = yield* embed({ @@ -1478,8 +1565,8 @@ export const run = ( pendingChunks = pendingChunks.filter( (ref) => !embeddedKeys.has(`${ref.path}:${ref.chunkId}`), ); - for (const path of embedded.paths) { - yield* commit({ ...input, runId: input.runId, path }); + if (embedded.paths.length > 0) { + yield* commit({ ...input, runId: input.runId, paths: embedded.paths }); } } } From ee65498eb450884828fd051383a4199f3b357698 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sat, 20 Jun 2026 02:42:02 +0530 Subject: [PATCH 50/57] perf(semantic-search): batch index storage operations (#45) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Batches semantic-search index storage operations so queue messages do fewer plugin-storage and D1 calls while preserving the existing indexing lifecycle. ## Changes - Adds typed bulk `getMany`, `getManyForOwner`, `putMany`, and `removeMany` operations to plugin storage collections. - Reworks scan, chunk, embed, and commit indexing paths to bulk-load and bulk-write jobs, chunks, and fingerprints. - Updates semantic-search test fixtures and plugin-storage tests to cover the new bulk facade methods. ## Tests - `bun run --cwd packages/core/sdk test -- src/plugin-storage.test.ts` - `bun run --cwd packages/plugins/semantic-search test -- src/sdk/tool-search-index.test.ts` - `bun run --cwd packages/core/sdk typecheck` - `bun run --cwd packages/plugins/semantic-search typecheck` - `bun run format:check` - `oxlint -c .oxlintrc.jsonc packages/core/sdk/src/executor.ts packages/core/sdk/src/plugin-storage.ts packages/core/sdk/src/plugin-storage.test.ts packages/plugins/semantic-search/src/sdk/tool-search-index.ts packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts --deny-warnings` ## Known tradeoffs - Root `bun run lint` still fails on pre-existing `.scratchpad` files outside this diff. ### [Stack](https://github.com/kitlangton/stack) 1. #44 2. **#45** πŸ‘ˆ current --- packages/core/sdk/src/executor.ts | 93 ++++ packages/core/sdk/src/plugin-storage.test.ts | 33 +- packages/core/sdk/src/plugin-storage.ts | 56 ++ .../src/sdk/tool-search-index.test.ts | 41 +- .../src/sdk/tool-search-index.ts | 515 ++++++++++-------- 5 files changed, 511 insertions(+), 227 deletions(-) diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 8e96cf6138..c38adfb4e3 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -1292,6 +1292,36 @@ const makePluginStorageFacade = (input: { Effect.map((row) => (row ? pluginStorageEntryFromRow(row) : null)), ); + const getManyVisible = (collection: string, keys: readonly string[]) => + Effect.gen(function* () { + const entries = new Map>(); + const uniqueKeys = [...new Set(keys)]; + for ( + let offset = 0; + offset < uniqueKeys.length; + offset += PLUGIN_STORAGE_DELETE_KEY_BATCH_SIZE + ) { + const batchKeys = uniqueKeys.slice(offset, offset + PLUGIN_STORAGE_DELETE_KEY_BATCH_SIZE); + const rows = yield* input.core.findMany("plugin_storage", { + where: (b) => + b.and( + b("plugin_id", "=", input.pluginId), + b("collection", "=", collection), + b("key", "in", batchKeys), + ), + }); + for (const row of sortByOwnerPrecedence(rows)) { + if (!entries.has(row.key)) entries.set(row.key, pluginStorageEntryFromRow(row)); + } + } + return new Map( + uniqueKeys.flatMap((key) => { + const entry = entries.get(key); + return entry === undefined ? [] : [[key, entry] as const]; + }), + ); + }); + const getForOwnerImpl = (owner: Owner, collection: string, key: string) => input.core .findFirst("plugin_storage", { @@ -1299,6 +1329,38 @@ const makePluginStorageFacade = (input: { }) .pipe(Effect.map((row) => (row ? pluginStorageEntryFromRow(row) : null))); + const getManyForOwnerImpl = (owner: Owner, collection: string, keys: readonly string[]) => + Effect.gen(function* () { + const entries = new Map>(); + const uniqueKeys = [...new Set(keys)]; + const os = ownerSubject(owner); + const subject = os ? os.subject : ORG_SUBJECT; + for ( + let offset = 0; + offset < uniqueKeys.length; + offset += PLUGIN_STORAGE_DELETE_KEY_BATCH_SIZE + ) { + const batchKeys = uniqueKeys.slice(offset, offset + PLUGIN_STORAGE_DELETE_KEY_BATCH_SIZE); + const rows = yield* input.core.findMany("plugin_storage", { + where: (b) => + b.and( + b("plugin_id", "=", input.pluginId), + b("collection", "=", collection), + b("key", "in", batchKeys), + b("owner", "=", owner), + b("subject", "=", subject), + ), + }); + for (const row of rows) entries.set(row.key, pluginStorageEntryFromRow(row)); + } + return new Map( + uniqueKeys.flatMap((key) => { + const entry = entries.get(key); + return entry === undefined ? [] : [[key, entry] as const]; + }), + ); + }); + const putImpl = (owner: Owner, collection: string, key: string, data: unknown) => Effect.gen(function* () { const os = ownerSubject(owner); @@ -1520,11 +1582,25 @@ const makePluginStorageFacade = (input: { PluginStorageEntry> | null, StorageFailure >, + getMany: (storageInput) => + getManyVisible(definition.name, storageInput.keys) as Effect.Effect< + ReadonlyMap>>, + StorageFailure + >, getForOwner: (storageInput) => getForOwnerImpl(storageInput.owner, definition.name, storageInput.key) as Effect.Effect< PluginStorageEntry> | null, StorageFailure >, + getManyForOwner: (storageInput) => + getManyForOwnerImpl( + storageInput.owner, + definition.name, + storageInput.keys, + ) as Effect.Effect< + ReadonlyMap>>, + StorageFailure + >, list: (storageInput) => queryCollection(definition, { keyPrefix: storageInput?.keyPrefix }), put: (storageInput) => putImpl( @@ -1536,6 +1612,15 @@ const makePluginStorageFacade = (input: { PluginStorageEntry>, StorageFailure >, + putMany: (storageInput) => + putManyImpl( + storageInput.owner, + storageInput.entries.map((entry) => ({ + collection: definition.name, + key: entry.key, + data: entry.data, + })), + ), query: (storageInput) => queryCollection(definition, storageInput), count: (storageInput) => queryCollection(definition, storageInput).pipe(Effect.map((rows) => rows.length)), @@ -1654,10 +1739,18 @@ const makePluginStorageFacade = (input: { }), }, remove: (storageInput) => removeImpl(storageInput.owner, definition.name, storageInput.key), + removeMany: (storageInput) => + removeManyImpl( + storageInput.owner, + storageInput.keys.map((key) => ({ collection: definition.name, key })), + ), }), get: (storageInput) => getVisible(storageInput.collection, storageInput.key), + getMany: (storageInput) => getManyVisible(storageInput.collection, storageInput.keys), getForOwner: (storageInput) => getForOwnerImpl(storageInput.owner, storageInput.collection, storageInput.key), + getManyForOwner: (storageInput) => + getManyForOwnerImpl(storageInput.owner, storageInput.collection, storageInput.keys), list: (storageInput) => Effect.gen(function* () { const rows = yield* input.core.findMany("plugin_storage", { diff --git a/packages/core/sdk/src/plugin-storage.test.ts b/packages/core/sdk/src/plugin-storage.test.ts index c8d8cb972a..41f7e1d1b9 100644 --- a/packages/core/sdk/src/plugin-storage.test.ts +++ b/packages/core/sdk/src/plugin-storage.test.ts @@ -65,21 +65,20 @@ const executionHistoryPlugin = definePlugin(() => ({ owner: Owner, rows: readonly { readonly key: string; readonly data: ToolCall }[], ) => - ctx.pluginStorage.putMany({ + ctx.storage.toolCalls.putMany({ owner, - entries: rows.map((row) => ({ - collection: toolCalls.name, - key: row.key, - data: row.data, - })), + entries: rows, }), removeMany: (owner: Owner, keys: readonly string[]) => - ctx.pluginStorage.removeMany({ + ctx.storage.toolCalls.removeMany({ owner, - entries: keys.map((key) => ({ collection: toolCalls.name, key })), + keys, }), get: (key: string) => ctx.storage.toolCalls.get({ key }), + getMany: (keys: readonly string[]) => ctx.storage.toolCalls.getMany({ keys }), getForOwner: (owner: Owner, key: string) => ctx.storage.toolCalls.getForOwner({ owner, key }), + getManyForOwner: (owner: Owner, keys: readonly string[]) => + ctx.storage.toolCalls.getManyForOwner({ owner, keys }), query: (input?: PluginStorageCollectionQueryInput) => ctx.storage.toolCalls.query(input), count: ( @@ -213,6 +212,15 @@ describe("plugin storage collections", () => { expect(stored[0]?.key).toBe("call-000"); expect(stored[0]?.data.status).toBe("failed"); + const selected = yield* executor.executionHistory.getManyForOwner("org", [ + "call-000", + "call-001", + "missing", + ]); + expect([...selected.keys()]).toEqual(["call-000", "call-001"]); + expect(selected.get("call-000")?.data.status).toBe("failed"); + expect(selected.get("call-001")?.data.toolId).toBe("shell"); + yield* executor.executionHistory.removeMany( "org", rows.map((row) => row.key), @@ -256,6 +264,15 @@ describe("plugin storage collections", () => { expect(visibleShared?.owner).toBe("user"); expect(visibleShared?.data.toolId).toBe("browser"); + const visibleRows = yield* executor.executionHistory.getMany(["shared", "missing"]); + expect([...visibleRows.keys()]).toEqual(["shared"]); + expect(visibleRows.get("shared")?.owner).toBe("user"); + expect(visibleRows.get("shared")?.data.toolId).toBe("browser"); + + const orgRows = yield* executor.executionHistory.getManyForOwner("org", ["shared"]); + expect(orgRows.get("shared")?.owner).toBe("org"); + expect(orgRows.get("shared")?.data.toolId).toBe("shell"); + const scopedRows = yield* executor.executionHistory.query({ where: { runId: "run-scope" }, orderBy: [{ field: "startedAt" }], diff --git a/packages/core/sdk/src/plugin-storage.ts b/packages/core/sdk/src/plugin-storage.ts index b926965b65..7c010406b0 100644 --- a/packages/core/sdk/src/plugin-storage.ts +++ b/packages/core/sdk/src/plugin-storage.ts @@ -104,6 +104,15 @@ export interface PluginStorageScopedKeyInput extends PluginStorageKeyInput { readonly owner: Owner; } +export interface PluginStorageKeysInput { + readonly collection: string; + readonly keys: readonly string[]; +} + +export interface PluginStorageScopedKeysInput extends PluginStorageKeysInput { + readonly owner: Owner; +} + export interface PluginStorageListInput { readonly collection: string; readonly keyPrefix?: string; @@ -135,6 +144,14 @@ export interface PluginStorageCollectionScopedKeyInput extends PluginStorageColl readonly owner: Owner; } +export interface PluginStorageCollectionKeysInput { + readonly keys: readonly string[]; +} + +export interface PluginStorageCollectionScopedKeysInput extends PluginStorageCollectionKeysInput { + readonly owner: Owner; +} + export interface PluginStorageCollectionListInput { readonly keyPrefix?: string; } @@ -145,6 +162,21 @@ export interface PluginStorageCollectionPutInput< readonly data: TData; } +export interface PluginStorageCollectionPutManyEntry { + readonly key: string; + readonly data: TData; +} + +export interface PluginStorageCollectionPutManyInput { + readonly owner: Owner; + readonly entries: readonly PluginStorageCollectionPutManyEntry[]; +} + +export interface PluginStorageCollectionRemoveManyInput { + readonly owner: Owner; + readonly keys: readonly string[]; +} + export interface PluginStorageCollectionQueryInput { readonly keyPrefix?: string; readonly where?: PluginStorageCollectionWhere; @@ -270,12 +302,24 @@ export interface PluginStorageCollectionFacade< PluginStorageEntry> | null, StorageFailure >; + readonly getMany: ( + input: PluginStorageCollectionKeysInput, + ) => Effect.Effect< + ReadonlyMap>>, + StorageFailure + >; readonly getForOwner: ( input: PluginStorageCollectionScopedKeyInput, ) => Effect.Effect< PluginStorageEntry> | null, StorageFailure >; + readonly getManyForOwner: ( + input: PluginStorageCollectionScopedKeysInput, + ) => Effect.Effect< + ReadonlyMap>>, + StorageFailure + >; readonly list: ( input?: PluginStorageCollectionListInput, ) => Effect.Effect< @@ -285,6 +329,9 @@ export interface PluginStorageCollectionFacade< readonly put: ( input: PluginStorageCollectionPutInput>, ) => Effect.Effect>, StorageFailure>; + readonly putMany: ( + input: PluginStorageCollectionPutManyInput>, + ) => Effect.Effect; readonly query: ( input?: PluginStorageCollectionQueryInput, ) => Effect.Effect< @@ -306,6 +353,9 @@ export interface PluginStorageCollectionFacade< readonly remove: ( input: PluginStorageCollectionScopedKeyInput, ) => Effect.Effect; + readonly removeMany: ( + input: PluginStorageCollectionRemoveManyInput, + ) => Effect.Effect; } export interface PluginStorageFacade { @@ -315,9 +365,15 @@ export interface PluginStorageFacade { readonly get: ( input: PluginStorageKeyInput, ) => Effect.Effect | null, StorageFailure>; + readonly getMany: ( + input: PluginStorageKeysInput, + ) => Effect.Effect>, StorageFailure>; readonly getForOwner: ( input: PluginStorageScopedKeyInput, ) => Effect.Effect | null, StorageFailure>; + readonly getManyForOwner: ( + input: PluginStorageScopedKeysInput, + ) => Effect.Effect>, StorageFailure>; readonly list: ( input: PluginStorageListInput, ) => Effect.Effect[], StorageFailure>; diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts index ed0a86c941..7d48dcfb65 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -102,22 +102,53 @@ const makeCollection = (collection: string): TestCollection }); const matches = (value: T, where: Record | undefined): boolean => { if (!where) return true; - return Object.entries(where).every( - ([key, expected]) => (value as Record)[key] === expected, - ); + return Object.entries(where).every(([key, expected]) => { + const actual = (value as Record)[key]; + if ( + typeof expected === "object" && + expected !== null && + "in" in expected && + Array.isArray((expected as { readonly in?: unknown }).in) + ) { + return (expected as { readonly in: readonly unknown[] }).in.includes(actual); + } + return actual === expected; + }); }; const facade = { data, get: ({ key }: { key: string }) => Effect.succeed(data.has(key) ? entry(key, data.get(key)!) : null), + getMany: ({ keys }: { keys: readonly string[] }) => + Effect.succeed( + new Map( + keys.flatMap((key) => { + const value = data.get(key); + return value === undefined ? [] : [[key, entry(key, value)] as const]; + }), + ), + ), getForOwner: ({ key }: { key: string }) => Effect.succeed(data.has(key) ? entry(key, data.get(key)!) : null), + getManyForOwner: ({ keys }: { keys: readonly string[] }) => + Effect.succeed( + new Map( + keys.flatMap((key) => { + const value = data.get(key); + return value === undefined ? [] : [[key, entry(key, value)] as const]; + }), + ), + ), list: () => Effect.succeed([...data.entries()].map(([key, value]) => entry(key, value))), put: ({ key, data: value }: { key: string; data: T }) => Effect.sync(() => { data.set(key, value); return entry(key, value); }), + putMany: ({ entries }: { entries: readonly { readonly key: string; readonly data: T }[] }) => + Effect.sync(() => { + for (const item of entries) data.set(item.key, item.data); + }), query: (input?: { where?: Record; orderBy?: readonly { readonly field: string; readonly direction?: "asc" | "desc" }[]; @@ -150,6 +181,10 @@ const makeCollection = (collection: string): TestCollection stats: () => Effect.succeed({ count: 0, min: null, max: null, percentiles: [] }), }, remove: ({ key }: { key: string }) => Effect.sync(() => data.delete(key)), + removeMany: ({ keys }: { keys: readonly string[] }) => + Effect.sync(() => { + for (const key of keys) data.delete(key); + }), }; // oxlint-disable-next-line executor/no-double-cast -- test fixture implements the storage facade methods exercised by these indexer tests return facade as unknown as TestCollection; diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts index d2f068c555..3a483eed36 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts @@ -280,6 +280,47 @@ const queryChunksForJob = ( ), ); +const queryChunksByPaths = ( + deps: IndexCollections, + input: { + readonly runId: string; + readonly paths: readonly string[]; + readonly status?: IndexChunk["status"]; + }, +): Effect.Effect< + ReadonlyMap, + SemanticSearchError +> => { + const paths = [...new Set(input.paths)]; + if (paths.length === 0) return Effect.succeed(new Map()); + return deps.chunks + .query({ + where: { + runId: input.runId, + path: { in: paths }, + ...(input.status === undefined ? {} : { status: input.status }), + }, + orderBy: [{ field: "path", direction: "asc" }], + }) + .pipe( + Effect.map((entries) => { + const grouped = new Map(); + for (const entry of entries) { + const group = grouped.get(entry.data.path); + if (group) group.push(entry); + else grouped.set(entry.data.path, [entry]); + } + for (const group of grouped.values()) { + group.sort((a, b) => a.data.chunkIndex - b.data.chunkIndex); + } + return grouped; + }), + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to query index chunks.", cause }), + ), + ); +}; + const getChunksByRefs = ( deps: IndexCollections, input: { @@ -290,19 +331,22 @@ const getChunksByRefs = ( readonly { readonly key: string; readonly data: IndexChunk }[], SemanticSearchError > => - Effect.forEach( - input.chunkRefs, - (ref) => - deps.chunks - .getForOwner({ owner: deps.owner, key: chunkKey(input.runId, ref.path, ref.chunkId) }) - .pipe(Effect.map((entry) => entry ?? null)), - { concurrency: INDEX_STORAGE_CONCURRENCY }, - ).pipe( - Effect.map((entries) => entries.flatMap((entry) => (entry === null ? [] : [entry]))), - Effect.mapError( - (cause) => new SemanticSearchError({ message: "Failed to load index chunks.", cause }), - ), - ); + deps.chunks + .getManyForOwner({ + owner: deps.owner, + keys: input.chunkRefs.map((ref) => chunkKey(input.runId, ref.path, ref.chunkId)), + }) + .pipe( + Effect.map((entries) => + input.chunkRefs.flatMap((ref) => { + const entry = entries.get(chunkKey(input.runId, ref.path, ref.chunkId)); + return entry === undefined ? [] : [entry]; + }), + ), + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to load index chunks.", cause }), + ), + ); interface EmbedBudget { readonly maxChunks: number; @@ -450,18 +494,8 @@ const loadFingerprints = ( deps: IndexCollections, paths: readonly string[], ): Effect.Effect, SemanticSearchError> => - Effect.forEach( - paths, - (path) => - deps.fingerprints - .getForOwner({ owner: deps.owner, key: path }) - .pipe(Effect.map((entry) => [path, entry?.data ?? null] as const)), - { concurrency: INDEX_STORAGE_CONCURRENCY }, - ).pipe( - Effect.map( - (pairs) => - new Map(pairs.flatMap(([path, data]) => (data === null ? [] : [[path, data] as const]))), - ), + deps.fingerprints.getManyForOwner({ owner: deps.owner, keys: paths }).pipe( + Effect.map((entries) => new Map([...entries].map(([path, entry]) => [path, entry.data]))), Effect.mapError( (cause) => new SemanticSearchError({ @@ -483,28 +517,24 @@ const putJob = (deps: IndexCollections, job: IndexJob): Effect.Effect => - queryChunksForJob(deps, job).pipe( - Effect.flatMap((entries) => - Effect.forEach( - entries, - (entry) => - deps.chunks.remove({ owner: deps.owner, key: entry.key }).pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ - message: `Failed to remove index chunk "${entry.key}".`, - cause, - }), - ), - ), - { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + deps.jobs + .putMany({ + owner: deps.owner, + entries: jobs.map((job) => ({ key: jobKey(job.runId, job.path), data: job })), + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to persist index jobs.", + cause, + }), ), - ), - ); + ); const getJobsByPaths = ( deps: IndexCollections, @@ -518,23 +548,24 @@ const getJobsByPaths = ( readonly { readonly key: string; readonly data: IndexJob }[], SemanticSearchError > => - Effect.forEach( - input.paths, - (path) => - deps.jobs - .getForOwner({ owner: deps.owner, key: jobKey(input.runId, path) }) - .pipe(Effect.map((entry) => (entry?.data.status === input.status ? entry : null))), - { concurrency: INDEX_STORAGE_CONCURRENCY }, - ).pipe( - Effect.map((entries) => - entries - .flatMap((entry) => (entry === null ? [] : [entry])) - .slice(0, input.limit ?? input.paths.length), - ), - Effect.mapError( - (cause) => new SemanticSearchError({ message: "Failed to load index jobs.", cause }), - ), - ); + deps.jobs + .getManyForOwner({ + owner: deps.owner, + keys: input.paths.map((path) => jobKey(input.runId, path)), + }) + .pipe( + Effect.map((entries) => + input.paths + .flatMap((path) => { + const entry = entries.get(jobKey(input.runId, path)); + return entry !== undefined && entry.data.status === input.status ? [entry] : []; + }) + .slice(0, input.limit ?? input.paths.length), + ), + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Failed to load index jobs.", cause }), + ), + ); const queryJobsByStatus = ( deps: IndexCollections, @@ -927,59 +958,57 @@ export const scan = ( let changed = 0; let skipped = 0; const changedPaths: string[] = []; + const jobs: IndexJob[] = []; const updatedAt = nowIso(); - yield* Effect.forEach( - selected, - ({ manifest, ordinal }) => { - const storedRow = stored.get(manifest.path); - const next: IndexJob = - storedRow !== undefined && storedRow.fingerprint === manifest.indexFingerprint - ? { - runId: input.runId, - namespace: input.namespace, - partition: input.partition, - ordinal, - address: String(manifest.address), - path: manifest.path, - name: manifest.name, - integration: manifest.integration, - description: manifest.description, - status: "skipped", - fingerprint: manifest.indexFingerprint, - oldFingerprint: storedRow.fingerprint, - oldChunkIds: storedRow.chunkIds, - chunkIds: storedRow.chunkIds, - updatedAt, - createdAt: updatedAt, - } - : { - runId: input.runId, - namespace: input.namespace, - partition: input.partition, - ordinal, - address: String(manifest.address), - path: manifest.path, - name: manifest.name, - integration: manifest.integration, - description: manifest.description, - status: "pendingChunk", - fingerprint: manifest.indexFingerprint, - oldFingerprint: storedRow?.fingerprint, - oldChunkIds: storedRow?.chunkIds ?? [], - chunkIds: [], - updatedAt, - createdAt: updatedAt, - }; - if (next.status === "skipped") skipped++; - else { - changed++; - changedPaths.push(next.path); - } - return putJob(input, next); - }, - { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, - ); + for (const { manifest, ordinal } of selected) { + const storedRow = stored.get(manifest.path); + const next: IndexJob = + storedRow !== undefined && storedRow.fingerprint === manifest.indexFingerprint + ? { + runId: input.runId, + namespace: input.namespace, + partition: input.partition, + ordinal, + address: String(manifest.address), + path: manifest.path, + name: manifest.name, + integration: manifest.integration, + description: manifest.description, + status: "skipped", + fingerprint: manifest.indexFingerprint, + oldFingerprint: storedRow.fingerprint, + oldChunkIds: storedRow.chunkIds, + chunkIds: storedRow.chunkIds, + updatedAt, + createdAt: updatedAt, + } + : { + runId: input.runId, + namespace: input.namespace, + partition: input.partition, + ordinal, + address: String(manifest.address), + path: manifest.path, + name: manifest.name, + integration: manifest.integration, + description: manifest.description, + status: "pendingChunk", + fingerprint: manifest.indexFingerprint, + oldFingerprint: storedRow?.fingerprint, + oldChunkIds: storedRow?.chunkIds ?? [], + chunkIds: [], + updatedAt, + createdAt: updatedAt, + }; + if (next.status === "skipped") skipped++; + else { + changed++; + changedPaths.push(next.path); + } + jobs.push(next); + } + yield* putJobs(input, jobs); return { runId: input.runId, @@ -1009,14 +1038,14 @@ export const chunk = ( paths: input.paths, limit: input.limit, }); - const existing = yield* Effect.forEach( - pendingEmbeddingJobs, - (entry) => - queryChunksForJob(input, entry.data).pipe( - Effect.map((chunkEntries) => ({ job: entry.data, chunkEntries })), - ), - { concurrency: INDEX_STORAGE_CONCURRENCY }, - ); + const chunksByPath = yield* queryChunksByPaths(input, { + runId: input.runId, + paths: pendingEmbeddingJobs.map((entry) => entry.data.path), + }); + const existing = pendingEmbeddingJobs.map((entry) => ({ + job: entry.data, + chunkEntries: chunksByPath.get(entry.data.path) ?? [], + })); const chunkRefs = existing.flatMap(({ chunkEntries }) => chunkEntries .filter((entry) => entry.data.status === "pendingEmbedding") @@ -1043,11 +1072,12 @@ export const chunk = ( 1, Math.floor(input.concurrency ?? DEFAULT_CHUNK_CONCURRENCY), ); - const paths: string[] = []; - const chunkRefs: ToolSearchIndex.ChunkRef[] = []; - const commitPaths: string[] = []; + const oldChunksByPath = yield* queryChunksByPaths(input, { + runId: input.runId, + paths: jobs.map((entry) => entry.data.path), + }); - const chunkCounts = yield* Effect.forEach( + const chunkedJobs = yield* Effect.forEach( jobs, (entry) => Effect.gen(function* () { @@ -1066,77 +1096,107 @@ export const chunk = ( const chunks = input.chunker.chunk(input.namespace, doc); const lexicalText = doc.lexicalText ?? buildLexicalText(doc); const lexicalTextKey = yield* putPayloadText(input, "lexical-text", lexicalText); - yield* removeChunksForJob(input, job); - yield* Effect.forEach(chunks, (chunk) => putChunk(input, job, chunk, updatedAt), { - concurrency: INDEX_STORAGE_CONCURRENCY, - discard: true, - }); - for (const chunk of chunks) { - chunkRefs.push({ path: job.path, chunkId: chunk.id }); - } - yield* putJob(input, { + const chunkEntries = yield* Effect.forEach( + chunks, + (chunk) => makeChunkEntry(input, job, chunk, updatedAt), + { concurrency: INDEX_STORAGE_CONCURRENCY }, + ); + const updatedJob: IndexJob = { ...job, status: "pendingEmbedding", chunkIds: chunks.map((chunk) => chunk.id), lexicalTextKey, updatedAt, - }); - paths.push(job.path); - if (chunks.length === 0) { - commitPaths.push(job.path); - } - return chunks.length; + }; + return { + path: job.path, + chunks: chunks.length, + oldChunkKeys: (oldChunksByPath.get(job.path) ?? []).map((chunk) => chunk.key), + chunkEntries, + chunkRefs: chunks.map((chunk) => ({ path: job.path, chunkId: chunk.id })), + updatedJob, + commitPath: chunks.length === 0 ? job.path : undefined, + }; }), { concurrency: chunkConcurrency }, ); - const chunkCount = chunkCounts.reduce((sum, count) => sum + count, 0); + const oldChunkKeys = chunkedJobs.flatMap((job) => job.oldChunkKeys); + if (oldChunkKeys.length > 0) { + yield* input.chunks.removeMany({ owner: input.owner, keys: oldChunkKeys }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to remove stale index chunks.", + cause, + }), + ), + ); + } + const chunkEntries = chunkedJobs.flatMap((job) => job.chunkEntries); + if (chunkEntries.length > 0) { + yield* input.chunks.putMany({ owner: input.owner, entries: chunkEntries }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to persist index chunks.", + cause, + }), + ), + ); + } + yield* putJobs( + input, + chunkedJobs.map((job) => job.updatedJob), + ); + const chunkCount = chunkedJobs.reduce((sum, job) => sum + job.chunks, 0); return { runId: input.runId, processed: jobs.length, chunks: chunkCount, - paths, - chunkRefs, - commitPaths, + paths: chunkedJobs.map((job) => job.path), + chunkRefs: chunkedJobs.flatMap((job) => job.chunkRefs), + commitPaths: chunkedJobs.flatMap((job) => + job.commitPath === undefined ? [] : [job.commitPath], + ), }; }); -const putChunk = ( +const makeChunkEntry = ( deps: IndexCollections, job: IndexJob, chunk: ToolChunk, timestamp: string, -): Effect.Effect => +): Effect.Effect<{ readonly key: string; readonly data: IndexChunk }, SemanticSearchError> => Effect.gen(function* () { const embeddingTextKey = yield* putPayloadText(deps, "embedding-text", chunk.embeddingText); - yield* deps.chunks.put({ - owner: deps.owner, + const data: IndexChunk = { + runId: job.runId, + namespace: job.namespace, + partition: job.partition, + path: job.path, + chunkId: chunk.id, + facet: chunk.facet, + chunkIndex: chunk.chunkIndex, + embeddingTextKey, + embeddingTextBytes: utf8Bytes(chunk.embeddingText), + embeddingTextTokens: estimateTokens(chunk.embeddingText), + name: chunk.name, + integration: chunk.integration, + description: job.description, + status: "pendingEmbedding", + createdAt: timestamp, + updatedAt: timestamp, + }; + return { key: chunkKey(job.runId, job.path, chunk.id), - data: { - runId: job.runId, - namespace: job.namespace, - partition: job.partition, - path: job.path, - chunkId: chunk.id, - facet: chunk.facet, - chunkIndex: chunk.chunkIndex, - embeddingTextKey, - embeddingTextBytes: utf8Bytes(chunk.embeddingText), - embeddingTextTokens: estimateTokens(chunk.embeddingText), - name: chunk.name, - integration: chunk.integration, - description: job.description, - status: "pendingEmbedding", - createdAt: timestamp, - updatedAt: timestamp, - }, - }); + data, + }; }).pipe( Effect.mapError( (cause) => - new SemanticSearchError({ message: `Failed to persist chunk "${chunk.id}".`, cause }), + new SemanticSearchError({ message: `Failed to prepare chunk "${chunk.id}".`, cause }), ), - Effect.asVoid, ); export const embed = ( @@ -1233,26 +1293,23 @@ export const embed = ( } yield* input.store.upsert(records); - yield* Effect.forEach( - group, - (entry) => - input.chunks - .put({ - owner: input.owner, - key: entry.key, - data: { ...entry.data, status: "indexed", updatedAt }, - }) - .pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ - message: `Failed to mark chunk "${entry.data.chunkId}" indexed.`, - cause, - }), - ), - ), - { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, - ); + yield* input.chunks + .putMany({ + owner: input.owner, + entries: group.map((entry) => ({ + key: entry.key, + data: { ...entry.data, status: "indexed", updatedAt }, + })), + }) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to mark chunks indexed.", + cause, + }), + ), + ); indexedChunks += records.length; for (const entry of group) { @@ -1283,7 +1340,11 @@ export const commit = ( return { runId: input.runId, processed: 0, committed: 0, paths: [] }; } - const committed = yield* finalizeCompletedEmbedJobs(input, entries, undefined, nowIso()); + const chunksByPath = yield* queryChunksByPaths(input, { + runId: input.runId, + paths: entries.map((entry) => entry.data.path), + }); + const committed = yield* finalizeCompletedEmbedJobs(input, entries, chunksByPath, nowIso()); return { runId: input.runId, processed: entries.length, @@ -1301,7 +1362,15 @@ const finalizeCompletedEmbedJobs = ( updatedAt: string, ): Effect.Effect => Effect.gen(function* () { - let finalized = 0; + const fingerprintRows: { + readonly key: string; + readonly data: FingerprintRow; + }[] = []; + const indexedJobs: IndexJob[] = []; + const oldChunkIds: string[] = []; + const oldDocumentFingerprints: string[] = []; + const lexicalDocs: FtsDocumentInput[] = []; + for (const { data: job } of jobs) { const chunkEntries = preloadedChunks?.get(job.path) ?? (yield* queryChunksForJob(input, job)); if (chunkEntries.some((entry) => entry.data.status !== "indexed")) { @@ -1315,32 +1384,21 @@ const finalizeCompletedEmbedJobs = ( }); } - yield* input.fingerprints - .put({ - owner: input.owner, - key: job.path, - data: { - path: job.path, - integration: job.integration, - fingerprint, - chunkIds: job.chunkIds, - }, - }) - .pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ - message: `Failed to persist fingerprint row for "${job.path}".`, - cause, - }), - ), - ); + fingerprintRows.push({ + key: job.path, + data: { + path: job.path, + integration: job.integration, + fingerprint, + chunkIds: job.chunkIds, + }, + }); if (job.oldChunkIds.length > 0) { - yield* input.store.deleteByIds(job.oldChunkIds); + oldChunkIds.push(...job.oldChunkIds); } if (job.oldFingerprint !== undefined && job.oldFingerprint !== fingerprint) { - yield* deleteCachedToolDocument(input, job.oldFingerprint); + oldDocumentFingerprints.push(job.oldFingerprint); } if (input.lexicalStore) { @@ -1357,13 +1415,38 @@ const finalizeCompletedEmbedJobs = ( integration: job.integration, lexicalText, }; - yield* input.lexicalStore.upsert([lexicalDoc]); + lexicalDocs.push(lexicalDoc); } - yield* putJob(input, { ...job, status: "indexed", updatedAt }); - finalized++; + indexedJobs.push({ ...job, status: "indexed", updatedAt }); + } + + if (fingerprintRows.length > 0) { + yield* input.fingerprints.putMany({ owner: input.owner, entries: fingerprintRows }).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to persist fingerprint rows.", + cause, + }), + ), + ); + } + if (oldChunkIds.length > 0) { + yield* input.store.deleteByIds([...new Set(oldChunkIds)]); + } + yield* Effect.forEach( + [...new Set(oldDocumentFingerprints)], + (fingerprint) => deleteCachedToolDocument(input, fingerprint), + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ); + if (input.lexicalStore && lexicalDocs.length > 0) { + yield* input.lexicalStore.upsert(lexicalDocs); + } + if (indexedJobs.length > 0) { + yield* putJobs(input, indexedJobs); } - return finalized; + return indexedJobs.length; }); export const status = ( From ae221f3c2ad0be6d5be11b40cc9e8481082777f4 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sat, 20 Jun 2026 19:12:58 +0530 Subject: [PATCH 51/57] perf(storage): batch plugin storage upserts (#46) ## Summary Adds a bulk upsert path through FumaDB and uses it for plugin-storage `putMany`, removing the delete-then-insert write pattern from bulk plugin storage updates. ## Changes - Add `upsertMany` to FumaDB query surfaces, with native Drizzle `ON CONFLICT DO UPDATE` support for SQLite/Postgres and memory/fallback adapter behavior. - Route SDK plugin-storage `putMany` through `core.upsertMany` using the plugin-storage unique key. - Reduce semantic-search index status reads by grouping job counts with plugin-storage aggregation and dedupe commit paths before job loads. - Cover policy-aware bulk upserts and update semantic-search test fixtures for grouped counts. ## Tests - `bun run --cwd packages/core/fumadb typecheck` - `bun run --cwd packages/core/sdk typecheck` - `bun run --cwd packages/plugins/semantic-search typecheck` - `node ../../../node_modules/vitest/vitest.mjs run src/query/table-policy.test.ts` from `packages/core/fumadb` - `bun run --cwd packages/core/sdk test src/plugin-storage.test.ts` - `bun run --cwd packages/plugins/semantic-search test src/sdk/tool-search-index.test.ts` - `./node_modules/.bin/oxlint -c .oxlintrc.jsonc --deny-warnings` --- .../core/fumadb/src/adapters/drizzle/query.ts | 71 +++++++++++ .../core/fumadb/src/adapters/memory/index.ts | 25 ++++ packages/core/fumadb/src/query/index.ts | 16 +++ packages/core/fumadb/src/query/orm/index.ts | 118 +++++++++++++++++- .../fumadb/src/query/table-policy.test.ts | 68 +++++++++- packages/core/sdk/src/executor.ts | 32 ++++- packages/core/sdk/src/fuma-runtime.ts | 1 + packages/core/sdk/src/test-config.ts | 9 ++ .../src/sdk/tool-search-index.test.ts | 22 +++- .../src/sdk/tool-search-index.ts | 79 +++++++----- 10 files changed, 403 insertions(+), 38 deletions(-) diff --git a/packages/core/fumadb/src/adapters/drizzle/query.ts b/packages/core/fumadb/src/adapters/drizzle/query.ts index 076b09c29b..c6d63a63fe 100644 --- a/packages/core/fumadb/src/adapters/drizzle/query.ts +++ b/packages/core/fumadb/src/adapters/drizzle/query.ts @@ -128,6 +128,16 @@ function buildWhere( ); } +function countConditionParameters(condition: Condition): number { + if (condition.type === ConditionType.Compare) { + if (condition.b instanceof Column) return 0; + if (Array.isArray(condition.b)) return condition.b.length; + return 1; + } + if (condition.type === ConditionType.Not) return countConditionParameters(condition.item); + return condition.items.reduce((count, item) => count + countConditionParameters(item), 0); +} + function mapValues( values: Record, table: AnyTable @@ -423,6 +433,67 @@ export function fromDrizzle( await this.createMany(table, [v.create]); } }, + async upsertMany(table, v) { + if (v.values.length === 0) return; + if (v.update.length === 0) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: adapter rejects invalid upsert shape + throw new Error("[FumaDB] upsertMany requires at least one update column."); + } + if (provider !== "sqlite" && provider !== "postgresql") { + for (const value of v.values) { + await this.upsert(table, { + where: { + type: ConditionType.And, + items: v.target.map((column) => ({ + type: ConditionType.Compare, + a: column, + operator: "=", + b: value[column.ormName], + })), + }, + update: Object.fromEntries( + v.update.map((column) => [column.ormName, value[column.ormName]]), + ), + create: value, + }); + } + return; + } + + const drizzleTable = toDrizzle(table); + const values = v.values.map((value) => mapValues(value, table)); + const where = v.where ? buildWhere(toDrizzleColumn, v.where) : undefined; + const whereParameters = v.where ? countConditionParameters(v.where) : 0; + const columnsPerRow = values.length > 0 ? Math.max(1, Object.keys(values[0]!).length) : 1; + const batchSize = maxBoundParameters + ? Math.max( + 1, + Math.min( + CREATE_MANY_BATCH_SIZE, + Math.floor(Math.max(1, maxBoundParameters - whereParameters) / columnsPerRow), + ), + ) + : CREATE_MANY_BATCH_SIZE; + const target = v.target.map((column) => drizzleTable[column.names.drizzle]); + const set = Object.fromEntries( + v.update.map((column) => [ + column.names.drizzle, + Drizzle.sql.raw(`excluded.${column.names.sql}`), + ]), + ); + + for (let i = 0; i < values.length; i += batchSize) { + const batch = values.slice(i, i + batchSize); + await (db as any) + .insert(drizzleTable) + .values(batch) + .onConflictDoUpdate({ + target, + set, + ...(where === undefined ? {} : { where }), + }); + } + }, async findMany(table, v) { return ( await db.query[table.names.drizzle].findMany(buildQueryConfig(table, v)) diff --git a/packages/core/fumadb/src/adapters/memory/index.ts b/packages/core/fumadb/src/adapters/memory/index.ts index 7b931407b9..1c8c3d59e5 100644 --- a/packages/core/fumadb/src/adapters/memory/index.ts +++ b/packages/core/fumadb/src/adapters/memory/index.ts @@ -183,6 +183,31 @@ export function memoryAdapter(options: MemoryAdapterOptions = {}): FumaDBAdapter } await this.create(table, v.create); }, + async upsertMany(table, v) { + if (v.update.length === 0) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: adapter rejects invalid upsert shape + throw new Error("[FumaDB] upsertMany requires at least one update column."); + } + for (const value of v.values) { + const existing = tableRows(db, table).find( + (row) => + matchesCondition(row, v.where) && + v.target.every((column) => row[column.ormName] === value[column.ormName]), + ); + if (existing) { + Object.assign( + existing, + cloneValue( + Object.fromEntries( + v.update.map((column) => [column.ormName, value[column.ormName]]), + ), + ), + ); + continue; + } + await this.create(table, value); + } + }, async create(table, values) { const row = applyDefaults(table, values); tableRows(db, table).push(row); diff --git a/packages/core/fumadb/src/query/index.ts b/packages/core/fumadb/src/query/index.ts index a339b8b868..478c7f0cda 100644 --- a/packages/core/fumadb/src/query/index.ts +++ b/packages/core/fumadb/src/query/index.ts @@ -185,6 +185,22 @@ export interface AbstractQuery { } ) => Promise; + /** + * Bulk upsert rows by a unique column target. + * + * Adapters with native conflict support should implement this as one or more + * `INSERT ... ON CONFLICT ... DO UPDATE` statements. Other adapters may fall + * back to per-row `upsert`. + */ + upsertMany: ( + table: TableName, + v: { + target: (keyof S["tables"][TableName]["columns"])[]; + update: (keyof S["tables"][TableName]["columns"])[]; + values: TableToInsertValues[]; + } + ) => Promise; + /** * Note: you cannot update the id of a row, some databases don't support that (including MongoDB). */ diff --git a/packages/core/fumadb/src/query/orm/index.ts b/packages/core/fumadb/src/query/orm/index.ts index 1d76a758fe..0426e08c79 100644 --- a/packages/core/fumadb/src/query/orm/index.ts +++ b/packages/core/fumadb/src/query/orm/index.ts @@ -4,6 +4,7 @@ import type { AnySchema, AnyTable, } from "../../schema"; +import { Column } from "../../schema"; import type { AbstractQuery, AnySelectClause, @@ -22,7 +23,12 @@ import type { JsonTimeBucketAdapterOptions, JsonTimeBucketRow, } from "../aggregate"; -import { buildCondition, createBuilder, type Condition } from "../condition-builder"; +import { + buildCondition, + createBuilder, + type Condition, + ConditionType, +} from "../condition-builder"; export interface CompiledJoin { relation: AnyRelation; @@ -241,6 +247,27 @@ const applyUpdatePolicies = async ( return nextWhere; }; +const conditionKey = (condition: Condition | undefined): string => { + if (!condition) return "none"; + if (condition.type === ConditionType.Compare) { + const right = + condition.b instanceof Column ? { column: condition.b.ormName } : { value: condition.b }; + return JSON.stringify({ + type: "compare", + left: condition.a.ormName, + operator: condition.operator, + right, + }); + } + if (condition.type === ConditionType.Not) { + return JSON.stringify({ type: "not", item: conditionKey(condition.item) }); + } + return JSON.stringify({ + type: condition.type === ConditionType.And ? "and" : "or", + items: condition.items.map(conditionKey), + }); +}; + const applyDeletePolicies = async ( table: AnyTable, where: Condition | undefined, @@ -294,6 +321,16 @@ export interface ORMAdapter { }, ) => Promise; + upsertMany?: ( + table: AnyTable, + v: { + target: AnyColumn[]; + update: AnyColumn[]; + values: Record[]; + where?: Condition; + }, + ) => Promise; + create: ( table: AnyTable, values: Record, @@ -423,6 +460,85 @@ export function toORM( ...options, }); }, + async upsertMany(name, { target, update, values }) { + const table = toTable(name); + if (values.length === 0) return; + + const targetColumns = target.map((columnName) => { + const column = table.columns[columnName as string]; + if (!column) throw new Error(`[FumaDB] unknown column name ${String(columnName)}.`); + return column; + }); + const updateColumns = update.map((columnName) => { + const column = table.columns[columnName as string]; + if (!column) throw new Error(`[FumaDB] unknown column name ${String(columnName)}.`); + return column; + }); + + const builder = createBuilder(table.columns); + const permittedRows: { + readonly value: Record; + readonly where: Condition | undefined; + }[] = []; + for (const value of values) { + const updateValues = Object.fromEntries( + updateColumns.map((column) => [column.ormName, value[column.ormName]]), + ); + const constrainedWhere = await applyUpdatePolicies( + table, + undefined, + updateValues, + context, + "upsert", + value, + ); + if (constrainedWhere === false) continue; + await runCreatePolicies(table, value, context); + permittedRows.push({ value, where: constrainedWhere }); + } + if (permittedRows.length === 0) return; + + if (internal.upsertMany) { + const groups = new Map< + string, + { readonly where: Condition | undefined; readonly values: Record[] } + >(); + for (const row of permittedRows) { + const key = conditionKey(row.where); + const group = groups.get(key); + if (group) { + group.values.push(row.value); + } else { + groups.set(key, { where: row.where, values: [row.value] }); + } + } + for (const group of groups.values()) { + await internal.upsertMany(table, { + target: targetColumns, + update: updateColumns, + values: group.values, + where: group.where, + }); + } + return; + } + + for (const row of permittedRows) { + const value = row.value; + const targetWhere = builder.and( + ...targetColumns.map((column) => builder(column.ormName, "=", value[column.ormName])), + ); + const where = builder.and(targetWhere, row.where ?? true); + if (where === false) continue; + await internal.upsert(table, { + where: where === true ? undefined : where, + update: Object.fromEntries( + updateColumns.map((column) => [column.ormName, value[column.ormName]]), + ), + create: value, + }); + } + }, async create(name, values) { const table = toTable(name); await runCreatePolicies(table, values, context); diff --git a/packages/core/fumadb/src/query/table-policy.test.ts b/packages/core/fumadb/src/query/table-policy.test.ts index 51d0c2b8c5..3ff5a81524 100644 --- a/packages/core/fumadb/src/query/table-policy.test.ts +++ b/packages/core/fumadb/src/query/table-policy.test.ts @@ -383,6 +383,25 @@ describe("FumaDB table policies", () => { }, }); + await tenantA.upsertMany("posts", { + target: ["id"], + update: ["title"], + values: [ + { + id: "post-a-1", + tenantId: "tenant-a", + authorId: "author-a", + title: "tenant-a-bulk-upserted", + }, + { + id: "post-a-4", + tenantId: "tenant-a", + authorId: "author-a", + title: "A Four", + }, + ], + }); + await expect( tenantA.findMany("posts", { select: ["id", "title"], @@ -391,7 +410,7 @@ describe("FumaDB table policies", () => { ).resolves.toEqual([ { id: "post-a-1", - title: "tenant-a-updated", + title: "tenant-a-bulk-upserted", }, { id: "post-a-2", @@ -401,6 +420,10 @@ describe("FumaDB table policies", () => { id: "post-a-3", title: "A Three", }, + { + id: "post-a-4", + title: "A Four", + }, ]); expect(tenantAContext.observed).toEqual( @@ -471,7 +494,7 @@ describe("FumaDB table policies", () => { ); it.effect( - "rejects out-of-context writes across createMany, updateMany, upsert, and transactions", + "rejects out-of-context writes across createMany, updateMany, upsert, upsertMany, and transactions", () => useHarness(async (orm) => { await seedTenants(orm); @@ -524,6 +547,47 @@ describe("FumaDB table policies", () => { }), ).rejects.toThrow("tenant tenant-b is not allowed for posts"); + await expect( + tenantA.upsertMany("posts", { + target: ["id"], + update: ["title"], + values: [ + { + id: "post-a-bulk-upsert", + tenantId: "tenant-a", + authorId: "author-a", + title: "A bulk upsert", + }, + { + id: "post-b-bulk-upsert", + tenantId: "tenant-b", + authorId: "author-b", + title: "B bulk upsert", + }, + ], + }), + ).rejects.toThrow("tenant tenant-b is not allowed for posts"); + await expect( + tenantA.findFirst("posts", { + where: (builder) => builder("id", "=", "post-a-bulk-upsert"), + }), + ).resolves.toBeNull(); + + await expect( + tenantA.upsertMany("posts", { + target: ["id"], + update: ["tenantId"], + values: [ + { + id: "post-a-1", + tenantId: "tenant-b", + authorId: "author-b", + title: "tenant move", + }, + ], + }), + ).rejects.toThrow("tenant tenant-b is not allowed for posts"); + await expect( tenantA.transaction(async (tx) => { await tx.create("posts", { diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index c38adfb4e3..a6b6a1f05d 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -847,6 +847,14 @@ type LooseStorageDb = { tableName: string, rows: readonly Record[], ) => Promise; + readonly upsertMany: ( + tableName: string, + options: { + readonly target: readonly string[]; + readonly update: readonly string[]; + readonly values: readonly Record[]; + }, + ) => Promise; readonly deleteMany: (tableName: string, options?: unknown) => Promise; readonly findFirst: ( tableName: string, @@ -897,6 +905,19 @@ const makeCoreDb = (fuma: ReturnType) => ({ : fuma .use(`${tableName}.createMany`, (db) => asLooseStorageDb(db).createMany(tableName, rows)) .pipe(Effect.asVoid), + upsertMany: ( + tableName: TName, + options: { + readonly target: readonly string[]; + readonly update: readonly string[]; + readonly values: readonly Record[]; + }, + ): Effect.Effect => + options.values.length === 0 + ? Effect.void + : fuma.use(`${tableName}.upsertMany`, (db) => + asLooseStorageDb(db).upsertMany(tableName, options), + ), deleteMany: ( tableName: TName, options: { readonly where?: CoreWhere } = {}, @@ -1485,12 +1506,11 @@ const makePluginStorageFacade = (input: { const uniqueEntries = [...entriesById.values()]; if (uniqueEntries.length === 0) return; - yield* deleteManyImpl(owner, os.subject, uniqueEntries); - const now = new Date(); - yield* input.core.createMany( - "plugin_storage", - uniqueEntries.map((entry) => ({ + yield* input.core.upsertMany("plugin_storage", { + target: ["tenant", "owner", "subject", "plugin_id", "collection", "key"], + update: ["data", "updated_at"], + values: uniqueEntries.map((entry) => ({ tenant, owner: os.owner, subject: os.subject, @@ -1501,7 +1521,7 @@ const makePluginStorageFacade = (input: { created_at: now, updated_at: now, })), - ); + }); }); const removeManyImpl = ( diff --git a/packages/core/sdk/src/fuma-runtime.ts b/packages/core/sdk/src/fuma-runtime.ts index 89f7b7e162..25b71b376c 100644 --- a/packages/core/sdk/src/fuma-runtime.ts +++ b/packages/core/sdk/src/fuma-runtime.ts @@ -148,6 +148,7 @@ const makeSafeFumaQuery = ( db.transaction((transactionDb) => run(makeSafeFumaQuery(transactionDb, options))), updateMany: (name, value) => db.updateMany(table(name), value), upsert: (name, value) => db.upsert(table(name), value), + upsertMany: (name, value) => db.upsertMany(table(name), value), }; return Object.freeze(query); diff --git a/packages/core/sdk/src/test-config.ts b/packages/core/sdk/src/test-config.ts index c1adadaac7..5a35a08e48 100644 --- a/packages/core/sdk/src/test-config.ts +++ b/packages/core/sdk/src/test-config.ts @@ -54,6 +54,14 @@ const makeLazyTestFumaDb = (options: { transaction: async (run) => (await start()).db.internal.transaction(run), updateMany: async (table, value) => (await start()).db.internal.updateMany(table, value), upsert: async (table, value) => (await start()).db.internal.upsert(table, value), + upsertMany: async (table, value) => { + const actual = await start(); + if (!actual.db.internal.upsertMany) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: lazy test DB must expose the current FumaDB adapter surface + throw new Error("[FumaDB] upsertMany is not supported by this adapter."); + } + return actual.db.internal.upsertMany(table, value); + }, }; const queryMethods = new Set([ @@ -71,6 +79,7 @@ const makeLazyTestFumaDb = (options: { "transaction", "updateMany", "upsert", + "upsertMany", ]); const makeDb = (context?: ExecutorOwnerPolicyContext): FumaDb => diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts index 7d48dcfb65..47d278f3c0 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -87,6 +87,8 @@ type TestCollection = PluginStorageCollectionFacade & { readonly data: Map; }; +type TestAggregateValue = string | number | boolean | null; + const makeCollection = (collection: string): TestCollection => { const data = new Map(); let id = 0; @@ -176,7 +178,25 @@ const makeCollection = (collection: string): TestCollection queryKeyset: () => Effect.succeed({ entries: [], nextCursor: null }), aggregate: { count: () => Effect.succeed(data.size), - groupCount: () => Effect.succeed([]), + groupCount: (input: { field: string; where?: Record }) => + Effect.succeed( + [ + ...[...data.values()] + .filter((value) => matches(value, input.where)) + .reduce((counts, value) => { + const fieldValue = (value as Record)[input.field]; + if ( + typeof fieldValue === "string" || + typeof fieldValue === "number" || + typeof fieldValue === "boolean" || + fieldValue === null + ) { + counts.set(fieldValue, (counts.get(fieldValue) ?? 0) + 1); + } + return counts; + }, new Map()), + ].map(([value, count]) => ({ value, count })), + ), timeBuckets: () => Effect.succeed([]), stats: () => Effect.succeed({ count: 0, min: null, max: null, percentiles: [] }), }, diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts index 3a483eed36..cd8815ee34 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts @@ -639,6 +639,42 @@ const loadLatestChunkUpdatedAt = ( ), ); +const countJobsByStatus = ( + collection: PluginStorageCollectionFacade, + runId: string, +): Effect.Effect, SemanticSearchError> => + collection.aggregate + .groupCount({ + field: "status", + valueType: "text", + where: { runId }, + }) + .pipe( + Effect.map((rows) => { + const counts: Record = { + pendingScan: 0, + skipped: 0, + pendingChunk: 0, + pendingEmbedding: 0, + indexed: 0, + failed: 0, + }; + for (const row of rows) { + if (typeof row.value === "string" && row.value in counts) { + counts[row.value as IndexJob["status"]] = row.count; + } + } + return counts; + }), + Effect.mapError( + (cause) => + new SemanticSearchError({ + message: "Failed to count index jobs by status.", + cause, + }), + ), + ); + export const fail = ( input: IndexCollections & ToolSearchIndex.FailInput, ): Effect.Effect => @@ -1331,10 +1367,11 @@ export const commit = ( input: IndexDeps & ToolSearchIndex.CommitInput, ): Effect.Effect => Effect.gen(function* () { + const paths = [...new Set(input.paths)]; const entries = yield* getJobsByPaths(input, { runId: input.runId, status: "pendingEmbedding", - paths: input.paths, + paths, }); if (entries.length === 0) { return { runId: input.runId, processed: 0, committed: 0, paths: [] }; @@ -1460,37 +1497,23 @@ export const status = ( (cause) => new SemanticSearchError({ message: "Failed to load index run.", cause }), ), ); - const count = (status: IndexJob["status"]) => - input.jobs - .count({ where: { runId: input.runId, status } }) - .pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ message: `Failed to count ${status} jobs.`, cause }), - ), - ); + const statusCounts = yield* countJobsByStatus(input.jobs, input.runId); const counts = yield* Effect.all( { - storedPendingScan: count("pendingScan"), - skipped: count("skipped"), - pendingChunk: count("pendingChunk"), - pendingEmbedding: count("pendingEmbedding"), - indexed: count("indexed"), - failed: count("failed"), latestJobUpdatedAt: loadLatestJobUpdatedAt(input.jobs, input.runId), latestChunkUpdatedAt: loadLatestChunkUpdatedAt(input.chunks, input.runId), }, { concurrency: INDEX_STORAGE_CONCURRENCY }, ); const observed = - counts.storedPendingScan + - counts.skipped + - counts.pendingChunk + - counts.pendingEmbedding + - counts.indexed + - counts.failed; + statusCounts.pendingScan + + statusCounts.skipped + + statusCounts.pendingChunk + + statusCounts.pendingEmbedding + + statusCounts.indexed + + statusCounts.failed; const total = run?.data.total ?? observed; - const pendingScan = Math.max(0, total - observed + counts.storedPendingScan); + const pendingScan = Math.max(0, total - observed + statusCounts.pendingScan); const lastProgressAt = [ run?.data.updatedAt, counts.latestJobUpdatedAt, @@ -1505,11 +1528,11 @@ export const status = ( status: run?.data.status ?? "running", total, pendingScan, - skipped: counts.skipped, - pendingChunk: counts.pendingChunk, - pendingEmbedding: counts.pendingEmbedding, - indexed: counts.indexed, - failed: counts.failed, + skipped: statusCounts.skipped, + pendingChunk: statusCounts.pendingChunk, + pendingEmbedding: statusCounts.pendingEmbedding, + indexed: statusCounts.indexed, + failed: statusCounts.failed, updatedAt: run?.data.updatedAt, lastProgressAt, }; From 95e0724bd4a21883529b74d933e093eeb92c71b2 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sat, 20 Jun 2026 19:57:10 +0530 Subject: [PATCH 52/57] fix(semantic-search): preserve maxTools across resume (#47) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Preserve a reindex run's `maxTools` cap across `resume`. Previously `resume` re-enqueued scan partitions with no limit, so a run started capped (e.g. a dev/debug pass) would run **uncapped** after a resume. ## Change - `create()` stores `maxTools` on the index-run record (`indexRuns` collection; optional β€” absent = uncapped). - `ReconcileResult` exposes `maxTools`; `reconcile()` returns `run.data.maxTools`. - A host can now forward `reconciled.maxTools` when re-enqueuing scan partitions on resume. ## Test - New: `reconcile` preserves the run's `maxTools` (capped β†’ value, uncapped β†’ undefined). - `tsgo --noEmit` βœ“ Β· `oxlint --deny-warnings` (changed files) βœ“ Β· `vitest run` 9 passed βœ“ ## Follow-up The dogfood host (`AryaLabsHQ/executor`) `resume` route forwards `reconciled.maxTools` once this lands (tracked in the host backlog). Surfaced originally by a Greptile P1 on the host queue PR. --- .../semantic-search/src/sdk/collections.ts | 3 +++ .../src/sdk/tool-search-index.test.ts | 21 +++++++++++++++++++ .../src/sdk/tool-search-index.ts | 5 +++++ 3 files changed, 29 insertions(+) diff --git a/packages/plugins/semantic-search/src/sdk/collections.ts b/packages/plugins/semantic-search/src/sdk/collections.ts index 25e26d4fb2..d17064e4ec 100644 --- a/packages/plugins/semantic-search/src/sdk/collections.ts +++ b/packages/plugins/semantic-search/src/sdk/collections.ts @@ -37,6 +37,9 @@ export const IndexRun = Schema.Struct({ status: IndexRunStatus, partitionCount: Schema.Number, total: Schema.Number, + /** Original reindex cap (`maxTools`), preserved so `resume` re-enqueues scan + * partitions with the same limit instead of running uncapped. Absent = no cap. */ + maxTools: Schema.optional(Schema.Number), createdAt: Schema.String, updatedAt: Schema.String, error: Schema.optional(Schema.String), diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts index 47d278f3c0..1232ef0d19 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -403,6 +403,27 @@ describe("ToolSearchIndex", () => { }), ); + it.effect("preserves the run's maxTools cap through reconcile", () => + Effect.gen(function* () { + const counters = { raw: 0, codegen: 0 }; + const executor = makeExecutor(counters); + const runs = makeCollection(indexRuns.name); + const jobs = makeCollection(indexJobs.name); + const chunks = makeCollection(indexChunks.name); + const fingerprints = makeCollection(toolFingerprints.name); + const blobs = makeBlobs(); + const base = { namespace, executor, runs, jobs, chunks, fingerprints, blobs, owner }; + + yield* create({ ...base, runId: "run-capped", partitionCount: 1, maxTools: 1 }); + const capped = yield* reconcile({ ...base, runId: "run-capped" }); + expect(capped.maxTools).toBe(1); + + yield* create({ ...base, runId: "run-uncapped", partitionCount: 1 }); + const uncapped = yield* reconcile({ ...base, runId: "run-uncapped" }); + expect(uncapped.maxTools).toBeUndefined(); + }), + ); + it.effect( "reconstructs embed messages when a chunk message retries after persisting chunks", () => diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts index cd8815ee34..47f6f5bfe6 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts @@ -164,6 +164,9 @@ export declare namespace ToolSearchIndex { export interface ReconcileResult { readonly runId: string; + /** The run's original `maxTools` cap (absent = uncapped), so the host can + * re-enqueue scan partitions with the same limit on resume. */ + readonly maxTools?: number; readonly scanPartitions: readonly number[]; readonly pendingChunkPaths: readonly string[]; readonly pendingEmbeddingChunks: readonly ChunkRef[]; @@ -892,6 +895,7 @@ export const reconcile = ( return { runId: input.runId, + maxTools: run.data.maxTools, scanPartitions, pendingChunkPaths: [...new Set(pendingChunkJobs.map((entry) => entry.data.path))], pendingEmbeddingChunks: pendingEmbeddingChunks.map((entry) => ({ @@ -920,6 +924,7 @@ export const create = ( status: "running", partitionCount, total: manifests.length, + ...(input.maxTools === undefined ? {} : { maxTools: input.maxTools }), createdAt, updatedAt: createdAt, }, From d1e35b069503f777634c582f1c1ce14443821b98 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Sun, 21 Jun 2026 14:53:25 +0530 Subject: [PATCH 53/57] fix(semantic-search): don't default the search prefix filter to the tenant namespace (#48) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `executor.semanticSearch.search` (the operator `tools.search` surface) returned **zero results for every query**, even against a fully populated vector + lexical index. Root cause: a single `namespace` value was overloaded as two different concepts. ## Root cause The search method conflated: - **`deps.namespace`** β€” the *tenant* namespace (e.g. `"default"`), used for Vectorize storage isolation inside `store.query`. - **`input.namespace`** β€” an *optional integration/path-prefix filter* the operator can pass to narrow results (e.g. `"github"`). With no operator-supplied prefix, it defaulted the prefix filter to the tenant namespace: ```ts const namespace = input.namespace ?? deps.namespace; // ← bug ``` The provider's `matchesNamespace` then required every tool's integration/path to *start with* the tenant id (`"github_api".startsWith("default")` β†’ `false`), so it dropped every match. This affected both the vector and lexical paths, since they share `matchesNamespace`. ## Fix Apply the integration prefix only when explicitly provided. The tenant namespace drives the store query alone and is still reported on the result page. ## Verification - New regression tests at the plugin-search level: with no prefix the provider receives `undefined` (no filter) and tenant-tagged results survive; an explicit prefix still passes through. - Suite green: `vitest` (20 passed), `tsgo --noEmit`, `oxlint --deny-warnings`. - Verified live on a self-host deploy β€” before: `{"items":[]}` for every query; after: `"query my postgres database"` β†’ Neon/Planetscale SQL tools, `"github"` β†’ `github_api` tools, and `?namespace=stripe_api` still narrows correctly. ## Notes - `makeSemanticSearchExtension` is exported for the unit test but **not** re-exported from the package index, so the public API is unchanged. - Changeset included (`patch`). --- ...semantic-search-tenant-namespace-prefix.md | 14 +++ .../semantic-search/src/sdk/plugin.test.ts | 85 +++++++++++++++++++ .../plugins/semantic-search/src/sdk/plugin.ts | 20 ++++- 3 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 .changeset/semantic-search-tenant-namespace-prefix.md create mode 100644 packages/plugins/semantic-search/src/sdk/plugin.test.ts diff --git a/.changeset/semantic-search-tenant-namespace-prefix.md b/.changeset/semantic-search-tenant-namespace-prefix.md new file mode 100644 index 0000000000..c9f4253013 --- /dev/null +++ b/.changeset/semantic-search-tenant-namespace-prefix.md @@ -0,0 +1,14 @@ +--- +"@executor-js/plugin-semantic-search": patch +--- + +fix: don't default the operator search prefix filter to the tenant namespace + +`executor.semanticSearch.search` (the operator `tools.search` surface) overloaded a +single `namespace` value as both the Vectorize tenant namespace (storage isolation) +and the optional integration/path-prefix filter. With no operator-supplied prefix it +defaulted the filter to the tenant namespace (e.g. `"default"`), so the provider's +`matchesNamespace` required every tool path to start with the tenant id and dropped +every result β€” search returned nothing for all queries despite a fully populated +index. The integration prefix is now applied only when explicitly provided; the +tenant namespace drives the store query alone and is still reported on the result page. diff --git a/packages/plugins/semantic-search/src/sdk/plugin.test.ts b/packages/plugins/semantic-search/src/sdk/plugin.test.ts new file mode 100644 index 0000000000..e5da652264 --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/plugin.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import type { ToolDiscoveryProvider, ToolDiscoveryResult } from "@executor-js/sdk/core"; + +import { makeWholeChunker } from "./chunker"; +import { makeSemanticSearchExtension } from "./plugin"; + +// A result whose integration/path is NOT the tenant id. This is exactly the case +// the bug broke: the operator search defaulted the integration-prefix filter to +// the tenant namespace ("default"), and the provider's `matchesNamespace` then +// dropped every tool whose path didn't start with "default" β€” i.e. all of them. +const githubItem: ToolDiscoveryResult = { + path: "github_api.repos.get", + name: "repos.get", + description: "Get a repository", + integration: "github_api", + score: 0.9, +}; + +// Build the extension with only what `search` needs: a tenant namespace and a +// provider that records the prefix it is handed. The remaining deps are unused +// by `search` (they drive indexing), so they stay undefined. +const extensionWith = (provider: ToolDiscoveryProvider) => + makeSemanticSearchExtension({ + namespace: "default", // tenant id β€” NOT a tool prefix + embedder: undefined, + store: undefined, + chunker: makeWholeChunker(), + fingerprints: undefined, + indexRuns: undefined, + indexJobs: undefined, + indexChunks: undefined, + blobs: undefined, + owner: undefined, + lexicalStore: undefined, + provider, + }); + +describe("makeSemanticSearchExtension β€” search namespace handling", () => { + it.effect("does not default the integration prefix to the tenant namespace", () => + Effect.gen(function* () { + let calls = 0; + let receivedPrefix: string | undefined; + const ext = extensionWith({ + searchTools: (input) => { + calls += 1; + receivedPrefix = input.namespace; + return Effect.succeed({ + items: [githubItem], + total: 1, + hasMore: false, + nextOffset: null, + }); + }, + }); + + const page = yield* ext.search(undefined as never, { query: "github" }); + + // Regression: with no operator prefix, the provider must receive `undefined` + // (no filter) β€” NOT the tenant namespace, which would drop every result. + expect(calls).toBe(1); + expect(receivedPrefix).toBeUndefined(); + // The tenant-tagged result survives, and the page reports the tenant. + expect(page.items).toEqual([githubItem]); + expect(page.namespace).toBe("default"); + }), + ); + + it.effect("passes an explicit integration prefix through to the provider", () => + Effect.gen(function* () { + let receivedPrefix: string | undefined; + const ext = extensionWith({ + searchTools: (input) => { + receivedPrefix = input.namespace; + return Effect.succeed({ items: [], total: 0, hasMore: false, nextOffset: null }); + }, + }); + + yield* ext.search(undefined as never, { query: "x", namespace: "github_api" }); + + expect(receivedPrefix).toBe("github_api"); + }), + ); +}); diff --git a/packages/plugins/semantic-search/src/sdk/plugin.ts b/packages/plugins/semantic-search/src/sdk/plugin.ts index 3afa876b5b..47f9c0bfd9 100644 --- a/packages/plugins/semantic-search/src/sdk/plugin.ts +++ b/packages/plugins/semantic-search/src/sdk/plugin.ts @@ -99,7 +99,9 @@ export interface SemanticSearchStatus { * scoped executor because only the request/API layer holds it. `reindex`/`search` * are inert β€” failing clearly β€” until both a vector store and an embedder are * present. */ -const makeSemanticSearchExtension = (deps: { +// Exported for unit testing the `search` namespace handling; not re-exported +// from the package index (see sdk/index.ts), so the public API is unchanged. +export const makeSemanticSearchExtension = (deps: { readonly namespace: string; readonly embedder: ToolEmbedder | undefined; readonly store: VectorStore | undefined; @@ -203,18 +205,28 @@ const makeSemanticSearchExtension = (deps: { executor: Executor, input: { readonly query: string; readonly namespace?: string; readonly limit?: number }, ): Effect.Effect => { - const namespace = input.namespace ?? deps.namespace; + // `input.namespace` is an OPTIONAL integration/path-prefix filter β€” distinct + // from the tenant `deps.namespace` (Vectorize storage isolation, applied + // inside the provider's store.query). It must NOT default to the tenant + // namespace: a tenant id like "default" is not a tool prefix, so the + // provider's `matchesNamespace` would drop every result. Absent β†’ no prefix + // filter; the response still reports the tenant namespace. + const prefix = input.namespace; return deps.provider ? deps.provider .searchTools({ executor, query: input.query, - namespace, + namespace: prefix, limit: input.limit ?? DEFAULT_SEARCH_LIMIT, offset: 0, }) .pipe( - Effect.map((page) => ({ namespace, query: input.query, items: page.items })), + Effect.map((page) => ({ + namespace: deps.namespace, + query: input.query, + items: page.items, + })), Effect.mapError( (cause) => new SemanticSearchError({ message: "Semantic search query failed.", cause }), From 52d38768dbfdc8719373b98ff80e5ef3498edefb Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 22 Jun 2026 03:19:18 +0530 Subject: [PATCH 54/57] perf(sdk): precompile tool-schema TypeScript during indexing (#49) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Production traces showed the web UI's tool-detail path (`GET /api/tools/schema`, `includeTypeScript: true`) paying the json-schema β†’ TypeScript compile on the first view of every tool. The reindex pipeline already visits every tool's schema, but it called `tools.schema({ includeTypeScript: false })` specifically to *skip* that compile β€” so it warmed the wrong cache variant and the UI's variant stayed cold until first manual view. ## What - `executor.tools.schema` always renders the TypeScript preview; the `includeTypeScript` option and the now-empty `ToolSchemaOptions` type are removed. - The reindex pipeline (`collectDocForTool`) calls `tools.schema` plainly, so every reindex pre-warms exactly the KV schema-view the UI reads β†’ first tool-detail views become cache hits instead of a cold compile. - `includeTypeScript` is dropped from the schema-view cache key (content + manifest), collapsing the two variants into one. ## Safety - Embedding/lexical text is unchanged: `collectDocForTool` only reads the raw schema facets (`inputSchema` / `outputSchema` / `schemaDefinitions`), never the rendered TypeScript β€” so no re-embedding / index churn. - The cache-key change supersedes existing entries; they re-warm on the next reindex or view (regenerable derived data). ## Gate tsgo (43/43), `oxlint --deny-warnings`, oxfmt, and vitest for `@executor-js/sdk` + `@executor-js/plugin-semantic-search` all green. Follow-up: a reindex on the deployed instance warms the new unified variant. ### [Stack](https://github.com/kitlangton/stack) 1. **#49** πŸ‘ˆ current 2. #50 --- .../tools-schema-precompile-typescript.md | 15 +++++ packages/core/sdk/src/executor.test.ts | 2 +- packages/core/sdk/src/executor.ts | 57 +++++++------------ packages/core/sdk/src/index.ts | 1 - packages/core/sdk/src/promise.ts | 1 - packages/core/sdk/src/shared.ts | 1 - .../sdk/src/tool-schema-view-cache.test.ts | 3 - .../core/sdk/src/tool-schema-view-cache.ts | 4 -- packages/core/sdk/src/types.ts | 12 ---- .../semantic-search/src/sdk/documents.ts | 10 ++-- .../src/sdk/tool-search-index.test.ts | 12 ++-- 11 files changed, 47 insertions(+), 71 deletions(-) create mode 100644 .changeset/tools-schema-precompile-typescript.md diff --git a/.changeset/tools-schema-precompile-typescript.md b/.changeset/tools-schema-precompile-typescript.md new file mode 100644 index 0000000000..817d475411 --- /dev/null +++ b/.changeset/tools-schema-precompile-typescript.md @@ -0,0 +1,15 @@ +--- +"@executor-js/sdk": minor +"@executor-js/plugin-semantic-search": patch +--- + +Always render the TypeScript preview in `executor.tools.schema` and warm it during +indexing. The `includeTypeScript` option (and the now-empty `ToolSchemaOptions` type) is +removed β€” the only caller that passed `false` was the reindex pipeline, which did so to +skip the CPU-heavy JSON-schema β†’ TypeScript codegen. It now calls `tools.schema` plainly, +so every reindex pre-warms exactly the schema view the web UI reads on first open (the +codegen is content-addressed in KV), turning first tool-detail views into cache hits +instead of a cold compile. The embedding text is unchanged β€” the indexer only ever read +the raw schema facets, never the rendered TypeScript. Because `includeTypeScript` is +dropped from the schema-view cache key, existing cached entries are superseded on the next +reindex or view (regenerable derived data). diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 9030a08351..614f73bf56 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -492,7 +492,7 @@ describe("createExecutor", () => { expect(inspect?.definitionSetHash).toHaveLength(64); expect(inspect?.indexFingerprint).toHaveLength(64); - const schema = yield* executor.tools.schema(addr("inspect"), { includeTypeScript: false }); + const schema = yield* executor.tools.schema(addr("inspect")); expect(schema?.schemaDefinitions).not.toHaveProperty("Unused"); }), ); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index a6b6a1f05d..0cac230f17 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -150,12 +150,7 @@ import { ORG_SUBJECT, type ExecutorOwnerPolicyContext, } from "./owner-policy"; -import { - ToolSchemaManifest, - ToolSchemaView, - type IntegrationDetectionResult, - type ToolSchemaOptions, -} from "./types"; +import { ToolSchemaManifest, ToolSchemaView, type IntegrationDetectionResult } from "./types"; import { type Tool, type ToolAnnotations, type ToolDef, type ToolListFilter } from "./tool"; import { buildToolTypeScriptPreview, type ToolTypeScriptPreview } from "./schema-types"; import { collectReferencedDefinitions } from "./schema-refs"; @@ -329,10 +324,7 @@ export type Executor = { readonly manifest: ( filter?: ToolListFilter, ) => Effect.Effect; - readonly schema: ( - address: ToolAddress, - options?: ToolSchemaOptions, - ) => Effect.Effect; + readonly schema: (address: ToolAddress) => Effect.Effect; }; readonly providers: { @@ -3216,14 +3208,12 @@ export const createExecutor = => Effect.gen(function* () { - // The JSON-schema β†’ TypeScript codegen is the CPU-heavy part of this - // surface; callers that only need the schema's content (a reindex - // computing a content fingerprint) opt out and skip it. Raw schema roots - // + referenced `$defs` are returned either way. - const includeTypeScript = options?.includeTypeScript ?? true; + // Always renders the JSON-schema β†’ TypeScript preview. The codegen is the + // CPU-heavy part of this surface, but the rendered view is content-addressed + // in KV, so the reindex pipeline warms exactly the view the web UI reads on + // first open. Raw schema roots + referenced `$defs` are returned alongside. const staticEntry = staticTools.get(String(address)); if (staticEntry) { const tool = staticToolToTool(staticEntry); @@ -3234,22 +3224,19 @@ export const createExecutor = Effect.succeed(Option.none()))); if (Option.isSome(cached)) return cached.value.view; - const preview = includeTypeScript - ? Option.getOrUndefined( - yield* buildCachedToolTypeScriptPreview({ - inputSchema: tool.inputSchema, - outputSchema: tool.outputSchema, - defs: new Map(), - }).pipe(Effect.option), - ) - : undefined; + const preview = Option.getOrUndefined( + yield* buildCachedToolTypeScriptPreview({ + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + defs: new Map(), + }).pipe(Effect.option), + ); const view = ToolSchemaView.make({ address, name: tool.name, @@ -3290,7 +3277,6 @@ export const createExecutor = (Object.entries(referenced)); - const preview = includeTypeScript - ? Option.getOrUndefined( - yield* buildCachedToolTypeScriptPreview({ - inputSchema: tool.inputSchema, - outputSchema: tool.outputSchema, - defs: referencedDefs, - }).pipe(Effect.option), - ) - : undefined; + const preview = Option.getOrUndefined( + yield* buildCachedToolTypeScriptPreview({ + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + defs: referencedDefs, + }).pipe(Effect.option), + ); const view = ToolSchemaView.make({ address, diff --git a/packages/core/sdk/src/index.ts b/packages/core/sdk/src/index.ts index 87bb80370f..f86e8a5f68 100644 --- a/packages/core/sdk/src/index.ts +++ b/packages/core/sdk/src/index.ts @@ -99,7 +99,6 @@ export type { CredentialProvider, ProviderEntry } from "./provider"; // Public projections / detection. export { ToolSchemaManifest, ToolSchemaView, IntegrationDetectionResult } from "./types"; -export type { ToolSchemaOptions } from "./types"; // Core schema. export { diff --git a/packages/core/sdk/src/promise.ts b/packages/core/sdk/src/promise.ts index 309e950f91..49c455292a 100644 --- a/packages/core/sdk/src/promise.ts +++ b/packages/core/sdk/src/promise.ts @@ -44,7 +44,6 @@ export type { UpdateToolPolicyInput, } from "./policies"; export { ToolSchemaManifest, ToolSchemaView, IntegrationDetectionResult } from "./types"; -export type { ToolSchemaOptions } from "./types"; export type { Tool, ToolDef, ToolListFilter, ToolAnnotations } from "./tool"; export type { AnyPlugin, PluginExtensions } from "./plugin"; export type { diff --git a/packages/core/sdk/src/shared.ts b/packages/core/sdk/src/shared.ts index ef1588f631..e4939ef60f 100644 --- a/packages/core/sdk/src/shared.ts +++ b/packages/core/sdk/src/shared.ts @@ -97,7 +97,6 @@ export type { ToolPolicyAction } from "./core-schema"; // Schema-side views + onboarding autodetect. export { ToolSchemaManifest, ToolSchemaView, IntegrationDetectionResult } from "./types"; -export type { ToolSchemaOptions } from "./types"; // OAuth wire contracts (data + tagged errors; the flow impl is server-only). export { diff --git a/packages/core/sdk/src/tool-schema-view-cache.test.ts b/packages/core/sdk/src/tool-schema-view-cache.test.ts index 425b987b1c..9dc1a05b95 100644 --- a/packages/core/sdk/src/tool-schema-view-cache.test.ts +++ b/packages/core/sdk/src/tool-schema-view-cache.test.ts @@ -69,9 +69,6 @@ describe("toolSchemaViewCacheKey", () => { inputSchema: { type: "object", properties: { pet: { type: "string" } } }, }), ).not.toBe(sameKey); - expect(yield* toolSchemaViewCacheKey({ ...base, includeTypeScript: false })).not.toBe( - sameKey, - ); }), ); }); diff --git a/packages/core/sdk/src/tool-schema-view-cache.ts b/packages/core/sdk/src/tool-schema-view-cache.ts index ca385781b3..ce99b289f6 100644 --- a/packages/core/sdk/src/tool-schema-view-cache.ts +++ b/packages/core/sdk/src/tool-schema-view-cache.ts @@ -25,14 +25,12 @@ interface ToolSchemaViewCacheKeyInput { readonly inputSchema?: unknown; readonly outputSchema?: unknown; readonly definitions: Readonly>; - readonly includeTypeScript?: boolean; } interface ToolSchemaViewManifestCacheKeyInput { readonly address: string; readonly indexFingerprint: string; readonly fingerprintVersion: string; - readonly includeTypeScript?: boolean; } const cachePayload = (input: ToolSchemaViewCacheKeyInput): string => @@ -46,7 +44,6 @@ const cachePayload = (input: ToolSchemaViewCacheKeyInput): string => inputSchema: input.inputSchema, outputSchema: input.outputSchema, definitions: input.definitions, - includeTypeScript: input.includeTypeScript ?? true, }); const manifestCachePayload = (input: ToolSchemaViewManifestCacheKeyInput): string => @@ -57,7 +54,6 @@ const manifestCachePayload = (input: ToolSchemaViewManifestCacheKeyInput): strin address: input.address, indexFingerprint: input.indexFingerprint, fingerprintVersion: input.fingerprintVersion, - includeTypeScript: input.includeTypeScript ?? true, }); export const toolSchemaViewCacheKey = (input: ToolSchemaViewCacheKeyInput): Effect.Effect => diff --git a/packages/core/sdk/src/types.ts b/packages/core/sdk/src/types.ts index 05a9f4b34a..3963046066 100644 --- a/packages/core/sdk/src/types.ts +++ b/packages/core/sdk/src/types.ts @@ -54,18 +54,6 @@ export const ToolSchemaManifest = Schema.Struct({ }); export type ToolSchemaManifest = typeof ToolSchemaManifest.Type; -/** Options for `executor.tools.schema`. */ -export interface ToolSchemaOptions { - /** Generate the TypeScript preview strings (`inputTypeScript` / - * `outputTypeScript` / `typeScriptDefinitions`). Defaults to `true`. Pass - * `false` to skip the CPU-heavy JSON-schema β†’ TypeScript codegen and return - * only the raw JSON schema roots plus referenced `$defs` β€” the cheap tier for - * callers that need the schema's content but not its rendered types (e.g. a - * reindex computing a content fingerprint). Mirrors `tools.list`'s - * `includeAnnotations` opt-out of an expensive per-tool derivation. */ - readonly includeTypeScript?: boolean; -} - // --------------------------------------------------------------------------- // Integration detection β€” optional capability on `PluginSpec.detect`. When a // user pastes a URL in the onboarding UI, `executor.integrations.detect(url)` diff --git a/packages/plugins/semantic-search/src/sdk/documents.ts b/packages/plugins/semantic-search/src/sdk/documents.ts index 5250b10c84..20ea2beffe 100644 --- a/packages/plugins/semantic-search/src/sdk/documents.ts +++ b/packages/plugins/semantic-search/src/sdk/documents.ts @@ -145,7 +145,7 @@ export const listToolDescriptors = ( * * This is the manifest tier for indexing: it carries precomputed raw-schema * fingerprints from source refresh, so scan/diff can avoid per-tool - * `tools.schema(includeTypeScript: false)` reads entirely. */ + * `tools.schema` reads entirely. */ export const listToolManifests = ( executor: Executor, options?: ListToolDescriptorsOptions, @@ -219,8 +219,10 @@ const schemaDefinitionsFacetText = ( /** Collect `ToolDocumentInput` for a single tool descriptor. * - * Attempts to fetch raw schema via `tools.schema(address, { includeTypeScript: - * false })` to populate compact schema facets; on failure it degrades + * Fetches the tool's schema via `tools.schema(address)` to populate compact + * schema facets from the raw input/output schema + `$defs`. The TypeScript + * preview it also renders is unused here, but the call warms the shared + * schema-view cache the web UI reads on first open. On failure it degrades * gracefully to an identity-only document. */ export const collectDocForTool = ( @@ -234,7 +236,7 @@ export const collectDocForTool = ( integration: String(tool.integration), description: stripHtml(String(tool.description ?? "")), }; - return executor.tools.schema(tool.address as Tool["address"], { includeTypeScript: false }).pipe( + return executor.tools.schema(tool.address as Tool["address"]).pipe( Effect.map((view): ToolDocumentInput => { if (view === null) return { ...base, lexicalText: buildLexicalText(base) }; diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts index 1232ef0d19..b09e0a2c77 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -68,14 +68,12 @@ const makeExecutor = ( tools: { list: () => Effect.succeed([tool]), manifest: () => Effect.succeed([manifestForTool(tool)]), - schema: (address, options) => { - const includeTypeScript = options?.includeTypeScript ?? true; - if (includeTypeScript) counters.codegen++; - else counters.raw++; + schema: (address) => { + counters.codegen++; return Effect.succeed({ address, inputSchema: { type: "object", properties: { owner: { type: "string" } } }, - inputTypeScript: includeTypeScript ? "{ owner: string }" : undefined, + inputTypeScript: "{ owner: string }", }); }, }, @@ -298,7 +296,7 @@ describe("ToolSearchIndex", () => { expect(chunked.processed).toBe(1); expect(chunked.chunks).toBeGreaterThan(0); expect(embedded).toMatchObject({ processed: chunked.chunks, chunks: chunked.chunks }); - expect(counters).toEqual({ raw: 1, codegen: 0 }); + expect(counters).toEqual({ raw: 0, codegen: 1 }); expect(upserted).toHaveLength(chunked.chunks); for (const vector of upserted) { expect( @@ -478,7 +476,7 @@ describe("ToolSearchIndex", () => { expect(first.chunkRefs.length).toBeGreaterThan(0); expect(retry.chunkRefs).toEqual(first.chunkRefs); expect(retry.commitPaths).toEqual([]); - expect(counters).toEqual({ raw: 1, codegen: 0 }); + expect(counters).toEqual({ raw: 0, codegen: 1 }); }), ); From 81e65df8135872184e2d177846253d15a84f3fca Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 22 Jun 2026 03:29:25 +0530 Subject: [PATCH 55/57] perf(sdk): cache tools.list by catalog + policy revision (#50) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Phase 1** of Tier-2 tool-catalog caching β€” stacked on #49. Caches `executor.tools.list` (today a full `tool` scan + `tool_policy` read on every call) behind a **content-derived revision key**, so the list stays warm and only re-reads D1 when the catalog or policies actually change. No TTL. ## Invalidation - `produceConnectionTools` writes a per-connection **`toolset_revision`** = `sha256` of the connection's manifest rows' `(path, index_fingerprint)`, in **`plugin_storage`** (owner-scoped, **no schema migration**), at all 5 catalog-mutation sites (produce, both clear branches, connection-remove, integration-remove). - `catalogGen` combines the in-scope connections' revisions; `policyGen` combines the `tool_policy` rows. Key = `sha256(owner, integration, connection, includeBlocked, catalogGen, policyGen)`. - **Token rotation does not bust** (never calls `produceConnectionTools`); add/remove/change a tool, or any policy edit, does. ## Scope - Cached value = the DB-derived, policy-filtered list. **Static tools unioned live** (never cached); **`query` applied post-cache** so distinct queries share one cached set. - Hand-authored `ToolListItem` Schema (list projection only β€” no input/output schema); branded ids stored as strings, rebuilt on read. ## Notes (intentional, single-tenant self-host) - `catalogGen`/`policyGen` reads are tenant-wide β†’ per-tenant-coarse invalidation (a catalog change re-warms all owners' list caches). Correct, just coarse. ## Gate tsgo (sdk) Β· `oxlint --deny-warnings` Β· oxfmt Β· vitest **sdk 350/350** (new cache suite: add/remove/content invalidation, policy bust, query post-cache, static-union-on-hit). Phase 2 (stacked) moves the UI list onto the compact manifest tier. ### [Stack](https://github.com/kitlangton/stack) 1. #49 2. **#50** πŸ‘ˆ current --- .changeset/tools-list-revision-cache.md | 14 ++ packages/core/sdk/src/executor.test.ts | 138 +++++++++++++ packages/core/sdk/src/executor.ts | 239 ++++++++++++++++++++--- packages/core/sdk/src/tool-list-cache.ts | 78 ++++++++ 4 files changed, 441 insertions(+), 28 deletions(-) create mode 100644 .changeset/tools-list-revision-cache.md create mode 100644 packages/core/sdk/src/tool-list-cache.ts diff --git a/.changeset/tools-list-revision-cache.md b/.changeset/tools-list-revision-cache.md new file mode 100644 index 0000000000..03663dd4b0 --- /dev/null +++ b/.changeset/tools-list-revision-cache.md @@ -0,0 +1,14 @@ +--- +"@executor-js/sdk": minor +--- + +Cache `tools.list` by a content-derived catalog revision. `produceConnectionTools` now +records a per-connection `toolset_revision` β€” a hash of the connection's manifest +`(path, index_fingerprint)` pairs β€” in `plugin_storage`, and `tools.list` keys a KV cache of +the DB-derived, policy-filtered list on the combined catalog revision plus a policy +revision (both read live, O(connections)/O(policies), not the full `tool` scan). Static tools +are unioned live and the `query` substring filter is applied post-cache, so a distinct query +reuses the same cached set. The cache stays warm through token rotation and unrelated writes +and invalidates exactly when a connection's tool set, a tool's schema, or a `tool_policy` +changes β€” including OpenAPI/MCP tool additions on refresh (a new manifest row) and +connection/integration removal. diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 614f73bf56..d81f415b65 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -652,3 +652,141 @@ describe("createExecutor", () => { }), ); }); + +describe("tools.list cache", () => { + const names = (tools: readonly { readonly name: ToolName }[]): string[] => + tools.map((tool) => String(tool.name)).sort(); + + const dynamicPlugin = (resolve: () => readonly { name: ToolName; description: string }[]) => + definePlugin(() => ({ + id: "listCache" as const, + credentialProviders: [memoryProvider()], + storage: () => ({}), + resolveTools: () => Effect.succeed({ tools: resolve(), definitions: {} }), + invokeTool: ({ toolRow }) => Effect.succeed({ ran: toolRow.name }), + extension: (ctx) => ({ + seed: () => + ctx.core.integrations.register({ slug: INTEG, description: "Demo", config: {} }), + }), + }))(); + + it.effect("reflects tool-set additions and removals across a refresh", () => + Effect.gen(function* () { + let tools = [ + { name: ToolName.make("inspect"), description: "inspect" }, + { name: ToolName.make("run"), description: "run" }, + ]; + const executor = yield* makeTestExecutor({ + plugins: [dynamicPlugin(() => tools)] as const, + }); + yield* executor.listCache.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + + // Prime the cache. + expect(names(yield* executor.tools.list({ integration: INTEG }))).toEqual(["inspect", "run"]); + + // Remove `inspect`, add `status` β€” a stale cache would still return [inspect, run]. + tools = [ + { name: ToolName.make("run"), description: "run" }, + { name: ToolName.make("status"), description: "status" }, + ]; + yield* executor.connections.refresh({ owner: "org", integration: INTEG, name: CONN }); + expect(names(yield* executor.tools.list({ integration: INTEG }))).toEqual(["run", "status"]); + }), + ); + + it.effect("reflects a tool content change across a refresh", () => + Effect.gen(function* () { + let description = "v1"; + const executor = yield* makeTestExecutor({ + plugins: [dynamicPlugin(() => [{ name: ToolName.make("inspect"), description }])] as const, + }); + yield* executor.listCache.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + + expect((yield* executor.tools.list({ integration: INTEG }))[0]?.description).toBe("v1"); + description = "v2"; + yield* executor.connections.refresh({ owner: "org", integration: INTEG, name: CONN }); + expect((yield* executor.tools.list({ integration: INTEG }))[0]?.description).toBe("v2"); + }), + ); + + it.effect("invalidates when a block policy is added", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [ + dynamicPlugin(() => [{ name: ToolName.make("inspect"), description: "inspect" }]), + ] as const, + }); + yield* executor.listCache.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + + expect(names(yield* executor.tools.list({ integration: INTEG }))).toEqual(["inspect"]); + // A stale cache would still surface the now-blocked tool. + yield* executor.policies.create({ owner: "org", pattern: `${INTEG}.*`, action: "block" }); + expect(yield* executor.tools.list({ integration: INTEG })).toEqual([]); + }), + ); + + it.effect("applies the query filter post-cache so distinct queries share a cached set", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [ + dynamicPlugin(() => [ + { name: ToolName.make("inspect"), description: "inspect" }, + { name: ToolName.make("run"), description: "run" }, + ]), + ] as const, + }); + yield* executor.listCache.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + + expect(names(yield* executor.tools.list({ integration: INTEG, query: "inspect" }))).toEqual([ + "inspect", + ]); + expect(names(yield* executor.tools.list({ integration: INTEG, query: "run" }))).toEqual([ + "run", + ]); + expect(names(yield* executor.tools.list({ integration: INTEG }))).toEqual(["inspect", "run"]); + }), + ); + + it.effect("unions static tools live on a cache hit", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + coreTools: { webBaseUrl: "http://localhost:3000" }, + }); + const first = yield* executor.tools.list({}); + const second = yield* executor.tools.list({}); + const addresses = (tools: readonly { readonly address: ToolAddress }[]) => + tools.map((tool) => String(tool.address)).sort(); + expect(addresses(first)).toEqual(addresses(second)); + expect(first.length).toBeGreaterThan(0); + expect(first.some((tool) => tool.static === true)).toBe(true); + }), + ); +}); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 0cac230f17..8bb7b30848 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -166,6 +166,15 @@ import { toolSchemaViewCacheKey, toolSchemaViewManifestCacheKey, } from "./tool-schema-view-cache"; +import { + CATALOG_REVISION_COLLECTION, + CATALOG_REVISION_PLUGIN_ID, + catalogRevisionStorageKey, + TOOL_LIST_CACHE_VERSION, + ToolListCacheEntry, + toolListCacheKey, + type ToolListCacheItem, +} from "./tool-list-cache"; import { refreshAccessToken, exchangeClientCredentials, @@ -1928,6 +1937,7 @@ export const createExecutor = (effect: Effect.Effect) => fuma.transaction(effect); // Populated once, never mutated after startup. @@ -2512,6 +2522,14 @@ export const createExecutor = + b.and( + b("plugin_id", "=", CATALOG_REVISION_PLUGIN_ID), + b("collection", "=", CATALOG_REVISION_COLLECTION), + b("key", "starts with", `${String(slug)}:`), + ), + }); yield* core.deleteMany("connection", { where }); yield* core.deleteMany("integration", { where: (b: AnyCb) => b("slug", "=", String(slug)), @@ -2557,6 +2575,14 @@ export const createExecutor = [manifestRow.path, manifestRow.index_fingerprint] as const) + .sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)), + ), + ); + yield* transaction( Effect.gen(function* () { yield* core.deleteMany("tool", { where }); @@ -2688,6 +2727,23 @@ export const createExecutor = + b.and( + byOwner(ref.owner)(b), + b("plugin_id", "=", CATALOG_REVISION_PLUGIN_ID), + b("collection", "=", CATALOG_REVISION_COLLECTION), + b("key", "=", catalogRevisionStorageKey(String(ref.integration), String(ref.name))), + ), + }); yield* core.deleteMany("connection", { where: (b: AnyCb) => b.and( @@ -3108,44 +3173,160 @@ export const createExecutor = { + const owner = item.owner as Owner; + const integration = IntegrationSlug.make(item.integration); + const connection = ConnectionName.make(item.connection); + const name = ToolName.make(item.name); + return { + address: toolAddress(owner, integration, connection, name), + owner, + integration, + connection, + name, + pluginId: item.pluginId, + description: item.description, + annotations: item.annotations as ToolAnnotations | undefined, + ...(item.static === undefined ? {} : { static: item.static }), + }; + }; + const toolToCachedItem = (tool: Tool): ToolListCacheItem => ({ + owner: tool.owner, + integration: tool.integration, + connection: tool.connection, + name: tool.name, + pluginId: tool.pluginId, + description: tool.description, + ...(tool.annotations === undefined ? {} : { annotations: tool.annotations }), + ...(tool.static === undefined ? {} : { static: tool.static }), + }); + const toolsList = (filter?: ToolListFilter): Effect.Effect => Effect.gen(function* () { - // Projected: the list surface is metadata (address, description, - // annotations) β€” loading every tool's input/output schema JSON made - // an unbounded list scale with schema bytes, not tool count. - const rows = yield* core.findMany("tool", { + const includeBlocked = filter?.includeBlocked ?? false; + const policyRows = yield* core.findMany("tool_policy", {}); + // Content-derived revisions, read live each call (cheap: `tool_policy` is + // small and the revision rows are O(connections)). The catalog only moves + // when a connection is (re)produced or removed, so the cached DB list stays + // warm through token rotation and unrelated writes. + const policyGen = yield* sha256Hex( + cacheKeyPayload( + policyRows + .map((policy) => ({ + id: policy.id, + owner: policy.owner, + pattern: policy.pattern, + action: policy.action, + position: policy.position, + })) + .sort((a, b) => + a.owner === b.owner + ? a.id < b.id + ? -1 + : a.id > b.id + ? 1 + : 0 + : a.owner < b.owner + ? -1 + : 1, + ), + ), + ); + const revisionRows = yield* core.findMany("plugin_storage", { where: (b: AnyCb) => b.and( - filter?.integration === undefined - ? true - : b("integration", "=", String(filter.integration)), filter?.owner === undefined ? true : b("owner", "=", filter.owner), - filter?.connection === undefined - ? true - : b("connection", "=", String(filter.connection)), + b("plugin_id", "=", CATALOG_REVISION_PLUGIN_ID), + b("collection", "=", CATALOG_REVISION_COLLECTION), ), - select: TOOL_INVOCATION_COLUMNS, }); - const includeBlocked = filter?.includeBlocked ?? false; - const policyRows = yield* core.findMany("tool_policy", {}); - const tools: Tool[] = []; - for (const row of rows) { - const tool = rowToTool(row); - if (!matchesToolFilter(tool, filter)) continue; - if (!includeBlocked) { - const effective = resolveEffectivePolicy( - normalizedPolicyId(tool), - policyRows, - ownerRankForRow, - tool.annotations?.requiresApproval, - ); - if (effective.action === "block") continue; + // Carry `owner` in both the tuple and the sort: the revision `key` is only + // `integration:connection`, so the all-owners read (filter.owner undefined) + // can surface same-`key` rows for different owners β€” sorting by `key` alone + // would order them nondeterministically and destabilize the cache key. + const catalogGen = yield* sha256Hex( + cacheKeyPayload( + revisionRows + .map((row) => ({ + owner: row.owner, + key: row.key, + revision: + (decodeJsonColumn(row.data) as { revision?: string } | undefined)?.revision ?? "", + })) + .sort((a, b) => + a.owner === b.owner + ? a.key < b.key + ? -1 + : a.key > b.key + ? 1 + : 0 + : a.owner < b.owner + ? -1 + : 1, + ), + ), + ); + const key = yield* toolListCacheKey({ + owner: filter?.owner, + integration: filter?.integration === undefined ? undefined : String(filter.integration), + connection: filter?.connection === undefined ? undefined : String(filter.connection), + includeBlocked, + catalogGen, + policyGen, + }); + + // The cached value is the DB-derived, policy-filtered list for this + // (owner, integration, connection, includeBlocked) scope β€” query-UNfiltered + // and static-EXcluded. Static tools are unioned live; `query` applied below. + const cached = yield* toolListCacheStore + .get(key) + .pipe(Effect.catch(() => Effect.succeed(Option.none()))); + let dbTools: Tool[]; + if (Option.isSome(cached)) { + dbTools = cached.value.tools.map(cachedItemToTool); + } else { + // Projected: the list surface is metadata (address, description, + // annotations) β€” loading every tool's input/output schema JSON made + // an unbounded list scale with schema bytes, not tool count. + const rows = yield* core.findMany("tool", { + where: (b: AnyCb) => + b.and( + filter?.integration === undefined + ? true + : b("integration", "=", String(filter.integration)), + filter?.owner === undefined ? true : b("owner", "=", filter.owner), + filter?.connection === undefined + ? true + : b("connection", "=", String(filter.connection)), + ), + select: TOOL_INVOCATION_COLUMNS, + }); + dbTools = []; + for (const row of rows) { + const tool = rowToTool(row); + if (!includeBlocked) { + const effective = resolveEffectivePolicy( + normalizedPolicyId(tool), + policyRows, + ownerRankForRow, + tool.annotations?.requiresApproval, + ); + if (effective.action === "block") continue; + } + dbTools.push(tool); } - tools.push(tool); + yield* toolListCacheStore + .set(key, { version: TOOL_LIST_CACHE_VERSION, tools: dbTools.map(toolToCachedItem) }) + .pipe(Effect.ignore); } + + const tools: Tool[] = [...dbTools]; + // Static tools are matched without `query` so they share the cached DB set; + // `query` is applied once to the union at the end. + const staticFilter = filter?.query === undefined ? filter : { ...filter, query: undefined }; for (const entry of staticTools.values()) { const tool = staticToolToTool(entry); - if (!matchesToolFilter(tool, filter)) continue; + if (!matchesToolFilter(tool, staticFilter)) continue; if (!includeBlocked) { const effective = resolveEffectivePolicy( normalizedPolicyId(tool), @@ -3157,7 +3338,9 @@ export const createExecutor = matchesToolFilter(tool, filter)); }); const toolsManifest = ( diff --git a/packages/core/sdk/src/tool-list-cache.ts b/packages/core/sdk/src/tool-list-cache.ts new file mode 100644 index 0000000000..e153cc2787 --- /dev/null +++ b/packages/core/sdk/src/tool-list-cache.ts @@ -0,0 +1,78 @@ +import { Effect, Schema } from "effect"; + +import { sha256Hex } from "./blob"; +import { cacheKeyPayload } from "./cache-key"; + +// --------------------------------------------------------------------------- +// `tools.list` cache. +// +// The list surface is a full `tool` scan + `tool_policy` read on every call. The +// catalog only changes when a connection is (re)produced or removed, so the +// DB-derived, policy-filtered list is cached and invalidated by a content-derived +// revision rather than a TTL: +// - catalogGen: combines the per-connection `toolset_revision` rows (below). +// - policyGen: combines the `tool_policy` rows. +// Static tools are unioned LIVE (never cached) and the `query` substring filter is +// applied post-cache, so distinct queries share one cached set. +// --------------------------------------------------------------------------- + +export const TOOL_LIST_CACHE_PREFIX = "tool-list/"; +export const TOOL_LIST_CACHE_VERSION = "v1"; + +/** plugin_storage coordinates for the per-connection toolset-revision rows that + * back the catalog generation. Owner-scoped like every other catalog row, and + * written inside the same transaction that rewrites the connection's tools. */ +export const CATALOG_REVISION_PLUGIN_ID = "executor.catalog-revision"; +export const CATALOG_REVISION_COLLECTION = "connection"; + +/** plugin_storage `key` for a connection's toolset-revision row. */ +export const catalogRevisionStorageKey = (integration: string, connection: string): string => + `${integration}:${connection}`; + +const ToolListItemAnnotations = Schema.Struct({ + requiresApproval: Schema.optional(Schema.Boolean), + approvalDescription: Schema.optional(Schema.String), + mayElicit: Schema.optional(Schema.Boolean), +}); + +/** The list-projection fields of a `Tool` (no input/output schema β€” the list + * never loads them). Branded id types are stored as plain strings and rebuilt + * with their `.make` constructors on read, so this needs no branded schemas. */ +const ToolListItem = Schema.Struct({ + owner: Schema.String, + integration: Schema.String, + connection: Schema.String, + name: Schema.String, + pluginId: Schema.String, + description: Schema.String, + annotations: Schema.optional(ToolListItemAnnotations), + static: Schema.optional(Schema.Boolean), +}); +export type ToolListCacheItem = typeof ToolListItem.Type; + +export const ToolListCacheEntry = Schema.Struct({ + version: Schema.Literal(TOOL_LIST_CACHE_VERSION), + tools: Schema.Array(ToolListItem), +}); + +interface ToolListCacheKeyInput { + readonly owner?: string; + readonly integration?: string; + readonly connection?: string; + readonly includeBlocked: boolean; + readonly catalogGen: string; + readonly policyGen: string; +} + +export const toolListCacheKey = (input: ToolListCacheKeyInput): Effect.Effect => + sha256Hex( + cacheKeyPayload({ + version: TOOL_LIST_CACHE_VERSION, + owner: input.owner, + integration: input.integration, + connection: input.connection, + includeBlocked: input.includeBlocked, + catalogGen: input.catalogGen, + policyGen: input.policyGen, + }), + ).pipe(Effect.map((hash) => `${TOOL_LIST_CACHE_PREFIX}${TOOL_LIST_CACHE_VERSION}/${hash}`)); From 4bb027a3f87eceb603f8d8377ced5311a0aae852 Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 22 Jun 2026 03:32:18 +0530 Subject: [PATCH 56/57] perf(web): virtualize long lists behind a shared windowing primitive (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Introduces a shared windowed-list primitive (`@executor-js/react/components/virtual-list`) and adopts it on the web-UI surfaces that mount unbounded or long lists, so they keep a bounded DOM instead of rendering every row at once. This is **M1 of a 2-PR stack**. The follow-up adds server-side cursor pagination to `tools.list` β€” the one list whose API still returns the whole set β€” and stacks on top of this branch. ## What's here (one commit per surface) - **VirtualList primitive** β€” `@tanstack/react-virtual` with dynamic measurement, an `onEndReached` infinite-scroll trigger (generalized from the runs shell's hand-rolled distance-from-bottom check), optional sticky section headers, and scroll restoration keyed by id. A `useWindowedRows` hook covers surfaces that can't use `
` rows (real ``s). - **Tool source-list** (worst case) β€” a broad source could mount thousands of tool rows at once, each carrying a dropdown-menu portal. The grouped account sections and dotted-name trees are flattened into one windowed list; section headers become sticky virtual items; only near-viewport rows mount. - **Runs table** β€” windowed via `useWindowedRows`; the real `
` + sticky `` stay, with top/bottom spacer ``s for off-screen rows. The two variable-width columns are already capped, so windowing a subset doesn't make `table-layout: auto` jitter. - **Run-detail drawer** β€” the logs and tool-calls tabs get their own windowed scroll containers; tool-call expanded state is lifted so it survives a row scrolling out of the window and back. - **Sticky-header range logic** extracted to a pure function with unit tests. ## Deliberately not virtualized The integrations catalog, the service-tokens table, and the org-members list are **page-scrolled** lists that hold dozens of rows in practice. Virtualizing them would mean introducing an independent bounded scroller β€” a UX change for negligible benefit. The primitive is ready if any ever grows unbounded. ## Gates - `typecheck`: 43/43 packages - tests: shared React package 162, execution-history 19 (including new windowing-logic unit tests) - `oxfmt` + `oxlint --deny-warnings` clean on all changed files ## Verification still to do (live browser) Unit tests cover the windowing math; the visual behaviour needs a real browser (the e2e browser surface is cloud-only). Worth exercising by hand: - Sticky section headers pin and swap correctly while scrolling a grouped tool list. - Runs-table column widths stay stable while scrolling deep into the keyset list. - Detail-drawer logs / tool-calls scroll and measure correctly; expanded tool calls keep their state across scroll. --- bun.lock | 5 + .../execution-history/src/react/RunsPage.tsx | 71 ++-- .../src/react/detail-drawer.tsx | 221 ++++++----- .../execution-history/src/react/shell.tsx | 104 +++-- packages/react/package.json | 1 + packages/react/src/components/tool-tree.tsx | 371 +++++++++++------- .../react/src/components/virtual-list.test.ts | 75 ++++ .../react/src/components/virtual-list.tsx | 343 ++++++++++++++++ 8 files changed, 879 insertions(+), 312 deletions(-) create mode 100644 packages/react/src/components/virtual-list.test.ts create mode 100644 packages/react/src/components/virtual-list.tsx diff --git a/bun.lock b/bun.lock index 0509cc8701..7f6bb4006a 100644 --- a/bun.lock +++ b/bun.lock @@ -1197,6 +1197,7 @@ "@shikijs/langs": "^4.0.2", "@shikijs/themes": "^4.0.2", "@tanstack/react-router": "catalog:", + "@tanstack/react-virtual": "^3.14.3", "@tanstack/virtual-file-routes": "^1.162.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -2888,6 +2889,8 @@ "@tanstack/react-store": ["@tanstack/react-store@0.9.3", "", { "dependencies": { "@tanstack/store": "0.9.3", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-y2iHd/N9OkoQbFJLUX1T9vbc2O9tjH0pQRgTcx1/Nz4IlwLvkgpuglXUx+mXt0g5ZDFrEeDnONPqkbfxXJKwRg=="], + "@tanstack/react-virtual": ["@tanstack/react-virtual@3.14.3", "", { "dependencies": { "@tanstack/virtual-core": "3.17.1" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-k/cnHPVaOfn46hSbiY6n4Dzf4QjCGWSF40zR5QIIYUqPAjpA6TN7InfYmcMiDVQGP2iUn9xsRbAl8u1v3UmeVQ=="], + "@tanstack/router-cli": ["@tanstack/router-cli@1.167.17", "", { "dependencies": { "@tanstack/router-generator": "1.167.17", "chokidar": "^5.0.0", "yargs": "^17.7.2" }, "bin": { "tsr": "bin/tsr.cjs" } }, "sha512-kws9PNdspkHbeZG6aLDjjpgH/hg7Q565BdSW9hHrjKmXPq/at7OKxQnA76aw9dJiJqy9OmST2RdOKlMEog43+g=="], "@tanstack/router-core": ["@tanstack/router-core@1.168.15", "", { "dependencies": { "@tanstack/history": "1.161.6", "cookie-es": "^3.0.0", "seroval": "^1.5.0", "seroval-plugins": "^1.5.0" }, "bin": { "intent": "bin/intent.js" } }, "sha512-Wr0424NDtD8fT/uALobMZ9DdcfsTyXtW5IPR++7zvW8/7RaIOeaqXpVDId8ywaGtqPWLWOfaUg2zUtYtukoXYA=="], @@ -2910,6 +2913,8 @@ "@tanstack/store": ["@tanstack/store@0.9.3", "", {}, "sha512-8reSzl/qGWGGVKhBoxXPMWzATSbZLZFWhwBAFO9NAyp0TxzfBP0mIrGb8CP8KrQTmvzXlR/vFPPUrHTLBGyFyw=="], + "@tanstack/virtual-core": ["@tanstack/virtual-core@3.17.1", "", {}, "sha512-VZyW2Uiml5tmBZwPGrSD3Sz73OxzljQMCmzYHsUTPEuTsERf5xwa+uWb01xEzkz3ZSYTjj8NEb/mKHvgKxyZdA=="], + "@tanstack/virtual-file-routes": ["@tanstack/virtual-file-routes@1.162.0", "", {}, "sha512-uhOeFyxLcU41HzvrxsGpiWdcMbScY1EDgbZ5K7DVRMYInbLYWAC0EA/kx9wXAoSM8q82bUG2hRl8+EAjE6XAbA=="], "@tokenizer/inflate": ["@tokenizer/inflate@0.4.1", "", { "dependencies": { "debug": "^4.4.3", "token-types": "^6.1.1" } }, "sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA=="], diff --git a/packages/plugins/execution-history/src/react/RunsPage.tsx b/packages/plugins/execution-history/src/react/RunsPage.tsx index 6c4173e0f7..be95ed4d1e 100644 --- a/packages/plugins/execution-history/src/react/RunsPage.tsx +++ b/packages/plugins/execution-history/src/react/RunsPage.tsx @@ -10,7 +10,7 @@ import { LiveButton } from "./live-button"; import { LiveDivider } from "./live-row"; import { RefreshButton } from "./refresh-button"; import { RunListRow } from "./run-row"; -import { RunsShell, RUNS_TABLE_COLSPAN } from "./shell"; +import { RunsShell, RUNS_TABLE_COLSPAN, type RunsShellRow } from "./shell"; import { RunsTimelineChart } from "./timeline-chart"; import { emptyRunsFilters, @@ -146,37 +146,40 @@ export function RunsPage() { [isDescending, view.rows, view.liveCutoffId], ); - const rowItems: React.ReactNode[] = []; + // Run rows + the interleaved live divider, as keyed entries the shell windows. + // Elements for off-screen runs are cheap descriptors β€” only the rows the + // windowing mounts actually run their per-row hooks / portals. + const rows: RunsShellRow[] = []; view.rows.forEach((run, index) => { if (index === cutoffIndex && cutoffIndex > 0) { - rowItems.push(); + rows.push({ key: "live-divider", node: }); } - rowItems.push( - 0 && index >= cutoffIndex} - columns={columns} - onSelect={() => setSelected(run.executionId)} - />, - ); + rows.push({ + key: run.executionId, + node: ( + 0 && index >= cutoffIndex} + columns={columns} + onSelect={() => setSelected(run.executionId)} + /> + ), + }); }); - if (view.isLoadMoreError) { - rowItems.push( - - - , - ); - } + const loadMoreError = view.isLoadMoreError ? ( + + + + ) : undefined; const toolbar = (
@@ -226,7 +229,7 @@ export function RunsPage() { /> ) : undefined; - const body = view.isLoading ? ( + const statusRow = view.isLoading ? (
- ) : ( - rowItems - ); + ) : undefined; // Prev/next drawer navigation mirrors the ArrowUp/ArrowDown keyboard handler: // index-1 is the row above (newer in descending sort), index+1 the row below. @@ -286,9 +287,11 @@ export function RunsPage() { } railCollapsed={railCollapsed} - > - {body} - + rows={rows} + statusRow={statusRow} + loadMoreError={loadMoreError} + scrollRestoreId="execution-history-runs" + /> - {lines.map((line, index) => ( + `${index}-${line.slice(0, 24)}`} + estimateSize={() => 28} + overscan={16} + className="min-h-0 flex-1 px-5 py-4" + aria-label="Run logs" + renderItem={(line) => ( 
           {line}
- ))} - + )} + /> ); } @@ -117,14 +123,15 @@ function ToolCallItem(props: { readonly call: ToolCallRow; readonly windowStart: number; readonly windowEnd: number; + readonly expanded: boolean; + readonly onToggle: () => void; }) { - const [expanded, setExpanded] = useState(false); return (
- {expanded && ( + {props.expanded && (
{props.call.status === "failed" && props.call.errorText ? ( @@ -157,9 +164,20 @@ function ToolCallItem(props: { } function ToolCallsTab(props: { readonly run: RunRow; readonly toolCalls: readonly ToolCallRow[] }) { + // Expanded state is lifted out of the row so it survives windowing (a row + // scrolled out and back keeps its expansion). + const [expanded, setExpanded] = useState>(() => new Set()); const windowStart = props.run.startedAt; + const toggle = useCallback((id: string) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + }, []); if (props.toolCalls.length === 0) { - return

No tool calls recorded.

; + return

No tool calls recorded.

; } // Derive the window end from the tool-call data as well as `run.completedAt` // so a fresher tool call can't outrun a stale run end and clamp its bar to a @@ -175,16 +193,25 @@ function ToolCallsTab(props: { readonly run: RunRow; readonly toolCalls: readonl ); const windowEnd = Math.max(props.run.completedAt ?? Date.now(), callsEnd); return ( -
- {props.toolCalls.map((call) => ( - - ))} -
+ call.toolCallId} + estimateSize={() => 52} + overscan={12} + className="min-h-0 flex-1 px-5 py-4" + aria-label="Tool calls" + renderItem={(call) => ( +
+ toggle(call.toolCallId)} + /> +
+ )} + /> ); } @@ -313,85 +340,87 @@ function DetailContent(props: { Logs - - -
- - - - {statusLabel(run.status)} - - - - {run.durationMs != null ? formatDuration(run.durationMs) : "β€”"} - - - {run.actorId !== null ? ( - - {actorLabel} - {run.actorKind !== null ? ( - Β· {run.actorKind} - ) : null} + + +
+
+ + + + {statusLabel(run.status)} - ) : ( - β€” - )} - - - - - - {run.completedAt != null ? ( - - ) : ( - formatDateTime(run.completedAt) - )} - -
- -
- - - via {trigger.label} - - Β· tools {run.toolCallCount} -
- - - - {props.errorText && ( -
-

Error

- 
-                  {props.errorText}
-
+ + + {run.durationMs != null ? formatDuration(run.durationMs) : "β€”"} + + + {run.actorId !== null ? ( + + {actorLabel} + {run.actorKind !== null ? ( + Β· {run.actorKind} + ) : null} + + ) : ( + β€” + )} + + + + + + {run.completedAt != null ? ( + + ) : ( + formatDateTime(run.completedAt) + )} +
- )} - {props.interactions.length > 0 && ( -
-

- {interactionsHeading} -

- {props.interactions.map((interaction) => ( - - ))} +
+ + + via {trigger.label} + + Β· tools {run.toolCallCount}
- )} - - - - - - - - + + + + {props.errorText && ( +
+

Error

+ 
+                    {props.errorText}
+
+
+ )} + + {props.interactions.length > 0 && ( +
+

+ {interactionsHeading} +

+ {props.interactions.map((interaction) => ( + + ))} +
+ )} +
+ + + + + + + +
); diff --git a/packages/plugins/execution-history/src/react/shell.tsx b/packages/plugins/execution-history/src/react/shell.tsx index 31300e7f47..abdf1ffe9b 100644 --- a/packages/plugins/execution-history/src/react/shell.tsx +++ b/packages/plugins/execution-history/src/react/shell.tsx @@ -1,18 +1,21 @@ import * as React from "react"; import { cn } from "@executor-js/react/lib/utils"; +import { useWindowedRows } from "@executor-js/react/components/virtual-list"; // --------------------------------------------------------------------------- -// RunsShell β€” two-column layout (filter rail + main pane) with infinite scroll. +// RunsShell β€” two-column layout (filter rail + main pane) with a windowed, +// infinite-scrolling table. // // The left
- - Failed to load more - - -
+ + Failed to load more + + +
Loading runs… @@ -238,9 +241,7 @@ export function RunsPage() { Unable to load runs.
with content-fit columns (table-layout: auto): the sticky -// and the rows share one column model, so columns size to their -// widest cell and stay aligned for free β€” no fixed widths. The scroller fires -// onLoadMore when the user scrolls within 320px of the bottom, guarded against -// re-firing while loading. +// above a single table scroller that owns both axes. The table stays a real +// semantic
with a sticky ; only the rows near the viewport are +// mounted, with top/bottom spacer s standing in for the rest (useWindowedRows +// from the shared virtualization primitive). The two columns that can run long β€” +// actor and code β€” are width-capped in run-row.tsx, so windowing a subset doesn't +// make `table-layout: auto` columns jitter as rows scroll in. The scroller fires +// onLoadMore within 320px of the bottom, guarded against re-firing while loading. // --------------------------------------------------------------------------- // Upper bound on the visible column count (dot + started + status + up to six @@ -20,48 +23,63 @@ import { cn } from "@executor-js/react/lib/utils"; // with this; an over-count colSpan is harmless (the browser clamps it). export const RUNS_TABLE_COLSPAN = 12; +// Approximate keyset row height (run-row.tsx: px-3 py-2 + border β‰ˆ 33px). Rows +// are near-uniform, so a fixed estimate keeps the spacer math stable. +const RUNS_ROW_ESTIMATE = 33; + +export interface RunsShellRow { + readonly key: React.Key; + readonly node: React.ReactNode; +} + export interface RunsShellProps { readonly filterRail: React.ReactNode; readonly toolbar: React.ReactNode; readonly chart?: React.ReactNode; readonly columnHeader: React.ReactNode; - readonly children: React.ReactNode; + /** Windowed body rows β€” run rows plus the interleaved live divider. */ + readonly rows: readonly RunsShellRow[]; + readonly estimateRowSize?: number; + readonly scrollRestoreId?: string; readonly onLoadMore: () => void; readonly hasMore: boolean; readonly isLoadingMore: boolean; readonly isEmpty: boolean; readonly emptyState?: React.ReactNode; + /** A full-width shown instead of the windowed rows (loading / error). */ + readonly statusRow?: React.ReactNode; + /** A appended after the rows β€” e.g. the load-more retry prompt. */ + readonly loadMoreError?: React.ReactNode; readonly railCollapsed?: boolean; } -const SCROLL_THRESHOLD = 320; - export function RunsShell(props: RunsShellProps) { const { filterRail, toolbar, chart, columnHeader, - children, + rows, + estimateRowSize, + scrollRestoreId, onLoadMore, hasMore, isLoadingMore, isEmpty, emptyState, + statusRow, + loadMoreError, railCollapsed, } = props; - const onScroll = React.useCallback( - (event: React.UIEvent) => { - if (!hasMore || isLoadingMore || isEmpty) return; - const target = event.currentTarget; - const distanceFromBottom = target.scrollHeight - target.scrollTop - target.clientHeight; - if (distanceFromBottom < SCROLL_THRESHOLD) { - onLoadMore(); - } - }, - [hasMore, isLoadingMore, isEmpty, onLoadMore], - ); + const canLoadMore = hasMore && !isLoadingMore && !isEmpty; + const { scrollRef, virtualItems, paddingTop, paddingBottom, onScroll } = useWindowedRows({ + count: rows.length, + estimateSize: estimateRowSize ?? RUNS_ROW_ESTIMATE, + overscan: 16, + onEndReached: canLoadMore ? onLoadMore : undefined, + scrollRestoreId, + }); return (
@@ -84,16 +102,13 @@ export function RunsShell(props: RunsShellProps) { {chart != null &&
{chart}
}
- {/* Table scroller β€” owns both axes. The sticky sticks to the top - of this scroller (below the toolbar/chart) and rides the same - horizontal scroll as the rows. `border-separate border-spacing-0` - keeps cell borders rendering on the sticky header (a known - border-collapse + sticky browser bug). No table width is forced, so - columns content-fit: narrower than the pane β†’ left-aligned with slack - on the right; wider β†’ horizontal scroll. */} -
+ {/* Table scroller β€” owns both axes and is the virtualizer's scroll + element. The sticky
rides the same scroll; only near-viewport + rows mount, bracketed by spacer s. `border-separate + border-spacing-0` keeps cell borders on the sticky header. */} +
{/* oxlint-disable-next-line react/forbid-elements -- the runs table needs - this custom scroll container (the onScroll infinite-scroll handler) + + this custom scroll container (the windowed infinite-scroll handler) + a sticky thead inside it; the shared
wraps its own overflow-x-auto div, which would conflict. Same suppression the shared Table component itself uses on its raw
. */} @@ -110,9 +125,34 @@ export function RunsShell(props: RunsShellProps) { )} + ) : statusRow != null ? ( + statusRow ) : ( <> - {children} + {paddingTop > 0 && ( + + + )} + {virtualItems.map((item) => ( + + {rows[item.index]!.node} + + ))} + {paddingBottom > 0 && ( + + + )} + {loadMoreError} {isLoadingMore && (
+
+
{ // --------------------------------------------------------------------------- // ToolTree β€” main export +// +// One windowed scroller over a single flattened item list. In grouped mode the +// per-account sections and their dotted-name trees are flattened together β€” +// section headers become sticky items β€” so a source with thousands of tools +// keeps a bounded DOM (only near-viewport rows mount). Expand/collapse state is +// lifted here and keyed by (section, path) so sections stay independent. // --------------------------------------------------------------------------- +// Flat (ungrouped) mode has no account sections; its rows share one empty key. +const FLAT_SECTION_KEY = ""; +// NUL separates section key from path - it can't appear in a `${owner}:${connection}` +// key or a dotted tool path, so prefix matching never bleeds across sections. +const COMPOSITE_SEP = "\u0000"; +const compositeKey = (sectionKey: string, path: string) => `${sectionKey}${COMPOSITE_SEP}${path}`; + +type ToolTreeItem = + | { readonly kind: "section"; readonly sectionKey: string; readonly group: AccountGroup } + | { + readonly kind: "leaf"; + readonly sectionKey: string; + readonly depth: number; + readonly path: string; + readonly tool: ToolSummary; + } + | { + readonly kind: "group"; + readonly sectionKey: string; + readonly depth: number; + readonly path: string; + readonly segment: string; + readonly count: number; + readonly open: boolean; + }; + +// The open group paths for one section: every group when searching (so matches +// are visible), otherwise the user's manual toggles plus the selected tool's +// ancestors (best-effort, based on its dotted name). +const resolveSectionOpenSet = ( + tree: TreeNode, + manualOpen: ReadonlySet, + sectionKey: string, + terms: readonly string[], + selectedToolId: string | null, +): Set => { + if (terms.length > 0) { + const all = new Set(); + collectGroupPaths(tree, all); + return all; + } + const set = new Set(); + const prefix = `${sectionKey}${COMPOSITE_SEP}`; + for (const composite of manualOpen) { + if (composite.startsWith(prefix)) set.add(composite.slice(prefix.length)); + } + if (selectedToolId) { + const parts = selectedToolId.split("."); + let acc = ""; + for (let i = 0; i < parts.length - 1; i++) { + acc = acc ? `${acc}.${parts[i]}` : parts[i]!; + set.add(acc); + } + } + return set; +}; + +function SectionHeader(props: { + group: AccountGroup; + ownerDisplay: ReturnType; +}) { + const { group, ownerDisplay } = props; + return ( +
+ {ownerDisplay.showOwnerLabels ? ( + + {ownerLabel(group.owner)} + + ) : null} + + {group.connection || ownerDisplay.label(group.owner)} + + + {group.tools.length} + +
+ ); +} + export function ToolTree(props: { tools: readonly ToolSummary[]; selectedToolId: string | null; @@ -311,10 +397,9 @@ export function ToolTree(props: { }, [policies]); const [search, setSearch] = useState(""); const searchRef = useRef(null); - const selectedRowRef = useRef(null); const ownerDisplay = useOwnerDisplay(); - const terms = search.trim().toLowerCase().split(/\s+/).filter(Boolean); + const terms = useMemo(() => search.trim().toLowerCase().split(/\s+/).filter(Boolean), [search]); const filteredTools = useMemo(() => { if (terms.length === 0) return tools; @@ -331,6 +416,46 @@ export function ToolTree(props: { [groupByConnection, filteredTools], ); + // Expand/collapse state, lifted out of the per-section bodies so account + // sections stay independent. Keys are `${sectionKey}\u0000${path}`. + const [manualOpen, setManualOpen] = useState>(() => new Set()); + + // The single flat item list fed to the virtualizer: section headers plus the + // flattened dotted-name rows of each section (or one flat tree when ungrouped). + const items = useMemo(() => { + const acc: ToolTreeItem[] = []; + const pushRows = (sectionTools: readonly ToolSummary[], sectionKey: string) => { + const tree = buildTree(sectionTools); + const openSet = resolveSectionOpenSet(tree, manualOpen, sectionKey, terms, selectedToolId); + const rows: Row[] = []; + flattenTree(tree, 0, openSet, rows); + for (const row of rows) { + if (row.kind === "leaf") { + acc.push({ kind: "leaf", sectionKey, depth: row.depth, path: row.path, tool: row.tool }); + } else { + acc.push({ + kind: "group", + sectionKey, + depth: row.depth, + path: row.path, + segment: row.segment, + count: row.count, + open: row.open, + }); + } + } + }; + if (groupByConnection) { + for (const group of accountGroups) { + acc.push({ kind: "section", sectionKey: group.key, group }); + pushRows(group.tools, group.key); + } + } else { + pushRows(filteredTools, FLAT_SECTION_KEY); + } + return acc; + }, [groupByConnection, accountGroups, filteredTools, manualOpen, terms, selectedToolId]); + // Keyboard shortcuts β€” `/` focuses search, Escape clears useEffect(() => { const handler = (e: KeyboardEvent) => { @@ -347,22 +472,92 @@ export function ToolTree(props: { return () => document.removeEventListener("keydown", handler); }, [search]); - // Scroll the selected row into view when it changes + // Scroll the selected row into view when it changes. Reads the latest items + // through a ref so re-flattening (e.g. expanding a group) doesn't re-trigger. + const listRef = useRef(null); + const itemsRef = useRef(items); + itemsRef.current = items; useEffect(() => { if (!selectedToolId) return; - selectedRowRef.current?.scrollIntoView({ block: "nearest" }); + const index = itemsRef.current.findIndex( + (it) => it.kind === "leaf" && it.tool.id === selectedToolId, + ); + if (index >= 0) listRef.current?.scrollToIndex(index, { align: "center" }); }, [selectedToolId]); - const bodyProps = { - selectedToolId, - onSelect, - onSetPolicy, - onClearPolicy, - exactPatterns, - search, - terms, - selectedRowRef, - }; + const toggleGroup = useCallback((sectionKey: string, path: string) => { + setManualOpen((prev) => { + const next = new Set(prev); + const key = compositeKey(sectionKey, path); + if (next.has(key)) next.delete(key); + else next.add(key); + return next; + }); + }, []); + + const isSticky = useCallback((index: number) => items[index]?.kind === "section", [items]); + const estimateSize = useCallback( + (index: number) => (items[index]?.kind === "section" ? 30 : 34), + [items], + ); + const getKey = useCallback((item: ToolTreeItem) => { + if (item.kind === "section") return `section:${item.sectionKey}`; + if (item.kind === "leaf") return `${item.sectionKey}:leaf:${item.tool.id}`; + return `${item.sectionKey}:group:${item.path}`; + }, []); + + const renderItem = useCallback( + (item: ToolTreeItem) => { + if (item.kind === "section") { + return ; + } + if (item.kind === "leaf") { + return ( + { + const nameParts = item.tool.name.split("."); + trackEvent("tool_selected", { + integration_slug: nameParts[0] ?? item.tool.name, + tool_name: nameParts.slice(1).join(".") || item.tool.name, + }); + onSelect(item.tool.id); + }} + search={search} + onSetPolicy={onSetPolicy} + onClearPolicy={onClearPolicy} + exactRule={exactPatterns.get(toPolicyPattern(item.tool.name))} + /> + ); + } + return ( + toggleGroup(item.sectionKey, item.path)} + search={search} + onSetPolicy={onSetPolicy} + onClearPolicy={onClearPolicy} + exactRule={exactPatterns.get(toPolicyPattern(`${item.path}.*`))} + /> + ); + }, + [ + ownerDisplay, + selectedToolId, + search, + onSelect, + onSetPolicy, + onClearPolicy, + exactPatterns, + toggleGroup, + ], + ); return (
@@ -389,149 +584,25 @@ export function ToolTree(props: { )}
-
- {filteredTools.length === 0 ? ( + (
{terms.length > 0 ? "No tools match your filter" : "No tools available"}
- ) : groupByConnection ? ( - accountGroups.map((group) => ( -
-
- {ownerDisplay.showOwnerLabels ? ( - - {ownerLabel(group.owner)} - - ) : null} - - {group.connection || ownerDisplay.label(group.owner)} - - - {group.tools.length} - -
- -
- )) - ) : ( - )} -
+ />
); } -// --------------------------------------------------------------------------- -// ToolTreeBody β€” one dotted-name tree for a tools subset. Owns its own -// expand/collapse state so account sections (grouped mode) expand independently -// and never share path keys. Already-filtered tools are passed in. -// --------------------------------------------------------------------------- - -function ToolTreeBody(props: { - tools: readonly ToolSummary[]; - selectedToolId: string | null; - onSelect: (toolId: string) => void; - onSetPolicy?: (pattern: string, action: ToolPolicyAction) => void; - onClearPolicy?: (pattern: string) => void; - exactPatterns: ReadonlyMap; - search: string; - terms: readonly string[]; - selectedRowRef: React.Ref; -}) { - const { - tools, - selectedToolId, - onSelect, - onSetPolicy, - onClearPolicy, - exactPatterns, - search, - terms, - selectedRowRef, - } = props; - const [manualOpen, setManualOpen] = useState>(() => new Set()); - - const tree = useMemo(() => buildTree(tools), [tools]); - - // When searching, expand everything so matches are visible. - // Also auto-expand groups that contain the selected tool. - const openSet = useMemo(() => { - if (terms.length > 0) { - const all = new Set(); - collectGroupPaths(tree, all); - return all; - } - const set = new Set(manualOpen); - if (selectedToolId) { - const parts = selectedToolId.split("."); - // Progressively add ancestor paths (best-effort, based on dotted name). - let acc = ""; - for (let i = 0; i < parts.length - 1; i++) { - acc = acc ? `${acc}.${parts[i]}` : parts[i]!; - set.add(acc); - } - } - return set; - }, [tree, manualOpen, selectedToolId, terms.length]); - - const rows = useMemo(() => { - const acc: Row[] = []; - flattenTree(tree, 0, openSet, acc); - return acc; - }, [tree, openSet]); - - const toggleGroup = (path: string) => { - setManualOpen((prev) => { - const next = new Set(prev); - if (next.has(path)) next.delete(path); - else next.add(path); - return next; - }); - }; - - return ( - <> - {rows.map((row) => - row.kind === "leaf" ? ( - { - const nameParts = row.tool.name.split("."); - trackEvent("tool_selected", { - integration_slug: nameParts[0] ?? row.tool.name, - tool_name: nameParts.slice(1).join(".") || row.tool.name, - }); - onSelect(row.tool.id); - }} - search={search} - onSetPolicy={onSetPolicy} - onClearPolicy={onClearPolicy} - exactRule={exactPatterns.get(toPolicyPattern(row.tool.name))} - /> - ) : ( - toggleGroup(row.path)} - search={search} - onSetPolicy={onSetPolicy} - onClearPolicy={onClearPolicy} - exactRule={exactPatterns.get(toPolicyPattern(`${row.path}.*`))} - /> - ), - )} - - ); -} - // --------------------------------------------------------------------------- // Row renderers // --------------------------------------------------------------------------- diff --git a/packages/react/src/components/virtual-list.test.ts b/packages/react/src/components/virtual-list.test.ts new file mode 100644 index 0000000000..8d250e0de4 --- /dev/null +++ b/packages/react/src/components/virtual-list.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { + clearScrollPosition, + DEFAULT_END_REACHED_THRESHOLD, + isNearBottom, + readScrollPosition, + resolveActiveStickyIndex, + saveScrollPosition, +} from "./virtual-list"; + +describe("isNearBottom", () => { + it("is true exactly at the threshold and false just above it", () => { + // distanceFromBottom = scrollHeight - scrollTop - clientHeight + const metrics = { scrollHeight: 1000, clientHeight: 400, scrollTop: 600 - 320 }; + // distance = 1000 - 280 - 400 = 320 === threshold β†’ near + expect(isNearBottom(metrics, 320)).toBe(true); + expect(isNearBottom({ ...metrics, scrollTop: metrics.scrollTop - 1 }, 320)).toBe(false); + }); + + it("is true once scrolled to the very bottom", () => { + expect(isNearBottom({ scrollHeight: 5000, clientHeight: 800, scrollTop: 4200 }, 320)).toBe( + true, + ); + }); + + it("is false near the top of a long list", () => { + expect(isNearBottom({ scrollHeight: 5000, clientHeight: 800, scrollTop: 0 }, 320)).toBe(false); + }); + + it("exposes the runs-shell default threshold", () => { + expect(DEFAULT_END_REACHED_THRESHOLD).toBe(320); + }); +}); + +describe("scroll position store", () => { + it("round-trips a saved offset by id", () => { + clearScrollPosition("runs"); + expect(readScrollPosition("runs")).toBeUndefined(); + saveScrollPosition("runs", 742); + expect(readScrollPosition("runs")).toBe(742); + }); + + it("keeps ids independent and clears one without touching the other", () => { + saveScrollPosition("a", 10); + saveScrollPosition("b", 20); + clearScrollPosition("a"); + expect(readScrollPosition("a")).toBeUndefined(); + expect(readScrollPosition("b")).toBe(20); + }); +}); + +describe("resolveActiveStickyIndex", () => { + // Sections at rows 0, 4 and 9 (e.g. the tool tree's per-account headers). + const sticky = [0, 4, 9]; + + it("pins the first header while scrolled within the first section", () => { + expect(resolveActiveStickyIndex(sticky, 0)).toBe(0); + expect(resolveActiveStickyIndex(sticky, 3)).toBe(0); + }); + + it("switches to the header at the section boundary", () => { + expect(resolveActiveStickyIndex(sticky, 4)).toBe(4); + expect(resolveActiveStickyIndex(sticky, 8)).toBe(4); + expect(resolveActiveStickyIndex(sticky, 9)).toBe(9); + expect(resolveActiveStickyIndex(sticky, 50)).toBe(9); + }); + + it("returns null when no header is at or above the first visible row", () => { + // A list whose first sticky header isn't at the very top. + expect(resolveActiveStickyIndex([2, 5], 0)).toBeNull(); + expect(resolveActiveStickyIndex([2, 5], 1)).toBeNull(); + expect(resolveActiveStickyIndex([], 7)).toBeNull(); + }); +}); diff --git a/packages/react/src/components/virtual-list.tsx b/packages/react/src/components/virtual-list.tsx new file mode 100644 index 0000000000..b225ade187 --- /dev/null +++ b/packages/react/src/components/virtual-list.tsx @@ -0,0 +1,343 @@ +import * as React from "react"; +import { + defaultRangeExtractor, + useVirtualizer, + type Range, + type VirtualItem, +} from "@tanstack/react-virtual"; + +import { cn } from "../lib/utils"; + +// --------------------------------------------------------------------------- +// VirtualList β€” the shared windowed-list primitive. +// +// Renders only the rows near the viewport for an arbitrary item array, so a +// list of thousands (the worst case today is a single source's ~3k tools) keeps +// a bounded DOM instead of mounting every row at once. Three concerns the web +// UI re-implements ad hoc are folded in here so every surface behaves the same: +// +// 1. Windowing β€” @tanstack/react-virtual with dynamic measurement, so rows of +// uneven height (collapsible log/tool-call blocks) work without a fixed +// itemHeight. +// 2. End-reached β€” the infinite-scroll trigger generalized from the runs +// shell's hand-rolled `scrollHeight - scrollTop - clientHeight < 320`. +// 3. Scroll restoration β€” net-new: a list's scroll offset survives unmount / +// remount (route change, drawer closeβ†’open) keyed by `scrollRestoreId`. +// +// Optional sticky headers (e.g. the tool tree's per-account section headers) +// are supported by marking those indices via `isSticky`: the active header is +// always kept in the render window and pinned to the top of the scroller. +// +// The runs table can't use this component directly (a real needs the +// scroller to wrap +); it composes the same building blocks +// exported below β€” `isNearBottom`, the scroll-position store, and the shared +// constants β€” against its own spacer rows. +// --------------------------------------------------------------------------- + +export const DEFAULT_END_REACHED_THRESHOLD = 320; +export const DEFAULT_ESTIMATE_SIZE = 36; +export const DEFAULT_OVERSCAN = 10; + +// --- scroll restoration ---------------------------------------------------- +// Module-level map of last-seen scrollTop keyed by a caller-provided id. Lives +// for the lifetime of the page so a list re-mounted on the same id (back nav, +// reopened drawer) lands where the user left it. Pure data β€” unit-testable. +const scrollPositions = new Map(); + +export function saveScrollPosition(id: string, top: number): void { + scrollPositions.set(id, top); +} + +export function readScrollPosition(id: string): number | undefined { + return scrollPositions.get(id); +} + +export function clearScrollPosition(id: string): void { + scrollPositions.delete(id); +} + +// --- end-reached math (pure, unit-testable) -------------------------------- +export interface ScrollMetrics { + readonly scrollHeight: number; + readonly scrollTop: number; + readonly clientHeight: number; +} + +/** True when the scroller is within `threshold` px of the bottom. */ +export function isNearBottom(metrics: ScrollMetrics, threshold: number): boolean { + return metrics.scrollHeight - metrics.scrollTop - metrics.clientHeight <= threshold; +} + +/** The sticky header that should be pinned at the top of the window: the last + * sticky index at or above the first visible row, or null if none precedes it. + * `stickyIndices` must be ascending. Pure β€” unit-testable. */ +export function resolveActiveStickyIndex( + stickyIndices: readonly number[], + startIndex: number, +): number | null { + let active: number | null = null; + for (const idx of stickyIndices) { + if (idx <= startIndex) active = idx; + else break; + } + return active; +} + +// --- imperative handle ----------------------------------------------------- +export interface VirtualListHandle { + scrollToIndex: (index: number, options?: { align?: "start" | "center" | "end" }) => void; + scrollToOffset: (offset: number) => void; +} + +export interface VirtualListProps { + readonly items: readonly T[]; + readonly renderItem: (item: T, index: number) => React.ReactNode; + readonly getKey: (item: T, index: number) => React.Key; + /** Estimated row height in px, used before a row is measured. Default 36. */ + readonly estimateSize?: (index: number) => number; + readonly overscan?: number; + /** Classes for the scroll container (must establish the scroll viewport). */ + readonly className?: string; + /** Mark an index as a sticky section header β€” pinned to the top while its + * section scrolls beneath it, and always kept in the render window. */ + readonly isSticky?: (index: number) => boolean; + /** Fired when the viewport reaches within `endReachedThreshold` px of the + * bottom. Re-entrancy is the caller's responsibility (guard on + * hasMore/isLoading), matching the existing runs-shell contract. */ + readonly onEndReached?: () => void; + readonly endReachedThreshold?: number; + /** Preserve/restore scroll offset across unmount under this id. */ + readonly scrollRestoreId?: string; + readonly isLoading?: boolean; + readonly renderLoading?: () => React.ReactNode; + readonly renderEmpty?: () => React.ReactNode; + /** Rendered in normal flow below the windowed rows β€” e.g. a "Loading more…" + * / "End of history" sentinel. */ + readonly footer?: React.ReactNode; + readonly apiRef?: React.Ref; + readonly "aria-label"?: string; +} + +export function VirtualList(props: VirtualListProps) { + const { + items, + renderItem, + getKey, + estimateSize, + overscan = DEFAULT_OVERSCAN, + className, + isSticky, + onEndReached, + endReachedThreshold = DEFAULT_END_REACHED_THRESHOLD, + scrollRestoreId, + isLoading, + renderLoading, + renderEmpty, + footer, + apiRef, + } = props; + + const scrollRef = React.useRef(null); + + // Indices that pin to the top. Recomputed when the row set changes. + const stickyIndices = React.useMemo(() => { + if (!isSticky) return [] as number[]; + const acc: number[] = []; + for (let i = 0; i < items.length; i++) { + if (isSticky(i)) acc.push(i); + } + return acc; + }, [items, isSticky]); + + // The header currently pinned β€” the last sticky index at or above the top of + // the window. Tracked in a ref so the range extractor can keep it mounted. + const activeStickyRef = React.useRef(null); + + const rangeExtractor = React.useCallback( + (range: Range): number[] => { + if (stickyIndices.length === 0) { + activeStickyRef.current = null; + return defaultRangeExtractor(range); + } + const active = resolveActiveStickyIndex(stickyIndices, range.startIndex); + activeStickyRef.current = active; + const indices = new Set(defaultRangeExtractor(range)); + if (active !== null) indices.add(active); + return [...indices].sort((a, b) => a - b); + }, + [stickyIndices], + ); + + const virtualizer = useVirtualizer({ + count: items.length, + getScrollElement: () => scrollRef.current, + estimateSize: estimateSize ?? (() => DEFAULT_ESTIMATE_SIZE), + overscan, + rangeExtractor, + // Key the measurement cache by stable item identity, not index, so measured + // heights follow rows when the list reorders (the tool tree re-flattens on + // expand/collapse/filter). Without this the virtualizer caches sizes by + // index and a reorder leaves stale heights until the next re-measure. + getItemKey: (index) => getKey(items[index]!, index), + }); + + React.useImperativeHandle( + apiRef, + () => ({ + scrollToIndex: (index, options) => virtualizer.scrollToIndex(index, options), + scrollToOffset: (offset) => virtualizer.scrollToOffset(offset), + }), + [virtualizer], + ); + + // Restore on mount; persist on unmount. Restore runs in a layout effect so the + // jump happens before paint. The estimate-based total size is in place by then, + // which is close enough for keyset lists; exact dynamic-measure drift is minor. + React.useLayoutEffect(() => { + const el = scrollRef.current; + if (!scrollRestoreId || !el) return; + const saved = readScrollPosition(scrollRestoreId); + if (saved !== undefined && saved > 0) { + el.scrollTop = saved; + } + // `el` is captured here so the cleanup saves the same node it restored β€” the + // scroll container is stable for the component's life (single container, + // inner content swaps for loading/empty states). + return () => saveScrollPosition(scrollRestoreId, el.scrollTop); + }, [scrollRestoreId]); + + const handleScroll = React.useCallback( + (event: React.UIEvent) => { + const el = event.currentTarget; + if (scrollRestoreId) saveScrollPosition(scrollRestoreId, el.scrollTop); + if (onEndReached && isNearBottom(el, endReachedThreshold)) onEndReached(); + }, + [onEndReached, endReachedThreshold, scrollRestoreId], + ); + + // One stable scroll container across loading / empty / populated states so the + // virtualizer's scroll element and the restore ref never swap nodes underfoot. + const showLoading = isLoading && items.length === 0; + const showEmpty = !showLoading && items.length === 0; + const virtualItems = virtualizer.getVirtualItems(); + + return ( +
+ {showLoading ? ( + renderLoading?.() + ) : showEmpty ? ( + <> + {renderEmpty?.()} + {footer} + + ) : ( + <> +
+ {virtualItems.map((virtualItem) => { + const index = virtualItem.index; + const pinned = activeStickyRef.current === index && isSticky?.(index) === true; + return ( +
+ {renderItem(items[index]!, index)} +
+ ); + })} +
+ {footer} + + )} +
+ ); +} + +// --------------------------------------------------------------------------- +// useWindowedRows β€” windowing for surfaces that can't use
rows. +// +// A real
needs its scroll container to wrap +, so the runs +// table can't drop in . This hook exposes the same engine: attach +// `scrollRef` to the scroll container, render a top spacer row of height +// `paddingTop`, the `virtualItems`, then a bottom spacer of `paddingBottom`. +// `onScroll` fires end-reached and persists scroll position, mirroring the +// component. Rows are assumed near-uniform height (fixed `estimateSize`); the +// sticky-header offset is absorbed by `overscan`. +// --------------------------------------------------------------------------- +export interface WindowedRows { + readonly scrollRef: React.RefObject; + readonly virtualItems: readonly VirtualItem[]; + readonly paddingTop: number; + readonly paddingBottom: number; + readonly onScroll: (event: React.UIEvent) => void; +} + +export function useWindowedRows(options: { + readonly count: number; + readonly estimateSize?: number; + readonly overscan?: number; + readonly onEndReached?: () => void; + readonly endReachedThreshold?: number; + readonly scrollRestoreId?: string; +}): WindowedRows { + const { + count, + estimateSize = DEFAULT_ESTIMATE_SIZE, + overscan = DEFAULT_OVERSCAN, + onEndReached, + endReachedThreshold = DEFAULT_END_REACHED_THRESHOLD, + scrollRestoreId, + } = options; + + const scrollRef = React.useRef(null); + + const virtualizer = useVirtualizer({ + count, + getScrollElement: () => scrollRef.current, + estimateSize: () => estimateSize, + overscan, + }); + + React.useLayoutEffect(() => { + const el = scrollRef.current; + if (!scrollRestoreId || !el) return; + const saved = readScrollPosition(scrollRestoreId); + if (saved !== undefined && saved > 0) el.scrollTop = saved; + return () => saveScrollPosition(scrollRestoreId, el.scrollTop); + }, [scrollRestoreId]); + + const onScroll = React.useCallback( + (event: React.UIEvent) => { + const el = event.currentTarget; + if (scrollRestoreId) saveScrollPosition(scrollRestoreId, el.scrollTop); + if (onEndReached && isNearBottom(el, endReachedThreshold)) onEndReached(); + }, + [onEndReached, endReachedThreshold, scrollRestoreId], + ); + + const virtualItems = virtualizer.getVirtualItems(); + const paddingTop = virtualItems.length > 0 ? virtualItems[0]!.start : 0; + const paddingBottom = + virtualItems.length > 0 + ? virtualizer.getTotalSize() - virtualItems[virtualItems.length - 1]!.end + : 0; + + return { scrollRef, virtualItems, paddingTop, paddingBottom, onScroll }; +} From 871c29a80ca605e03484af41908026d0f90cacdb Mon Sep 17 00:00:00 2001 From: Saatvik Arya Date: Mon, 22 Jun 2026 13:34:41 +0530 Subject: [PATCH 57/57] perf(semantic-search): snapshot the tool manifest per run (KV-only scan) (#52) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Production traces showed the reindex **scan** phase dominating wall-time: ~30 min for an all-skip pass over ~5,400 tools, ~3 tools/s. The cause is read amplification β€” `scan` calls `tools.manifest()` (a full `tool_schema_manifest` table read) **once per partition page**, then filters to its partition and slices `SCAN_PAGE` rows. With `PARTITION_COUNT=12` and `SCAN_PAGE=25` that's ~216 scan messages, each re-reading all ~5,400 rows cross-region (SINβ†’NRT): `O(NΒ²/pageLimit)` β‰ˆ **1.17M row-transfers** to scan 5,400 tools. `partition` is computed in-app (`hash(path) % count`), so it can't be pushed down to SQL without a migration. ## What - **`@executor-js/sdk`** β€” `createExecutor` exposes its cache `KeyValueStore` on the instance as **`executor.cache`** (Cloudflare KV in prod, in-memory fallback otherwise). Plugins reach durable cache storage without re-threading a binding; wrap with `KeyValueStore.toSchemaStore` like the internal `tools.list` / schema-view caches. - **`@executor-js/plugin-semantic-search`** β€” `create` reads the manifest **once**, partitions it (preserving each entry's full-list ordinal), and writes **one snapshot per partition** to `executor.cache` (`index-manifest/{runId}/{partition}`). `scan` reads only its partition's snapshot β€” **KV-only, no D1 fallback on the hot path**. A miss fails the message so the queue retries; the host delays the *initial* scan fan-out once to let the write propagate (separate host PR). Snapshots are deleted on run completion. ## Result - D1 manifest reads: `~1.17M β†’ ~5,400` (a single pass at `create`). - Per-scan-message memory: full manifest (~5 MB) β†’ one partition (~400 KB). - Scan wall-time: **~30 min β†’ seconds–minutes** (and the host delay PR keeps the fast path certain). ## Safety / notes - KV-only by design: a snapshot miss surfaces loudly (queue retry / DLQ) rather than silently degrading to the slow D1 path. - Snapshots are partition-sharded so no single KV value approaches the 25 MB cap (scales to ~300k tools). - KV has no native TTL through the abstraction, so cleanup is explicit on completion; a stalled / terminally-failed run leaves its (small) snapshot until overwritten β€” a known, bounded leak. - Behavior preserved: the scan's skip-vs-changed classification, ordinal ordering, pagination, and `maxTools` cap are unchanged (the snapshot is built from the same `listToolManifests` call). ## Gate tsgo **43/43** Β· `oxlint --deny-warnings` 0/0 Β· oxfmt Β· vitest **sdk 350/350**, **plugin-semantic-search 78/78** (new: manifest-read-once-per-create, KV-miss-fails-no-fallback, snapshot-cleared-on-complete). Stacked deploy: needs the host PR that delays the initial scan fan-out for KV propagation. --- .../semantic-search-manifest-snapshot.md | 10 + packages/core/sdk/src/executor.ts | 8 + .../src/sdk/manifest-snapshot.ts | 44 +++++ .../src/sdk/tool-search-index.test.ts | 176 +++++++++++++++++- .../src/sdk/tool-search-index.ts | 119 +++++++++--- 5 files changed, 333 insertions(+), 24 deletions(-) create mode 100644 .changeset/semantic-search-manifest-snapshot.md create mode 100644 packages/plugins/semantic-search/src/sdk/manifest-snapshot.ts diff --git a/.changeset/semantic-search-manifest-snapshot.md b/.changeset/semantic-search-manifest-snapshot.md new file mode 100644 index 0000000000..6bbf6cb21a --- /dev/null +++ b/.changeset/semantic-search-manifest-snapshot.md @@ -0,0 +1,10 @@ +--- +"@executor-js/sdk": minor +"@executor-js/plugin-semantic-search": minor +--- + +Snapshot the tool manifest per reindex run instead of re-reading it from D1 on every scan page. + +`createExecutor` now exposes its cache `KeyValueStore` on the executor instance as `executor.cache`, so plugins can reach durable cache storage (Cloudflare KV in production) without re-threading a binding β€” wrap it with `KeyValueStore.toSchemaStore` for typed entries, the same way the internal `tools.list` / schema-view caches do. + +The semantic-search scan phase used to call `tools.manifest()` β€” a full `tool_schema_manifest` table read β€” once per partition page, so an unchanged catalog cost `O(NΒ² / pageLimit)` cross-region reads. Now `create` reads the manifest once, partitions it, and writes one snapshot per partition to `executor.cache`; `scan` reads only its partition's snapshot (KV-only, with no D1 fallback on the hot path β€” a miss fails the message so the queue retries). Snapshots are deleted when the run completes. This collapses the scan's manifest reads from `O(NΒ² / pageLimit)` to a single pass and cuts per-message memory from the whole manifest to one partition's slice. diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 8bb7b30848..6708f4f477 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -361,6 +361,13 @@ export type Executor = { * machinery (e.g. execution observers) can attribute work to an owner * without re-threading identity through every call site. */ readonly owner: OwnerBinding; + + /** The host-provided key/value cache primitive (Cloudflare KV in prod, an + * in-memory fallback otherwise). Plugins reach durable cache storage through + * this without re-threading a binding β€” wrap it with + * `KeyValueStore.toSchemaStore` for typed entries (mirrors the internal + * `tools.list` / schema-view caches). */ + readonly cache: KeyValueStore.KeyValueStore; } & PluginExtensions; export interface ExecutorDb { @@ -4215,6 +4222,7 @@ export const createExecutor = => value as Executor; diff --git a/packages/plugins/semantic-search/src/sdk/manifest-snapshot.ts b/packages/plugins/semantic-search/src/sdk/manifest-snapshot.ts new file mode 100644 index 0000000000..26efa6cbf4 --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/manifest-snapshot.ts @@ -0,0 +1,44 @@ +import { ToolSchemaManifest } from "@executor-js/sdk/core"; +import { Schema } from "effect"; + +// --------------------------------------------------------------------------- +// Per-run tool-manifest snapshot. +// +// The scan phase used to re-read the entire `tool_schema_manifest` table from D1 +// on every page (12 partitions Γ— ~N/limit pages), so an unchanged catalog cost +// O(NΒ²/pageLimit) cross-region reads. Instead, `create` reads the manifest once, +// partitions it, and writes one snapshot per partition to the executor's KV cache. +// `scan` then reads only its partition's snapshot β€” KV-only, no D1 fallback on the +// hot path; a miss fails the message so the queue retries (the initial scan fan-out +// is delayed once to let the write propagate). Snapshots are deleted when the run +// reaches a terminal state (`complete` or terminal `fail`). +// +// Only `create` writes the snapshot β€” `reconcile` (resume) does not. The snapshot is +// durable (no TTL; removed only at terminal state), so a resumed stalled run still +// finds it. If a snapshot is ever genuinely absent at resume (e.g. a version-key bump +// or KV rollback), its scans DLQ and the recovery path is a fresh reindex, not resume. +// --------------------------------------------------------------------------- + +export const MANIFEST_SNAPSHOT_PREFIX = "index-manifest/"; +export const MANIFEST_SNAPSHOT_VERSION = "v1"; + +/** A manifest paired with its ordinal in the full (pre-partition) manifest list. + * The ordinal is preserved so partitioned jobs keep the same stable ordering + * they had when the scan filtered the full list in-process. */ +const ManifestSnapshotItem = Schema.Struct({ + ordinal: Schema.Number, + manifest: ToolSchemaManifest, +}); +export type ManifestSnapshotItem = typeof ManifestSnapshotItem.Type; + +/** One partition's slice of the run's manifest snapshot. Sharded per partition so + * a scan message loads only its ~N/partitions rows, not the whole catalog, and + * no single KV value approaches the 25 MB cap. */ +export const ManifestSnapshotEntry = Schema.Struct({ + version: Schema.Literal(MANIFEST_SNAPSHOT_VERSION), + items: Schema.Array(ManifestSnapshotItem), +}); +export type ManifestSnapshotEntry = typeof ManifestSnapshotEntry.Type; + +export const manifestSnapshotKey = (runId: string, partition: number): string => + `${MANIFEST_SNAPSHOT_PREFIX}${MANIFEST_SNAPSHOT_VERSION}/${runId}/${partition}`; diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts index b09e0a2c77..1c1b77bb22 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -9,6 +9,7 @@ import type { } from "@executor-js/sdk/core"; import { makeInMemoryBlobStore, pluginBlobStore } from "@executor-js/sdk/core"; import { Effect } from "effect"; +import * as KeyValueStore from "effect/unstable/persistence/KeyValueStore"; import { makeFacetChunker } from "./chunker"; import { @@ -22,7 +23,18 @@ import { toolFingerprints, } from "./collections"; import type { ToolEmbedder } from "./embedder"; -import { chunk, commit, create, embed, fail, reconcile, scan, status } from "./tool-search-index"; +import { SemanticSearchError } from "./errors"; +import { + chunk, + commit, + complete, + create, + embed, + fail, + reconcile, + scan, + status, +} from "./tool-search-index"; import type { VectorInput, VectorStore } from "./store"; const owner: Owner = "org" as Owner; @@ -31,6 +43,27 @@ const namespace = "test-ns"; const makeBlobs = () => pluginBlobStore(makeInMemoryBlobStore(), { org: "test-org", user: null }, "semanticSearch"); +/** In-memory cache primitive for the executor mock β€” durable within a single test + * so a `create` snapshot write is visible to the `scan` reads that follow. */ +const makeMemoryCache = (): Executor["cache"] => { + const rows = new Map(); + return KeyValueStore.makeStringOnly({ + get: (key) => Effect.sync(() => rows.get(key)), + set: (key, value) => + Effect.sync(() => { + rows.set(key, value); + }), + remove: (key) => + Effect.sync(() => { + rows.delete(key); + }), + clear: Effect.sync(() => { + rows.clear(); + }), + size: Effect.sync(() => rows.size), + }); +}; + const manifestForTool = ( tool: Tool, fingerprint = `fingerprint:${String(tool.address)}`, @@ -64,7 +97,7 @@ const makeExecutor = ( connection: "default" as never, pluginId: "test", }; - const executor: Pick = { + const executor: Pick = { tools: { list: () => Effect.succeed([tool]), manifest: () => Effect.succeed([manifestForTool(tool)]), @@ -77,6 +110,7 @@ const makeExecutor = ( }); }, }, + cache: makeMemoryCache(), }; return executor as Executor; }; @@ -511,12 +545,13 @@ describe("ToolSearchIndex", () => { pluginId: "test", }, ]; - const executor: Pick = { + const executor: Pick = { tools: { list: () => Effect.succeed(tools), manifest: () => Effect.succeed(tools.map((tool) => manifestForTool(tool))), schema: () => Effect.succeed(null), }, + cache: makeMemoryCache(), }; const runs = makeCollection(indexRuns.name); const jobs = makeCollection(indexJobs.name); @@ -852,3 +887,138 @@ describe("ToolSearchIndex", () => { }), ); }); + +describe("ToolSearchIndex manifest snapshot", () => { + const makeTools = (count: number): Tool[] => + Array.from({ length: count }, (_, i) => ({ + address: `tools.github.repos.get${i}` as never, + name: `repos.get${i}` as never, + integration: "github" as never, + description: `Get repository ${i}`, + owner, + connection: "default" as never, + pluginId: "test", + })); + + /** Executor mock that counts `tools.manifest()` calls so a test can prove the + * scan reads the KV snapshot rather than re-querying the manifest per page. */ + const makeCountingExecutor = (tools: Tool[]) => { + const counters = { manifest: 0 }; + const executor: Pick = { + tools: { + list: () => Effect.succeed(tools), + manifest: () => { + counters.manifest++; + return Effect.succeed(tools.map((tool) => manifestForTool(tool))); + }, + schema: () => Effect.succeed(null), + }, + cache: makeMemoryCache(), + }; + return { executor: executor as Executor, counters }; + }; + + const makeBase = (executor: Executor) => ({ + namespace, + executor, + runs: makeCollection(indexRuns.name), + jobs: makeCollection(indexJobs.name), + chunks: makeCollection(indexChunks.name), + fingerprints: makeCollection(toolFingerprints.name), + blobs: makeBlobs(), + owner, + }); + + it.effect("reads the manifest once at create and pages scans from the KV snapshot", () => + Effect.gen(function* () { + const { executor, counters } = makeCountingExecutor(makeTools(5)); + const base = makeBase(executor); + + yield* create({ ...base, runId: "run-snap", partitionCount: 1 }); + expect(counters.manifest).toBe(1); + + let pages = 0; + let hasMore = true; + while (hasMore) { + const result = yield* scan({ ...base, runId: "run-snap", partition: 0, limit: 2 }); + hasMore = result.hasMore; + pages++; + } + + expect(pages).toBe(3); // ceil(5 / 2) pages, each from the snapshot + expect(counters.manifest).toBe(1); // never re-read during scan + expect(base.jobs.data.size).toBe(5); // every tool scanned exactly once + }), + ); + + it.effect("fails the scan when the run snapshot is missing (no D1 fallback)", () => + Effect.gen(function* () { + const { executor, counters } = makeCountingExecutor(makeTools(2)); + const base = makeBase(executor); + + // No `create`, so nothing was written to the KV snapshot. + const error = yield* Effect.flip( + scan({ ...base, runId: "run-missing", partition: 0, limit: 10 }), + ); + + expect(error).toBeInstanceOf(SemanticSearchError); // KV miss β†’ fail, not a live read + expect(counters.manifest).toBe(0); // the scan never falls back to a live manifest read + }), + ); + + it.effect("clears the partition snapshots when the run completes", () => + Effect.gen(function* () { + const { executor } = makeCountingExecutor(makeTools(3)); + const base = makeBase(executor); + + yield* create({ ...base, runId: "run-clear", partitionCount: 1 }); + let hasMore = true; + while (hasMore) { + const result = yield* scan({ ...base, runId: "run-clear", partition: 0, limit: 10 }); + hasMore = result.hasMore; + } + + const store: VectorStore = { + maxTopK: 100, + query: () => Effect.succeed([]), + upsert: () => Effect.void, + deleteByIds: () => Effect.void, + }; + const embedder: ToolEmbedder = { + model: "test", + dimensions: 3, + embedDocuments: (texts) => Effect.succeed(texts.map(() => [0.1, 0.2, 0.3])), + embedQuery: () => Effect.succeed([0.1, 0.2, 0.3]), + }; + yield* complete({ + ...base, + store, + embedder, + chunker: makeFacetChunker(), + runId: "run-clear", + }); + + // The snapshot is gone, so a post-completion scan can no longer read it. + const error = yield* Effect.flip( + scan({ ...base, runId: "run-clear", partition: 0, limit: 10 }), + ); + expect(error).toBeInstanceOf(SemanticSearchError); // snapshot cleared on completion + }), + ); + + it.effect("clears the partition snapshots when the run terminally fails", () => + Effect.gen(function* () { + const { executor } = makeCountingExecutor(makeTools(3)); + const base = makeBase(executor); + + yield* create({ ...base, runId: "run-failclear", partitionCount: 1 }); + const failed = yield* fail({ ...base, runId: "run-failclear", error: "queue exhausted" }); + expect(failed.runFailed).toBe(true); // no pending paths/chunks β†’ run is terminally failed + + const error = yield* Effect.flip( + scan({ ...base, runId: "run-failclear", partition: 0, limit: 10 }), + ); + expect(error).toBeInstanceOf(SemanticSearchError); // snapshot cleared on terminal failure + }), + ); +}); diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts index 47f6f5bfe6..2a2b539fa7 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts @@ -3,9 +3,11 @@ import type { Owner, PluginBlobStore, PluginStorageCollectionFacade, + ToolSchemaManifest, } from "@executor-js/sdk/core"; import { sha256Hex } from "@executor-js/sdk/core"; -import { Context, Effect, Predicate, Schema } from "effect"; +import { Context, Effect, Option, Predicate, Schema } from "effect"; +import * as KeyValueStore from "effect/unstable/persistence/KeyValueStore"; import { type Chunker, type ToolChunk, ToolDocumentInput } from "./chunker"; import { @@ -28,6 +30,12 @@ import { } from "./documents"; import type { ToolEmbedder } from "./embedder"; import { SemanticSearchError } from "./errors"; +import { + ManifestSnapshotEntry, + type ManifestSnapshotItem, + MANIFEST_SNAPSHOT_VERSION, + manifestSnapshotKey, +} from "./manifest-snapshot"; import { cyrb53 } from "./fingerprint"; import type { VectorInput, VectorStore } from "./store"; import type { FtsDocumentInput, FtsLexicalStore } from "./store-fts"; @@ -679,7 +687,7 @@ const countJobsByStatus = ( ); export const fail = ( - input: IndexCollections & ToolSearchIndex.FailInput, + input: IndexStores & ToolSearchIndex.FailInput, ): Effect.Effect => Effect.gen(function* () { const updatedAt = nowIso(); @@ -826,6 +834,7 @@ export const fail = ( new SemanticSearchError({ message: "Failed to mark index run failed.", cause }), ), ); + yield* deleteManifestSnapshot(input.executor, input.runId, run.data.partitionCount); } return { @@ -906,6 +915,85 @@ export const reconcile = ( }; }); +const manifestSnapshotStore = (executor: Executor) => + KeyValueStore.toSchemaStore(executor.cache, ManifestSnapshotEntry); + +/** Partition the run's full manifest list (preserving each entry's ordinal in the + * full list) and write one snapshot per partition to the executor cache. Written + * once at run creation so the scan phase reads KV instead of re-querying the whole + * `tool_schema_manifest` table on every page. Fails loudly: the scan path has no + * D1 fallback, so a run must not start without its snapshot. */ +const writeManifestSnapshot = ( + executor: Executor, + runId: string, + partitionCount: number, + manifests: readonly ToolSchemaManifest[], +): Effect.Effect => + Effect.gen(function* () { + const store = manifestSnapshotStore(executor); + const byPartition: ManifestSnapshotItem[][] = Array.from({ length: partitionCount }, () => []); + manifests.forEach((manifest, ordinal) => { + byPartition[partitionForPath(manifest.path, partitionCount)]?.push({ ordinal, manifest }); + }); + yield* Effect.forEach( + byPartition, + (items, partition) => + store.set(manifestSnapshotKey(runId, partition), { + version: MANIFEST_SNAPSHOT_VERSION, + items, + }), + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ).pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Failed to write tool manifest snapshot.", cause }), + ), + ); + }); + +/** Read a partition's manifest snapshot. KV-only by design: a miss fails the scan + * so the queue retries (the initial scan fan-out is delayed once to let the write + * propagate) β€” there is deliberately no D1 fallback on this hot path. */ +const readManifestSnapshot = ( + executor: Executor, + runId: string, + partition: number, +): Effect.Effect => + manifestSnapshotStore(executor) + .get(manifestSnapshotKey(runId, partition)) + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Failed to read tool manifest snapshot.", cause }), + ), + Effect.flatMap( + Option.match({ + onNone: () => + Effect.fail( + new SemanticSearchError({ + message: `Tool manifest snapshot for run "${runId}" partition ${partition} is missing.`, + }), + ), + onSome: (entry) => Effect.succeed(entry.items), + }), + ), + ); + +/** Best-effort delete of every partition snapshot once a run reaches a terminal + * state (`complete` or terminal `fail`). A stalled run that never reaches a + * terminal state leaves its snapshot until overwritten β€” a small, bounded leak; + * KV has no native TTL through this abstraction. */ +const deleteManifestSnapshot = ( + executor: Executor, + runId: string, + partitionCount: number, +): Effect.Effect => + Effect.forEach( + Array.from({ length: partitionCount }, (_, partition) => partition), + (partition) => executor.cache.remove(manifestSnapshotKey(runId, partition)), + { concurrency: INDEX_STORAGE_CONCURRENCY, discard: true }, + ).pipe(Effect.ignore); + export const create = ( input: IndexStores & ToolSearchIndex.CreateInput, ): Effect.Effect => @@ -914,6 +1002,8 @@ export const create = ( const manifests = yield* listToolManifests(input.executor, { maxTools: input.maxTools }); const createdAt = nowIso(); + yield* writeManifestSnapshot(input.executor, input.runId, partitionCount, manifests); + yield* input.runs .put({ owner: input.owner, @@ -947,26 +1037,12 @@ export const scan = ( input: IndexStores & ToolSearchIndex.ScanInput, ): Effect.Effect => Effect.gen(function* () { - const run = yield* input.runs - .getForOwner({ owner: input.owner, key: input.runId }) - .pipe( - Effect.mapError( - (cause) => new SemanticSearchError({ message: "Failed to load index run.", cause }), - ), - ); - if (run === null) { - return yield* new SemanticSearchError({ - message: `Index run "${input.runId}" does not exist.`, - }); - } + const partitionManifests = yield* readManifestSnapshot( + input.executor, + input.runId, + input.partition, + ); - const manifests = yield* listToolManifests(input.executor, { maxTools: input.maxTools }); - const partitionManifests = manifests - .map((manifest, ordinal) => ({ manifest, ordinal })) - .filter( - ({ manifest }) => - partitionForPath(manifest.path, run.data.partitionCount) === input.partition, - ); const existing = yield* input.jobs .count({ where: { runId: input.runId, partition: input.partition } }) .pipe( @@ -1631,6 +1707,7 @@ export const complete = ( new SemanticSearchError({ message: "Failed to mark index run completed.", cause }), ), ); + yield* deleteManifestSnapshot(input.executor, input.runId, existing.data.partitionCount); } return { runId: input.runId, removed: result.removed }; });