feat: add the execution-metrics plugin

.changeset/execution-metrics-plugin.md

@@ -0,0 +1,13 @@
+---
+"@executor-js/plugin-execution-metrics": minor
+---
+
+Add the `@executor-js/plugin-execution-metrics` plugin — an opt-in metrics sink
+built on the ExecutionObserver foundation. The base plugin drives Effect Metric
+counters and an execution-duration histogram from the event stream; `./api`'s
+`executionMetricsHttpPlugin` exposes them as an auth-gated `GET
+/execution-metrics/metrics` Prometheus scrape for long-lived hosts; and
+`./cloudflare`'s `createWaeMetricsObserver` writes Workers Analytics Engine data
+points on the Cloudflare host (per-isolate Effect metrics being meaningless on a
+Worker fleet). The base `.` entry never loads `@executor-js/api` or
+`@cloudflare/workers-types`.

apps/host-cloudflare/package.json

@@ -24,6 +24,7 @@
     "@executor-js/fumadb": "workspace:*",
     "@executor-js/host-mcp": "workspace:*",
     "@executor-js/plugin-encrypted-secrets": "workspace:*",
+    "@executor-js/plugin-execution-metrics": "workspace:*",
     "@executor-js/plugin-graphql": "workspace:*",
     "@executor-js/plugin-mcp": "workspace:*",
     "@executor-js/plugin-openapi": "workspace:*",

apps/host-cloudflare/src/app.ts

@@ -33,7 +33,7 @@ import { preloadQuickJs } from "./quickjs";
 
 export const makeCloudflareApp = async (env: CloudflareEnv) => {
   const config = loadConfig(env);
-  const plugins = makeCloudflarePlugins(config.secretKey);
+  const plugins = makeCloudflarePlugins(config.secretKey, env.ANALYTICS);
 
   // Load the Workers-compatible (WASM-inlined) QuickJS variant before any
   // executor is built — the default variant can't fetch its .wasm on Workers.

apps/host-cloudflare/src/config.ts

@@ -1,4 +1,5 @@
 import type { D1Database, DurableObjectNamespace, R2Bucket } from "@cloudflare/workers-types";
+import type { AnalyticsEngineDataset } from "@executor-js/plugin-execution-metrics/cloudflare";
 
 import { isValidOrgSlug } from "@executor-js/api";
 
@@ -18,6 +19,10 @@ export interface CloudflareEnv {
   readonly DB: D1Database;
   /** R2 bucket binding — holds values too large for a D1 row (~1-2MB cap). */
   readonly BLOBS?: R2Bucket;
+  /** Workers Analytics Engine binding — opt-in sink for execution metrics. When
+   *  bound (uncomment `analytics_engine_datasets` in wrangler.jsonc), each
+   *  finished execution/tool call writes a data point; absent, metrics are off. */
+  readonly ANALYTICS?: AnalyticsEngineDataset;
   /** MCP session Durable Object namespace — one addressable isolate per MCP
    *  session (the DO id IS the session id), so a session survives across the
    *  Worker's stateless isolates. */

apps/host-cloudflare/src/plugins.ts

@@ -2,6 +2,12 @@ import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
 import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";
 import { graphqlHttpPlugin } from "@executor-js/plugin-graphql/api";
 import { encryptedSecretsPlugin } from "@executor-js/plugin-encrypted-secrets";
+import { executionMetricsPlugin } from "@executor-js/plugin-execution-metrics";
+import {
+  createWaeMetricsObserver,
+  type AnalyticsEngineDataset,
+} from "@executor-js/plugin-execution-metrics/cloudflare";
+import { noopExecutionObserver } from "@executor-js/sdk";
 
 // ---------------------------------------------------------------------------
 // The Cloudflare host's plugin list — the same protocol/provider plugins as
@@ -12,14 +18,25 @@ import { encryptedSecretsPlugin } from "@executor-js/plugin-encrypted-secrets";
 //
 // `dangerouslyAllowStdioMCP` is false: a multi-user instance must not let a user
 // spawn arbitrary stdio MCP processes.
+//
+// Execution metrics ship to Workers Analytics Engine — opt-in via the wrangler
+// `ANALYTICS` binding. The plugin is always in the tuple (it has no tables/API,
+// so the shape is stable), but its observer is a no-op until `env.ANALYTICS` is
+// bound. Effect's Metric registry is per-isolate (meaningless on a Worker
+// fleet), so the local Prometheus scrape is deliberately NOT mounted here; WAE
+// is the durable sink. To enable: uncomment `analytics_engine_datasets` in
+// wrangler.jsonc.
 // ---------------------------------------------------------------------------
 
-export const makeCloudflarePlugins = (secretKey: string) =>
+export const makeCloudflarePlugins = (secretKey: string, analytics?: AnalyticsEngineDataset) =>
   [
     openApiHttpPlugin(),
     mcpHttpPlugin({ dangerouslyAllowStdioMCP: false }),
     graphqlHttpPlugin(),
     encryptedSecretsPlugin({ key: secretKey }),
+    executionMetricsPlugin({
+      observer: () => (analytics ? createWaeMetricsObserver(analytics) : noopExecutionObserver),
+    }),
   ] as const;
 
 export type CloudflarePlugins = ReturnType<typeof makeCloudflarePlugins>;

apps/host-cloudflare/wrangler.jsonc

@@ -33,6 +33,13 @@
       "bucket_name": "executor-blobs",
     },
   ],
+  // Workers Analytics Engine — the opt-in sink for execution metrics. Uncomment
+  // to enable: each finished execution/tool call writes a data point (queryable
+  // via Cloudflare's GraphQL Analytics API). The plugin is inert until this
+  // binding is present. Requires a paid Workers plan.
+  // "analytics_engine_datasets": [
+  //   { "binding": "ANALYTICS", "dataset": "executor_executions" },
+  // ],
   // The MCP session Durable Object: one addressable isolate per MCP session (the
   // DO id IS the session id) so a session survives across the Worker's stateless
   // isolates — without it, `tools/list` after `initialize` can land on a fresh

packages/plugins/execution-metrics/CHANGELOG.md

@@ -0,0 +1 @@
+# @executor-js/plugin-execution-metrics

packages/plugins/execution-metrics/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@executor-js/plugin-execution-metrics",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/sdk/index.ts",
+    "./api": "./src/api/index.ts",
+    "./cloudflare": "./src/cloudflare/index.ts"
+  },
+  "scripts": {
+    "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src || true)",
+    "typecheck": "tsgo --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "dependencies": {
+    "@executor-js/sdk": "workspace:*",
+    "effect": "catalog:"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20250620.0",
+    "@effect/vitest": "catalog:",
+    "@executor-js/api": "workspace:*",
+    "@types/node": "catalog:",
+    "bun-types": "catalog:",
+    "tsup": "catalog:",
+    "vitest": "catalog:"
+  },
+  "peerDependencies": {
+    "@executor-js/api": "workspace:*"
+  },
+  "peerDependenciesMeta": {
+    "@executor-js/api": {
+      "optional": true
+    }
+  }
+}

packages/plugins/execution-metrics/src/api/group.ts

@@ -0,0 +1,28 @@
+import { HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/unstable/httpapi";
+import { Schema } from "effect";
+import { InternalError } from "@executor-js/sdk/shared";
+
+// ---------------------------------------------------------------------------
+// Responses
+//
+// The scrape endpoint returns Prometheus text-exposition, not JSON.
+// `HttpApiSchema.asText()` flips the success body to `text/plain` so the
+// renderer's string is written verbatim.
+// ---------------------------------------------------------------------------
+
+const PrometheusResponse = Schema.String.pipe(HttpApiSchema.asText());
+
+// ---------------------------------------------------------------------------
+// Group — the execution-metrics HTTP surface.
+//
+// One GET endpoint. Auth-gated automatically by the host's execution-stack
+// middleware, like every other plugin route. `InternalError` is the shared
+// opaque 500 surface translated at the HTTP edge by `capture`.
+// ---------------------------------------------------------------------------
+
+export const ExecutionMetricsGroup = HttpApiGroup.make("execution-metrics").add(
+  HttpApiEndpoint.get("metrics", "/execution-metrics/metrics", {
+    success: PrometheusResponse,
+    error: InternalError,
+  }),
+);

packages/plugins/execution-metrics/src/api/handlers.ts

@@ -0,0 +1,27 @@
+import { HttpApiBuilder } from "effect/unstable/httpapi";
+
+import { addGroup } from "@executor-js/api";
+import { ExecutionMetricsGroup } from "./group";
+import { renderPrometheus } from "./prometheus";
+
+// ---------------------------------------------------------------------------
+// Composed API — core + execution-metrics group
+// ---------------------------------------------------------------------------
+
+const ExecutorApiWithMetrics = addGroup(ExecutionMetricsGroup);
+
+// ---------------------------------------------------------------------------
+// Handlers
+//
+// The scrape reads the global Effect Metric registry directly (no plugin
+// extension state), so there is no `extensionService` to require — the handler
+// just renders the snapshot. `renderPrometheus` has no error channel, so no
+// `capture` translation is needed; the group's `InternalError` covers defects
+// the edge middleware captures.
+// ---------------------------------------------------------------------------
+
+export const ExecutionMetricsHandlers = HttpApiBuilder.group(
+  ExecutorApiWithMetrics,
+  "execution-metrics",
+  (handlers) => handlers.handle("metrics", () => renderPrometheus),
+);

packages/plugins/execution-metrics/src/api/index.ts

@@ -0,0 +1,24 @@
+import { definePlugin } from "@executor-js/sdk/core";
+
+import { executionMetricsPlugin, type ExecutionMetricsPluginOptions } from "../sdk/plugin";
+import { ExecutionMetricsGroup } from "./group";
+import { ExecutionMetricsHandlers } from "./handlers";
+
+export { ExecutionMetricsGroup } from "./group";
+export { ExecutionMetricsHandlers } from "./handlers";
+export { renderPrometheus } from "./prometheus";
+
+// HTTP-augmented variant of `executionMetricsPlugin`. The returned plugin
+// carries the HTTP `routes` and `handlers` so a host can mount the Prometheus
+// scrape (`GET /execution-metrics/metrics`). The scrape reads the global Effect
+// Metric registry, so there is no plugin extension state and no
+// `extensionService`. Hosts that compose an `HttpApi` import this; SDK-only
+// consumers stay on `@executor-js/plugin-execution-metrics` and never load
+// `@executor-js/api`.
+export const executionMetricsHttpPlugin = definePlugin(
+  (options?: ExecutionMetricsPluginOptions) => ({
+    ...executionMetricsPlugin(options),
+    routes: () => ExecutionMetricsGroup,
+    handlers: () => ExecutionMetricsHandlers,
+  }),
+);

packages/plugins/execution-metrics/src/api/prometheus.test.ts

@@ -0,0 +1,109 @@
+import { describe, expect, it } from "@effect/vitest";
+import { Effect } from "effect";
+
+import {
+  ExecutionFinished,
+  ExecutionId,
+  ExecutionStarted,
+  ExecutionToolCallId,
+  Subject,
+  Tenant,
+  ToolCallFinished,
+  ToolCallStarted,
+} from "@executor-js/sdk/core";
+
+import { createExecutionMetricsObserver } from "../sdk/observer";
+import { renderPrometheus } from "./prometheus";
+
+const owner = { tenant: Tenant.make("tenant_test"), subject: Subject.make("subject_test") };
+
+const executionId = ExecutionId.make("exec_metrics_test");
+const toolCallId = ExecutionToolCallId.make("toolcall_metrics_test");
+
+const startedAt = new Date("2026-06-13T00:00:00.000Z");
+const completedAt = new Date("2026-06-13T00:00:00.250Z");
+
+describe("execution-metrics observer + renderPrometheus", () => {
+  it.effect("records counters/histogram a Prometheus scrape can expose", () =>
+    Effect.gen(function* () {
+      const observer = createExecutionMetricsObserver();
+
+      yield* observer.handle(
+        new ExecutionStarted({
+          executionId,
+          owner,
+          code: "console.log('hi')",
+          trigger: { kind: "manual" },
+          startedAt,
+        }),
+      );
+      yield* observer.handle(
+        new ToolCallStarted({
+          executionId,
+          toolCallId,
+          owner,
+          path: "github.issues.create",
+          args: {},
+          startedAt,
+        }),
+      );
+      yield* observer.handle(
+        new ToolCallFinished({
+          executionId,
+          toolCallId,
+          owner,
+          path: "github.issues.create",
+          status: "completed",
+          completedAt,
+        }),
+      );
+      yield* observer.handle(
+        new ExecutionFinished({
+          executionId,
+          owner,
+          status: "completed",
+          result: "ok",
+          completedAt,
+        }),
+      );
+
+      const text = yield* renderPrometheus;
+
+      // Counter + histogram series names are exposed.
+      expect(text).toContain("# TYPE executor_execution_started_total counter");
+      expect(text).toContain("# TYPE executor_execution_completed_total counter");
+      expect(text).toContain("# TYPE executor_tool_call_started_total counter");
+      expect(text).toContain("# TYPE executor_tool_call_completed_total counter");
+      expect(text).toContain("# TYPE executor_execution_duration_ms histogram");
+      expect(text).toContain("executor_execution_duration_ms_bucket");
+      expect(text).toContain('le="+Inf"');
+
+      // Effect's histogram buckets already carry the terminal `+Inf` boundary,
+      // so the renderer must emit it exactly once per series — a duplicate
+      // `+Inf` bucket line is a duplicate label set and a scrape rejects it.
+      const infBuckets = text
+        .split("\n")
+        .filter(
+          (line) =>
+            line.startsWith("executor_execution_duration_ms_bucket{") && line.includes('le="+Inf"'),
+        );
+      expect(infBuckets).toHaveLength(1);
+
+      // The trigger_kind attribute rides the tagged series.
+      expect(text).toContain('trigger_kind="manual"');
+
+      // At least one started execution was counted (global registry — value ≥ 1).
+      const startedLine = text
+        .split("\n")
+        .find(
+          (line) =>
+            line.startsWith("executor_execution_started_total{") &&
+            line.includes('trigger_kind="manual"'),
+        );
+      const startedValue = startedLine
+        ? Number(startedLine.slice(startedLine.lastIndexOf(" ") + 1))
+        : 0;
+      expect(startedValue).toBeGreaterThanOrEqual(1);
+    }),
+  );
+});