[pull] main from UsefulSoftwareCo:main

.claude/skills/prod-telemetry/SKILL.md

@@ -29,6 +29,15 @@ join the same traces via traceparent).
 
 **Span names worth querying** (and their custom attrs):
 
+- `mcp.execute` / `mcp.execute.resume` — `mcp.execute.mode`
+  (`pausable`/`inline`), `mcp.execute.code_length`, and
+  `mcp.execute.outcome` (`ok`/`fail`/`paused`) with, on failures,
+  `mcp.execute.error_kind` (`type_error` | `reference_error` |
+  `syntax_error` | `range_error` | `tool_error` | `timeout` |
+  `resource_limit` | `serialization_error` | `thrown` | `unknown`).
+  Sandbox script failures ride the MCP success channel, so `status.code`
+  stays OK — filter on these attributes, not span status. Spans from
+  before the attributes shipped carry neither; absence is not success.
 - `executor.tool.execute` — `mcp.tool.name` (full address), and since
   PR #992: `executor.tool.outcome` (`ok`/`fail`),
   `executor.tool.error_code`, `executor.tool.error_status`,

e2e/local/cli-telemetry-optout.test.ts

@@ -0,0 +1,27 @@
+import { expect, it } from "@effect/vitest";
+import { Effect } from "effect";
+
+import { makeCliSurface } from "../src/surfaces/cli";
+
+it.live("terminal E2E sessions force product telemetry off", () =>
+  makeCliSurface()
+    .session(
+      [
+        process.execPath,
+        "-e",
+        "console.log([process.env.DO_NOT_TRACK, process.env.EXECUTOR_DISABLE_ANALYTICS, process.env.EXECUTOR_DISABLE_INTEGRATIONS_FETCH].join('|'))",
+      ],
+      (terminal) =>
+        terminal.screen
+          .waitUntil((screen) => screen.text.includes("1|1|1"), { timeoutMs: 5_000 })
+          .then((screen) => expect(screen.text).toContain("1|1|1")),
+      {
+        env: {
+          DO_NOT_TRACK: "0",
+          EXECUTOR_DISABLE_ANALYTICS: "0",
+          EXECUTOR_DISABLE_INTEGRATIONS_FETCH: "0",
+        },
+      },
+    )
+    .pipe(Effect.timeout("10 seconds")),
+);

e2e/src/surfaces/cli.ts

@@ -32,6 +32,15 @@ interface AsciicastEvent {
   bytes?: number[];
 }
 
+// E2E subprocesses must never report product usage or fetch the production
+// integrations catalog. Keep these after caller-provided values below so a
+// scenario cannot accidentally turn outbound telemetry back on.
+const E2E_OUTBOUND_OPT_OUT = {
+  DO_NOT_TRACK: "1",
+  EXECUTOR_DISABLE_ANALYTICS: "1",
+  EXECUTOR_DISABLE_INTEGRATIONS_FETCH: "1",
+} as const;
+
 /** terminal-control's JSONL recording → asciicast v2 (what asciinema plays). */
 const toAsciicast = (recording: Uint8Array): string => {
   const events = new TextDecoder()
@@ -74,7 +83,7 @@ export const makeCliSurface = (): CliSurface => ({
         const session: Session = await tc.launch({
           command,
           cwd: options?.cwd,
-          env: options?.env,
+          env: { ...options?.env, ...E2E_OUTBOUND_OPT_OUT },
           record: options?.record ? true : undefined,
           viewport: options?.viewport,
         });

packages/core/execution/src/engine.ts

@@ -62,6 +62,25 @@ export type ResumeResponse = {
 // caller is itself the human approver (the operator-facing Run/Test panel).
 const acceptAllHandler: ElicitationHandler = () => Effect.succeed({ action: "accept" });
 
+/**
+ * Stamp the current `mcp.execute` / `mcp.execute.resume` span with how the
+ * execution ended. Sandbox failures ride the success channel as
+ * `ExecuteResult.error`, so without this the span reads OK and the failure
+ * class is unqueryable. Attributes stay enumerable identifiers — never the
+ * error message itself.
+ */
+const annotateExecuteOutcome = (result: ExecuteResult) =>
+  Effect.annotateCurrentSpan(
+    result.error
+      ? { "mcp.execute.outcome": "fail", "mcp.execute.error_kind": result.errorKind ?? "unknown" }
+      : { "mcp.execute.outcome": "ok" },
+  );
+
+const annotateExecutionOutcome = (execution: ExecutionResult) =>
+  execution.status === "paused"
+    ? Effect.annotateCurrentSpan({ "mcp.execute.outcome": "paused" })
+    : annotateExecuteOutcome(execution.result);
+
 // ---------------------------------------------------------------------------
 // Result formatting
 // ---------------------------------------------------------------------------
@@ -529,6 +548,7 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
     if (options?.autoApprove) {
       yield* Effect.annotateCurrentSpan({ "mcp.execute.auto_approve": true });
       const result = yield* runInlineExecution(code, { onElicitation: acceptAllHandler });
+      yield* annotateExecuteOutcome(result);
       return { status: "completed", result } satisfies ExecutionResult;
     }
 
@@ -596,7 +616,9 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
       ),
     );
 
-    return (yield* awaitCompletionOrPause(fiber, pauseQueue)) as ExecutionResult;
+    const outcome = (yield* awaitCompletionOrPause(fiber, pauseQueue)) as ExecutionResult;
+    yield* annotateExecutionOutcome(outcome);
+    return outcome;
   });
 
   /**
@@ -619,13 +641,17 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
     const settled = settledOutcomes.get(executionId);
     if (settled) {
       yield* Effect.annotateCurrentSpan({ "mcp.execute.resume.replayed": true });
-      return (yield* settled) as ExecutionResult;
+      const replayed = (yield* settled) as ExecutionResult;
+      yield* annotateExecutionOutcome(replayed);
+      return replayed;
     }
 
     const pending = pendingResumes.get(executionId);
     if (pending) {
       yield* Effect.annotateCurrentSpan({ "mcp.execute.resume.joined_inflight": true });
-      return (yield* Deferred.await(pending)) as ExecutionResult;
+      const joined = (yield* Deferred.await(pending)) as ExecutionResult;
+      yield* annotateExecutionOutcome(joined);
+      return joined;
     }
 
     const paused = pausedExecutions.get(executionId);
@@ -640,7 +666,7 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
       content: response.content,
     });
 
-    return (yield* awaitCompletionOrPause(paused.fiber, paused.pauseQueue).pipe(
+    const outcome = (yield* awaitCompletionOrPause(paused.fiber, paused.pauseQueue).pipe(
       Effect.onExit((exit) =>
         Effect.gen(function* () {
           recordSettledOutcome(executionId, exit);
@@ -649,6 +675,8 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
         }),
       ),
     )) as ExecutionResult;
+    yield* annotateExecutionOutcome(outcome);
+    return outcome;
   });
 
   /**
@@ -670,7 +698,11 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
       },
       toolDiscoveryProvider,
     );
-    return yield* codeExecutor.execute(code, invoker).pipe(Effect.withSpan("executor.code.exec"));
+    const result = yield* codeExecutor
+      .execute(code, invoker)
+      .pipe(Effect.withSpan("executor.code.exec"));
+    yield* annotateExecuteOutcome(result);
+    return result;
   });
 
   return {

packages/kernel/core/src/error-kind.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "@effect/vitest";
+
+import { classifyThrownExecuteError } from "./error-kind";
+
+describe("classifyThrownExecuteError", () => {
+  it("maps built-in JS error names to their kinds", () => {
+    expect(classifyThrownExecuteError("SyntaxError")).toBe("syntax_error");
+    expect(classifyThrownExecuteError("TypeError")).toBe("type_error");
+    expect(classifyThrownExecuteError("ReferenceError")).toBe("reference_error");
+    expect(classifyThrownExecuteError("RangeError")).toBe("range_error");
+    expect(classifyThrownExecuteError("DataCloneError")).toBe("serialization_error");
+  });
+
+  it("maps runtime-tagged names to their kinds", () => {
+    expect(classifyThrownExecuteError("ExecutionToolError")).toBe("tool_error");
+    expect(classifyThrownExecuteError("ExecutionTimeoutError")).toBe("timeout");
+  });
+
+  it("treats anything else as the script's own throw", () => {
+    expect(classifyThrownExecuteError("Error")).toBe("thrown");
+    expect(classifyThrownExecuteError("CustomDomainError")).toBe("thrown");
+    expect(classifyThrownExecuteError(null)).toBe("thrown");
+    expect(classifyThrownExecuteError(undefined)).toBe("thrown");
+  });
+});

packages/kernel/core/src/error-kind.ts

@@ -0,0 +1,37 @@
+/**
+ * Enumerable classification of a failed sandbox execution, derived from the
+ * structured error the runtime already holds (the thrown error's `name`, or
+ * the runtime's own typed failure). Carried on `ExecuteResult.errorKind`
+ * beside the descriptive `error` string so telemetry can count failure
+ * classes as identifiers without ever recording message content.
+ */
+export type ExecuteErrorKind =
+  | "syntax_error"
+  | "type_error"
+  | "reference_error"
+  | "range_error"
+  | "tool_error"
+  | "timeout"
+  | "resource_limit"
+  | "serialization_error"
+  | "thrown";
+
+const KIND_BY_ERROR_NAME: Readonly<Record<string, ExecuteErrorKind>> = {
+  SyntaxError: "syntax_error",
+  TypeError: "type_error",
+  ReferenceError: "reference_error",
+  RangeError: "range_error",
+  ExecutionToolError: "tool_error",
+  ExecutionTimeoutError: "timeout",
+  DataCloneError: "serialization_error",
+};
+
+/**
+ * Classify an error thrown inside the sandbox by its `name`. Runtimes tag
+ * their own conditions with dedicated names (`ExecutionToolError` for a
+ * failed tool-dispatch rethrow, `ExecutionTimeoutError` for the in-sandbox
+ * deadline); anything unrecognized is the script's own `throw`.
+ */
+export const classifyThrownExecuteError = (
+  errorName: string | null | undefined,
+): ExecuteErrorKind => (errorName != null ? KIND_BY_ERROR_NAME[errorName] : undefined) ?? "thrown";

packages/kernel/core/src/index.ts

@@ -1,4 +1,5 @@
 export * from "./types";
+export * from "./error-kind";
 export * from "./validation";
 export * from "./json-schema";
 export * from "./effect-errors";

packages/kernel/core/src/types.ts

@@ -3,6 +3,7 @@ import type * as Cause from "effect/Cause";
 import type * as Effect from "effect/Effect";
 
 import type { CodeExecutionError } from "./effect-errors";
+import type { ExecuteErrorKind } from "./error-kind";
 
 /** Branded tool path */
 export type ToolPath = string & { readonly __toolPath: unique symbol };
@@ -42,6 +43,8 @@ export type ExecuteResult = {
   result: unknown;
   output?: ExecuteOutputItem[];
   error?: string;
+  /** Enumerable failure class for telemetry; never carries message content. */
+  errorKind?: ExecuteErrorKind;
   logs?: string[];
 };
 

packages/kernel/runtime-dynamic-worker/src/executor.ts

@@ -14,12 +14,14 @@ import * as Data from "effect/Data";
 import * as Effect from "effect/Effect";
 
 import {
+  classifyThrownExecuteError,
   CodeCompilationError,
   recoverExecutionBody,
   SandboxHostTimeoutError,
   SandboxRuntimeError,
   stripTypeScript,
   type CodeExecutor,
+  type ExecuteErrorKind,
   type ExecuteOutputItem,
   type ExecuteResult,
   type SandboxToolInvoker,
@@ -231,6 +233,23 @@ const RUNTIME_SIGNATURES = [
 
 export type SandboxFailureKind = "compilation" | "runtime" | "internal";
 
+/**
+ * Signatures of the serialization subset of `RUNTIME_SIGNATURES`, used to
+ * split `SandboxRuntimeError` into its two telemetry classes: the user
+ * returned something that can't cross the sandbox boundary vs the isolate
+ * hit a CPU/memory/capacity limit.
+ */
+const SERIALIZATION_SIGNATURES = [
+  "could not be cloned",
+  "does not support serialization",
+  "Could not serialize",
+] as const;
+
+const sandboxRuntimeErrorKind = (message: string): ExecuteErrorKind =>
+  SERIALIZATION_SIGNATURES.some((signature) => message.includes(signature))
+    ? "serialization_error"
+    : "resource_limit";
+
 /**
  * Classify a sandbox rejection so the runtime knows whether to surface
  * its message descriptively (the user's mistake or a transient,
@@ -785,6 +804,9 @@ const evaluate = (
     return {
       result: error ? null : response.result,
       error,
+      ...(response.error
+        ? { errorKind: classifyThrownExecuteError(serializedErrorName(response.error.primary)) }
+        : {}),
       output:
         Array.isArray(response.output) && response.output.length > 0 ? response.output : undefined,
       logs: response.logs,
@@ -812,16 +834,28 @@ const runInDynamicWorker = (
     // `DynamicWorkerExecutionError` and remain opaque.
     Effect.catchTags({
       CodeCompilationError: (error) =>
-        Effect.succeed({ result: null, error: error.message } satisfies ExecuteResult),
+        Effect.succeed({
+          result: null,
+          error: error.message,
+          errorKind: "syntax_error",
+        } satisfies ExecuteResult),
       SandboxRuntimeError: (error) =>
-        Effect.succeed({ result: null, error: error.message } satisfies ExecuteResult),
+        Effect.succeed({
+          result: null,
+          error: error.message,
+          errorKind: sandboxRuntimeErrorKind(error.message),
+        } satisfies ExecuteResult),
       // A wedged isolate that the in-sandbox timer failed to catch is
       // unrecoverable, but reporting it as a delivered, descriptive error beats
       // the original symptom (open-ended silence). Fold it into the success
       // channel like the other safe-to-report conditions so it reaches the
       // model instead of collapsing to an opaque internal error.
       SandboxHostTimeoutError: (error) =>
-        Effect.succeed({ result: null, error: error.message } satisfies ExecuteResult),
+        Effect.succeed({
+          result: null,
+          error: error.message,
+          errorKind: "timeout",
+        } satisfies ExecuteResult),
     }),
     Effect.withSpan("executor.code.exec.dynamic_worker", {
       attributes: { "executor.runtime": "dynamic-worker" },