[pull] main from UsefulSoftwareCo:main

.changeset/graph-slice-urls-first-class.md

@@ -0,0 +1,5 @@
+---
+"@executor-js/plugin-openapi": patch
+---
+
+Make Microsoft Graph slice URLs first-class spec sources instead of a hidden substitution. Catalog tiles now point directly at the slice release assets, the stored specUrl is exactly what gets fetched, and selection narrowing travels visibly in the URL fragment; requesting the upstream monolith URL fetches the monolith, never a silently swapped slice.

.changeset/graph-spec-slices.md

@@ -0,0 +1,5 @@
+---
+"@executor-js/plugin-openapi": patch
+---
+
+Serve Microsoft Graph preset selections from precomputed slice release assets instead of the 43MB upstream monolith. The monolith fetch almost never survives a 128MB Workers isolate (production traces show one completion in 30 days), so covered selections — every catalog preset, plus any combination within the default bundle — now read a 4–19MB filtered document built offline by the graph-slices workflow, with the monolith path kept only as a fallback and for full-graph/custom-scope selections.

.changeset/openapi-streaming-preview.md

@@ -0,0 +1,5 @@
+---
+"@executor-js/plugin-openapi": patch
+---
+
+Preview OpenAPI spec-format selections (Microsoft Graph) through the streaming structural-split path instead of a whole-document parse, and guard generic whole-document parses by parsed-tree size (line count for block YAML, text size for JSON). Previewing a Graph preset URL previously parsed the 43MB source whole and killed the 128MB Workers isolate mid-request, surfacing as an empty 503; it now streams within budget, and oversized generic specs fail with an actionable error instead of taking down the isolate.

.changeset/slim-search-tools.md

@@ -0,0 +1,6 @@
+---
+"@executor-js/execution": patch
+"executor": patch
+---
+
+Slim the per-integration `search_<integration>` tool definitions to under half their size: one shared one-line description (the tool name already carries the namespace) and a single bare `query` parameter, dropping the `limit`/`offset` knobs. A session pays for these definitions once per connected integration, so the surface now costs ~2k tokens instead of ~5k at 30 integrations; paging through a namespace belongs in `execute`.

.github/workflows/graph-slices.yml

@@ -0,0 +1,47 @@
+# Refresh the Microsoft Graph slice release assets.
+#
+# The Graph OpenAPI monolith (~43MB) cannot be processed inside a Workers
+# isolate, so the runtime reads per-selection slices published on the
+# `graph-slices` release tag (see packages/plugins/openapi/src/providers/
+# microsoft/slices.ts). This workflow rebuilds the slices from the current
+# upstream spec on a schedule and on demand.
+name: Graph slices
+
+on:
+  schedule:
+    # Weekly; Microsoft's msgraph-metadata automation lands upstream refreshes
+    # on a similar cadence. A failed run leaves the previous assets serving.
+    - cron: "17 6 * * 1"
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+
+jobs:
+  slices:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.11
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Generate slices
+        working-directory: packages/plugins/openapi
+        run: bun scripts/generate-graph-slices.ts --out "$RUNNER_TEMP/graph-slices"
+
+      - name: Publish to the graph-slices release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release view graph-slices --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 || \
+            gh release create graph-slices --repo "$GITHUB_REPOSITORY" \
+              --title "Microsoft Graph slices" --latest=false \
+              --notes "Generated per-preset Microsoft Graph OpenAPI slices. Data release consumed by the openapi plugin's Microsoft adapter; refreshed by the graph-slices workflow."
+          gh release upload graph-slices "$RUNNER_TEMP/graph-slices"/* \
+            --repo "$GITHUB_REPOSITORY" --clobber

.oxlintrc.jsonc

@@ -64,6 +64,7 @@
         "apps/desktop/src/main.ts",
         "scripts/**/*.{ts,js}",
         "apps/*/scripts/**/*.{ts,js}",
+        "packages/*/*/scripts/**/*.{ts,js}",
         "packages/kernel/runtime-*/src/**/*.{ts,tsx,js,mjs}",
       ],
       "rules": {

apps/cloud/src/routes/app/billing_.plans.tsx

@@ -47,7 +47,17 @@ function useRefreshAfterCheckout(plans: Plan[] | undefined, refetch: () => void)
   const [finalizingPlan, setFinalizingPlan] = useState<string | null>(null);
   const plansRef = useRef(plans);
   plansRef.current = plans;
-
+  const refetchRef = useRef(refetch);
+  refetchRef.current = refetch;
+  const armedAtRef = useRef(0);
+
+  // One-shot: move the URL marker into state. Only this step is single-shot —
+  // the poll below keys off the STATE, because effects do not live as long as
+  // state: the shell re-runs effects shortly after mount (auth seeds a frame
+  // after first paint and the suspense boundary re-reveals), and when the poll
+  // lived inside this effect its re-run saw the already-stripped URL, so the
+  // cleanup killed the interval and nothing ever refetched — the page sat on
+  // "Activating" forever while the webhook had long landed.
   useEffect(() => {
     const params = new URLSearchParams(window.location.search);
     const attachedPlanId = params.get(CHECKOUT_RETURN_PARAM);
@@ -56,25 +66,34 @@ function useRefreshAfterCheckout(plans: Plan[] | undefined, refetch: () => void)
     params.delete(CHECKOUT_RETURN_PARAM);
     const query = params.toString();
     window.history.replaceState({}, "", `${window.location.pathname}${query ? `?${query}` : ""}`);
+    armedAtRef.current = Date.now();
     setFinalizingPlan(attachedPlanId);
+  }, []);
+
+  // Poll while a plan is finalizing. Re-armable: if the effect is recycled the
+  // state still says which plan is settling, so the interval comes right back.
+  // The timeout is measured from when the marker was consumed, not from the
+  // current effect run, so recycles cannot extend it. refetch goes through a
+  // ref so the interval always calls the CURRENT client's refetch, even after
+  // the billing provider rebuilds.
+  useEffect(() => {
+    if (!finalizingPlan) return;
 
     const reflected = () =>
-      plansRef.current?.find((p) => p.id === attachedPlanId)?.customerEligibility?.status ===
+      plansRef.current?.find((p) => p.id === finalizingPlan)?.customerEligibility?.status ===
       "active";
 
-    let elapsed = 0;
-    refetch();
+    refetchRef.current();
     const interval = setInterval(() => {
-      elapsed += 1500;
-      if (reflected() || elapsed >= 20_000) {
+      if (reflected() || Date.now() - armedAtRef.current >= 20_000) {
         clearInterval(interval);
         setFinalizingPlan(null);
         return;
       }
-      refetch();
+      refetchRef.current();
     }, 1500);
     return () => clearInterval(interval);
-  }, [refetch]);
+  }, [finalizingPlan]);
 
   // Drop the optimistic state the moment the refetched data reflects the plan,
   // so it does not linger until the next poll tick after the webhook lands.

e2e/cloud/repro-listener-supersede.test.ts

@@ -0,0 +1,130 @@
+// Cloud: regression test for the standalone-listener eviction storm.
+//
+// The MCP spec allows a client to hold a standalone SSE listener (a bare GET)
+// for server-initiated messages. The patched agents SDK applies
+// latest-listener-wins to that stream:
+//
+//   packages/... -> agents/dist/mcp/index.js:880
+//   supersedePriorStreamConnections(agent, connection.id, STANDALONE_STREAM_ID)
+//     -> for every OTHER connection on the same streamId: close(1000, ...)
+//
+// STANDALONE_STREAM_ID is shared by every listener, so a session with two
+// listeners has each arrival kill the other. Any SSE client reconnects when
+// its stream closes, so two listeners evict each other indefinitely: the
+// session can never hold a listener open and the client sees a permanent
+// "disconnected / reconnecting" churn.
+//
+// Observed in production: one session produced ~55,600 listener GETs over
+// 6.75 hours against only 106 POSTs, arriving 3-4 within milliseconds every
+// ~1.2s, each answered 200 in ~130ms.
+//
+// This test drives N listeners that reconnect when closed, exactly like a real
+// client, and asserts the session does NOT churn.
+import { expect } from "@effect/vitest";
+import { Effect } from "effect";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+
+import { scenario } from "../src/scenario";
+import { Mcp, Target } from "../src/services";
+import type { Identity } from "../src/target";
+
+const emailOf = (identity: Identity): string => identity.credentials?.email ?? identity.label;
+
+const RUN_MS = 15_000;
+const SCENARIO_TIMEOUT_MS = RUN_MS + 120_000;
+
+interface Trial {
+  readonly opens: number;
+  readonly medianLifeMs: number;
+}
+
+/** Hold `count` standalone listeners for RUN_MS, reconnecting whenever the
+ *  server closes one — the behaviour of any real SSE client. */
+const driveListeners = async (
+  mcpUrl: string,
+  bearer: string,
+  sessionId: string,
+  count: number,
+): Promise<Trial> => {
+  const deadline = Date.now() + RUN_MS;
+  const lifetimes: number[] = [];
+  let opens = 0;
+
+  const listener = async (): Promise<void> => {
+    while (Date.now() < deadline) {
+      const started = Date.now();
+      const controller = new AbortController();
+      const guard = setTimeout(() => controller.abort(), Math.max(500, deadline - Date.now()));
+      try {
+        const res = await fetch(mcpUrl, {
+          method: "GET",
+          headers: {
+            authorization: `Bearer ${bearer}`,
+            accept: "text/event-stream",
+            "mcp-session-id": sessionId,
+            "mcp-protocol-version": "2025-06-18",
+          },
+          signal: controller.signal,
+        });
+        opens += 1;
+        const reader = res.body?.getReader();
+        // Read until the SERVER closes the stream.
+        if (reader) {
+          for (;;) {
+            const { done } = await reader.read();
+            if (done) break;
+          }
+        }
+      } catch {
+        // aborted at the deadline, or the socket was torn down
+      }
+      clearTimeout(guard);
+      lifetimes.push(Date.now() - started);
+    }
+  };
+
+  await Promise.all(Array.from({ length: count }, () => listener()));
+  const sorted = [...lifetimes].sort((a, b) => a - b);
+  return { opens, medianLifeMs: sorted[Math.floor(sorted.length / 2)] ?? 0 };
+};
+
+scenario(
+  "REGRESSION · a second standalone listener must not evict the first",
+  { timeout: SCENARIO_TIMEOUT_MS },
+  Effect.gen(function* () {
+    const target = yield* Target;
+    const mcp = yield* Mcp;
+    const identity = yield* target.newIdentity();
+    const bearer = yield* mcp.mintBearer(emailOf(identity));
+
+    const client = new Client({ name: "e2e-supersede", version: "0.0.1" }, { capabilities: {} });
+    const transport = new StreamableHTTPClientTransport(new URL(target.mcpUrl), {
+      requestInit: { headers: { authorization: `Bearer ${bearer}` } },
+    });
+    yield* Effect.promise(() => client.connect(transport));
+    const sessionId = transport.sessionId;
+    expect(sessionId, "the client got a session id").toEqual(expect.any(String));
+    if (sessionId === undefined) return yield* Effect.die("missing session id");
+    yield* Effect.promise(() => client.close().catch(() => undefined));
+
+    // Control: a single listener should simply stay open for the whole window.
+    const single = yield* Effect.promise(() => driveListeners(target.mcpUrl, bearer, sessionId, 1));
+    console.log(JSON.stringify({ event: "supersede_single", ...single }));
+
+    // Two listeners, each reconnecting when closed.
+    const double = yield* Effect.promise(() => driveListeners(target.mcpUrl, bearer, sessionId, 2));
+    console.log(JSON.stringify({ event: "supersede_double", ...double }));
+
+    // One listener over a 15s window opens roughly once (a max-age rotation
+    // could add one more). Anything beyond a handful is the eviction storm.
+    expect(single.opens, "a single listener does not churn").toBeLessThanOrEqual(3);
+
+    // The regression: two listeners must not evict each other. Without the fix
+    // this is in the hundreds, with a median stream life of ~200ms.
+    expect(double.opens, "two listeners do not evict each other").toBeLessThanOrEqual(6);
+    expect(double.medianLifeMs, "streams stay open rather than dying instantly").toBeGreaterThan(
+      2_000,
+    );
+  }),
+);

e2e/scenarios/microsoft-graph-default.test.ts

@@ -6,10 +6,10 @@ import { composePluginApi } from "@executor-js/api/server";
 import {
   MICROSOFT_AUTH_TEMPLATE_SLUG,
   MICROSOFT_AUTHORIZATION_URL,
-  MICROSOFT_GRAPH_OPENAPI_URL,
   MICROSOFT_TOKEN_URL,
   microsoftCatalog,
   microsoftGraphAdapter,
+  microsoftGraphSliceUrl,
 } from "@executor-js/plugin-openapi/providers/microsoft";
 import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
 import {
@@ -55,11 +55,29 @@ scenario(
 
     yield* Effect.ensuring(
       Effect.gen(function* () {
+        // The add dialog previews before registering. This must stream: the
+        // whole-document parse of the Graph source OOMs the dev workerd
+        // isolate the same way it did the production one.
+        const preview = yield* client.openapi.previewSpec({
+          payload: {
+            spec: microsoftGraphSliceUrl(MICROSOFT_FILES_PRESET_ID),
+            specFormat: "microsoft-graph",
+          },
+        });
+        expect(
+          preview.operationCount,
+          "the preview streams the files selection without a whole-document parse",
+        ).toBeGreaterThan(10);
+        expect(
+          preview.healthCheckCandidates.length,
+          "the preview carries ranked health-check candidates for the selection",
+        ).toBeGreaterThan(0);
+
         const added = yield* client.openapi.addSpec({
           payload: {
             spec: {
               kind: "url",
-              url: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`,
+              url: microsoftGraphSliceUrl(MICROSOFT_FILES_PRESET_ID),
             },
             slug: integration,
             name: "Microsoft Graph Files",

e2e/scenarios/microsoft-graph-full.test.ts

@@ -5,9 +5,9 @@ import { Effect } from "effect";
 import { composePluginApi } from "@executor-js/api/server";
 import {
   MICROSOFT_AUTH_TEMPLATE_SLUG,
-  MICROSOFT_GRAPH_OPENAPI_URL,
   microsoftCatalog,
   microsoftGraphAdapter,
+  microsoftGraphSliceUrl,
 } from "@executor-js/plugin-openapi/providers/microsoft";
 import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
 import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
@@ -57,7 +57,7 @@ scenario(
           payload: {
             spec: {
               kind: "url",
-              url: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`,
+              url: microsoftGraphSliceUrl(MICROSOFT_FILES_PRESET_ID),
             },
             slug: integration,
             name: "Microsoft Graph Files",

e2e/scenarios/namespace-search-tools.test.ts

@@ -104,12 +104,17 @@ scenario(
         // The core surface is untouched.
         expect(names, "execute still works on an opted-in session").toContain("execute");
         expect(names, "skills still works on an opted-in session").toContain("skills");
-        // The description is minimal and points back at the execute flow.
+        // The NAME carries the namespace; the description is one shared line
+        // that points back at the execute flow, kept tiny because a session
+        // pays for it once per connected integration.
         const described = optedInTools.find((tool) => tool.name === searchTool);
-        expect(described?.description, "the tool description names its namespace").toContain(slug);
         expect(described?.description, "the tool description points at execute").toContain(
           "execute",
         );
+        expect(
+          (described?.description ?? "").length,
+          "the tool description stays lean",
+        ).toBeLessThan(120);
 
         // A keyword call returns the matching tool, exactly as
         // `tools.search({ query, namespace })` inside execute would.

e2e/scenarios/provider-plugins-ui.test.ts

@@ -57,11 +57,13 @@ scenario(
 
       await step("A Microsoft service preset opens the OpenAPI add flow", async () => {
         await page.goto(
-          "/integrations/add/openapi?preset=microsoft-files&url=https%3A%2F%2Fraw.githubusercontent.com%2Fmicrosoftgraph%2Fmsgraph-metadata%2Fmaster%2Fopenapi%2Fv1.0%2Fopenapi.yaml%23preset%3Dfiles",
+          "/integrations/add/openapi?preset=microsoft-files&url=https%3A%2F%2Fgithub.com%2FUsefulSoftwareCo%2Fexecutor%2Freleases%2Fdownload%2Fgraph-slices%2Ffiles.yaml",
           { waitUntil: "domcontentloaded" },
         );
         await page.getByRole("heading", { name: "Add OpenAPI integration" }).waitFor();
-        await expect.poll(() => page.locator("textarea").inputValue()).toContain("preset=files");
+        await expect
+          .poll(() => page.locator("textarea").inputValue())
+          .toContain("graph-slices/files.yaml");
       });
     });
   }),