[pull] main from UsefulSoftwareCo:main

.changeset/lazy-mcp-client-module.md

@@ -0,0 +1,5 @@
+---
+"@executor-js/plugin-mcp": patch
+---
+
+Load the MCP client SDK lazily on first outbound connection instead of at module evaluation. Runtimes that bundle the plugin (notably Cloudflare Workers) no longer pay the client package's module-eval memory and CPU on startup or on code paths that never dial an MCP server.

.changeset/mcp-namespace-search-tools.md

@@ -0,0 +1,8 @@
+---
+"@executor-js/execution": patch
+"executor": patch
+---
+
+**Opt-in per-integration search tools on the MCP surface**
+
+Connecting with `?search_tools=true` (stdio: `executor mcp --search-tools`) adds one minimally-described `search_<integration>` MCP tool per connected integration, so the integration namespaces reach the model as tool names it can see without calling anything. Each call routes through the same flow as `tools.search({ namespace })` inside `execute`, and the tool list comes from the same inventory the `execute` description shows. Off by default; a clean endpoint URL is unchanged.

apps/cli/src/main.ts

@@ -1362,6 +1362,7 @@ const mcpUrlForActiveLocalServer = (input: {
   readonly connection: ExecutorServerConnection;
   readonly elicitationMode: "browser" | "model";
   readonly artifacts: boolean;
+  readonly searchTools: boolean;
 }): URL => {
   const url = new URL("/mcp", input.connection.origin);
   if (input.elicitationMode === "browser") {
@@ -1372,6 +1373,11 @@ const mcpUrlForActiveLocalServer = (input: {
   if (!input.artifacts) {
     url.searchParams.set("artifacts", "false");
   }
+  // Per-integration search tools are off by default; only the opt-in is
+  // spelled out.
+  if (input.searchTools) {
+    url.searchParams.set("search_tools", "true");
+  }
   return url;
 };
 
@@ -1387,6 +1393,7 @@ const runMcpHttpBridge = async (input: {
   readonly manifest: ExecutorLocalServerManifest;
   readonly elicitationMode: "browser" | "model";
   readonly artifacts: boolean;
+  readonly searchTools: boolean;
 }): Promise<void> => {
   const stdio = new StdioServerTransport();
   const authorization = getExecutorServerAuthorizationHeader(input.manifest.connection);
@@ -1395,6 +1402,7 @@ const runMcpHttpBridge = async (input: {
       connection: input.manifest.connection,
       elicitationMode: input.elicitationMode,
       artifacts: input.artifacts,
+      searchTools: input.searchTools,
     }),
     authorization ? { requestInit: { headers: { Authorization: authorization } } } : undefined,
   );
@@ -1473,6 +1481,7 @@ const runMcpHttpBridge = async (input: {
 const runStdioMcpSession = (input: {
   readonly elicitationMode: "browser" | "model";
   readonly artifacts: boolean;
+  readonly searchTools: boolean;
 }) =>
   Effect.gen(function* () {
     // `executor mcp` never owns the local database. If a local server is already
@@ -1489,6 +1498,7 @@ const runStdioMcpSession = (input: {
           manifest: active,
           elicitationMode: input.elicitationMode,
           artifacts: input.artifacts,
+          searchTools: input.searchTools,
         }),
       );
       return;
@@ -1515,6 +1525,7 @@ const runStdioMcpSession = (input: {
         manifest: elected,
         elicitationMode: input.elicitationMode,
         artifacts: input.artifacts,
+        searchTools: input.searchTools,
       }),
     );
   });
@@ -2880,11 +2891,18 @@ const mcpCommand = Command.make(
           "Withhold the artifact surface from this connection: the artifact tools, the app shell resource, and the artifact skills. Served by default.",
         ),
       ),
+    searchTools: Options.boolean("search-tools")
+      .pipe(Options.withDefault(false))
+      .pipe(
+        Options.withDescription(
+          "Serve one search_<integration> tool per connected integration. Off by default; each routes through the same flow as tools.search inside execute.",
+        ),
+      ),
   },
-  ({ scope, elicitationMode, noArtifacts }) =>
+  ({ scope, elicitationMode, noArtifacts, searchTools }) =>
     Effect.gen(function* () {
       applyScope(scope);
-      yield* runStdioMcpSession({ elicitationMode, artifacts: !noArtifacts });
+      yield* runStdioMcpSession({ elicitationMode, artifacts: !noArtifacts, searchTools });
     }),
 ).pipe(Command.withDescription("Start an MCP server over stdio"));
 

apps/cloud/scripts/dev-db.ts

@@ -15,6 +15,7 @@ import { PGlite } from "@electric-sql/pglite";
 import { PGLiteSocketServer } from "@electric-sql/pglite-socket";
 import { drizzle } from "drizzle-orm/pglite";
 import { migrate } from "drizzle-orm/pglite/migrator";
+import postgres from "postgres";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 // Port + data dir default to the dev values but are env-overridable so a second
@@ -118,27 +119,32 @@ await migrate(drizzle(db), { migrationsFolder: MIGRATIONS_FOLDER });
 //      transaction or pipeline affinity they can no longer release.
 // src/db/dev-db-socket-concurrency.node.test.ts is the regression test for
 // all of the above.
-const server = new PGLiteSocketServer({
-  db,
-  port: PORT,
-  host: "127.0.0.1",
-  maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000),
-  // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse
-  // sent, no Sync) with its socket still OPEN would hold the queue's handler
-  // affinity forever and starve every other connection, since affinity only
-  // releases on detach and detach needs close/error/idle-timeout. In ms; the
-  // timer resets on every data event. The patch scopes the reap to connections
-  // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest
-  // connection is the normal state of a healthy postgres.js pool held by a
-  // long-lived scope (SSE), and reaping those raced live queries into
-  // sporadic `write CONNECTION_ENDED` 500s.
-  idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000),
-});
+const makeServer = () =>
+  new PGLiteSocketServer({
+    db,
+    port: PORT,
+    host: "127.0.0.1",
+    maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000),
+    // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse
+    // sent, no Sync) with its socket still OPEN would hold the queue's handler
+    // affinity forever and starve every other connection, since affinity only
+    // releases on detach and detach needs close/error/idle-timeout. In ms; the
+    // timer resets on every data event. The patch scopes the reap to connections
+    // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest
+    // connection is the normal state of a healthy postgres.js pool held by a
+    // long-lived scope (SSE), and reaping those raced live queries into
+    // sporadic `write CONNECTION_ENDED` 500s.
+    idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000),
+  });
 
+let server = makeServer();
 await server.start();
 console.log(`[dev-db] Listening on postgresql://postgres:postgres@127.0.0.1:${PORT}/postgres`);
 
+let stopping = false;
+
 const shutdown = async () => {
+  stopping = true;
   console.log("\n[dev-db] Shutting down");
   await server.stop();
   await db.close();
@@ -147,3 +153,112 @@ const shutdown = async () => {
 
 process.on("SIGINT", shutdown);
 process.on("SIGTERM", shutdown);
+
+// ---------------------------------------------------------------------------
+// Wedge watchdog
+// ---------------------------------------------------------------------------
+//
+// Twice the socket server has shipped a state machine that could stop
+// answering NEW connections while the process, the port, and PGlite all stayed
+// up (the CI e2e "cloud signIn: callback set no session (500)" cascades: every
+// in-flight query dies once, then every fresh connection's startup packet
+// times out — CONNECT_TIMEOUT — for the rest of the shard). The known paths
+// are patched with regression tests, but each recurrence so far has found a
+// new path, and a wedged front-end turns ONE infra hiccup into a failure of
+// every remaining test in the shard.
+//
+// So: probe the server the way the app does — a fresh TCP connection, real
+// startup handshake, `select 1` — and when several consecutive probes fail,
+// dump the server's internals to the boot log and swap in a fresh socket
+// server on the same PGlite instance (all state is in PGlite; the front-end is
+// stateless, so this drops only already-doomed connections). If a restart
+// doesn't restore service, exit non-zero: the boot supervisor logs the exit
+// loudly and the run fails fast with an attributable cause instead of minutes
+// of anonymous CONNECT_TIMEOUTs. The wedge itself stays visible in the
+// server-logs artifact via the [dev-db][watchdog] lines.
+const WATCHDOG_INTERVAL_MS = Number(process.env.DEV_DB_WATCHDOG_INTERVAL_MS ?? 5_000);
+// 3 consecutive failures ≈ 15s+ of hard unavailability. PGlite serves queries
+// in milliseconds; even a deep queue clears in well under one probe interval,
+// so consecutive startup failures this sustained only happen wedged.
+const WATCHDOG_FAILURES_TO_RESTART = 3;
+const WATCHDOG_MAX_RESTARTS = 3;
+
+const probe = async (): Promise<void> => {
+  const sql = postgres(`postgres://postgres:postgres@127.0.0.1:${PORT}/postgres`, {
+    max: 1,
+    idle_timeout: 0,
+    connect_timeout: 5,
+    fetch_types: false,
+    prepare: false,
+    onnotice: () => undefined,
+  });
+  try {
+    // connect_timeout only bounds the handshake; race the query too so a
+    // post-startup wedge cannot hang the watchdog itself.
+    await Promise.race([
+      sql.unsafe("select 1"),
+      sleep(10_000).then(() => {
+        throw new Error("probe query timed out after 10s");
+      }),
+    ]);
+  } finally {
+    await sql.end({ timeout: 5 }).catch(() => {});
+  }
+};
+
+const watchdog = async () => {
+  let consecutiveFailures = 0;
+  let restarts = 0;
+  for (;;) {
+    await sleep(WATCHDOG_INTERVAL_MS);
+    if (stopping) return;
+    try {
+      await probe();
+      consecutiveFailures = 0;
+    } catch (cause) {
+      consecutiveFailures += 1;
+      console.error(
+        `[dev-db][watchdog] probe failed (${consecutiveFailures}/${WATCHDOG_FAILURES_TO_RESTART}): ${String(cause)}`,
+      );
+      if (consecutiveFailures < WATCHDOG_FAILURES_TO_RESTART) continue;
+      console.error(
+        `[dev-db][watchdog] socket server wedged; stats: ${JSON.stringify(server.getStats())}`,
+      );
+      if (restarts >= WATCHDOG_MAX_RESTARTS) {
+        console.error(
+          `[dev-db][watchdog] still wedged after ${restarts} restarts — giving up so the boot supervisor reports it`,
+        );
+        process.exit(1);
+      }
+      restarts += 1;
+      consecutiveFailures = 0;
+      console.error(
+        `[dev-db][watchdog] restarting socket server (${restarts}/${WATCHDOG_MAX_RESTARTS})`,
+      );
+      // stop() itself goes through the query queue (detach rolls back open
+      // transactions), so a wedge deep enough can hang the restart too —
+      // bound it and treat that as fatal rather than hanging the watchdog.
+      const restart = async () => {
+        await server.stop();
+        server = makeServer();
+        await server.start();
+      };
+      try {
+        await Promise.race([
+          restart(),
+          sleep(15_000).then(() => {
+            throw new Error("restart timed out after 15s");
+          }),
+        ]);
+        console.error(`[dev-db][watchdog] socket server restarted`);
+      } catch (restartCause) {
+        console.error(
+          `[dev-db][watchdog] restart failed (${String(restartCause)}) — exiting so the boot supervisor reports it`,
+        );
+        process.exit(1);
+      }
+    }
+  }
+};
+
+void watchdog();

apps/cloud/src/auth/ssr-gate.ts → apps/cloud/src/auth/doc-gate.ts

@@ -1,19 +1,23 @@
 // ---------------------------------------------------------------------------
-// SSR auth gate — the server-side session check for DOCUMENT requests.
+// Document auth gate — the server-side session check for DOCUMENT requests.
 //
 // The sealed `wos-session` cookie is verified right here in the worker
 // (unseal + JWT check against cached JWKS — no per-request WorkOS round trip
-// except token refresh), so by the time the SPA is served the server KNOWS
-// who it's serving:
+// except token refresh), so by the time the SPA shell is served the server
+// KNOWS who it's serving:
 //
 // - signed out → 302 /login (carrying ?returnTo=) before any app HTML exists
 // - org-less   → 302 /create-org (onboarding owns those sessions)
-// - signed in  → the document is served WITH the verified identity: the
-//   auth-hint travels to the SSR render via request-middleware context (the
-//   root loader picks it up), and is minted as a cookie when the browser
-//   doesn't hold a current one — so the very first paint is the real app
-//   shell, never a skeleton. The hint is display-only; /account/me remains
-//   the authority and the client keeps it fresh from then on.
+// - signed in  → the prerendered shell is served, and the auth-hint cookie is
+//   minted when the browser doesn't hold a current one — the client seeds its
+//   auth state from that cookie right after mount (there is no per-request
+//   render to dehydrate it into). The hint is display-only; /account/me
+//   remains the authority and the client keeps it fresh from then on.
+//
+// Beyond redirects, this gate is load-bearing for SESSION LIFETIME: WorkOS
+// refresh tokens are single-use, and a verify that rotates the sealed session
+// must deliver the new cookie to the browser or the next expiry logs the user
+// out. Document navigations are where that rotation lands.
 //
 // Scope: GET/HEAD requests that are document navigations (sec-fetch-dest /
 // accept), excluding app-owned paths (/api, /mcp — they answer for themselves
@@ -39,7 +43,6 @@ import { parseCookie } from "./cookies";
 import { LAST_ORG_COOKIE } from "./last-org-cookie";
 import { sealedSessionDisplayName } from "./middleware";
 import { authorizeOrganizationSelector } from "./organization";
-import { browserOriginFromRequest } from "./request-origin";
 import { loginPath, safeReturnTo } from "./return-to";
 import { ONBOARDING_PATHS, PUBLIC_PATHS } from "./route-paths";
 import { WorkOSClient } from "./workos";
@@ -279,19 +282,11 @@ export const authGateMiddleware = createMiddleware({ type: "request" }).server(
       }
     }
 
-    // Serve the document WITH the verified identity: the hint rides to the
-    // SSR render through middleware context (the root loader reads it), so
-    // the server paints the real authenticated shell — no loading state, no
-    // skeleton. The request origin rides along too: it's what the connect
-    // card's MCP URL is built from, and the server knows it (the SPA only
-    // learns `window.location.origin` after mount), so passing it here lets
-    // SSR render the real `https://…/<org>/mcp` instead of the client-side
-    // `http://127.0.0.1:4000` default — which would otherwise flash until
-    // hydration corrected it. Set-cookie writes ride on the rendered response.
+    // Serve the shell, minting the auth-hint cookie when the browser lacks a
+    // current one so the client's post-mount cookie read seeds the verified
+    // identity. Set-cookie writes ride on the shell response.
     const { hint, mint } = await resolveAuthHint(session, cookieHeader);
-    const result = await next({
-      context: { authHint: hint, origin: browserOriginFromRequest(request) },
-    });
+    const result = await next();
     if (!mint && !session.refreshedSession) return result;
 
     const response = new Response(result.response.body, result.response);

apps/cloud/src/auth/last-org-cookie.ts

@@ -8,7 +8,7 @@
 // cookie fills the gap: the client records the slug of the org it's verifiably
 // viewing, and the two bare-entry deciders honor it —
 //
-//   - the SSR auth gate redirects bare document paths onto it (ssr-gate.ts)
+//   - the document auth gate redirects bare document paths onto it (doc-gate.ts)
 //   - the login callback prefers it when picking the org for a fresh session
 //     with a bare returnTo (handlers.ts)
 //

apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts

@@ -259,6 +259,57 @@ describe("dev-db PGlite socket under concurrent connections", () => {
     },
   );
 
+  // Regression for the reap SLOT LEAK: detach(true) removes the socket's
+  // listeners before destroying it, so a server-initiated teardown (the idle
+  // backstop) never fired the server's 'close' bookkeeping — the reaped
+  // handler stayed in the server's handlers set forever, burning one
+  // maxConnections slot per reap. Enough reaps over a long run and the server
+  // answers every NEW connection with "Too many connections" while the
+  // process, the port, and PGlite are all healthy — postgres.js surfaces that
+  // as the same CONNECT_TIMEOUT cascade as the queue wedges. The server now
+  // drops the handler when it dispatches its terminal error.
+  it("reaped handlers release their connection slots", { timeout: 30_000 }, async () => {
+    const port = 45993;
+    const db = await PGlite.create();
+    const server = new PGLiteSocketServer({
+      db,
+      port,
+      host: "127.0.0.1",
+      maxConnections: 2,
+      idleTimeout: 250,
+    });
+    await server.start();
+
+    // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: sockets must be closed on every path
+    try {
+      // Burn through more reaps than there are slots: each staller opens a
+      // pipeline and goes silent, so the idle backstop reaps it (the server
+      // destroys the socket — its 'close' marks that reap complete).
+      for (let i = 0; i < 3; i++) {
+        const staller = await openWireClient(port);
+        staller.write(parseFrame(`select ${i + 1}`));
+        await new Promise<void>((res) => staller.once("close", res));
+      }
+
+      expect(
+        server.getStats().activeConnections,
+        "reaped handlers stay counted against maxConnections",
+      ).toBe(0);
+
+      const sql = makeClient(port);
+      // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: sockets must be closed on every path
+      try {
+        expect((await sql.unsafe(`select 6 as six`))[0]).toEqual({ six: 6 });
+      } finally {
+        // oxlint-disable-next-line executor/no-promise-catch -- test boundary: a failed teardown must not mask the assertion
+        await sql.end({ timeout: 5 }).catch(() => {});
+      }
+    } finally {
+      await server.stop();
+      await db.close();
+    }
+  });
+
   // Regression for the second wedge mode behind the same CI cascade: a client
   // whose socket dies WHILE its pipeline-opening entry is executing. detach()
   // clears pipeline affinity before the entry finishes, so the queue then