From d4afe0c79f146dd169a00988a2d5d0469297be19 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:09:56 -0700 Subject: [PATCH 1/6] Slim per-integration search tool definitions (#1749) --- .changeset/slim-search-tools.md | 6 +++ e2e/scenarios/namespace-search-tools.test.ts | 9 ++++- .../mcp/src/namespace-search-tools.test.ts | 34 +++++++++++++---- packages/hosts/mcp/src/tool-server.ts | 37 ++++++++----------- 4 files changed, 55 insertions(+), 31 deletions(-) create mode 100644 .changeset/slim-search-tools.md diff --git a/.changeset/slim-search-tools.md b/.changeset/slim-search-tools.md new file mode 100644 index 0000000000..05d45ea03a --- /dev/null +++ b/.changeset/slim-search-tools.md @@ -0,0 +1,6 @@ +--- +"@executor-js/execution": patch +"executor": patch +--- + +Slim the per-integration `search_` tool definitions to under half their size: one shared one-line description (the tool name already carries the namespace) and a single bare `query` parameter, dropping the `limit`/`offset` knobs. A session pays for these definitions once per connected integration, so the surface now costs ~2k tokens instead of ~5k at 30 integrations; paging through a namespace belongs in `execute`. diff --git a/e2e/scenarios/namespace-search-tools.test.ts b/e2e/scenarios/namespace-search-tools.test.ts index 8bec55df8c..51a0d954e3 100644 --- a/e2e/scenarios/namespace-search-tools.test.ts +++ b/e2e/scenarios/namespace-search-tools.test.ts @@ -104,12 +104,17 @@ scenario( // The core surface is untouched. expect(names, "execute still works on an opted-in session").toContain("execute"); expect(names, "skills still works on an opted-in session").toContain("skills"); - // The description is minimal and points back at the execute flow. + // The NAME carries the namespace; the description is one shared line + // that points back at the execute flow, kept tiny because a session + // pays for it once per connected integration. const described = optedInTools.find((tool) => tool.name === searchTool); - expect(described?.description, "the tool description names its namespace").toContain(slug); expect(described?.description, "the tool description points at execute").toContain( "execute", ); + expect( + (described?.description ?? "").length, + "the tool description stays lean", + ).toBeLessThan(120); // A keyword call returns the matching tool, exactly as // `tools.search({ query, namespace })` inside execute would. diff --git a/packages/hosts/mcp/src/namespace-search-tools.test.ts b/packages/hosts/mcp/src/namespace-search-tools.test.ts index cbbd4a2251..6e798f0f89 100644 --- a/packages/hosts/mcp/src/namespace-search-tools.test.ts +++ b/packages/hosts/mcp/src/namespace-search-tools.test.ts @@ -118,16 +118,38 @@ describe("MCP host — per-integration search tools", () => { // session. expect(names.filter((name) => name.startsWith("search_"))).toHaveLength(2); - // Minimal descriptions: one line that names the namespace and points - // back at the execute flow. + // Minimal descriptions: the NAME carries the namespace; the shared + // one-line description only points back at the execute flow. const gmail = tools.find((tool) => tool.name === "search_google_gmail"); - expect(gmail?.description).toContain("google_gmail"); expect(gmail?.description).toContain("execute"); expect(gmail?.description?.includes("\n")).toBe(false); }, ); }); + it("keeps each serialized definition small — the whole point is cheap context", async () => { + // A session serves one of these per connected integration (up to 50), so + // definition bytes multiply. 300 serialized chars/tool keeps the full + // surface around ~2k tokens; the original shipped shape was 551 chars/tool + // (~5k tokens for 30 integrations), which defeated the feature's purpose. + const { engine } = makeRecordingEngine(); + await withClient( + { engine, description: DESCRIPTION_WITH_INVENTORY, searchToolsEnabled: true }, + async (client) => { + const tools = (await client.listTools()).tools.filter((tool) => + tool.name.startsWith("search_"), + ); + expect(tools.length).toBeGreaterThan(0); + for (const tool of tools) { + expect( + JSON.stringify(tool).length, + `${tool.name} definition must stay lean`, + ).toBeLessThan(300); + } + }, + ); + }); + it("registers none when the description carries no inventory", async () => { const { engine } = makeRecordingEngine(); await withClient( @@ -149,11 +171,9 @@ describe("MCP host — per-integration search tools", () => { async (client) => { const result = await client.callTool({ name: "search_github", - arguments: { query: "issues", limit: 5 }, + arguments: { query: "issues" }, }); - expect(executed).toEqual([ - 'return tools.search({"query":"issues","namespace":"github","limit":5})', - ]); + expect(executed).toEqual(['return tools.search({"query":"issues","namespace":"github"})']); expect(result.isError ?? false).toBe(false); }, ); diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index 0589317217..f8c6a908b9 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -1267,20 +1267,12 @@ export const createExecutorMcpServer = ( // `search_` is `execute` running `tools.search` with the // namespace pinned. The code is built HERE, from the slug the tool was - // registered under and JSON-encoded arguments — never concatenated from + // registered under and a JSON-encoded query — never concatenated from // raw model input — and then takes the exact `executeCode` path, so the // results, formatting, and telemetry match a hand-written // `tools.search({ namespace })` call. - const searchNamespaceCode = ( - integration: string, - args: { readonly query?: string; readonly limit?: number; readonly offset?: number }, - ): string => - `return tools.search(${JSON.stringify({ - query: args.query ?? "", - namespace: integration, - ...(args.limit === undefined ? {} : { limit: args.limit }), - ...(args.offset === undefined ? {} : { offset: args.offset }), - })})`; + const searchNamespaceCode = (integration: string, query: string | undefined): string => + `return tools.search(${JSON.stringify({ query: query ?? "", namespace: integration })})`; /** What the caller could bind an unresolved role to. Best effort: the * connections port is optional, and a failure to enumerate must not @@ -1627,6 +1619,13 @@ export const createExecutorMcpServer = ( // `tools.search({ namespace })` inside `execute` (see searchNamespaceCode). // The inventory comes from the same built description the model reads, so // the two surfaces cannot list different integrations. + // + // A session serves up to 50 of these, so every definition byte is paid ~50 + // times in the client's context. The NAME is the payload; everything else + // stays as small as it can: one shared description sentence (the slug + // would only repeat the name) and a single bare `query` parameter — no + // paging knobs, because anything past the first page belongs in `execute`. + // `namespace-search-tools.test.ts` pins the serialized size. if (searchToolsEnabled) { // The MCP tool-name grammar ([A-Za-z0-9_-]). Integration slugs already // conform (they are `tools.` property names in sandbox code); one @@ -1640,19 +1639,13 @@ export const createExecutorMcpServer = ( server.registerTool( `search_${integration}`, { - description: `Find \`${integration}\` tools. Same results as \`tools.search({ query, namespace: "${integration}" })\` inside execute; run what you find with execute.`, - inputSchema: { - query: z - .string() - .optional() - .describe("Keywords to match. Omit to list the whole namespace."), - limit: z.number().optional().describe("Max results per page."), - offset: z.number().optional().describe("Pagination offset."), - }, + description: + "Search this integration's tools; empty query lists all. Run results with execute.", + inputSchema: { query: z.string().optional() }, }, - ({ query, limit, offset }, extra) => + ({ query }, extra) => runToolEffect( - executeCode(searchNamespaceCode(integration, { query, limit, offset }), extra).pipe( + executeCode(searchNamespaceCode(integration, query), extra).pipe( Effect.withSpan("mcp.host.tool.namespace_search", { attributes: { "mcp.tool.name": `search_${integration}`, From b429102fb94b1bc3c9848a125b8515ac243f3197 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:12:30 -0700 Subject: [PATCH 2/6] Re-arm the post-checkout billing poll across effect recycles (#1750) --- apps/cloud/src/routes/app/billing_.plans.tsx | 35 +++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/apps/cloud/src/routes/app/billing_.plans.tsx b/apps/cloud/src/routes/app/billing_.plans.tsx index a94969e81f..820020e84a 100644 --- a/apps/cloud/src/routes/app/billing_.plans.tsx +++ b/apps/cloud/src/routes/app/billing_.plans.tsx @@ -47,7 +47,17 @@ function useRefreshAfterCheckout(plans: Plan[] | undefined, refetch: () => void) const [finalizingPlan, setFinalizingPlan] = useState(null); const plansRef = useRef(plans); plansRef.current = plans; - + const refetchRef = useRef(refetch); + refetchRef.current = refetch; + const armedAtRef = useRef(0); + + // One-shot: move the URL marker into state. Only this step is single-shot — + // the poll below keys off the STATE, because effects do not live as long as + // state: the shell re-runs effects shortly after mount (auth seeds a frame + // after first paint and the suspense boundary re-reveals), and when the poll + // lived inside this effect its re-run saw the already-stripped URL, so the + // cleanup killed the interval and nothing ever refetched — the page sat on + // "Activating" forever while the webhook had long landed. useEffect(() => { const params = new URLSearchParams(window.location.search); const attachedPlanId = params.get(CHECKOUT_RETURN_PARAM); @@ -56,25 +66,34 @@ function useRefreshAfterCheckout(plans: Plan[] | undefined, refetch: () => void) params.delete(CHECKOUT_RETURN_PARAM); const query = params.toString(); window.history.replaceState({}, "", `${window.location.pathname}${query ? `?${query}` : ""}`); + armedAtRef.current = Date.now(); setFinalizingPlan(attachedPlanId); + }, []); + + // Poll while a plan is finalizing. Re-armable: if the effect is recycled the + // state still says which plan is settling, so the interval comes right back. + // The timeout is measured from when the marker was consumed, not from the + // current effect run, so recycles cannot extend it. refetch goes through a + // ref so the interval always calls the CURRENT client's refetch, even after + // the billing provider rebuilds. + useEffect(() => { + if (!finalizingPlan) return; const reflected = () => - plansRef.current?.find((p) => p.id === attachedPlanId)?.customerEligibility?.status === + plansRef.current?.find((p) => p.id === finalizingPlan)?.customerEligibility?.status === "active"; - let elapsed = 0; - refetch(); + refetchRef.current(); const interval = setInterval(() => { - elapsed += 1500; - if (reflected() || elapsed >= 20_000) { + if (reflected() || Date.now() - armedAtRef.current >= 20_000) { clearInterval(interval); setFinalizingPlan(null); return; } - refetch(); + refetchRef.current(); }, 1500); return () => clearInterval(interval); - }, [refetch]); + }, [finalizingPlan]); // Drop the optimistic state the moment the refetched data reflects the plan, // so it does not linger until the next poll tick after the webhook lands. From 0007474602d8da3642648f216bfdb0f09eb0914f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:22:49 -0700 Subject: [PATCH 3/6] Stream the OpenAPI preview for Graph-sized spec selections (#1751) * Guard whole-document OpenAPI parses by parsed-tree size * Stream the OpenAPI preview for spec-format selections * Cover the Graph preset preview in the catalog e2e scenario --- .changeset/openapi-streaming-preview.md | 5 + e2e/scenarios/microsoft-graph-default.test.ts | 18 ++ .../microsoft/spec-format-adapter.test.ts | 20 ++ packages/plugins/openapi/src/sdk/extract.ts | 187 +++++++++++++++++- .../plugins/openapi/src/sdk/parse.test.ts | 86 +++++++- packages/plugins/openapi/src/sdk/parse.ts | 77 ++++++++ packages/plugins/openapi/src/sdk/plugin.ts | 34 +++- .../openapi/src/sdk/preview-streaming.test.ts | 183 +++++++++++++++++ packages/plugins/openapi/src/sdk/preview.ts | 148 +++++++++++++- 9 files changed, 736 insertions(+), 22 deletions(-) create mode 100644 .changeset/openapi-streaming-preview.md create mode 100644 packages/plugins/openapi/src/sdk/preview-streaming.test.ts diff --git a/.changeset/openapi-streaming-preview.md b/.changeset/openapi-streaming-preview.md new file mode 100644 index 0000000000..fb7d142fb6 --- /dev/null +++ b/.changeset/openapi-streaming-preview.md @@ -0,0 +1,5 @@ +--- +"@executor-js/plugin-openapi": patch +--- + +Preview OpenAPI spec-format selections (Microsoft Graph) through the streaming structural-split path instead of a whole-document parse, and guard generic whole-document parses by parsed-tree size (line count for block YAML, text size for JSON). Previewing a Graph preset URL previously parsed the 43MB source whole and killed the 128MB Workers isolate mid-request, surfacing as an empty 503; it now streams within budget, and oversized generic specs fail with an actionable error instead of taking down the isolate. diff --git a/e2e/scenarios/microsoft-graph-default.test.ts b/e2e/scenarios/microsoft-graph-default.test.ts index 88619e1191..499a25af92 100644 --- a/e2e/scenarios/microsoft-graph-default.test.ts +++ b/e2e/scenarios/microsoft-graph-default.test.ts @@ -55,6 +55,24 @@ scenario( yield* Effect.ensuring( Effect.gen(function* () { + // The add dialog previews before registering. This must stream: the + // whole-document parse of the Graph source OOMs the dev workerd + // isolate the same way it did the production one. + const preview = yield* client.openapi.previewSpec({ + payload: { + spec: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`, + specFormat: "microsoft-graph", + }, + }); + expect( + preview.operationCount, + "the preview streams the files selection without a whole-document parse", + ).toBeGreaterThan(10); + expect( + preview.healthCheckCandidates.length, + "the preview carries ranked health-check candidates for the selection", + ).toBeGreaterThan(0); + const added = yield* client.openapi.addSpec({ payload: { spec: { diff --git a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts index 1dbdfd6ba6..d3da64cd65 100644 --- a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts +++ b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts @@ -3,6 +3,7 @@ import { Effect, Layer } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { structuralSplit } from "@executor-js/plugin-openapi"; +import { previewSpecTextStreaming } from "../../sdk/preview"; import { microsoftGraphAdapter } from "./spec-format-adapter"; import { MICROSOFT_GRAPH_OPENAPI_URL } from "./presets"; @@ -90,3 +91,22 @@ it.effect("uses catalog URL fragments to select one Graph workload", () => expect(keepPathItem("/irrelevant", { get: { operationId: "irrelevant.Get" } })).toBeNull(); }), ); + +it.effect("stream-previews a Graph selection without a whole-document parse", () => + Effect.gen(function* () { + const converted = yield* microsoftGraphAdapter.fetch({ + urls: [`${MICROSOFT_GRAPH_OPENAPI_URL}#preset=profile`], + httpClientLayer: graphHttpClientLayer, + }); + const preview = yield* previewSpecTextStreaming(converted.specText, converted.keepPathItem); + + expect(preview.operationCount).toBe(1); + expect(preview.operations.map((operation) => operation.operationId)).toEqual(["me.GetUser"]); + expect(preview.healthCheckCandidates).toHaveLength(1); + expect(preview.healthCheckCandidates[0]?.method).toBe("get"); + expect(preview.oauth2Presets).toHaveLength(1); + expect(preview.servers.map((server) => server.url)).toEqual([ + "https://graph.microsoft.com/v1.0", + ]); + }), +); diff --git a/packages/plugins/openapi/src/sdk/extract.ts b/packages/plugins/openapi/src/sdk/extract.ts index 956e923e81..77a2202607 100644 --- a/packages/plugins/openapi/src/sdk/extract.ts +++ b/packages/plugins/openapi/src/sdk/extract.ts @@ -742,6 +742,23 @@ export const streamOperationBindings = ( const isPathItemValue = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); +/** Parse one path-item range to its kept (optionally trimmed) value. Applied + * identically wherever a structure is walked more than once, so per-operation + * indexes stay aligned across passes. */ +const parseKeptPathItem = ( + structure: SpecStructure, + range: ByteRange, + keepPathItem: KeepPathItem | undefined, +): readonly [string, PathItemObject] | null => { + const entry = parseEntry(structure.text, range, 2); + if (!entry) return null; + const [path, rawValue] = entry; + if (!isPathItemValue(rawValue)) return null; + if (!keepPathItem) return [path, rawValue as PathItemObject]; + const kept = keepPathItem(path, rawValue); + return kept ? [path, kept as PathItemObject] : null; +}; + /** * Stream invocation bindings straight from a `SpecStructure` (the structural * split of a large spec) without ever materializing the whole-document tree. @@ -775,15 +792,8 @@ export const streamOperationBindingsFromStructure = ( // Parse one path-item range to its kept (optionally trimmed) value, applying // `keepPathItem` identically in both passes so the operation index aligns. - const keptPathItem = (range: ByteRange): readonly [string, PathItemObject] | null => { - const entry = parseEntry(structure.text, range, 2); - if (!entry) return null; - const [path, rawValue] = entry; - if (!isPathItemValue(rawValue)) return null; - if (!keepPathItem) return [path, rawValue as PathItemObject]; - const kept = keepPathItem(path, rawValue); - return kept ? [path, kept as PathItemObject] : null; - }; + const keptPathItem = (range: ByteRange): readonly [string, PathItemObject] | null => + parseKeptPathItem(structure, range, keepPathItem); // Pass 1 (light): collect schema-free tool-path planning metadata in // document order. No bindings, no schemas; one path-item resident at a time. @@ -874,3 +884,162 @@ export const streamOperationBindingsFromStructure = ( return { toolCount: plans.length, toolNames: plans.map((plan) => plan.toolPath) }; }).pipe(Effect.withSpan("OpenApi.streamOperationBindingsFromStructure")); + +// --------------------------------------------------------------------------- +// Streaming preview extraction +// --------------------------------------------------------------------------- + +export interface StreamedPreviewParameter { + readonly name: string; + readonly location: ParameterLocation; + readonly required: boolean; + readonly description?: string; +} + +/** Schema-free per-operation metadata for the preview path: everything the + * add screen's operation list and health-check candidate ranking need, and + * nothing that scales with schema size. */ +export interface StreamedPreviewOperation { + readonly operationId: string; + /** Tool path planned over the full kept operation set, so preview candidates + * match the names registration will assign. */ + readonly toolPath: string; + readonly method: HttpMethod; + readonly pathTemplate: string; + readonly summary: string | undefined; + readonly description: string | undefined; + readonly tags: readonly string[]; + readonly deprecated: boolean; + readonly parameters: readonly StreamedPreviewParameter[]; + /** Position in kept-document order; key for `streamOutputSchemas`. */ + readonly operationIndex: number; +} + +export interface StreamedPreviewExtraction { + /** Parsed document head (openapi, info, servers, tags, security, ...). */ + readonly head: Record; + /** Schema-free components (parameters / requestBodies / responses / + * securitySchemes / ...) for `$ref` resolution and auth extraction. */ + readonly components: Record; + readonly servers: readonly ServerInfo[]; + readonly operations: readonly StreamedPreviewOperation[]; +} + +/** + * Streaming twin of `extract` for the preview path: walk a `SpecStructure` + * path-item by path-item (each parsed in isolation and discarded) and keep only + * schema-free per-operation metadata, so previewing a Graph-sized spec never + * materializes the whole-document tree that OOMs a 128MB Workers isolate. + * `keepPathItem` applies the same selection filter as the streaming compile, so + * the preview describes exactly the operation set registration would persist. + */ +export const streamPreviewOperations = ( + structure: SpecStructure, + keepPathItem?: KeepPathItem, +): StreamedPreviewExtraction => { + const head = parseHead(structure); + const components = parseSmallComponents(structure); + // oxlint-disable-next-line executor/no-double-cast -- boundary: same schema-free resolver doc as `streamOperationBindingsFromStructure` (head + small components, empty paths), read only for .servers and `$ref` resolution. + const resolverDoc = { ...head, paths: {}, components } as unknown as ParsedDocument; + const r = new DocResolver(resolverDoc); + const docServers = extractServers(resolverDoc); + + const inputs: OperationPathInput[] = []; + const metas: Omit[] = []; + for (const range of structure.pathItems) { + const kept = parseKeptPathItem(structure, range, keepPathItem); + if (!kept) continue; + const [path, pathItem] = kept; + for (const method of HTTP_METHODS) { + const operation = pathItem[method]; + if (!operation) continue; + const resolvedPathTemplate = explicitPathTemplate(operation) ?? path; + const tags = (operation.tags ?? []).filter((t) => t.trim().length > 0); + const operationId = deriveOperationId(method, path, operation); + inputs.push({ + operationId, + explicitToolPath: explicitToolPath(operation), + method, + pathTemplate: resolvedPathTemplate, + tag0: tags[0], + }); + const parameters = extractParameters(pathItem, operation, r).map( + (parameter): StreamedPreviewParameter => ({ + name: parameter.name, + location: parameter.location, + required: parameter.required, + ...(Option.isSome(parameter.description) + ? { description: parameter.description.value } + : {}), + }), + ); + metas.push({ + operationId, + method, + pathTemplate: resolvedPathTemplate, + summary: operation.summary, + description: operation.description, + tags, + deprecated: operation.deprecated === true, + parameters, + operationIndex: metas.length, + }); + } + } + + const plans = planToolPaths(inputs); + const toolPathByOpIndex: (string | undefined)[] = new Array(inputs.length); + for (const plan of plans) toolPathByOpIndex[plan.operationIndex] = plan.toolPath; + + return { + head, + components, + servers: docServers, + operations: metas.flatMap((meta) => { + const toolPath = toolPathByOpIndex[meta.operationIndex]; + return toolPath === undefined ? [] : [{ ...meta, toolPath }]; + }), + }; +}; + +/** + * Re-walk the structure and build the raw output schema (component `$ref`s + * intact) for just the operations in `wanted` — the bounded response-schema + * walk behind the preview's typed identity picker. Iteration order and + * `keepPathItem` application match `streamPreviewOperations`, so the indexes + * line up. + */ +export const streamOutputSchemas = ( + structure: SpecStructure, + wanted: ReadonlySet, + keepPathItem?: KeepPathItem, +): ReadonlyMap => { + const result = new Map(); + if (wanted.size === 0) return result; + // oxlint-disable-next-line executor/no-double-cast -- boundary: same schema-free resolver doc as `streamPreviewOperations`. + const resolverDoc = { + ...parseHead(structure), + paths: {}, + components: parseSmallComponents(structure), + } as unknown as ParsedDocument; + const r = new DocResolver(resolverDoc); + + let opIndex = 0; + for (const range of structure.pathItems) { + if (result.size >= wanted.size) break; + const kept = parseKeptPathItem(structure, range, keepPathItem); + if (!kept) continue; + const [, pathItem] = kept; + for (const method of HTTP_METHODS) { + const operation = pathItem[method]; + if (!operation) continue; + const index = opIndex; + opIndex += 1; + if (!wanted.has(index)) continue; + const responseBody = extractResponseBody(operation, r); + const outputSchema = responseBody ? outputSchemaFromResponseBody(responseBody) : undefined; + if (outputSchema !== undefined) result.set(index, outputSchema); + } + } + return result; +}; diff --git a/packages/plugins/openapi/src/sdk/parse.test.ts b/packages/plugins/openapi/src/sdk/parse.test.ts index 9990c1ce5f..039f51534a 100644 --- a/packages/plugins/openapi/src/sdk/parse.test.ts +++ b/packages/plugins/openapi/src/sdk/parse.test.ts @@ -1,8 +1,15 @@ import { describe, expect, it } from "@effect/vitest"; -import { Effect } from "effect"; +import { Effect, Layer } from "effect"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { OpenApiParseError } from "./errors"; -import { parse } from "./parse"; +import { + MAX_JSON_SPEC_CHARS, + MAX_SPEC_TEXT_CHARS, + MAX_YAML_SPEC_LINES, + fetchSpecText, + parse, +} from "./parse"; describe("OpenAPI parse", () => { it.effect("parses JSON OpenAPI documents", () => @@ -82,4 +89,79 @@ paths: {} expect(doc.info.description).toHaveLength(largeDescription.length); }), ); + + it.effect("rejects structure-dense YAML documents above the line cap", () => + Effect.gen(function* () { + // Three lines per path-item; well past the cap while only a few MB of + // text — the shape (not the size) is what a whole parse cannot survive. + const pathItems = " /a:\n get: {}\n".repeat(Math.ceil(MAX_YAML_SPEC_LINES / 2)); + const error = yield* parse( + `openapi: 3.0.0\ninfo:\n title: Dense\n version: 1.0.0\npaths:\n${pathItems}`, + ).pipe(Effect.flip); + + expect(error).toBeInstanceOf(OpenApiParseError); + expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse whole/)); + expect(error).toHaveProperty("message", expect.stringMatching(/lines/)); + }), + ); + + it.effect("rejects JSON documents above the JSON size cap", () => + Effect.gen(function* () { + const padded = `{"openapi":"3.1.0","x-pad":"${"x".repeat(MAX_JSON_SPEC_CHARS)}"}`; + const error = yield* parse(padded).pipe(Effect.flip); + + expect(error).toBeInstanceOf(OpenApiParseError); + expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse/)); + }), + ); + + it.effect("rejects any document above the text ceiling", () => + Effect.gen(function* () { + const padded = `openapi: 3.0.0\ninfo:\n description: "${"x".repeat(MAX_SPEC_TEXT_CHARS)}"\n`; + const error = yield* parse(padded).pipe(Effect.flip); + + expect(error).toBeInstanceOf(OpenApiParseError); + expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse/)); + }), + ); +}); + +describe("OpenAPI fetchSpecText", () => { + const specUrl = "https://example.com/openapi.yaml"; + + const layerWithResponse = (response: Response) => + Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.succeed(HttpClientResponse.fromWeb(request, response)), + ), + ); + + it.effect("rejects a document whose declared length is above the text ceiling", () => + Effect.gen(function* () { + const error = yield* fetchSpecText(specUrl).pipe( + Effect.provide( + layerWithResponse( + new Response("openapi: 3.0.0", { + status: 200, + headers: { "content-length": String(MAX_SPEC_TEXT_CHARS + 1) }, + }), + ), + ), + Effect.flip, + ); + + expect(error).toBeInstanceOf(OpenApiParseError); + expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse/)); + }), + ); + + it.effect("fetches a document with an in-range declared length", () => + Effect.gen(function* () { + const specText = yield* fetchSpecText(specUrl).pipe( + Effect.provide(layerWithResponse(new Response("openapi: 3.0.0", { status: 200 }))), + ); + + expect(specText).toBe("openapi: 3.0.0"); + }), + ); }); diff --git a/packages/plugins/openapi/src/sdk/parse.ts b/packages/plugins/openapi/src/sdk/parse.ts index 77b788dae9..8f2557dcb0 100644 --- a/packages/plugins/openapi/src/sdk/parse.ts +++ b/packages/plugins/openapi/src/sdk/parse.ts @@ -7,6 +7,53 @@ import { OpenApiExtractionError, OpenApiParseError } from "./errors"; export type ParsedDocument = OpenAPIV3.Document | OpenAPIV3_1.Document; +const MiB = 1024 * 1024; + +/** + * Whole-document parse guards. What kills a 128MB Cloudflare Workers isolate is + * the parsed TREE, not the text: the 43MB / ~1.6M-line Microsoft Graph YAML + * builds a ~300MB tree and dies mid-request with an empty 503 (measured + * 2026-08), while a same-order text whose bulk is one flat scalar parses fine + * (see "parses Graph-sized YAML" in parse.test.ts). So the guards measure tree + * size by proxy, per input shape, and turn the isolate death into an + * actionable error: + * + * - Any text above `MAX_SPEC_TEXT_CHARS` is rejected outright — the string + * plus any parse output cannot fit regardless of shape. + * - Block YAML builds roughly one node per line (~190 bytes of tree per line + * measured on Graph), so YAML is capped by newline count. + * - JSON (and flow-style YAML, same sniff) concentrates structure without + * newlines, so it is capped by text size; the 16MB Cloudflare JSON spec is + * known-good and must stay under the cap. + * + * Provider adapters that stream via `structuralSplit` (Microsoft Graph) never + * enter this path and are not capped. + */ +export const MAX_SPEC_TEXT_CHARS = 48 * MiB; +export const MAX_JSON_SPEC_CHARS = 32 * MiB; +export const MAX_YAML_SPEC_LINES = 400_000; + +const formatMiB = (chars: number): string => `${Math.ceil((chars / MiB) * 10) / 10}MB`; + +const specGuidance = + "Filter the spec to the operations you need before adding it, or use a curated " + + "provider preset that selects a workload server-side."; + +const specTooLargeMessage = (size: number, limit: number): string => + `OpenAPI document is too large to parse (${formatMiB(size)}, limit ${formatMiB(limit)}). ` + + specGuidance; + +const specTooDenseMessage = (lines: number): string => + `OpenAPI document is too large to parse whole (${lines.toLocaleString("en-US")} lines, ` + + `limit ${MAX_YAML_SPEC_LINES.toLocaleString("en-US")}). ` + + specGuidance; + +const countLines = (text: string): number => { + let count = 1; + for (let pos = text.indexOf("\n"); pos !== -1; pos = text.indexOf("\n", pos + 1)) count += 1; + return count; +}; + export interface SpecFetchCredentials { readonly headers?: Record; readonly queryParams?: Record; @@ -50,6 +97,16 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( message: `Failed to fetch OpenAPI document: HTTP ${response.status}`, }); } + // Reject documents the whole-parse path can never handle before downloading + // them. The declared byte length bounds the decoded text from above only for + // the coarse any-shape cap, so this never rejects a spec `parseSpecObject` + // would have accepted; the precise per-shape check still runs there. + const declaredLength = Number(response.headers["content-length"]); + if (Number.isFinite(declaredLength) && declaredLength > MAX_SPEC_TEXT_CHARS) { + return yield* new OpenApiParseError({ + message: specTooLargeMessage(declaredLength, MAX_SPEC_TEXT_CHARS), + }); + } const specText = yield* response.text.pipe( Effect.mapError( (_cause) => @@ -108,6 +165,26 @@ export const parseSpecObject = (text: string): Effect.Effect MAX_SPEC_TEXT_CHARS) { + return yield* new OpenApiParseError({ + message: specTooLargeMessage(trimmed.length, MAX_SPEC_TEXT_CHARS), + }); + } + if (trimmed.startsWith("{") || trimmed.startsWith("[")) { + if (trimmed.length > MAX_JSON_SPEC_CHARS) { + return yield* new OpenApiParseError({ + message: specTooLargeMessage(trimmed.length, MAX_JSON_SPEC_CHARS), + }); + } + } else { + const lines = countLines(trimmed); + if (lines > MAX_YAML_SPEC_LINES) { + return yield* new OpenApiParseError({ + message: specTooDenseMessage(lines), + }); + } + } + const parsed = yield* parseJsonLike(trimmed).pipe( Effect.mapError( () => diff --git a/packages/plugins/openapi/src/sdk/plugin.ts b/packages/plugins/openapi/src/sdk/plugin.ts index c2c4223886..fab5c3b1a7 100644 --- a/packages/plugins/openapi/src/sdk/plugin.ts +++ b/packages/plugins/openapi/src/sdk/plugin.ts @@ -37,6 +37,7 @@ import { OAuth2Preset, SecurityScheme, previewSpecText, + previewSpecTextStreaming, type SpecPreview, } from "./preview"; import { deriveAuthenticationTemplateFromPreview, firstBaseUrlForPreview } from "./derive-auth"; @@ -807,18 +808,25 @@ export const openApiPlugin = definePlugin< const explicitBaseUrl = config.baseUrl ?? resolved.baseUrl; const needsDerivedBaseUrl = explicitBaseUrl == null; const needsDerivedAuth = config.authenticationTemplate == null; + // Spec-format selections (resolved.keepPathItem) preview via the + // streaming path: the whole-document parse of a Graph-sized source is + // the measured isolate OOM. The OAuth-discovery enrich re-parses the + // full text for the same reason, and an adapter spec declares its + // auth (or the adapter supplies the template), so it is skipped. const preview = needsDerivedBaseUrl || needsDerivedAuth - ? yield* previewSpecText(resolved.specText).pipe( - Effect.flatMap((rawPreview) => - enrichPreviewWithDiscoveredOAuth({ - specText: resolved.specText, - preview: rawPreview, - specUrl: resolved.specUrl ?? specInputToSpecUrl(config.spec), - baseUrl: explicitBaseUrl, - }), - ), - ) + ? resolved.keepPathItem + ? yield* previewSpecTextStreaming(resolved.specText, resolved.keepPathItem) + : yield* previewSpecText(resolved.specText).pipe( + Effect.flatMap((rawPreview) => + enrichPreviewWithDiscoveredOAuth({ + specText: resolved.specText, + preview: rawPreview, + specUrl: resolved.specUrl ?? specInputToSpecUrl(config.spec), + baseUrl: explicitBaseUrl, + }), + ), + ) : undefined; const derivedBaseUrl = needsDerivedBaseUrl && preview ? firstBaseUrlForPreview(preview) : undefined; @@ -1101,6 +1109,12 @@ export const openApiPlugin = definePlugin< }, httpClientLayer, ); + // Spec-format selections stream (whole-parse of a Graph-sized + // source OOMs the isolate) and skip the OAuth-discovery enrich — + // same rationale as the addSpec derived preview above. + if (resolved.keepPathItem) { + return yield* previewSpecTextStreaming(resolved.specText, resolved.keepPathItem); + } const preview = yield* previewSpecText(resolved.specText); return yield* enrichPreviewWithDiscoveredOAuth({ specText: resolved.specText, diff --git a/packages/plugins/openapi/src/sdk/preview-streaming.test.ts b/packages/plugins/openapi/src/sdk/preview-streaming.test.ts new file mode 100644 index 0000000000..e85a3ee3c3 --- /dev/null +++ b/packages/plugins/openapi/src/sdk/preview-streaming.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { OpenApiExtractionError } from "./errors"; +import { previewSpecText, previewSpecTextStreaming } from "./preview"; +import type { SpecPreview } from "./preview"; + +// Streamable block-YAML fixture: parameter/schema `$ref`s, a path-level +// parameter, a deprecated operation, top-level security, and a transitive +// schema chain (WidgetList → Widget → Owner) for the response-field walk. +const fixture = `openapi: 3.0.4 +info: + title: Streamed Fixture + version: 1.2.3 + description: Fixture for streaming preview parity +servers: + - url: https://api.example.com/v1 +security: + - appAuth: [] +paths: + /widgets: + get: + operationId: widgets.list + summary: List widgets + tags: + - widgets + parameters: + - $ref: '#/components/parameters/PageSize' + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/WidgetList' + post: + operationId: widgets.create + summary: Create a widget + tags: + - widgets + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Widget' + responses: + "201": + description: Created + content: + application/json: + schema: + $ref: '#/components/schemas/Widget' + /widgets/{widget-id}: + parameters: + - name: widget-id + in: path + required: true + schema: + type: string + get: + operationId: widgets.get + deprecated: true + tags: + - widgets + - detail + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/Widget' +components: + parameters: + PageSize: + name: pageSize + in: query + required: false + description: Page size + schema: + type: integer + securitySchemes: + appAuth: + type: oauth2 + flows: + authorizationCode: + authorizationUrl: https://auth.example.com/authorize + tokenUrl: https://auth.example.com/token + scopes: + widgets.read: Read widgets + schemas: + WidgetList: + type: object + properties: + items: + type: array + items: + $ref: '#/components/schemas/Widget' + Widget: + type: object + properties: + id: + type: string + owner: + $ref: '#/components/schemas/Owner' + Owner: + type: object + properties: + name: + type: string +`; + +const sortedOperations = (preview: SpecPreview) => + [...preview.operations].sort((a, b) => + `${a.operationId}:${a.method}`.localeCompare(`${b.operationId}:${b.method}`), + ); + +describe("previewSpecTextStreaming", () => { + it.effect("matches the whole-document preview on a streamable spec", () => + Effect.gen(function* () { + const whole = yield* previewSpecText(fixture); + const streamed = yield* previewSpecTextStreaming(fixture); + + expect(streamed.title).toEqual(whole.title); + expect(streamed.description).toEqual(whole.description); + expect(streamed.version).toEqual(whole.version); + expect(streamed.servers).toEqual(whole.servers); + expect(streamed.operationCount).toBe(whole.operationCount); + expect(streamed.tags).toEqual(whole.tags); + expect(streamed.securitySchemes).toEqual(whole.securitySchemes); + expect(streamed.authStrategies).toEqual(whole.authStrategies); + expect(streamed.headerPresets).toEqual(whole.headerPresets); + expect(streamed.oauth2Presets).toEqual(whole.oauth2Presets); + expect(streamed.healthCheckCandidates).toEqual(whole.healthCheckCandidates); + expect(sortedOperations(streamed)).toEqual(sortedOperations(whole)); + }), + ); + + it.effect("projects response fields through the transitive schema closure", () => + Effect.gen(function* () { + const streamed = yield* previewSpecTextStreaming(fixture); + + const listCandidate = streamed.healthCheckCandidates.find( + (candidate) => candidate.method === "get" && candidate.operation.endsWith("list"), + ); + expect(listCandidate).toBeDefined(); + expect(listCandidate!.responseFields ?? []).not.toHaveLength(0); + }), + ); + + it.effect("applies the keep filter to counts, tags, and candidates", () => + Effect.gen(function* () { + const streamed = yield* previewSpecTextStreaming(fixture, (path, pathItem) => + path === "/widgets/{widget-id}" ? null : pathItem, + ); + + expect(streamed.operationCount).toBe(2); + expect(streamed.tags).toEqual(["widgets"]); + expect( + streamed.operations.every((operation) => operation.operationId !== "widgets.get"), + ).toBe(true); + expect( + streamed.healthCheckCandidates.every((candidate) => candidate.method !== "delete"), + ).toBe(true); + expect(streamed.healthCheckCandidates).toHaveLength(2); + }), + ); + + it.effect("fails cleanly on a spec outside the streamable profile", () => + Effect.gen(function* () { + const error = yield* previewSpecTextStreaming( + JSON.stringify({ + openapi: "3.1.0", + info: { title: "Inline", version: "1.0.0" }, + paths: {}, + }), + ).pipe(Effect.flip); + + expect(error).toBeInstanceOf(OpenApiExtractionError); + expect(error).toHaveProperty("message", expect.stringMatching(/streamable/)); + }), + ); +}); diff --git a/packages/plugins/openapi/src/sdk/preview.ts b/packages/plugins/openapi/src/sdk/preview.ts index 56fb3e7038..4266b02347 100644 --- a/packages/plugins/openapi/src/sdk/preview.ts +++ b/packages/plugins/openapi/src/sdk/preview.ts @@ -8,10 +8,22 @@ import { } from "@executor-js/sdk/core"; import { parse, resolveSpecText, type ParsedDocument } from "./parse"; -import { extract } from "./extract"; +import { + extract, + streamOutputSchemas, + streamPreviewOperations, + type StreamedPreviewOperation, +} from "./extract"; import { compileToolDefinitions } from "./definitions"; import { normalizeOpenApiRefs } from "./backing"; +import { OpenApiExtractionError } from "./errors"; import { DocResolver } from "./openapi-utils"; +import { + collectReferencedSchemas, + indexSchemas, + structuralSplit, + type KeepPathItem, +} from "./split"; import { HttpMethod, ServerInfo, type ExtractedOperation, type ExtractionResult } from "./types"; // Mutating HTTP methods: mirrors `REQUIRE_APPROVAL` in `./invoke` but kept @@ -565,6 +577,140 @@ export const previewSpecText = Effect.fn("OpenApi.previewSpecText")(function* (s }); }); +// --------------------------------------------------------------------------- +// Streaming preview (spec-format selections over Graph-sized specs) +// --------------------------------------------------------------------------- + +const streamedCandidate = (op: StreamedPreviewOperation): HealthCheckCandidate => { + const method = op.method.toLowerCase(); + return { + operation: op.toolPath, + method, + requiredArgCount: op.parameters.filter((parameter) => parameter.required).length, + destructive: DESTRUCTIVE_METHODS.has(method), + summary: op.summary ?? op.description ?? `${method.toUpperCase()} ${op.pathTemplate}`, + ...(op.parameters.length > 0 ? { parameters: op.parameters } : {}), + }; +}; + +/** + * Streaming twin of `previewSpecText` for spec-format selections (Microsoft + * Graph): never parses the document whole. The whole-document parse of the 43MB + * Graph source builds a ~300MB tree that kills a 128MB Workers isolate — the + * add/update path already streams via `structuralSplit`, and this brings the + * preview path onto the same primitive: head + schema-free components parse + * small; path-items parse one at a time (through `keepPathItem`, so the preview + * matches what registration persists); and only the top-ranked candidates get + * their response schema walked, against the transitive `$ref` closure rather + * than the full schema map. + */ +export const previewSpecTextStreaming = Effect.fn("OpenApi.previewSpecTextStreaming")(function* ( + specText: string, + keepPathItem?: KeepPathItem, +) { + const structure = structuralSplit(specText); + if (!structure) { + return yield* new OpenApiExtractionError({ + message: + "OpenAPI spec is not in the streamable block-YAML profile (no top-level `paths:` block); cannot stream-preview a spec this large in-band.", + }); + } + + const { head, components, servers, operations } = streamPreviewOperations( + structure, + keepPathItem, + ); + + // oxlint-disable-next-line executor/no-double-cast -- boundary: schema-free resolver doc (head + small components, empty paths), read only for `$ref` resolution into components. + const resolverDoc = { ...head, paths: {}, components } as unknown as ParsedDocument; + const resolver = new DocResolver(resolverDoc); + const rawSchemes = + components.securitySchemes && typeof components.securitySchemes === "object" + ? (components.securitySchemes as Record) + : {}; + const securitySchemes = extractSecuritySchemes(rawSchemes, resolver); + + const rawSecurity = (Array.isArray(head.security) ? head.security : []) as Array< + Record + >; + const declaredStrategies = rawSecurity.map((entry) => + AuthStrategy.make({ schemes: Object.keys(entry) }), + ); + const authStrategies = + declaredStrategies.length > 0 + ? declaredStrategies + : securitySchemes.map((scheme) => AuthStrategy.make({ schemes: [scheme.name] })); + + // Rank all kept operations, keep candidate metadata for the top slice, and + // walk response schemas only for the top survivors — same caps and ranking as + // the whole-document path. + const ranked = operations + .map((op) => ({ operationIndex: op.operationIndex, candidate: streamedCandidate(op) })) + .sort((a, b) => compareHealthCheckCandidates(a.candidate, b.candidate)) + .slice(0, MAX_PREVIEW_CANDIDATES); + + const fieldCandidates = ranked.slice(0, MAX_PREVIEW_RESPONSE_FIELD_CANDIDATES); + const outputSchemas = streamOutputSchemas( + structure, + new Set(fieldCandidates.map((entry) => entry.operationIndex)), + keepPathItem, + ); + // Hoisted `$defs` restricted to the transitive `$ref` closure of the walked + // output schemas — `projectResponseFields` resolves within this map, and the + // closure guarantees every reachable ref is present. + const hoistedDefs: Record = {}; + for (const [name, schema] of Object.entries( + collectReferencedSchemas(structure, indexSchemas(structure), [...outputSchemas.values()]), + )) { + hoistedDefs[name] = normalizeOpenApiRefs(schema); + } + const healthCheckCandidates = ranked.map(({ operationIndex, candidate }, index) => { + if (index >= MAX_PREVIEW_RESPONSE_FIELD_CANDIDATES) return candidate; + const outputSchema = outputSchemas.get(operationIndex); + if (outputSchema === undefined) return candidate; + const responseFields = projectResponseFields(normalizeOpenApiRefs(outputSchema), hoistedDefs); + return responseFields.length > 0 ? { ...candidate, responseFields } : candidate; + }); + + const info = + head.info && typeof head.info === "object" && !Array.isArray(head.info) + ? (head.info as Record) + : {}; + const infoString = (key: string): string | undefined => { + const value = info[key]; + return typeof value === "string" ? value : undefined; + }; + + const tagSet = new Set(); + for (const op of operations) { + for (const tag of op.tags) tagSet.add(tag); + } + + return SpecPreview.make({ + title: Option.fromNullishOr(infoString("title")), + description: Option.fromNullishOr(infoString("description")), + version: Option.fromNullishOr(infoString("version")), + servers, + operationCount: operations.length, + operations: operations.map((op) => + PreviewOperation.make({ + operationId: op.operationId, + method: op.method, + path: op.pathTemplate, + summary: Option.fromNullishOr(op.summary), + tags: op.tags, + deprecated: op.deprecated, + }), + ), + tags: [...tagSet].sort(), + securitySchemes, + authStrategies, + headerPresets: buildHeaderPresets(securitySchemes, authStrategies), + oauth2Presets: buildOAuth2Presets(securitySchemes), + healthCheckCandidates, + }); +}); + /** Preview an OpenAPI spec — extract metadata without registering anything. * Accepts either a URL or raw JSON/YAML text. */ export const previewSpec = Effect.fn("OpenApi.previewSpec")(function* (input: string) { From ddbf0feba38c8502d78fa20c3081391b8ba3d112 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:04:34 -0700 Subject: [PATCH 4/6] Serve Microsoft Graph selections from precomputed spec slices (#1753) --- .changeset/graph-spec-slices.md | 5 + .github/workflows/graph-slices.yml | 47 +++++ .oxlintrc.jsonc | 1 + .../openapi/scripts/generate-graph-slices.ts | 109 ++++++++++ .../openapi/src/providers/microsoft/graph.ts | 26 ++- .../providers/microsoft/slice-build.test.ts | 104 ++++++++++ .../src/providers/microsoft/slice-build.ts | 193 ++++++++++++++++++ .../src/providers/microsoft/slices.test.ts | 70 +++++++ .../openapi/src/providers/microsoft/slices.ts | 87 ++++++++ .../microsoft/spec-format-adapter.test.ts | 75 +++++++ 10 files changed, 714 insertions(+), 3 deletions(-) create mode 100644 .changeset/graph-spec-slices.md create mode 100644 .github/workflows/graph-slices.yml create mode 100644 packages/plugins/openapi/scripts/generate-graph-slices.ts create mode 100644 packages/plugins/openapi/src/providers/microsoft/slice-build.test.ts create mode 100644 packages/plugins/openapi/src/providers/microsoft/slice-build.ts create mode 100644 packages/plugins/openapi/src/providers/microsoft/slices.test.ts create mode 100644 packages/plugins/openapi/src/providers/microsoft/slices.ts diff --git a/.changeset/graph-spec-slices.md b/.changeset/graph-spec-slices.md new file mode 100644 index 0000000000..16345d752d --- /dev/null +++ b/.changeset/graph-spec-slices.md @@ -0,0 +1,5 @@ +--- +"@executor-js/plugin-openapi": patch +--- + +Serve Microsoft Graph preset selections from precomputed slice release assets instead of the 43MB upstream monolith. The monolith fetch almost never survives a 128MB Workers isolate (production traces show one completion in 30 days), so covered selections — every catalog preset, plus any combination within the default bundle — now read a 4–19MB filtered document built offline by the graph-slices workflow, with the monolith path kept only as a fallback and for full-graph/custom-scope selections. diff --git a/.github/workflows/graph-slices.yml b/.github/workflows/graph-slices.yml new file mode 100644 index 0000000000..35768a0ac4 --- /dev/null +++ b/.github/workflows/graph-slices.yml @@ -0,0 +1,47 @@ +# Refresh the Microsoft Graph slice release assets. +# +# The Graph OpenAPI monolith (~43MB) cannot be processed inside a Workers +# isolate, so the runtime reads per-selection slices published on the +# `graph-slices` release tag (see packages/plugins/openapi/src/providers/ +# microsoft/slices.ts). This workflow rebuilds the slices from the current +# upstream spec on a schedule and on demand. +name: Graph slices + +on: + schedule: + # Weekly; Microsoft's msgraph-metadata automation lands upstream refreshes + # on a similar cadence. A failed run leaves the previous assets serving. + - cron: "17 6 * * 1" + workflow_dispatch: {} + +permissions: + contents: write + +jobs: + slices: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.11 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Generate slices + working-directory: packages/plugins/openapi + run: bun scripts/generate-graph-slices.ts --out "$RUNNER_TEMP/graph-slices" + + - name: Publish to the graph-slices release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release view graph-slices --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 || \ + gh release create graph-slices --repo "$GITHUB_REPOSITORY" \ + --title "Microsoft Graph slices" --latest=false \ + --notes "Generated per-preset Microsoft Graph OpenAPI slices. Data release consumed by the openapi plugin's Microsoft adapter; refreshed by the graph-slices workflow." + gh release upload graph-slices "$RUNNER_TEMP/graph-slices"/* \ + --repo "$GITHUB_REPOSITORY" --clobber diff --git a/.oxlintrc.jsonc b/.oxlintrc.jsonc index da06c53517..c86ee9dc58 100644 --- a/.oxlintrc.jsonc +++ b/.oxlintrc.jsonc @@ -64,6 +64,7 @@ "apps/desktop/src/main.ts", "scripts/**/*.{ts,js}", "apps/*/scripts/**/*.{ts,js}", + "packages/*/*/scripts/**/*.{ts,js}", "packages/kernel/runtime-*/src/**/*.{ts,tsx,js,mjs}", ], "rules": { diff --git a/packages/plugins/openapi/scripts/generate-graph-slices.ts b/packages/plugins/openapi/scripts/generate-graph-slices.ts new file mode 100644 index 0000000000..a40c72789f --- /dev/null +++ b/packages/plugins/openapi/scripts/generate-graph-slices.ts @@ -0,0 +1,109 @@ +/** + * Generate the Microsoft Graph slice release assets. + * + * bun scripts/generate-graph-slices.ts [--source ] [--out ] + * + * Fetches (or reads) the Graph OpenAPI monolith, builds one slice per catalog + * preset plus the default bundle via `slice-build.ts`, validates every slice + * against the runtime's streamable profile, and writes `.yaml` files + * plus `manifest.json` to the output directory. The graph-slices workflow runs + * this and uploads the output to the `graph-slices` release tag; runtime + * resolution lives in `src/providers/microsoft/slices.ts`. + * + * Offline-only: this whole-parses the 43MB source, which only works where + * memory is free (CI runner / dev machine), never in a Workers isolate. + */ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { structuralSplit } from "../src/sdk/split"; +import { + MICROSOFT_GRAPH_DEFAULT_PRESET_IDS, + MICROSOFT_GRAPH_OPENAPI_URL, + microsoftGraphScopePresets, +} from "../src/providers/microsoft/presets"; +import { MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET } from "../src/providers/microsoft/slices"; +import { + buildGraphSliceDocument, + parseGraphSourceDocument, +} from "../src/providers/microsoft/slice-build"; + +const argValue = (flag: string): string | undefined => { + const index = process.argv.indexOf(flag); + return index !== -1 ? process.argv[index + 1] : undefined; +}; + +const source = argValue("--source") ?? MICROSOFT_GRAPH_OPENAPI_URL; +const outDir = argValue("--out") ?? "graph-slices-out"; + +const readSource = async (): Promise => { + if (source.startsWith("http://") || source.startsWith("https://")) { + const response = await fetch(source); + if (!response.ok) { + throw new Error(`Failed to fetch Graph source: HTTP ${response.status}`); + } + return response.text(); + } + return readFile(source, "utf8"); +}; + +const sourceText = await readSource(); +const sourceSha256 = createHash("sha256").update(sourceText).digest("hex"); +const doc = parseGraphSourceDocument(sourceText); +if (!doc) { + throw new Error("Microsoft Graph source did not parse to an object"); +} + +const selections: readonly { readonly asset: string; readonly presetIds: readonly string[] }[] = [ + ...microsoftGraphScopePresets.map((preset) => ({ asset: preset.id, presetIds: [preset.id] })), + { + asset: MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET, + presetIds: MICROSOFT_GRAPH_DEFAULT_PRESET_IDS, + }, +]; + +await mkdir(outDir, { recursive: true }); + +const manifestAssets: Record< + string, + { + readonly bytes: number; + readonly paths: number; + readonly operations: number; + readonly schemas: number; + } +> = {}; + +for (const { asset, presetIds } of selections) { + const slice = buildGraphSliceDocument(doc, presetIds); + if (slice.operationCount === 0) { + throw new Error(`Slice "${asset}" kept zero operations — preset filter or source drifted`); + } + const structure = structuralSplit(slice.specText); + if (!structure) { + throw new Error(`Slice "${asset}" is not in the streamable block-YAML profile`); + } + if (structure.pathItems.length !== slice.pathCount) { + throw new Error( + `Slice "${asset}" splitter sees ${structure.pathItems.length} path-items, expected ${slice.pathCount}`, + ); + } + await writeFile(join(outDir, `${asset}.yaml`), slice.specText); + manifestAssets[asset] = { + bytes: Buffer.byteLength(slice.specText), + paths: slice.pathCount, + operations: slice.operationCount, + schemas: slice.schemaCount, + }; + console.log( + `${asset}: ${(Buffer.byteLength(slice.specText) / 1024 / 1024).toFixed(2)}MB, ` + + `${slice.pathCount} paths, ${slice.operationCount} operations, ${slice.schemaCount} schemas`, + ); +} + +await writeFile( + join(outDir, "manifest.json"), + `${JSON.stringify({ source, sourceSha256, generatedAt: new Date().toISOString(), assets: manifestAssets }, null, 2)}\n`, +); +console.log(`wrote ${selections.length} slices + manifest.json to ${outDir}`); diff --git a/packages/plugins/openapi/src/providers/microsoft/graph.ts b/packages/plugins/openapi/src/providers/microsoft/graph.ts index 921f1fa1a9..36ac23dc4f 100644 --- a/packages/plugins/openapi/src/providers/microsoft/graph.ts +++ b/packages/plugins/openapi/src/providers/microsoft/graph.ts @@ -15,6 +15,7 @@ import { } from "../../sdk/split"; import type { Authentication } from "../../sdk/types"; +import { fetchMicrosoftGraphSlice, microsoftGraphSliceAssetForSelection } from "./slices"; import { MICROSOFT_AUTHORIZATION_URL, MICROSOFT_AUTH_TEMPLATE_SLUG, @@ -759,9 +760,28 @@ export const buildMicrosoftGraphOpenApiSpec = ( ): Effect.Effect => Effect.gen(function* () { const selection = yield* validateSelectionUrls(normalizeSelection(input), urlPolicy); - const sourceText = yield* fetchMicrosoftGraphOpenApiSpec(selection.specUrl).pipe( - Effect.provide(httpClientLayer), - ); + // Covered selections read a precomputed slice (sub-MB) instead of the 43MB + // monolith: in production, the monolith fetch alone almost never survives + // the 128MB isolate (once in the 30 days before 2026-08-26). Slices apply + // only to the pinned Microsoft URL — an override (local Graph emulators) + // serves its own document. A missing/failed slice (asset not yet published, + // release unreachable) falls back to the monolith path, which is the prior + // behavior for the selections a slice would have covered. + const sliceAsset = + selection.specUrl === MICROSOFT_GRAPH_OPENAPI_URL + ? microsoftGraphSliceAssetForSelection(selection) + : null; + const sourceText = + sliceAsset !== null + ? yield* fetchMicrosoftGraphSlice(sliceAsset).pipe( + Effect.catchTag("OpenApiParseError", () => + fetchMicrosoftGraphOpenApiSpec(selection.specUrl), + ), + Effect.provide(httpClientLayer), + ) + : yield* fetchMicrosoftGraphOpenApiSpec(selection.specUrl).pipe( + Effect.provide(httpClientLayer), + ); // Structural split is the only entry point: parsing the whole 37MB tree // OOMs the 128MB Workers isolate (measured: HTTP 503). No fallback. A spec diff --git a/packages/plugins/openapi/src/providers/microsoft/slice-build.test.ts b/packages/plugins/openapi/src/providers/microsoft/slice-build.test.ts new file mode 100644 index 0000000000..8425e0be19 --- /dev/null +++ b/packages/plugins/openapi/src/providers/microsoft/slice-build.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { structuralSplit } from "../../sdk/split"; +import { buildGraphSliceDocument, parseGraphSourceDocument } from "./slice-build"; + +// Graph-shaped source: a mail path, an unrelated path, and a schema chain +// where only part is reachable from the mail selection. +const source = `openapi: 3.0.4 +info: + title: Microsoft Graph Fixture + version: v1.0 +servers: + - url: https://graph.microsoft.com/v1.0 +paths: + /me/messages: + get: + operationId: me.ListMessages + security: + - azureAdDelegated: + - Mail.ReadWrite + parameters: + - $ref: '#/components/parameters/Top' + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/microsoft.graph.messageCollection' + /irrelevant: + get: + operationId: irrelevant.Get + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/microsoft.graph.unrelated' +components: + parameters: + Top: + name: $top + in: query + schema: + type: integer + securitySchemes: + azureAdDelegated: + type: oauth2 + flows: + authorizationCode: + authorizationUrl: https://login.microsoftonline.com/common/oauth2/v2.0/authorize + tokenUrl: https://login.microsoftonline.com/common/oauth2/v2.0/token + scopes: + Mail.ReadWrite: Read and write mail + schemas: + microsoft.graph.messageCollection: + type: object + properties: + value: + type: array + items: + $ref: '#/components/schemas/microsoft.graph.message' + microsoft.graph.message: + type: object + properties: + id: + type: string + microsoft.graph.unrelated: + type: object + properties: + name: + type: string +`; + +describe("buildGraphSliceDocument", () => { + it("keeps the selection's paths and prunes components to the reachable closure", () => { + const doc = parseGraphSourceDocument(source); + expect(doc).not.toBeNull(); + const slice = buildGraphSliceDocument(doc!, ["mail"]); + + expect(slice.pathCount).toBe(1); + expect(slice.operationCount).toBe(1); + expect(slice.specText).toContain("/me/messages"); + expect(slice.specText).not.toContain("/irrelevant"); + expect(slice.specText).toContain("microsoft.graph.messageCollection"); + expect(slice.specText).toContain("microsoft.graph.message"); + expect(slice.specText).not.toContain("microsoft.graph.unrelated"); + // Referenced small components survive; securitySchemes always survive. + expect(slice.specText).toContain("$top"); + expect(slice.specText).toContain("azureAdDelegated"); + }); + + it("emits the streamable block-YAML profile the runtime splitter accepts", () => { + const doc = parseGraphSourceDocument(source); + expect(doc).not.toBeNull(); + const slice = buildGraphSliceDocument(doc!, ["mail"]); + + const structure = structuralSplit(slice.specText); + expect(structure).not.toBeNull(); + expect(structure!.pathItems).toHaveLength(slice.pathCount); + expect(structure!.schemas).toHaveLength(slice.schemaCount); + }); +}); diff --git a/packages/plugins/openapi/src/providers/microsoft/slice-build.ts b/packages/plugins/openapi/src/providers/microsoft/slice-build.ts new file mode 100644 index 0000000000..8fde215908 --- /dev/null +++ b/packages/plugins/openapi/src/providers/microsoft/slice-build.ts @@ -0,0 +1,193 @@ +import { JSON_SCHEMA, dump as dumpYamlDocument, load as parseYamlDocument } from "js-yaml"; + +import { microsoftGraphKeepPathItem } from "./graph"; +import { + microsoftGraphExactPathsForPresetIds, + microsoftGraphPathPrefixesForPresetIds, + microsoftGraphTagPrefixesForPresetIds, +} from "./presets"; + +/** + * Offline Microsoft Graph slice construction. NOT part of the runtime import + * graph: the 43MB Graph source cannot be held in a 128MB Workers isolate (its + * fetch alone has completed once in the last 30 days of production traces), so + * slices are built where memory is free — the graph-slices workflow / a dev + * machine — published as release assets, and fetched per selection at runtime + * by `slices.ts`. Import this module only from scripts and tests. + */ + +/** `components` subkeys retained whole in a slice — mirrors the runtime + * splitter's `SMALL_COMPONENT_SECTIONS` (`sdk/split.ts`), which keeps these + * resident for `$ref` resolution. `schemas` is pruned to the transitive + * closure instead; `examples` is dropped. */ +const SMALL_COMPONENT_SECTIONS = [ + "parameters", + "requestBodies", + "responses", + "headers", + "links", + "securitySchemes", +] as const; + +const COMPONENT_REF_PREFIX = "#/components/"; + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const decodeRefSegment = (segment: string): string => + segment.replace(/~1/g, "/").replace(/~0/g, "~"); + +interface ComponentRef { + readonly section: string; + readonly name: string; +} + +const collectComponentRefs = (value: unknown, into: (ref: ComponentRef) => void): void => { + if (typeof value === "string") { + if (value.startsWith(COMPONENT_REF_PREFIX)) { + const rest = value.slice(COMPONENT_REF_PREFIX.length); + const slash = rest.indexOf("/"); + if (slash > 0) { + const section = rest.slice(0, slash); + const name = decodeRefSegment(rest.slice(slash + 1)); + if (name.length > 0) into({ section, name }); + } + } + return; + } + if (Array.isArray(value)) { + for (const item of value) collectComponentRefs(item, into); + return; + } + if (isRecord(value)) { + for (const item of Object.values(value)) collectComponentRefs(item, into); + } +}; + +/** + * Transitive `#/components/
/` closure over every component + * section, seeded from `roots` (the kept path-items). Only components actually + * reachable from a kept operation survive — Graph's ~8k schemas and its + * catch-all responses/parameters sections otherwise drag nearly the whole + * component graph into every slice. + */ +const componentClosure = ( + components: Record, + roots: readonly unknown[], +): Record> => { + const kept: Record> = {}; + const queue: ComponentRef[] = []; + const seen = new Set(); + const enqueue = (ref: ComponentRef): void => { + const key = `${ref.section}/${ref.name}`; + if (seen.has(key)) return; + seen.add(key); + queue.push(ref); + }; + + for (const root of roots) collectComponentRefs(root, enqueue); + for (let i = 0; i < queue.length; i += 1) { + const { section, name } = queue[i]!; + const sectionValues = components[section]; + if (!isRecord(sectionValues)) continue; + const component = sectionValues[name]; + if (component === undefined) continue; + (kept[section] ??= {})[name] = component; + collectComponentRefs(component, enqueue); + } + return kept; +}; + +export interface GraphSliceBuild { + /** The slice as streamable block YAML (the same profile `structuralSplit` + * accepts, so the runtime pipeline treats a slice exactly like a source). */ + readonly specText: string; + readonly pathCount: number; + readonly operationCount: number; + readonly schemaCount: number; +} + +const HTTP_METHODS = new Set(["delete", "get", "head", "options", "patch", "post", "put", "trace"]); + +const countOperations = (paths: Record): number => { + let count = 0; + for (const pathItem of Object.values(paths)) { + if (!isRecord(pathItem)) continue; + for (const key of Object.keys(pathItem)) { + if (HTTP_METHODS.has(key.toLowerCase())) count += 1; + } + } + return count; +}; + +/** + * Build one preset selection's slice from the parsed Graph document: keep the + * selection's path-items (via the same `microsoftGraphKeepPathItem` filter the + * runtime applies), retain the small component sections whole, and prune + * `components.schemas` to the transitive `$ref` closure of everything kept. + */ +export const buildGraphSliceDocument = ( + doc: Record, + presetIds: readonly string[], +): GraphSliceBuild => { + const keepPathItem = microsoftGraphKeepPathItem({ + coversFullGraph: false, + presetIds, + customScopes: [], + exactPaths: microsoftGraphExactPathsForPresetIds(presetIds), + pathPrefixes: microsoftGraphPathPrefixesForPresetIds(presetIds), + tagPrefixes: microsoftGraphTagPrefixesForPresetIds(presetIds), + }); + + const sourcePaths = isRecord(doc.paths) ? doc.paths : {}; + const paths: Record = {}; + for (const [path, pathItem] of Object.entries(sourcePaths)) { + if (!isRecord(pathItem)) continue; + const kept = keepPathItem(path, pathItem); + if (kept) paths[path] = kept; + } + + const sourceComponents = isRecord(doc.components) ? doc.components : {}; + const closure = componentClosure(sourceComponents, [paths]); + const components: Record = {}; + for (const section of SMALL_COMPONENT_SECTIONS) { + if (closure[section]) components[section] = closure[section]; + } + // securitySchemes are never `$ref`'d from operations by name in Graph + // (operations reference them via `security` entries), so retain the section + // whole — it is tiny and the runtime reads OAuth endpoints from it. + if (isRecord(sourceComponents.securitySchemes)) { + components.securitySchemes = sourceComponents.securitySchemes; + } + const schemas = closure.schemas ?? {}; + components.schemas = schemas; + + const slice: Record = { + ...(doc.openapi !== undefined ? { openapi: doc.openapi } : {}), + ...(doc.info !== undefined ? { info: doc.info } : {}), + ...(doc.servers !== undefined ? { servers: doc.servers } : {}), + ...(doc.security !== undefined ? { security: doc.security } : {}), + paths, + components, + }; + + // noRefs duplicates shared subtrees instead of emitting YAML anchors, which + // the streamable block profile forbids; lineWidth -1 keeps scalars on one + // line so no wrapped line can be mistaken for structure. + const specText = dumpYamlDocument(slice, { noRefs: true, lineWidth: -1, schema: JSON_SCHEMA }); + + return { + specText, + pathCount: Object.keys(paths).length, + operationCount: countOperations(paths), + schemaCount: Object.keys(schemas).length, + }; +}; + +/** Parse Graph source YAML for slice building, or null when the source does + * not parse to an object. Offline-only: this is the whole-document parse the + * runtime can never do. */ +export const parseGraphSourceDocument = (sourceText: string): Record | null => { + const parsed = parseYamlDocument(sourceText, { json: true, schema: JSON_SCHEMA }); + return isRecord(parsed) ? parsed : null; +}; diff --git a/packages/plugins/openapi/src/providers/microsoft/slices.test.ts b/packages/plugins/openapi/src/providers/microsoft/slices.test.ts new file mode 100644 index 0000000000..6f94a2fceb --- /dev/null +++ b/packages/plugins/openapi/src/providers/microsoft/slices.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { MICROSOFT_GRAPH_ALL_PRESET_IDS, MICROSOFT_GRAPH_DEFAULT_PRESET_IDS } from "./presets"; +import { + MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET, + microsoftGraphSliceAssetForSelection, +} from "./slices"; + +describe("microsoftGraphSliceAssetForSelection", () => { + it("maps a single catalog preset to its asset", () => { + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["mail"], + customScopes: [], + }), + ).toBe("mail"); + }); + + it("maps the default bundle in any order to the default asset", () => { + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: [...MICROSOFT_GRAPH_DEFAULT_PRESET_IDS].reverse(), + customScopes: [], + }), + ).toBe(MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET); + }); + + it("serves combinations within the default bundle from the default slice", () => { + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["mail", "calendar"], + customScopes: [], + }), + ).toBe(MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET); + }); + + it("needs the monolith for full-graph, custom scopes, unknown presets, and combinations outside the default bundle", () => { + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: true, + presetIds: [...MICROSOFT_GRAPH_ALL_PRESET_IDS], + customScopes: [], + }), + ).toBeNull(); + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["mail"], + customScopes: ["Chat.Read"], + }), + ).toBeNull(); + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["not-a-preset"], + customScopes: [], + }), + ).toBeNull(); + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["mail", "users"], + customScopes: [], + }), + ).toBeNull(); + }); +}); diff --git a/packages/plugins/openapi/src/providers/microsoft/slices.ts b/packages/plugins/openapi/src/providers/microsoft/slices.ts new file mode 100644 index 0000000000..f129e36740 --- /dev/null +++ b/packages/plugins/openapi/src/providers/microsoft/slices.ts @@ -0,0 +1,87 @@ +import { Effect } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; + +import { OpenApiParseError } from "../../sdk/errors"; + +import { MICROSOFT_GRAPH_DEFAULT_PRESET_IDS, microsoftGraphPresetForId } from "./presets"; + +/** + * Runtime access to precomputed Microsoft Graph slices. + * + * The 43MB Graph monolith cannot be processed in a 128MB Workers isolate — in + * production its fetch alone completed once in the 30 days before 2026-08-26; + * every other preview/add died mid-download with an empty 503. Slices are + * built offline (`slice-build.ts`, refreshed by the graph-slices workflow), + * published as release assets, and fetched here per selection: the isolate + * only ever holds the sub-megabyte filtered document for the selection. + */ + +export const MICROSOFT_GRAPH_SLICE_RELEASE_TAG = "graph-slices"; + +export const MICROSOFT_GRAPH_SLICE_BASE_URL = `https://github.com/UsefulSoftwareCo/executor/releases/download/${MICROSOFT_GRAPH_SLICE_RELEASE_TAG}`; + +/** Asset name for the default catalog bundle (`MICROSOFT_GRAPH_DEFAULT_PRESET_IDS`). */ +export const MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET = "default"; + +/** + * The published asset covering a selection, or null when the selection needs + * the monolith. A slice may be a superset of the selection: the runtime always + * applies the selection's `keepPathItem` filter to whatever source it reads, so + * any combination within the default bundle can be served from the default + * slice and narrowed in-band. The monolith remains necessary for full-graph + * coverage, custom scopes (scope matching walks operations outside any + * preset's paths), and combinations reaching outside the default bundle + * (precomputing every combination is combinatorial). + */ +export const microsoftGraphSliceAssetForSelection = (selection: { + readonly coversFullGraph: boolean; + readonly presetIds: readonly string[]; + readonly customScopes: readonly string[]; +}): string | null => { + if (selection.coversFullGraph) return null; + if (selection.customScopes.length > 0) return null; + if (selection.presetIds.length === 0) return null; + if (selection.presetIds.some((presetId) => !microsoftGraphPresetForId(presetId))) return null; + if (selection.presetIds.length === 1) return selection.presetIds[0]!; + const defaultIds = new Set(MICROSOFT_GRAPH_DEFAULT_PRESET_IDS); + if (selection.presetIds.every((presetId) => defaultIds.has(presetId))) { + return MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET; + } + return null; +}; + +export const microsoftGraphSliceUrl = (asset: string): string => + `${MICROSOFT_GRAPH_SLICE_BASE_URL}/${encodeURIComponent(asset)}.yaml`; + +export const fetchMicrosoftGraphSlice = Effect.fn("Microsoft.fetchGraphSlice")(function* ( + asset: string, +) { + const client = yield* HttpClient.HttpClient; + const response = yield* client + .execute( + HttpClientRequest.get(microsoftGraphSliceUrl(asset)).pipe( + HttpClientRequest.setHeader("Accept", "application/yaml, text/yaml, */*"), + ), + ) + .pipe( + Effect.mapError( + () => + new OpenApiParseError({ + message: `Failed to fetch Microsoft Graph slice: ${asset}`, + }), + ), + ); + if (response.status < 200 || response.status >= 300) { + return yield* new OpenApiParseError({ + message: `Failed to fetch Microsoft Graph slice ${asset}: HTTP ${response.status}`, + }); + } + return yield* response.text.pipe( + Effect.mapError( + () => + new OpenApiParseError({ + message: `Failed to read Microsoft Graph slice body: ${asset}`, + }), + ), + ); +}); diff --git a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts index d3da64cd65..44c09c65d9 100644 --- a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts +++ b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts @@ -6,6 +6,7 @@ import { structuralSplit } from "@executor-js/plugin-openapi"; import { previewSpecTextStreaming } from "../../sdk/preview"; import { microsoftGraphAdapter } from "./spec-format-adapter"; import { MICROSOFT_GRAPH_OPENAPI_URL } from "./presets"; +import { microsoftGraphSliceUrl } from "./slices"; const graphFixture = ` openapi: 3.0.4 @@ -92,6 +93,80 @@ it.effect("uses catalog URL fragments to select one Graph workload", () => }), ); +// Distinct content at the slice URL so tests can tell which source was read. +const sliceFixture = `openapi: 3.0.4 +info: + title: Microsoft Graph Slice Fixture + version: v1.0 +servers: + - url: https://graph.microsoft.com/v1.0 +paths: + /me: + get: + operationId: me.GetUser + security: + - azureAdDelegated: + - User.Read + responses: + "200": + description: OK +components: + securitySchemes: + azureAdDelegated: + type: oauth2 + flows: + authorizationCode: + authorizationUrl: https://login.microsoftonline.com/common/oauth2/v2.0/authorize + tokenUrl: https://login.microsoftonline.com/common/oauth2/v2.0/token + scopes: + User.Read: Read user profile +`; + +const sliceAwareHttpClientLayer = Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => { + const body = + request.url === microsoftGraphSliceUrl("profile") + ? sliceFixture + : request.url === MICROSOFT_GRAPH_OPENAPI_URL + ? graphFixture + : null; + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response(body ?? "not found", { status: body === null ? 404 : 200 }), + ), + ); + }), +); + +it.effect("reads the published slice for a covered selection", () => + Effect.gen(function* () { + const converted = yield* microsoftGraphAdapter.fetch({ + urls: [`${MICROSOFT_GRAPH_OPENAPI_URL}#preset=profile`], + httpClientLayer: sliceAwareHttpClientLayer, + }); + + expect(converted.specText).toBe(sliceFixture); + // The catalog URL (with fragment stripped) stays canonical so refresh + // re-resolves through the adapter, not the slice hosting. + expect(converted.specUrl).toBe(MICROSOFT_GRAPH_OPENAPI_URL); + }), +); + +it.effect("falls back to the monolith when the slice asset is unavailable", () => + Effect.gen(function* () { + // graphHttpClientLayer 404s everything except the monolith URL, including + // the slice URL — the existing selection tests above exercise this same + // fallback implicitly. + const converted = yield* microsoftGraphAdapter.fetch({ + urls: [`${MICROSOFT_GRAPH_OPENAPI_URL}#preset=profile`], + httpClientLayer: graphHttpClientLayer, + }); + + expect(converted.specText).toBe(graphFixture); + }), +); + it.effect("stream-previews a Graph selection without a whole-document parse", () => Effect.gen(function* () { const converted = yield* microsoftGraphAdapter.fetch({ From 6574ab18f1ac077d2d5d9c259acdc07896770aff Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:23:58 -0700 Subject: [PATCH 5/6] Stop evicting standalone MCP listeners; fail fast when the transport is missing (#1754) --- e2e/cloud/repro-listener-supersede.test.ts | 130 ++++++++++++++++ patches/agents@0.17.3.patch | 172 +++++++++++++++++++-- 2 files changed, 285 insertions(+), 17 deletions(-) create mode 100644 e2e/cloud/repro-listener-supersede.test.ts diff --git a/e2e/cloud/repro-listener-supersede.test.ts b/e2e/cloud/repro-listener-supersede.test.ts new file mode 100644 index 0000000000..b225b8b596 --- /dev/null +++ b/e2e/cloud/repro-listener-supersede.test.ts @@ -0,0 +1,130 @@ +// Cloud: regression test for the standalone-listener eviction storm. +// +// The MCP spec allows a client to hold a standalone SSE listener (a bare GET) +// for server-initiated messages. The patched agents SDK applies +// latest-listener-wins to that stream: +// +// packages/... -> agents/dist/mcp/index.js:880 +// supersedePriorStreamConnections(agent, connection.id, STANDALONE_STREAM_ID) +// -> for every OTHER connection on the same streamId: close(1000, ...) +// +// STANDALONE_STREAM_ID is shared by every listener, so a session with two +// listeners has each arrival kill the other. Any SSE client reconnects when +// its stream closes, so two listeners evict each other indefinitely: the +// session can never hold a listener open and the client sees a permanent +// "disconnected / reconnecting" churn. +// +// Observed in production: one session produced ~55,600 listener GETs over +// 6.75 hours against only 106 POSTs, arriving 3-4 within milliseconds every +// ~1.2s, each answered 200 in ~130ms. +// +// This test drives N listeners that reconnect when closed, exactly like a real +// client, and asserts the session does NOT churn. +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; + +import { scenario } from "../src/scenario"; +import { Mcp, Target } from "../src/services"; +import type { Identity } from "../src/target"; + +const emailOf = (identity: Identity): string => identity.credentials?.email ?? identity.label; + +const RUN_MS = 15_000; +const SCENARIO_TIMEOUT_MS = RUN_MS + 120_000; + +interface Trial { + readonly opens: number; + readonly medianLifeMs: number; +} + +/** Hold `count` standalone listeners for RUN_MS, reconnecting whenever the + * server closes one — the behaviour of any real SSE client. */ +const driveListeners = async ( + mcpUrl: string, + bearer: string, + sessionId: string, + count: number, +): Promise => { + const deadline = Date.now() + RUN_MS; + const lifetimes: number[] = []; + let opens = 0; + + const listener = async (): Promise => { + while (Date.now() < deadline) { + const started = Date.now(); + const controller = new AbortController(); + const guard = setTimeout(() => controller.abort(), Math.max(500, deadline - Date.now())); + try { + const res = await fetch(mcpUrl, { + method: "GET", + headers: { + authorization: `Bearer ${bearer}`, + accept: "text/event-stream", + "mcp-session-id": sessionId, + "mcp-protocol-version": "2025-06-18", + }, + signal: controller.signal, + }); + opens += 1; + const reader = res.body?.getReader(); + // Read until the SERVER closes the stream. + if (reader) { + for (;;) { + const { done } = await reader.read(); + if (done) break; + } + } + } catch { + // aborted at the deadline, or the socket was torn down + } + clearTimeout(guard); + lifetimes.push(Date.now() - started); + } + }; + + await Promise.all(Array.from({ length: count }, () => listener())); + const sorted = [...lifetimes].sort((a, b) => a - b); + return { opens, medianLifeMs: sorted[Math.floor(sorted.length / 2)] ?? 0 }; +}; + +scenario( + "REGRESSION · a second standalone listener must not evict the first", + { timeout: SCENARIO_TIMEOUT_MS }, + Effect.gen(function* () { + const target = yield* Target; + const mcp = yield* Mcp; + const identity = yield* target.newIdentity(); + const bearer = yield* mcp.mintBearer(emailOf(identity)); + + const client = new Client({ name: "e2e-supersede", version: "0.0.1" }, { capabilities: {} }); + const transport = new StreamableHTTPClientTransport(new URL(target.mcpUrl), { + requestInit: { headers: { authorization: `Bearer ${bearer}` } }, + }); + yield* Effect.promise(() => client.connect(transport)); + const sessionId = transport.sessionId; + expect(sessionId, "the client got a session id").toEqual(expect.any(String)); + if (sessionId === undefined) return yield* Effect.die("missing session id"); + yield* Effect.promise(() => client.close().catch(() => undefined)); + + // Control: a single listener should simply stay open for the whole window. + const single = yield* Effect.promise(() => driveListeners(target.mcpUrl, bearer, sessionId, 1)); + console.log(JSON.stringify({ event: "supersede_single", ...single })); + + // Two listeners, each reconnecting when closed. + const double = yield* Effect.promise(() => driveListeners(target.mcpUrl, bearer, sessionId, 2)); + console.log(JSON.stringify({ event: "supersede_double", ...double })); + + // One listener over a 15s window opens roughly once (a max-age rotation + // could add one more). Anything beyond a handful is the eviction storm. + expect(single.opens, "a single listener does not churn").toBeLessThanOrEqual(3); + + // The regression: two listeners must not evict each other. Without the fix + // this is in the hundreds, with a median stream life of ~200ms. + expect(double.opens, "two listeners do not evict each other").toBeLessThanOrEqual(6); + expect(double.medianLifeMs, "streams stay open rather than dying instantly").toBeGreaterThan( + 2_000, + ); + }), +); diff --git a/patches/agents@0.17.3.patch b/patches/agents@0.17.3.patch index df7396f95d..8ac23d3bd2 100644 --- a/patches/agents@0.17.3.patch +++ b/patches/agents@0.17.3.patch @@ -1,3 +1,6 @@ +diff --git a/node_modules/agents/.bun-tag-61b2f1517ab5ced4 b/.bun-tag-61b2f1517ab5ced4 +new file mode 100644 +index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/node_modules/agents/.bun-tag-c0c639aa2299e502 b/.bun-tag-c0c639aa2299e502 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 @@ -38,7 +41,7 @@ index c8fad448e8797b89690a99d93490d1363851b225..77f9fe3f6f2375eadc9f7a2974f0d202 McpAgent, type McpAuthContext, diff --git a/dist/mcp/index.js b/dist/mcp/index.js -index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756cd72faef 100644 +index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..cab60ce74ac9430f06700436ac352341695c6666 100644 --- a/dist/mcp/index.js +++ b/dist/mcp/index.js @@ -28,13 +28,17 @@ import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/ @@ -494,9 +497,36 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 console.error("Error closing SSE connection:", error); } } -@@ -634,7 +857,23 @@ var StreamableHTTPServerTransport = class { +@@ -586,6 +809,7 @@ var StreamableHTTPServerTransport = class { + constructor(options) { + this._started = false; + this._streamResponseIds = /* @__PURE__ */ new Map(); ++ this._replayInFlight = /* @__PURE__ */ new Map(); + const { agent } = getCurrentAgent(); + if (!agent) throw new Error("McpAgent was not found in Transport constructor"); + this._agent = agent; +@@ -627,23 +851,74 @@ var StreamableHTTPServerTransport = class { + const resumedStreamId = await this._eventStore.getStreamIdForEventId?.(lastEventId); + if (resumedStreamId) { + const resumeState = { streamId: resumedStreamId }; +- if (resumedStreamId === STANDALONE_STREAM_ID) resumeState._standaloneSse = true; +- else { ++ if (resumedStreamId === STANDALONE_STREAM_ID) { ++ resumeState._standaloneSse = true; ++ resumeState._attachedAt = Date.now(); ++ } else { + const persistedReqs = await agent.getStreamRequestIds(resumedStreamId); + if (persistedReqs && persistedReqs.length > 0) resumeState.requestIds = persistedReqs; } - this.supersedePriorStreamConnections(agent, connection.id, resumedStreamId); +- this.supersedePriorStreamConnections(agent, connection.id, resumedStreamId); ++ // A POST stream has a single owner, so resuming it takes the ++ // stream over from any prior connection. The standalone stream ++ // id is shared by EVERY listener the client holds, so it must ++ // never be superseded: with two listeners (mcp-remote holds ++ // two), each eviction triggers the SDK's reconnect, which ++ // evicts the other listener, which reconnects — a permanent ++ // mutual-eviction storm. Standalone resumes attach alongside. ++ if (resumedStreamId !== STANDALONE_STREAM_ID) this.supersedePriorStreamConnections(agent, connection.id, resumedStreamId); connection.setState(resumeState); - await this.replayEvents(lastEventId); + const ackStreamIds = []; @@ -519,8 +549,18 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 return; } } -@@ -644,6 +883,26 @@ var StreamableHTTPServerTransport = class { - _standaloneSse: true +- this.supersedePriorStreamConnections(agent, connection.id, STANDALONE_STREAM_ID); ++ // A fresh listener attaches ALONGSIDE any existing ones — see the ++ // supersede note in the resume branch above for why evicting the ++ // incumbent (latest-listener-wins) storms with multi-listener ++ // clients. Dead listeners are reaped by bridge keepalive write ++ // failures and max-age rotation, and sendStandalone routes to the ++ // newest attachment. + const standaloneState = { + streamId: STANDALONE_STREAM_ID, +- _standaloneSse: true ++ _standaloneSse: true, ++ _attachedAt: Date.now() }; connection.setState(standaloneState); + const replayedStreamIds = await this.replayUndeliveredResponses(agent, connection); @@ -546,7 +586,17 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 } /** * Close any connection (other than `selfId`) currently bound to -@@ -664,12 +923,14 @@ var StreamableHTTPServerTransport = class { +@@ -651,6 +926,9 @@ var StreamableHTTPServerTransport = class { + * Closing rather than mutating sibling state mirrors how the SDK's + * single `_streamMapping` entry gives last-writer-wins for free, and + * keeps `send()` from routing to a stale bridge. ++ * ++ * Only ever called for single-owner POST streams. The shared ++ * STANDALONE_STREAM_ID must not go through here — see handleGetRequest. + */ + supersedePriorStreamConnections(agent, selfId, streamId) { + for (const other of agent.getConnections()) { +@@ -664,12 +942,14 @@ var StreamableHTTPServerTransport = class { * Only used when resumability is enabled */ async replayEvents(lastEventId) { @@ -562,7 +612,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 this.writeSSEEvent(connection, message, eventId); } catch (error) { this.onerror?.(error); -@@ -678,6 +939,33 @@ var StreamableHTTPServerTransport = class { +@@ -678,6 +958,44 @@ var StreamableHTTPServerTransport = class { } catch (error) { this.onerror?.(error); } @@ -578,9 +628,20 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 + async replayUndeliveredResponses(agent, connection, skipStreamId) { + const replayedStreamIds = []; + if (!this._eventStore?.replayEventsForStream) return replayedStreamIds; ++ const liveConnectionIds = new Set(Array.from(agent.getConnections(), (conn) => conn.id)); + const streamIds = await agent.getUndeliveredStreamIds(); + for (const streamId of streamIds) { + if (skipStreamId !== void 0 && streamId === skipStreamId) continue; ++ // Listeners now attach alongside each other, so two fresh GETs can ++ // arrive within milliseconds (a multi-listener client starting up) ++ // and both see the same undelivered markers. Without this guard ++ // each would replay the same response and the client would receive ++ // it twice. First live replayer wins; if it dies before its ack, ++ // its id drops out of getConnections() and a later GET replays the ++ // stream again — the at-least-once contract is preserved. ++ const replayingConnectionId = this._replayInFlight.get(streamId); ++ if (replayingConnectionId !== void 0 && replayingConnectionId !== connection.id && liveConnectionIds.has(replayingConnectionId)) continue; ++ this._replayInFlight.set(streamId, connection.id); + let replayedResponse = false; + await this._eventStore.replayEventsForStream(streamId, { send: async (eventId, message) => { + try { @@ -596,7 +657,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 } /** * Writes an event to the SSE stream with proper formatting -@@ -689,10 +977,65 @@ var StreamableHTTPServerTransport = class { +@@ -689,10 +1007,65 @@ var StreamableHTTPServerTransport = class { return connection.send(JSON.stringify({ type: "cf_mcp_agent_event", event: eventData, @@ -662,7 +723,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 * Handles POST requests containing JSON-RPC messages */ async handlePostRequest(req, parsedBody) { -@@ -733,6 +1076,22 @@ var StreamableHTTPServerTransport = class { +@@ -733,6 +1106,22 @@ var StreamableHTTPServerTransport = class { }; connection.setState(postState); if (this._eventStore) await agent.setStreamRequestIds(streamId, requestIds); @@ -685,7 +746,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 for (const message of messages) { if (this.messageInterceptor) { if (await this.messageInterceptor(message, { -@@ -760,7 +1119,22 @@ var StreamableHTTPServerTransport = class { +@@ -760,7 +1149,22 @@ var StreamableHTTPServerTransport = class { * when the originating WS has dropped. */ async sendOnStream(agent, streamId, relatedIds, liveConnection, message, requestId) { @@ -709,7 +770,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 let shouldClose = false; if (isJSONRPCResultResponse(message) || isJSONRPCErrorResponse(message)) { let responseIds = this._streamResponseIds.get(streamId); -@@ -777,9 +1151,11 @@ var StreamableHTTPServerTransport = class { +@@ -777,9 +1181,11 @@ var StreamableHTTPServerTransport = class { } catch (error) { this.onerror?.(error); } @@ -723,7 +784,31 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 } } async send(message, options) { -@@ -861,12 +1237,10 @@ var StreamableHTTPServerTransport = class { +@@ -798,14 +1204,19 @@ var StreamableHTTPServerTransport = class { + * + * Sent on exactly one stream, per MCP: "the server MUST send each of + * its JSON-RPC messages on only one of the connected streams; it MUST +- * NOT broadcast the same message across multiple streams." +- * `handleGetRequest` supersedes prior standalone connections, so +- * there is at most one to send on. ++ * NOT broadcast the same message across multiple streams." A session ++ * may hold several standalone listeners (they attach alongside each ++ * other); the newest attachment is the most likely to have a live ++ * client behind it, so the message goes there. + */ + async sendStandalone(message) { + const agent = this._agent; + const eventId = await this._eventStore?.storeEvent(STANDALONE_STREAM_ID, message); +- const standalone = Array.from(agent.getConnections()).find((conn) => conn.state?._standaloneSse); ++ let standalone; ++ for (const conn of agent.getConnections()) { ++ if (!conn.state?._standaloneSse) continue; ++ if (!standalone || (conn.state._attachedAt ?? 0) > (standalone.state?._attachedAt ?? 0)) standalone = conn; ++ } + if (standalone) this.writeSSEEvent(standalone, message, eventId); + } + /** +@@ -861,12 +1272,10 @@ var StreamableHTTPServerTransport = class { * * ## Lifecycle * @@ -740,7 +825,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 * * Standalone GET stream events (`_GET_stream`) are *not* cleared * automatically; they accumulate for the lifetime of the DO. Bounded -@@ -893,12 +1267,34 @@ var DurableObjectEventStore = class DurableObjectEventStore { +@@ -893,12 +1302,34 @@ var DurableObjectEventStore = class DurableObjectEventStore { } async storeEvent(streamId, message) { if (streamId.includes(":")) throw new Error(`DurableObjectEventStore: streamId must not contain ':' (got ${JSON.stringify(streamId)})`); @@ -775,7 +860,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 return eventId; } async getStreamIdForEventId(eventId) { -@@ -915,9 +1311,59 @@ var DurableObjectEventStore = class DurableObjectEventStore { +@@ -915,9 +1346,59 @@ var DurableObjectEventStore = class DurableObjectEventStore { start: startKey, limit: DurableObjectEventStore.REPLAY_LIMIT }); @@ -836,7 +921,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 /** * Drop the event log for a single stream. Called by the transport * immediately after a POST's final response has been written to the -@@ -973,6 +1419,13 @@ DurableObjectEventStore.EVENT_KEY_PREFIX = "__mcp_event__:"; +@@ -973,6 +1454,13 @@ DurableObjectEventStore.EVENT_KEY_PREFIX = "__mcp_event__:"; DurableObjectEventStore.SEQ_PAD = 16; DurableObjectEventStore.DELETE_CHUNK = 128; DurableObjectEventStore.REPLAY_LIMIT = 1e3; @@ -850,7 +935,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 //#endregion //#region src/mcp/client-transports.ts /** -@@ -1381,6 +1834,47 @@ var McpAgent = class McpAgent extends Agent { +@@ -1381,6 +1869,48 @@ var McpAgent = class McpAgent extends Agent { async deleteStreamRequestIds(streamId) { await this.ctx.storage.delete(`${McpAgent.STREAM_REQS_KEY_PREFIX}${streamId}`); } @@ -881,6 +966,7 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 + async acknowledgeDeliveredStream(streamId) { + await this.deleteStreamRequestIds(streamId); + await this.deleteUndeliveredStream(streamId); ++ this._transport?.["_replayInFlight"]?.delete(streamId); + const eventStore = this._transport?.["_eventStore"]; + if (eventStore && isClearableEventStore(eventStore)) await eventStore.clearStream(streamId); + } @@ -898,7 +984,59 @@ index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..444312ff41fbcff60e2ea681a9d5c756 /** * Reverse lookup: find which POST stream a given `requestId` belongs * to, and return the stream's full `requestIds` list in the same -@@ -1697,7 +2191,8 @@ var McpAgent = class McpAgent extends Agent { +@@ -1516,23 +2046,36 @@ var McpAgent = class McpAgent extends Agent { + return; + } + break; +- case "streamable-http": if (this._transport instanceof StreamableHTTPServerTransport) switch (req.headers.get(MCP_HTTP_METHOD_HEADER)) { +- case "POST": { +- const payloadHeader = req.headers.get(MCP_MESSAGE_HEADER); +- let rawPayload; +- if (!payloadHeader) rawPayload = "{}"; +- else try { +- rawPayload = Buffer.from(payloadHeader, "base64").toString("utf-8"); +- } catch (_error) { +- throw new Error("Internal Server Error: Failed to decode MCP message header"); ++ case "streamable-http": { ++ if (!(this._transport instanceof StreamableHTTPServerTransport)) { ++ // onStart failed or has not completed: without a transport ++ // the request is never dispatched, so the client would sit ++ // on a silent stream until its own timeout (~30s) and the ++ // session would look bricked. Close the WS so the worker ++ // surfaces an error immediately and the client's retry ++ // lands on a restarted DO that reruns onStart. ++ conn.close(1011, "MCP transport not initialized"); ++ return; ++ } ++ switch (req.headers.get(MCP_HTTP_METHOD_HEADER)) { ++ case "POST": { ++ const payloadHeader = req.headers.get(MCP_MESSAGE_HEADER); ++ let rawPayload; ++ if (!payloadHeader) rawPayload = "{}"; ++ else try { ++ rawPayload = Buffer.from(payloadHeader, "base64").toString("utf-8"); ++ } catch (_error) { ++ throw new Error("Internal Server Error: Failed to decode MCP message header"); ++ } ++ const parsedBody = JSON.parse(rawPayload); ++ this._transport.handlePostRequest(req, parsedBody); ++ break; + } +- const parsedBody = JSON.parse(rawPayload); +- this._transport?.handlePostRequest(req, parsedBody); +- break; ++ case "GET": ++ this._transport.handleGetRequest(req); ++ break; + } +- case "GET": +- this._transport?.handleGetRequest(req); +- break; ++ break; + } + } + } +@@ -1697,7 +2240,8 @@ var McpAgent = class McpAgent extends Agent { } }; McpAgent.STREAM_REQS_KEY_PREFIX = "__mcp_stream_reqs__:"; From 7c12aeea390225291ce4c97865b392237ee7934d Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:40:13 -0700 Subject: [PATCH 6/6] Make Microsoft Graph slice URLs first-class spec sources (#1755) --- .changeset/graph-slice-urls-first-class.md | 5 ++ e2e/scenarios/microsoft-graph-default.test.ts | 6 +- e2e/scenarios/microsoft-graph-full.test.ts | 4 +- e2e/scenarios/provider-plugins-ui.test.ts | 6 +- .../openapi/src/providers/microsoft/graph.ts | 48 ++++++----- .../openapi/src/providers/microsoft/index.ts | 8 ++ .../src/providers/microsoft/presets.ts | 8 +- .../src/providers/microsoft/slice-urls.ts | 37 +++++++++ .../src/providers/microsoft/slices.test.ts | 80 ++++++------------- .../openapi/src/providers/microsoft/slices.ts | 69 ++++++---------- .../microsoft/spec-format-adapter.test.ts | 56 ++++++++++--- .../microsoft/spec-format-adapter.ts | 31 +++++-- 12 files changed, 214 insertions(+), 144 deletions(-) create mode 100644 .changeset/graph-slice-urls-first-class.md create mode 100644 packages/plugins/openapi/src/providers/microsoft/slice-urls.ts diff --git a/.changeset/graph-slice-urls-first-class.md b/.changeset/graph-slice-urls-first-class.md new file mode 100644 index 0000000000..fd2ba2e367 --- /dev/null +++ b/.changeset/graph-slice-urls-first-class.md @@ -0,0 +1,5 @@ +--- +"@executor-js/plugin-openapi": patch +--- + +Make Microsoft Graph slice URLs first-class spec sources instead of a hidden substitution. Catalog tiles now point directly at the slice release assets, the stored specUrl is exactly what gets fetched, and selection narrowing travels visibly in the URL fragment; requesting the upstream monolith URL fetches the monolith, never a silently swapped slice. diff --git a/e2e/scenarios/microsoft-graph-default.test.ts b/e2e/scenarios/microsoft-graph-default.test.ts index 499a25af92..f0dc33acab 100644 --- a/e2e/scenarios/microsoft-graph-default.test.ts +++ b/e2e/scenarios/microsoft-graph-default.test.ts @@ -6,10 +6,10 @@ import { composePluginApi } from "@executor-js/api/server"; import { MICROSOFT_AUTH_TEMPLATE_SLUG, MICROSOFT_AUTHORIZATION_URL, - MICROSOFT_GRAPH_OPENAPI_URL, MICROSOFT_TOKEN_URL, microsoftCatalog, microsoftGraphAdapter, + microsoftGraphSliceUrl, } from "@executor-js/plugin-openapi/providers/microsoft"; import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; import { @@ -60,7 +60,7 @@ scenario( // isolate the same way it did the production one. const preview = yield* client.openapi.previewSpec({ payload: { - spec: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`, + spec: microsoftGraphSliceUrl(MICROSOFT_FILES_PRESET_ID), specFormat: "microsoft-graph", }, }); @@ -77,7 +77,7 @@ scenario( payload: { spec: { kind: "url", - url: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`, + url: microsoftGraphSliceUrl(MICROSOFT_FILES_PRESET_ID), }, slug: integration, name: "Microsoft Graph Files", diff --git a/e2e/scenarios/microsoft-graph-full.test.ts b/e2e/scenarios/microsoft-graph-full.test.ts index bba32a2c2a..527b0ddbc3 100644 --- a/e2e/scenarios/microsoft-graph-full.test.ts +++ b/e2e/scenarios/microsoft-graph-full.test.ts @@ -5,9 +5,9 @@ import { Effect } from "effect"; import { composePluginApi } from "@executor-js/api/server"; import { MICROSOFT_AUTH_TEMPLATE_SLUG, - MICROSOFT_GRAPH_OPENAPI_URL, microsoftCatalog, microsoftGraphAdapter, + microsoftGraphSliceUrl, } from "@executor-js/plugin-openapi/providers/microsoft"; import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; @@ -57,7 +57,7 @@ scenario( payload: { spec: { kind: "url", - url: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`, + url: microsoftGraphSliceUrl(MICROSOFT_FILES_PRESET_ID), }, slug: integration, name: "Microsoft Graph Files", diff --git a/e2e/scenarios/provider-plugins-ui.test.ts b/e2e/scenarios/provider-plugins-ui.test.ts index add10ccd15..fd8dda0efa 100644 --- a/e2e/scenarios/provider-plugins-ui.test.ts +++ b/e2e/scenarios/provider-plugins-ui.test.ts @@ -57,11 +57,13 @@ scenario( await step("A Microsoft service preset opens the OpenAPI add flow", async () => { await page.goto( - "/integrations/add/openapi?preset=microsoft-files&url=https%3A%2F%2Fraw.githubusercontent.com%2Fmicrosoftgraph%2Fmsgraph-metadata%2Fmaster%2Fopenapi%2Fv1.0%2Fopenapi.yaml%23preset%3Dfiles", + "/integrations/add/openapi?preset=microsoft-files&url=https%3A%2F%2Fgithub.com%2FUsefulSoftwareCo%2Fexecutor%2Freleases%2Fdownload%2Fgraph-slices%2Ffiles.yaml", { waitUntil: "domcontentloaded" }, ); await page.getByRole("heading", { name: "Add OpenAPI integration" }).waitFor(); - await expect.poll(() => page.locator("textarea").inputValue()).toContain("preset=files"); + await expect + .poll(() => page.locator("textarea").inputValue()) + .toContain("graph-slices/files.yaml"); }); }); }), diff --git a/packages/plugins/openapi/src/providers/microsoft/graph.ts b/packages/plugins/openapi/src/providers/microsoft/graph.ts index 36ac23dc4f..ebb9d900fe 100644 --- a/packages/plugins/openapi/src/providers/microsoft/graph.ts +++ b/packages/plugins/openapi/src/providers/microsoft/graph.ts @@ -15,7 +15,11 @@ import { } from "../../sdk/split"; import type { Authentication } from "../../sdk/types"; -import { fetchMicrosoftGraphSlice, microsoftGraphSliceAssetForSelection } from "./slices"; +import { + fetchMicrosoftGraphSlice, + microsoftGraphPresetIdsForSliceAsset, + microsoftGraphSliceAssetFromUrl, +} from "./slices"; import { MICROSOFT_AUTHORIZATION_URL, MICROSOFT_AUTH_TEMPLATE_SLUG, @@ -244,6 +248,10 @@ const normalizeMicrosoftGraphSpecUrl = ( policy?: MicrosoftGraphUrlPolicy, ): string | null => { if (value === MICROSOFT_GRAPH_OPENAPI_URL) return value; + const sliceAsset = microsoftGraphSliceAssetFromUrl(value); + if (sliceAsset !== null && microsoftGraphPresetIdsForSliceAsset(sliceAsset) !== null) { + return value; + } return allowUnsafeUrl(value, policy) ?? null; }; @@ -759,26 +767,26 @@ export const buildMicrosoftGraphOpenApiSpec = ( urlPolicy?: MicrosoftGraphUrlPolicy, ): Effect.Effect => Effect.gen(function* () { - const selection = yield* validateSelectionUrls(normalizeSelection(input), urlPolicy); - // Covered selections read a precomputed slice (sub-MB) instead of the 43MB - // monolith: in production, the monolith fetch alone almost never survives - // the 128MB isolate (once in the 30 days before 2026-08-26). Slices apply - // only to the pinned Microsoft URL — an override (local Graph emulators) - // serves its own document. A missing/failed slice (asset not yet published, - // release unreachable) falls back to the monolith path, which is the prior - // behavior for the selections a slice would have covered. - const sliceAsset = - selection.specUrl === MICROSOFT_GRAPH_OPENAPI_URL - ? microsoftGraphSliceAssetForSelection(selection) - : null; + // A slice URL carries its own selection: when the caller passes no preset + // ids, the asset's selection applies (rather than the default bundle). + const inputSliceAsset = input.specUrl + ? microsoftGraphSliceAssetFromUrl(input.specUrl.trim()) + : null; + const inputSlicePresetIds = + inputSliceAsset !== null ? microsoftGraphPresetIdsForSliceAsset(inputSliceAsset) : null; + const selectionInput = + inputSlicePresetIds !== null && (!input.presetIds || input.presetIds.length === 0) + ? { ...input, presetIds: inputSlicePresetIds } + : input; + const selection = yield* validateSelectionUrls(normalizeSelection(selectionInput), urlPolicy); + // The URL is the byte source, never substituted. Catalog selections point + // at precomputed slice URLs (the 43MB monolith cannot be processed in a + // 128MB isolate — its fetch completed once in the 30 days before + // 2026-08-26); the monolith and emulator-override URLs fetch exactly what + // they name. const sourceText = - sliceAsset !== null - ? yield* fetchMicrosoftGraphSlice(sliceAsset).pipe( - Effect.catchTag("OpenApiParseError", () => - fetchMicrosoftGraphOpenApiSpec(selection.specUrl), - ), - Effect.provide(httpClientLayer), - ) + microsoftGraphSliceAssetFromUrl(selection.specUrl) !== null + ? yield* fetchMicrosoftGraphSlice(selection.specUrl).pipe(Effect.provide(httpClientLayer)) : yield* fetchMicrosoftGraphOpenApiSpec(selection.specUrl).pipe( Effect.provide(httpClientLayer), ); diff --git a/packages/plugins/openapi/src/providers/microsoft/index.ts b/packages/plugins/openapi/src/providers/microsoft/index.ts index fe51454d3b..c9b7bacfc3 100644 --- a/packages/plugins/openapi/src/providers/microsoft/index.ts +++ b/packages/plugins/openapi/src/providers/microsoft/index.ts @@ -39,3 +39,11 @@ export { type MicrosoftGraphSpecBuild, } from "./graph"; export { microsoftGraphAdapter } from "./spec-format-adapter"; +export { + MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET, + MICROSOFT_GRAPH_SLICE_BASE_URL, + MICROSOFT_GRAPH_SLICE_RELEASE_TAG, + microsoftGraphPresetIdsForSliceAsset, + microsoftGraphSliceAssetFromUrl, + microsoftGraphSliceUrl, +} from "./slices"; diff --git a/packages/plugins/openapi/src/providers/microsoft/presets.ts b/packages/plugins/openapi/src/providers/microsoft/presets.ts index b1dabc9e2c..33cbcf0e6c 100644 --- a/packages/plugins/openapi/src/providers/microsoft/presets.ts +++ b/packages/plugins/openapi/src/providers/microsoft/presets.ts @@ -1,5 +1,7 @@ import type { IntegrationPreset } from "@executor-js/sdk/core"; +import { microsoftGraphSliceUrl } from "./slice-urls"; + export interface MicrosoftGraphPreset { readonly id: string; readonly name: string; @@ -528,8 +530,10 @@ export const microsoftGraphTagPrefixesForPresetIds = ( export const microsoftServiceSlug = (presetId: string): string => `microsoft_${presetId.replaceAll("-", "_")}`; -const microsoftGraphCatalogUrl = (presetId: string): string => - `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${encodeURIComponent(presetId)}`; +// Catalog tiles point at the slice URL itself: the URL a user sees (and the +// integration stores) is exactly what gets fetched — no server-side source +// substitution. The slice asset name carries the selection. +const microsoftGraphCatalogUrl = (presetId: string): string => microsoftGraphSliceUrl(presetId); const microsoftGraphCatalogAuthTemplate = (preset: MicrosoftGraphScopePreset) => [ { diff --git a/packages/plugins/openapi/src/providers/microsoft/slice-urls.ts b/packages/plugins/openapi/src/providers/microsoft/slice-urls.ts new file mode 100644 index 0000000000..1bd4c3e0b0 --- /dev/null +++ b/packages/plugins/openapi/src/providers/microsoft/slice-urls.ts @@ -0,0 +1,37 @@ +/** + * Microsoft Graph slice URL vocabulary. Leaf module (no preset imports) shared + * by the catalog (`presets.ts`), the runtime fetch (`slices.ts`), and the + * adapter's URL classification. + * + * Slice URLs are first-class spec sources: the catalog points at them + * directly, what the integration stores as `specUrl` is what was fetched, and + * any narrowing within a slice travels visibly in the URL fragment + * (`#preset=mail,calendar`). The 43MB upstream monolith URL is never silently + * substituted — requesting it fetches it. + */ + +export const MICROSOFT_GRAPH_SLICE_RELEASE_TAG = "graph-slices"; + +export const MICROSOFT_GRAPH_SLICE_BASE_URL = `https://github.com/UsefulSoftwareCo/executor/releases/download/${MICROSOFT_GRAPH_SLICE_RELEASE_TAG}`; + +/** Asset covering the default catalog bundle (`MICROSOFT_GRAPH_DEFAULT_PRESET_IDS`). */ +export const MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET = "default"; + +export const microsoftGraphSliceUrl = (asset: string): string => + `${MICROSOFT_GRAPH_SLICE_BASE_URL}/${encodeURIComponent(asset)}.yaml`; + +/** The asset a slice URL names, or null for any other URL. Fragment and query + * are ignored — callers strip the fragment into a selection separately. */ +export const microsoftGraphSliceAssetFromUrl = (url: string): string | null => { + if (!URL.canParse(url)) return null; + const parsed = new URL(url); + parsed.hash = ""; + const href = parsed.toString(); + if (!href.startsWith(`${MICROSOFT_GRAPH_SLICE_BASE_URL}/`) || !href.endsWith(".yaml")) { + return null; + } + const asset = decodeURIComponent( + href.slice(`${MICROSOFT_GRAPH_SLICE_BASE_URL}/`.length, -".yaml".length), + ); + return asset.length > 0 && !asset.includes("/") ? asset : null; +}; diff --git a/packages/plugins/openapi/src/providers/microsoft/slices.test.ts b/packages/plugins/openapi/src/providers/microsoft/slices.test.ts index 6f94a2fceb..b04fb0b9d7 100644 --- a/packages/plugins/openapi/src/providers/microsoft/slices.test.ts +++ b/packages/plugins/openapi/src/providers/microsoft/slices.test.ts @@ -1,70 +1,40 @@ import { describe, expect, it } from "@effect/vitest"; -import { MICROSOFT_GRAPH_ALL_PRESET_IDS, MICROSOFT_GRAPH_DEFAULT_PRESET_IDS } from "./presets"; +import { MICROSOFT_GRAPH_DEFAULT_PRESET_IDS, MICROSOFT_GRAPH_OPENAPI_URL } from "./presets"; import { MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET, - microsoftGraphSliceAssetForSelection, + microsoftGraphPresetIdsForSliceAsset, + microsoftGraphSliceAssetFromUrl, + microsoftGraphSliceUrl, } from "./slices"; -describe("microsoftGraphSliceAssetForSelection", () => { - it("maps a single catalog preset to its asset", () => { +describe("microsoftGraphSliceAssetFromUrl", () => { + it("round-trips slice URLs and ignores fragments", () => { + expect(microsoftGraphSliceAssetFromUrl(microsoftGraphSliceUrl("mail"))).toBe("mail"); expect( - microsoftGraphSliceAssetForSelection({ - coversFullGraph: false, - presetIds: ["mail"], - customScopes: [], - }), - ).toBe("mail"); + microsoftGraphSliceAssetFromUrl(`${microsoftGraphSliceUrl("default")}#preset=mail,calendar`), + ).toBe(MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET); }); - it("maps the default bundle in any order to the default asset", () => { - expect( - microsoftGraphSliceAssetForSelection({ - coversFullGraph: false, - presetIds: [...MICROSOFT_GRAPH_DEFAULT_PRESET_IDS].reverse(), - customScopes: [], - }), - ).toBe(MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET); + it("rejects non-slice URLs", () => { + expect(microsoftGraphSliceAssetFromUrl(MICROSOFT_GRAPH_OPENAPI_URL)).toBeNull(); + expect(microsoftGraphSliceAssetFromUrl("https://example.com/mail.yaml")).toBeNull(); + expect(microsoftGraphSliceAssetFromUrl("not a url")).toBeNull(); }); +}); - it("serves combinations within the default bundle from the default slice", () => { - expect( - microsoftGraphSliceAssetForSelection({ - coversFullGraph: false, - presetIds: ["mail", "calendar"], - customScopes: [], - }), - ).toBe(MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET); +describe("microsoftGraphPresetIdsForSliceAsset", () => { + it("maps a preset asset to its single preset", () => { + expect(microsoftGraphPresetIdsForSliceAsset("mail")).toEqual(["mail"]); }); - it("needs the monolith for full-graph, custom scopes, unknown presets, and combinations outside the default bundle", () => { - expect( - microsoftGraphSliceAssetForSelection({ - coversFullGraph: true, - presetIds: [...MICROSOFT_GRAPH_ALL_PRESET_IDS], - customScopes: [], - }), - ).toBeNull(); - expect( - microsoftGraphSliceAssetForSelection({ - coversFullGraph: false, - presetIds: ["mail"], - customScopes: ["Chat.Read"], - }), - ).toBeNull(); - expect( - microsoftGraphSliceAssetForSelection({ - coversFullGraph: false, - presetIds: ["not-a-preset"], - customScopes: [], - }), - ).toBeNull(); - expect( - microsoftGraphSliceAssetForSelection({ - coversFullGraph: false, - presetIds: ["mail", "users"], - customScopes: [], - }), - ).toBeNull(); + it("maps the default asset to the default bundle", () => { + expect(microsoftGraphPresetIdsForSliceAsset(MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET)).toEqual( + MICROSOFT_GRAPH_DEFAULT_PRESET_IDS, + ); + }); + + it("returns null for unknown assets", () => { + expect(microsoftGraphPresetIdsForSliceAsset("not-a-preset")).toBeNull(); }); }); diff --git a/packages/plugins/openapi/src/providers/microsoft/slices.ts b/packages/plugins/openapi/src/providers/microsoft/slices.ts index f129e36740..724ec2ffce 100644 --- a/packages/plugins/openapi/src/providers/microsoft/slices.ts +++ b/packages/plugins/openapi/src/providers/microsoft/slices.ts @@ -4,62 +4,41 @@ import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { OpenApiParseError } from "../../sdk/errors"; import { MICROSOFT_GRAPH_DEFAULT_PRESET_IDS, microsoftGraphPresetForId } from "./presets"; +import { MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET } from "./slice-urls"; + +export { + MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET, + MICROSOFT_GRAPH_SLICE_BASE_URL, + MICROSOFT_GRAPH_SLICE_RELEASE_TAG, + microsoftGraphSliceAssetFromUrl, + microsoftGraphSliceUrl, +} from "./slice-urls"; /** * Runtime access to precomputed Microsoft Graph slices. * * The 43MB Graph monolith cannot be processed in a 128MB Workers isolate — in - * production its fetch alone completed once in the 30 days before 2026-08-26; - * every other preview/add died mid-download with an empty 503. Slices are - * built offline (`slice-build.ts`, refreshed by the graph-slices workflow), - * published as release assets, and fetched here per selection: the isolate - * only ever holds the sub-megabyte filtered document for the selection. + * production its fetch alone completed once in the 30 days before 2026-08-26. + * Slices are built offline (`slice-build.ts`, refreshed by the graph-slices + * workflow) and published as release assets that the catalog references as + * ordinary spec URLs; the isolate only ever holds the filtered document it was + * actually asked to fetch. */ -export const MICROSOFT_GRAPH_SLICE_RELEASE_TAG = "graph-slices"; - -export const MICROSOFT_GRAPH_SLICE_BASE_URL = `https://github.com/UsefulSoftwareCo/executor/releases/download/${MICROSOFT_GRAPH_SLICE_RELEASE_TAG}`; - -/** Asset name for the default catalog bundle (`MICROSOFT_GRAPH_DEFAULT_PRESET_IDS`). */ -export const MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET = "default"; - -/** - * The published asset covering a selection, or null when the selection needs - * the monolith. A slice may be a superset of the selection: the runtime always - * applies the selection's `keepPathItem` filter to whatever source it reads, so - * any combination within the default bundle can be served from the default - * slice and narrowed in-band. The monolith remains necessary for full-graph - * coverage, custom scopes (scope matching walks operations outside any - * preset's paths), and combinations reaching outside the default bundle - * (precomputing every combination is combinatorial). - */ -export const microsoftGraphSliceAssetForSelection = (selection: { - readonly coversFullGraph: boolean; - readonly presetIds: readonly string[]; - readonly customScopes: readonly string[]; -}): string | null => { - if (selection.coversFullGraph) return null; - if (selection.customScopes.length > 0) return null; - if (selection.presetIds.length === 0) return null; - if (selection.presetIds.some((presetId) => !microsoftGraphPresetForId(presetId))) return null; - if (selection.presetIds.length === 1) return selection.presetIds[0]!; - const defaultIds = new Set(MICROSOFT_GRAPH_DEFAULT_PRESET_IDS); - if (selection.presetIds.every((presetId) => defaultIds.has(presetId))) { - return MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET; - } - return null; +/** The preset selection a slice asset carries when the URL has no narrowing + * fragment, or null for an unknown asset. */ +export const microsoftGraphPresetIdsForSliceAsset = (asset: string): readonly string[] | null => { + if (asset === MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET) return MICROSOFT_GRAPH_DEFAULT_PRESET_IDS; + return microsoftGraphPresetForId(asset) ? [asset] : null; }; -export const microsoftGraphSliceUrl = (asset: string): string => - `${MICROSOFT_GRAPH_SLICE_BASE_URL}/${encodeURIComponent(asset)}.yaml`; - export const fetchMicrosoftGraphSlice = Effect.fn("Microsoft.fetchGraphSlice")(function* ( - asset: string, + sliceUrl: string, ) { const client = yield* HttpClient.HttpClient; const response = yield* client .execute( - HttpClientRequest.get(microsoftGraphSliceUrl(asset)).pipe( + HttpClientRequest.get(sliceUrl).pipe( HttpClientRequest.setHeader("Accept", "application/yaml, text/yaml, */*"), ), ) @@ -67,20 +46,20 @@ export const fetchMicrosoftGraphSlice = Effect.fn("Microsoft.fetchGraphSlice")(f Effect.mapError( () => new OpenApiParseError({ - message: `Failed to fetch Microsoft Graph slice: ${asset}`, + message: "Failed to fetch the Microsoft Graph slice document", }), ), ); if (response.status < 200 || response.status >= 300) { return yield* new OpenApiParseError({ - message: `Failed to fetch Microsoft Graph slice ${asset}: HTTP ${response.status}`, + message: `Failed to fetch the Microsoft Graph slice document: HTTP ${response.status}`, }); } return yield* response.text.pipe( Effect.mapError( () => new OpenApiParseError({ - message: `Failed to read Microsoft Graph slice body: ${asset}`, + message: "Failed to read the Microsoft Graph slice document body", }), ), ); diff --git a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts index 44c09c65d9..9db13e8a06 100644 --- a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts +++ b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts @@ -139,31 +139,67 @@ const sliceAwareHttpClientLayer = Layer.succeed(HttpClient.HttpClient)( }), ); -it.effect("reads the published slice for a covered selection", () => +it.effect("fetches a slice URL as the byte source and derives its selection from the asset", () => Effect.gen(function* () { const converted = yield* microsoftGraphAdapter.fetch({ - urls: [`${MICROSOFT_GRAPH_OPENAPI_URL}#preset=profile`], + urls: [microsoftGraphSliceUrl("profile")], httpClientLayer: sliceAwareHttpClientLayer, }); expect(converted.specText).toBe(sliceFixture); - // The catalog URL (with fragment stripped) stays canonical so refresh - // re-resolves through the adapter, not the slice hosting. - expect(converted.specUrl).toBe(MICROSOFT_GRAPH_OPENAPI_URL); + // What is stored is what was fetched — no source substitution. + expect(converted.specUrl).toBe(microsoftGraphSliceUrl("profile")); + // The asset name carries the selection even with no fragment. + const keepPathItem = converted.keepPathItem!; + expect(keepPathItem("/me", { get: { operationId: "me.GetUser" } })).toEqual({ + get: { operationId: "me.GetUser" }, + }); + expect(keepPathItem("/irrelevant", { get: { operationId: "irrelevant.Get" } })).toBeNull(); + }), +); + +it.effect("narrows within a slice via the URL fragment", () => + Effect.gen(function* () { + const layer = Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response( + request.url === microsoftGraphSliceUrl("default") ? sliceFixture : "not found", + { status: request.url === microsoftGraphSliceUrl("default") ? 200 : 404 }, + ), + ), + ), + ), + ); + const converted = yield* microsoftGraphAdapter.fetch({ + urls: [`${microsoftGraphSliceUrl("default")}#preset=profile`], + httpClientLayer: layer, + }); + + expect(converted.specText).toBe(sliceFixture); + expect(converted.specUrl).toBe(microsoftGraphSliceUrl("default")); + const keepPathItem = converted.keepPathItem!; + // Profile keeps /me; a mail-only path from the default bundle is dropped. + expect(keepPathItem("/me", { get: { operationId: "me.GetUser" } })).toEqual({ + get: { operationId: "me.GetUser" }, + }); + expect(keepPathItem("/me/messages", { post: { operationId: "me.CreateMessage" } })).toBeNull(); }), ); -it.effect("falls back to the monolith when the slice asset is unavailable", () => +it.effect("fetches the monolith URL as given — no silent slice substitution", () => Effect.gen(function* () { - // graphHttpClientLayer 404s everything except the monolith URL, including - // the slice URL — the existing selection tests above exercise this same - // fallback implicitly. + // sliceAwareHttpClientLayer serves both URLs; requesting the monolith with + // a preset fragment must read the monolith bytes. const converted = yield* microsoftGraphAdapter.fetch({ urls: [`${MICROSOFT_GRAPH_OPENAPI_URL}#preset=profile`], - httpClientLayer: graphHttpClientLayer, + httpClientLayer: sliceAwareHttpClientLayer, }); expect(converted.specText).toBe(graphFixture); + expect(converted.specUrl).toBe(MICROSOFT_GRAPH_OPENAPI_URL); }), ); diff --git a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.ts b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.ts index 601b9607a1..16c762dc01 100644 --- a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.ts +++ b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.ts @@ -3,19 +3,40 @@ import { Effect } from "effect"; import type { SpecFormatAdapter } from "../../sdk/spec-format"; import { buildMicrosoftGraphOpenApiSpec, microsoftGraphKeepPathItem } from "./graph"; +import { microsoftGraphPresetIdsForSliceAsset, microsoftGraphSliceAssetFromUrl } from "./slices"; +const fragmentPresetIds = (hash: string): readonly string[] => + hash.startsWith("#preset=") + ? decodeURIComponent(hash.slice("#preset=".length)) + .split(",") + .map((presetId) => presetId.trim()) + .filter((presetId) => presetId.length > 0) + : []; + +/** + * Selection from a catalog URL. A slice URL is the byte source itself: its + * asset carries the selection, and a fragment may narrow within the asset. + * Any other URL (the upstream monolith, an emulator override) is fetched as + * given, with the fragment as the selection filter. + */ const graphCatalogSelection = ( rawUrl: string | undefined, ): { readonly specUrl?: string; readonly presetIds?: readonly string[] } => { if (!rawUrl || !URL.canParse(rawUrl)) return rawUrl ? { specUrl: rawUrl } : {}; const parsed = new URL(rawUrl); - const preset = parsed.hash.startsWith("#preset=") - ? decodeURIComponent(parsed.hash.slice("#preset=".length)) - : ""; + const fromFragment = fragmentPresetIds(parsed.hash); parsed.hash = ""; + const specUrl = parsed.toString(); + const sliceAsset = microsoftGraphSliceAssetFromUrl(specUrl); + const presetIds = + fromFragment.length > 0 + ? fromFragment + : sliceAsset !== null + ? (microsoftGraphPresetIdsForSliceAsset(sliceAsset) ?? []) + : []; return { - specUrl: parsed.toString(), - ...(preset.length > 0 ? { presetIds: [preset] } : {}), + specUrl, + ...(presetIds.length > 0 ? { presetIds } : {}), }; };