diff --git a/packages/cli/cli/src/cli.ts b/packages/cli/cli/src/cli.ts index a66b578cae0a..2afab04fd035 100644 --- a/packages/cli/cli/src/cli.ts +++ b/packages/cli/cli/src/cli.ts @@ -1020,7 +1020,8 @@ function addGenerateCommand(cli: Argv, cliContext: CliContext) return await generateAPIWorkspaces({ project: await loadProjectAndRegisterWorkspacesWithContext(cliContext, { commandLineApiWorkspace: argv.api, - defaultToAllApiWorkspaces: false + defaultToAllApiWorkspaces: false, + skipApiWorkspaces: argv.sdkConfig != null }), cliContext, version: argv.version, @@ -1087,7 +1088,8 @@ function addGenerateCommand(cli: Argv, cliContext: CliContext) return await generateAPIWorkspaces({ project: await loadProjectAndRegisterWorkspacesWithContext(cliContext, { commandLineApiWorkspace: argv.api, - defaultToAllApiWorkspaces: false + defaultToAllApiWorkspaces: false, + skipApiWorkspaces: argv.sdkConfig != null }), cliContext, version: argv.version, @@ -2803,7 +2805,7 @@ function addSdkCommand(cli: Argv, cliContext: CliContext) { function addSdkMigrateCommand(cli: Argv, cliContext: CliContext): void { cli.command( "migrate", - "Create a Postman SDK Config v1 file from one or more resolved Fern SDK groups", + "Create an SDK Config v1 file from one or more resolved Fern SDK groups", (yargs) => yargs .option("group", { diff --git a/packages/cli/cli/src/commands/generate/__test__/createFernSourceArchiveResolver.test.ts b/packages/cli/cli/src/commands/generate/__test__/createFernSourceArchiveResolver.test.ts index 370570f36b0e..0aa55c96c906 100644 --- a/packages/cli/cli/src/commands/generate/__test__/createFernSourceArchiveResolver.test.ts +++ b/packages/cli/cli/src/commands/generate/__test__/createFernSourceArchiveResolver.test.ts @@ -1,7 +1,10 @@ import { generatorsYml } from "@fern-api/configuration-loader"; import { AbsoluteFilePath, RelativeFilePath } from "@fern-api/fs-utils"; import { ConjureWorkspace, OSSWorkspace } from "@fern-api/lazy-fern-workspace"; -import { createGroupedSpecsTarGzArchiveSettled } from "@fern-api/local-workspace-runner"; +import { + createGroupedSpecsTarGzArchiveSettled, + validateSdkConfigImportSettings +} from "@fern-api/local-workspace-runner"; import { type FernSourceArchiveRequest } from "@fern-api/remote-workspace-runner"; import { createMockTaskContext } from "@fern-api/task-context"; import { FernFiddle } from "@fern-fern/fiddle-sdk"; @@ -11,7 +14,8 @@ import { createFernSourceArchiveResolver } from "../createFernSourceArchiveResol vi.mock("@fern-api/local-workspace-runner", async (importOriginal) => ({ ...(await importOriginal()), - createGroupedSpecsTarGzArchiveSettled: vi.fn() + createGroupedSpecsTarGzArchiveSettled: vi.fn(), + validateSdkConfigImportSettings: vi.fn() })); function makeGenerator(): generatorsYml.GeneratorInvocation { @@ -39,6 +43,7 @@ function makeGenerator(): generatorsYml.GeneratorInvocation { describe("createFernSourceArchiveResolver", () => { beforeEach(() => { vi.mocked(createGroupedSpecsTarGzArchiveSettled).mockReset(); + vi.mocked(validateSdkConfigImportSettings).mockReset(); }); it("returns an actionable error when the workspace cannot expose source specs", async () => { @@ -129,7 +134,7 @@ describe("createFernSourceArchiveResolver", () => { sdkName: "api", sdkVersion: "1.0.0", audiences: [], - targets: [{ language: "typescript" }] + targets: [{ language: "typescript", clientPathParameterStyle: "wrapped" }] } })([request]) ).rejects.toMatchObject({ @@ -139,5 +144,8 @@ describe("createFernSourceArchiveResolver", () => { expect(createGroupedSpecsTarGzArchiveSettled).toHaveBeenCalledWith( expect.objectContaining({ audiences: { type: "select", audiences: [] } }) ); + expect(validateSdkConfigImportSettings).toHaveBeenCalledWith([], { + clientPathParameterStyle: "wrapped" + }); }); }); diff --git a/packages/cli/cli/src/commands/generate/__test__/createSdkConfigWorkspace.test.ts b/packages/cli/cli/src/commands/generate/__test__/createSdkConfigWorkspace.test.ts new file mode 100644 index 000000000000..499cf32e9beb --- /dev/null +++ b/packages/cli/cli/src/commands/generate/__test__/createSdkConfigWorkspace.test.ts @@ -0,0 +1,222 @@ +import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { getLatestGeneratorVersion } from "@fern-api/configuration-loader"; +import { bundleRemoteOpenAPI } from "@fern-api/lazy-fern-workspace"; +import { createMockTaskContext } from "@fern-api/task-context"; +import { parseSdkConfigV1 } from "@postman/sdk-config/sdk-config/v1"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createSdkConfigWorkspace } from "../createSdkConfigWorkspace.js"; + +vi.mock("@fern-api/configuration-loader", async (importOriginal) => ({ + ...(await importOriginal()), + getLatestGeneratorVersion: vi.fn() +})); + +vi.mock("@fern-api/lazy-fern-workspace", async (importOriginal) => ({ + ...(await importOriginal()), + bundleRemoteOpenAPI: vi.fn() +})); + +describe("createSdkConfigWorkspace", () => { + const temporaryDirectories: string[] = []; + + afterEach(async () => { + vi.resetAllMocks(); + vi.unstubAllGlobals(); + await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true }))); + }); + + it("constructs sources and resolves an omitted generator version without generators.yml", async () => { + vi.mocked(getLatestGeneratorVersion).mockResolvedValue("4.1.0"); + const directory = await mkdtemp(path.join(tmpdir(), "fern-sdk-config-workspace-")); + temporaryDirectories.push(directory); + await mkdir(path.join(directory, "specs")); + await writeFile( + path.join(directory, "specs", "openapi.yml"), + "openapi: 3.0.0\ninfo:\n title: Payments\n version: 1.0.0\npaths: {}\n" + ); + + const { workspace, cleanup } = await createSdkConfigWorkspace({ + sdkConfig: parseSdkConfigV1({ + schemaVersion: "sdk-config/v1", + sdkName: "payments", + sdkVersion: "1.0.0", + source: { + specs: [ + { + id: "payments", + type: "openapi", + path: "./specs/openapi.yml", + namespace: "payments" + } + ] + }, + api: { audiences: [] }, + client: {}, + package: {}, + docs: {}, + generation: {}, + targets: [ + { + language: "typescript", + output: { delivery: "files", path: "./generated/typescript" } + } + ] + }), + absolutePathToConfig: path.join(directory, "sdk-config.yml"), + cliVersion: "0.0.0", + context: createMockTaskContext() + }); + + expect(workspace.allSpecs).toMatchObject([ + { + type: "openapi", + absoluteFilepath: path.join(directory, "specs", "openapi.yml"), + namespace: "payments" + } + ]); + expect(workspace.generatorsConfiguration?.groups).toMatchObject([ + { + groupName: "sdk-config", + generators: [ + { + name: "fernapi/fern-typescript-sdk", + version: "4.1.0", + language: "typescript", + absolutePathToLocalOutput: path.join(directory, "generated", "typescript") + } + ] + } + ]); + expect(workspace.generatorsConfiguration?.absolutePathToConfiguration).toBe( + path.join(directory, "sdk-config.yml") + ); + expect(getLatestGeneratorVersion).toHaveBeenCalledWith( + expect.objectContaining({ + generatorName: "fernapi/fern-typescript-sdk", + cliVersion: "0.0.0" + }) + ); + await cleanup(); + }); + + it("preserves an explicitly pinned generator version without resolving latest", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "fern-sdk-config-workspace-")); + temporaryDirectories.push(directory); + await writeFile( + path.join(directory, "openapi.yml"), + "openapi: 3.0.0\ninfo:\n title: Payments\n version: 1.0.0\npaths: {}\n" + ); + + const { workspace, cleanup } = await createSdkConfigWorkspace({ + sdkConfig: parseSdkConfigV1({ + schemaVersion: "sdk-config/v1", + sdkName: "payments", + source: { + specs: [{ id: "payments", type: "openapi", path: "./openapi.yml" }] + }, + api: {}, + client: {}, + package: {}, + docs: {}, + generation: {}, + targets: [ + { + language: "typescript", + generatorVersion: "4.0.0", + output: { delivery: "zip" } + } + ] + }), + absolutePathToConfig: path.join(directory, "sdk-config.yml"), + cliVersion: "0.0.0", + context: createMockTaskContext() + }); + + expect(workspace.generatorsConfiguration?.groups[0]?.generators[0]).toMatchObject({ + name: "fernapi/fern-typescript-sdk", + version: "4.0.0" + }); + expect(getLatestGeneratorVersion).not.toHaveBeenCalled(); + await cleanup(); + }); + + it("materializes a bundled OpenAPI URL source and cleans it up", async () => { + vi.mocked(bundleRemoteOpenAPI).mockResolvedValue({ + openapi: "3.0.0", + info: { title: "Payments", version: "1.0.0" }, + paths: {} + }); + const directory = await mkdtemp(path.join(tmpdir(), "fern-sdk-config-workspace-")); + temporaryDirectories.push(directory); + + const created = await createSdkConfigWorkspace({ + sdkConfig: parseSdkConfigV1({ + schemaVersion: "sdk-config/v1", + sdkName: "payments", + source: { + specs: [ + { + id: "payments", + type: "openapi", + url: "https://example.com/openapi.yaml" + } + ] + }, + api: {}, + client: {}, + package: {}, + docs: {}, + generation: {}, + targets: [{ language: "typescript", generatorVersion: "4.0.0", output: { delivery: "zip" } }] + }), + absolutePathToConfig: path.join(directory, "sdk-config.yml"), + cliVersion: "0.0.0", + context: createMockTaskContext() + }); + + const materialized = created.workspace.allSpecs[0]; + if (materialized?.type !== "openapi") { + throw new Error("Expected an OpenAPI specification"); + } + expect(bundleRemoteOpenAPI).toHaveBeenCalledWith("https://example.com/openapi.yaml"); + expect(JSON.parse(await readFile(materialized.absoluteFilepath, "utf-8"))).toMatchObject({ + info: { title: "Payments" } + }); + + await created.cleanup(); + await expect(access(materialized.absoluteFilepath)).rejects.toThrow(); + }); + + it.each(["asyncapi", "graphql"] as const)("rejects unsupported %s URL sources", async (type) => { + const directory = await mkdtemp(path.join(tmpdir(), "fern-sdk-config-workspace-")); + temporaryDirectories.push(directory); + + await expect( + createSdkConfigWorkspace({ + sdkConfig: parseSdkConfigV1({ + schemaVersion: "sdk-config/v1", + sdkName: "payments", + source: { + specs: [{ id: "payments", type, url: `https://example.com/${type}.yaml` }] + }, + api: {}, + client: {}, + package: {}, + docs: {}, + generation: {}, + targets: [{ language: "typescript", generatorVersion: "4.0.0", output: { delivery: "zip" } }] + }), + absolutePathToConfig: path.join(directory, "sdk-config.yml"), + cliVersion: "0.0.0", + context: { + failAndThrow: (message: string) => { + throw new Error(message); + } + } as never + }) + ).rejects.toThrow(`SDK Config ${type} URL source 'payments' is not supported`); + }); +}); diff --git a/packages/cli/cli/src/commands/generate/__test__/loadSdkConfigV1.test.ts b/packages/cli/cli/src/commands/generate/__test__/loadSdkConfigV1.test.ts index 6063c45b4241..9a860e848c34 100644 --- a/packages/cli/cli/src/commands/generate/__test__/loadSdkConfigV1.test.ts +++ b/packages/cli/cli/src/commands/generate/__test__/loadSdkConfigV1.test.ts @@ -57,7 +57,9 @@ describe("loadSdkConfigV1", () => { generatorVersion: "4.0.0", sdkName: "petstore-node", sdkVersion: "2.0.0", - clientPathParameterStyle: "language-default" + clientPathParameterStyle: "language-default", + requestedOutput: { type: "download" }, + absolutePathToLocalOutputArchive: join(directory, "generated", "typescript.zip") } ] }); @@ -95,4 +97,81 @@ describe("loadSdkConfigV1", () => { payload: { sdkName: "petstore", targets: [{ language: "typescript" }] } }); }); + + it("projects SDK Config GitHub delivery and package metadata for the remote request", async () => { + const directory = await mkdtemp(join(tmpdir(), "fern-sdk-config-")); + temporaryDirectories.push(directory); + const configPath = join(directory, "sdk-config.yml"); + await writeFile( + configPath, + YAML.stringify({ + schemaVersion: "sdk-config/v1", + sdkName: "petstore", + source: { specs: [{ id: "openapi", type: "openapi", path: "./openapi.yml" }] }, + api: {}, + client: {}, + package: { packageName: "@acme/sdk" }, + docs: {}, + generation: {}, + output: { + delivery: "github", + github: { repository: "acme/sdk", mode: "pull-request" }, + publish: { registry: "npm" } + }, + targets: [{ language: "typescript", generatorVersion: "4.0.0" }] + }) + ); + + await expect(loadSdkConfigV1(configPath)).resolves.toMatchObject({ + payload: { + targets: [ + { + package: { packageName: "@acme/sdk" }, + requestedOutput: { + type: "github", + repository: "acme/sdk", + mode: "pull-request", + publish: { registry: "npm" } + } + } + ] + } + }); + }); + + it("resolves a configured ZIP filename relative to sdk-config.yml", async () => { + const directory = await mkdtemp(join(tmpdir(), "fern-sdk-config-")); + temporaryDirectories.push(directory); + const configPath = join(directory, "sdk-config.yml"); + await writeFile( + configPath, + YAML.stringify({ + schemaVersion: "sdk-config/v1", + sdkName: "petstore", + source: { specs: [{ id: "openapi", type: "openapi", path: "./openapi.yml" }] }, + api: {}, + client: {}, + package: {}, + docs: {}, + generation: {}, + targets: [ + { + language: "typescript", + output: { delivery: "zip", fileName: "./artifacts/petstore.zip" } + } + ] + }) + ); + + await expect(loadSdkConfigV1(configPath)).resolves.toMatchObject({ + payload: { + targets: [ + { + requestedOutput: { type: "download" }, + absolutePathToLocalOutputArchive: join(directory, "artifacts", "petstore.zip") + } + ] + } + }); + }); }); diff --git a/packages/cli/cli/src/commands/generate/__test__/resolveGroupsForSdkConfig.test.ts b/packages/cli/cli/src/commands/generate/__test__/resolveGroupsForSdkConfig.test.ts deleted file mode 100644 index 109c811eba12..000000000000 --- a/packages/cli/cli/src/commands/generate/__test__/resolveGroupsForSdkConfig.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import type { FernSdkConfigV1Payload } from "@fern-api/remote-workspace-runner"; -import { createMockTaskContext } from "@fern-api/task-context"; -import type { AbstractAPIWorkspace } from "@fern-api/workspace-loader"; -import { describe, expect, it } from "vitest"; - -import { resolveGroupsForSdkConfig } from "../generateAPIWorkspaces.js"; - -describe("resolveGroupsForSdkConfig", () => { - it("selects the generator groups that exactly cover every SDK Config target", () => { - const workspace = createWorkspace([ - { name: "typescript", generators: ["fernapi/fern-typescript-sdk"] }, - { name: "python", generators: ["fernapi/fern-python-sdk"] }, - { name: "java", generators: ["fernapi/fern-java-sdk"] } - ]); - - expect( - resolveGroupsForSdkConfig({ - workspace, - sdkConfigV1: createSdkConfig(["typescript", "python"]), - context: createMockTaskContext() - }) - ).toEqual(["python", "typescript"]); - }); - - it("prefers one multi-language group over separate groups", () => { - const workspace = createWorkspace([ - { - name: "production", - generators: ["fernapi/fern-typescript-sdk", "fernapi/fern-python-sdk"] - }, - { name: "typescript", generators: ["fernapi/fern-typescript-sdk"] }, - { name: "python", generators: ["fernapi/fern-python-sdk"] } - ]); - - expect( - resolveGroupsForSdkConfig({ - workspace, - sdkConfigV1: createSdkConfig(["typescript", "python"]), - context: createMockTaskContext() - }) - ).toEqual(["production"]); - }); - - it("finds a valid exact cover instead of committing to a larger dead end", () => { - const workspace = createWorkspace([ - { - name: "typescript-python-dead-end", - generators: ["fernapi/fern-typescript-sdk", "fernapi/fern-python-sdk"] - }, - { name: "typescript", generators: ["fernapi/fern-typescript-sdk"] }, - { - name: "python-java", - generators: ["fernapi/fern-python-sdk", "fernapi/fern-java-sdk"] - } - ]); - - expect( - resolveGroupsForSdkConfig({ - workspace, - sdkConfigV1: createSdkConfig(["typescript", "python", "java"]), - context: createMockTaskContext() - }) - ).toEqual(["python-java", "typescript"]); - }); - - it("uses the default group to disambiguate equivalent target coverage", () => { - const workspace = createWorkspace( - [ - { name: "typescript-preview", generators: ["fernapi/fern-typescript-sdk"] }, - { name: "typescript-production", generators: ["fernapi/fern-typescript-sdk"] } - ], - "typescript-production" - ); - - expect( - resolveGroupsForSdkConfig({ - workspace, - sdkConfigV1: createSdkConfig(["typescript"]), - context: createMockTaskContext() - }) - ).toEqual(["typescript-production"]); - }); - - it("requires an explicit group when equivalent candidates are ambiguous", () => { - const workspace = createWorkspace([ - { name: "typescript-preview", generators: ["fernapi/fern-typescript-sdk"] }, - { name: "typescript-production", generators: ["fernapi/fern-typescript-sdk"] } - ]); - - expect(() => - resolveGroupsForSdkConfig({ - workspace, - sdkConfigV1: createSdkConfig(["typescript"]), - context: { - failAndThrow: (message: string) => { - throw new Error(message); - } - } as never - }) - ).toThrow("match multiple generator group combinations: typescript-preview; typescript-production"); - }); -}); - -function createWorkspace( - groups: Array<{ name: string; generators: string[] }>, - defaultGroup?: string -): AbstractAPIWorkspace { - return { - generatorsConfiguration: { - absolutePathToConfiguration: "/tmp/generators.yml", - defaultGroup, - groups: groups.map((group) => ({ - groupName: group.name, - generators: group.generators.map((name) => ({ name })) - })) - } - } as unknown as AbstractAPIWorkspace; -} - -function createSdkConfig(languages: string[]): FernSdkConfigV1Payload { - return { - body: Buffer.from("{}"), - sdkName: "api", - sdkVersion: "1.0.0", - targets: languages.map((language) => ({ language })) - } as FernSdkConfigV1Payload; -} diff --git a/packages/cli/cli/src/commands/generate/createSdkConfigWorkspace.ts b/packages/cli/cli/src/commands/generate/createSdkConfigWorkspace.ts new file mode 100644 index 000000000000..fa4d913095b0 --- /dev/null +++ b/packages/cli/cli/src/commands/generate/createSdkConfigWorkspace.ts @@ -0,0 +1,270 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { getOpenAPISettings, type OpenAPISpec, type Spec } from "@fern-api/api-workspace-commons"; +import { generatorsYml, getLatestGeneratorVersion } from "@fern-api/configuration-loader"; +import { AbsoluteFilePath } from "@fern-api/fs-utils"; +import { bundleRemoteOpenAPI, OSSWorkspace } from "@fern-api/lazy-fern-workspace"; +import { CliError, TaskContext } from "@fern-api/task-context"; +import { FernFiddle } from "@fern-fern/fiddle-sdk"; +import type { SdkConfigV1, SdkConfigV1SourceSpec } from "@postman/sdk-config/sdk-config/v1"; + +const SDK_CONFIG_GROUP = "sdk-config"; +const DEFAULT_LOCAL_OUTPUT_DIRECTORY = "generated"; + +const GENERATOR_BY_LANGUAGE: Record = { + typescript: "fernapi/fern-typescript-sdk", + python: "fernapi/fern-python-sdk", + java: "fernapi/fern-java-sdk", + kotlin: "fernapi/fern-kotlin-sdk", + go: "fernapi/fern-go-sdk", + csharp: "fernapi/fern-csharp-sdk", + php: "fernapi/fern-php-sdk", + ruby: "fernapi/fern-ruby-sdk-v2", + rust: "fernapi/fern-rust-sdk", + swift: "fernapi/fern-swift-sdk", + cli: "fernapi/fern-cli-generator", + mcp: "fernapi/fern-mcp-server" +}; + +export interface CreatedSdkConfigWorkspace { + workspace: OSSWorkspace; + cleanup: () => Promise; +} + +export async function createSdkConfigWorkspace({ + sdkConfig, + absolutePathToConfig, + cliVersion, + context +}: { + sdkConfig: SdkConfigV1; + absolutePathToConfig: string; + cliVersion: string; + context: TaskContext; +}): Promise { + const configDirectory = path.dirname(absolutePathToConfig); + const temporaryDirectories: string[] = []; + const cleanup = async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })) + ); + }; + try { + const specs: Spec[] = []; + for (const spec of sdkConfig.source.specs) { + specs.push(await createSpec({ spec, sdkConfig, configDirectory, context, temporaryDirectories })); + } + const group: generatorsYml.GeneratorGroup = { + groupName: SDK_CONFIG_GROUP, + audiences: + sdkConfig.api.audiences == null + ? { type: "all" } + : { type: "select", audiences: sdkConfig.api.audiences }, + generators: await Promise.all( + sdkConfig.targets.map(async (target) => { + const name = GENERATOR_BY_LANGUAGE[target.language]; + if (name == null) { + return context.failAndThrow( + `SDK Config target language '${target.language}' is not supported by the Fern remote generation bridge`, + undefined, + { code: CliError.Code.ConfigError } + ); + } + const version = + target.generatorVersion ?? + (await getLatestGeneratorVersion({ + generatorName: name, + cliVersion, + channel: undefined, + includeMajor: true, + context + })); + if (version == null) { + return context.failAndThrow( + `Could not resolve a generator version for SDK Config target '${target.language}'`, + undefined, + { code: CliError.Code.NetworkError } + ); + } + return createGeneratorInvocation({ + name, + version, + language: target.language, + output: target.output ?? sdkConfig.output, + configDirectory + }); + }) + ), + reviewers: undefined + }; + const generatorsConfiguration: generatorsYml.GeneratorsConfiguration = { + api: undefined, + defaultGroup: SDK_CONFIG_GROUP, + groupAliases: {}, + reviewers: undefined, + groups: [group], + whitelabel: undefined, + ai: undefined, + replay: undefined, + rawConfiguration: {} as generatorsYml.GeneratorsConfigurationSchema, + // This is an in-memory adapter. The path is retained only for diagnostics; it is never parsed as generators.yml. + absolutePathToConfiguration: AbsoluteFilePath.of(absolutePathToConfig) + }; + const workspace = new OSSWorkspace({ + allSpecs: specs, + specs: specs.filter((spec): spec is OpenAPISpec => spec.type === "openapi"), + generatorsConfiguration, + workspaceName: undefined, + cliVersion, + absoluteFilePath: AbsoluteFilePath.of(configDirectory) + }); + await workspace.processGraphQLSpecs(context); + return { workspace, cleanup }; + } catch (error) { + await cleanup(); + throw error; + } +} + +function createGeneratorInvocation({ + name, + version, + language, + output, + configDirectory +}: { + name: string; + version: string; + language: string; + output: SdkConfigV1["output"]; + configDirectory: string; +}): generatorsYml.GeneratorInvocation { + return { + name, + version, + config: {}, + outputMode: FernFiddle.remoteGen.OutputMode.downloadFiles({}), + automation: { generate: true, preview: true, upgrade: true, verify: true }, + containerImage: undefined, + irVersionOverride: undefined, + // SDK Config permits files delivery without a path. Keep those outputs separated by + // language under a stable directory next to sdk-config.yml. + absolutePathToLocalOutput: + output?.delivery === "files" + ? AbsoluteFilePath.of( + path.resolve(configDirectory, output.path ?? `${DEFAULT_LOCAL_OUTPUT_DIRECTORY}/${language}`) + ) + : undefined, + absolutePathToLocalSnippets: undefined, + keywords: undefined, + smartCasing: false, + smartCasingDigitWordBoundary: false, + disableExamples: false, + language: isLegacyGenerationLanguage(language) ? language : undefined, + publishMetadata: undefined, + readme: undefined, + settings: undefined + }; +} + +async function createSpec({ + spec, + sdkConfig, + configDirectory, + context, + temporaryDirectories +}: { + spec: SdkConfigV1SourceSpec; + sdkConfig: SdkConfigV1; + configDirectory: string; + context: TaskContext; + temporaryDirectories: string[]; +}): Promise { + if ((spec.overlays?.length ?? 0) > 1) { + return context.failAndThrow( + `SDK Config source '${spec.id}' declares multiple overlays, which the Fern source loader does not yet support`, + undefined, + { code: CliError.Code.ConfigError } + ); + } + const absoluteFilepath = await resolveSourcePath(spec, configDirectory, context, temporaryDirectories); + const absoluteFilepathToOverrides = spec.overrides?.map((value) => resolveTransformPath(value, configDirectory)); + if (spec.type === "graphql") { + return { + type: "graphql", + absoluteFilepath, + absoluteFilepathToOverrides, + absoluteFilepathToExamples: undefined, + namespace: spec.namespace + }; + } + const rootSettings = sdkConfig.source.apiImportSettings ?? {}; + const settings = { ...rootSettings, ...spec.apiImportSettings }; + return { + type: "openapi", + absoluteFilepath, + absoluteFilepathToOverrides, + absoluteFilepathToOverlays: + spec.overlays?.[0] == null ? undefined : resolveTransformPath(spec.overlays[0], configDirectory), + namespace: spec.namespace, + settings: getOpenAPISettings({ + overrides: { + ...settings, + useTitlesAsName: settings.titleAsSchemaName, + shouldUseIdiomaticRequestNames: settings.idiomaticRequestNames + } + }), + source: { + type: spec.type === "asyncapi" ? "asyncapi" : "openapi", + file: absoluteFilepath + } + }; +} + +async function resolveSourcePath( + spec: SdkConfigV1SourceSpec, + configDirectory: string, + context: TaskContext, + temporaryDirectories: string[] +): Promise { + if ("path" in spec) { + return AbsoluteFilePath.of(path.resolve(configDirectory, spec.path)); + } + if (spec.type !== "openapi") { + return context.failAndThrow( + `SDK Config ${spec.type} URL source '${spec.id}' is not supported by the Fern CLI yet. Download ${spec.url} into your project and use a path source instead.`, + undefined, + { code: CliError.Code.ConfigError } + ); + } + let bundled: unknown; + try { + bundled = await bundleRemoteOpenAPI(spec.url); + } catch (error) { + return context.failAndThrow( + `Could not resolve SDK Config OpenAPI source '${spec.id}' from ${spec.url}`, + error, + { + code: CliError.Code.NetworkError + } + ); + } + const directory = await mkdtemp(path.join(tmpdir(), "fern-sdk-config-source-")); + temporaryDirectories.push(directory); + const absolutePath = path.join(directory, `${sanitizeFilename(spec.id)}.json`); + await writeFile(absolutePath, `${JSON.stringify(bundled)}\n`); + return AbsoluteFilePath.of(absolutePath); +} + +function resolveTransformPath(value: string, configDirectory: string): AbsoluteFilePath { + return AbsoluteFilePath.of(path.resolve(configDirectory, value)); +} + +function sanitizeFilename(value: string): string { + return value.replace(/[^a-zA-Z0-9._-]/g, "-"); +} + +function isLegacyGenerationLanguage(language: string): language is generatorsYml.GenerationLanguage { + return Object.values(generatorsYml.GenerationLanguage).some((candidate) => candidate === language); +} diff --git a/packages/cli/cli/src/commands/generate/generateAPIWorkspaces.ts b/packages/cli/cli/src/commands/generate/generateAPIWorkspaces.ts index 4616b56eea6a..56d8cbe10c51 100644 --- a/packages/cli/cli/src/commands/generate/generateAPIWorkspaces.ts +++ b/packages/cli/cli/src/commands/generate/generateAPIWorkspaces.ts @@ -3,16 +3,13 @@ import { ContainerRunner, Values } from "@fern-api/core-utils"; import { AbsoluteFilePath, cwd, join, RelativeFilePath, resolve } from "@fern-api/fs-utils"; import { askToLogin } from "@fern-api/login"; import { Project } from "@fern-api/project-loader"; -import { - type AutomationRunOptions, - type FernSdkConfigV1Payload, - getFernSdkGenApiLanguage -} from "@fern-api/remote-workspace-runner"; +import { type AutomationRunOptions, type FernSdkConfigV1Payload } from "@fern-api/remote-workspace-runner"; import { CliError } from "@fern-api/task-context"; import { AbstractAPIWorkspace } from "@fern-api/workspace-loader"; import { CliContext } from "../../cli-context/CliContext.js"; import { PREVIEW_DIRECTORY } from "../../constants.js"; import { checkOutputDirectory } from "./checkOutputDirectory.js"; +import { createSdkConfigWorkspace } from "./createSdkConfigWorkspace.js"; import { expandGroupFilter } from "./expandGroupFilter.js"; import { filterGenerators } from "./filterGenerators.js"; import { generateWorkspace } from "./generateAPIWorkspace.js"; @@ -105,6 +102,7 @@ export async function generateAPIWorkspaces({ }): Promise { let token: FernToken | undefined = undefined; let sdkConfigV1: FernSdkConfigV1Payload | undefined; + let cleanupSdkConfigWorkspace: (() => Promise) | undefined; if (sdkConfigPath != null) { if (useLocalDocker) { @@ -114,135 +112,159 @@ export async function generateAPIWorkspaces({ { code: CliError.Code.ConfigError } ); } + if (groupNames != null && groupNames.length > 0) { + return cliContext.failAndThrow( + "--group cannot be used with --sdk-config because SDK Config targets are authoritative", + undefined, + { code: CliError.Code.ConfigError } + ); + } try { const loaded = await loadSdkConfigV1(sdkConfigPath); sdkConfigV1 = loaded.payload; + const createdWorkspace = await cliContext.runTask(async (context) => + createSdkConfigWorkspace({ + sdkConfig: loaded.config, + absolutePathToConfig: loaded.absolutePath, + cliVersion: cliContext.environment.packageVersion, + context + }) + ); + cleanupSdkConfigWorkspace = createdWorkspace.cleanup; + // SDK Config owns both sources and targets. Ignore any workspace assembled from + // generators.yml while retaining project-level organization configuration. + project = { ...project, apiWorkspaces: [createdWorkspace.workspace] }; cliContext.logger.info(`Using SDK Config v1 from ${loaded.absolutePath}`); } catch (error) { + await cleanupSdkConfigWorkspace?.(); return cliContext.failAndThrow(undefined, error, { code: CliError.Code.ConfigError }); } } - if (!useLocalDocker) { - const currentToken = await cliContext.runTask(async (context) => { - return askToLogin(context); - }); - if (currentToken.type === "user") { - await cliContext.runTask(async (context) => { - await createOrganizationIfDoesNotExist({ - organization: project.config.organization, - token: currentToken, - context - }); + try { + if (!useLocalDocker) { + const currentToken = await cliContext.runTask(async (context) => { + return askToLogin(context); }); + if (currentToken.type === "user") { + await cliContext.runTask(async (context) => { + await createOrganizationIfDoesNotExist({ + organization: project.config.organization, + token: currentToken, + context + }); + }); + } + token = currentToken; + } else { + // Local generation must stay non-interactive: silently pick up an existing + // token (FERN_TOKEN env var or saved login file) so Venus calls are + // authenticated when possible, and leave `token` undefined otherwise. + token = await getToken(); } - token = currentToken; - } else { - // Local generation must stay non-interactive: silently pick up an existing - // token (FERN_TOKEN env var or saved login file) so Venus calls are - // authenticated when possible, and leave `token` undefined otherwise. - token = await getToken(); - } - // Pre-flight: resolve groups for every selected workspace up front. If any workspace is - // misconfigured for this invocation (e.g. `--group foo` targets a group that doesn't exist - // in one of the `--api`-selected workspaces, or no `--group` was passed and one workspace - // lacks a `default-group`), `resolveGroupsOrFail` throws before we start any generation. - // We keep the resolved names so `generateWorkspace` doesn't need to re-run the resolver - // (and re-log "Using default group '…' from generators.yml"). - const resolvedGroupNamesByWorkspace = await resolveGroupsForAllWorkspaces({ - project, - groupNames, - sdkConfigV1, - automation, - cliContext - }); - if (sdkConfigV1 != null) { - const selectedWorkspaces = [...resolvedGroupNamesByWorkspace.entries()].filter( - ([, resolvedGroups]) => resolvedGroups.length > 0 - ); - if (selectedWorkspaces.length !== 1) { - return cliContext.failAndThrow( - `SDK Config v1 must resolve to exactly one API workspace; resolved ${selectedWorkspaces.length}`, - undefined, - { code: CliError.Code.ConfigError } + // Pre-flight: resolve groups for every selected workspace up front. If any workspace is + // misconfigured for this invocation (e.g. `--group foo` targets a group that doesn't exist + // in one of the `--api`-selected workspaces, or no `--group` was passed and one workspace + // lacks a `default-group`), `resolveGroupsOrFail` throws before we start any generation. + // We keep the resolved names so `generateWorkspace` doesn't need to re-run the resolver + // (and re-log "Using default group '…' from generators.yml"). + const resolvedGroupNamesByWorkspace = await resolveGroupsForAllWorkspaces({ + project, + groupNames, + sdkConfigV1, + automation, + cliContext + }); + if (sdkConfigV1 != null) { + const selectedWorkspaces = [...resolvedGroupNamesByWorkspace.entries()].filter( + ([, resolvedGroups]) => resolvedGroups.length > 0 ); + if (selectedWorkspaces.length !== 1) { + return cliContext.failAndThrow( + `SDK Config v1 must resolve to exactly one API workspace; resolved ${selectedWorkspaces.length}`, + undefined, + { code: CliError.Code.ConfigError } + ); + } } - } - await confirmOutputDirectoriesForEligibleGenerators({ - project, - resolvedGroupNamesByWorkspace, - generatorName, - generatorIndex, - automation, - cliContext, - force - }); + await confirmOutputDirectoriesForEligibleGenerators({ + project, + resolvedGroupNamesByWorkspace, + generatorName, + generatorIndex, + automation, + cliContext, + force + }); - cliContext.instrumentPostHogEvent({ - orgId: project.config.organization, - command: resolvePosthogCommandLabel(automation), - properties: { - workspaces: buildPosthogWorkspaces({ project, groupNames, generatorName }) - } - }); + cliContext.instrumentPostHogEvent({ + orgId: project.config.organization, + command: resolvePosthogCommandLabel(automation), + properties: { + workspaces: buildPosthogWorkspaces({ project, groupNames, generatorName }) + } + }); - await Promise.all( - project.apiWorkspaces.map(async (workspace) => { - const resolvedGroupNames = resolvedGroupNamesByWorkspace.get(workspace); - // Workspaces skipped by the pre-flight (no generators.yml or no configured groups) - // still need to run through `generateWorkspace` so the existing warning paths fire. - // An undefined entry means "skipped"; an empty array would mean "resolved to nothing". - await cliContext.runTaskForWorkspace(workspace, async (context) => { - const absolutePathToPreview = preview - ? outputDir != null - ? AbsoluteFilePath.of(resolve(cwd(), outputDir)) - : join(workspace.absoluteFilePath, RelativeFilePath.of(PREVIEW_DIRECTORY)) - : undefined; + await Promise.all( + project.apiWorkspaces.map(async (workspace) => { + const resolvedGroupNames = resolvedGroupNamesByWorkspace.get(workspace); + // Workspaces skipped by the pre-flight (no generators.yml or no configured groups) + // still need to run through `generateWorkspace` so the existing warning paths fire. + // An undefined entry means "skipped"; an empty array would mean "resolved to nothing". + await cliContext.runTaskForWorkspace(workspace, async (context) => { + const absolutePathToPreview = preview + ? outputDir != null + ? AbsoluteFilePath.of(resolve(cwd(), outputDir)) + : join(workspace.absoluteFilePath, RelativeFilePath.of(PREVIEW_DIRECTORY)) + : undefined; - if (absolutePathToPreview != null) { - context.logger.info(`Writing preview to ${absolutePathToPreview}`); - } + if (absolutePathToPreview != null) { + context.logger.info(`Writing preview to ${absolutePathToPreview}`); + } - await generateWorkspace({ - organization: project.config.organization, - workspace, - projectConfig: project.config, - context, - version, - resolvedGroupNames: resolvedGroupNames ?? [], - generatorName, - generatorIndex, - shouldLogS3Url, - token, - useLocalDocker, - keepDocker, - absolutePathToPreview, - mode, - runner, - inspect, - lfsOverride, - sdkConfigV1, - fernignorePath, - skipFernignore, - dynamicIrOnly, - noReplay, - verify, - retryRateLimited, - requireEnvVars, - automationMode, - autoMerge, - skipIfNoDiff, - generateTests, - automation, - pack, - packMode, - packOnly + await generateWorkspace({ + organization: project.config.organization, + workspace, + projectConfig: project.config, + context, + version, + resolvedGroupNames: resolvedGroupNames ?? [], + generatorName, + generatorIndex, + shouldLogS3Url, + token, + useLocalDocker, + keepDocker, + absolutePathToPreview, + mode, + runner, + inspect, + lfsOverride, + sdkConfigV1, + fernignorePath, + skipFernignore, + dynamicIrOnly, + noReplay, + verify, + retryRateLimited, + requireEnvVars, + automationMode, + autoMerge, + skipIfNoDiff, + generateTests, + automation, + pack, + packMode, + packOnly + }); }); - }); - }) - ); + }) + ); + } finally { + await cleanupSdkConfigWorkspace?.(); + } } /** @@ -272,8 +294,13 @@ async function resolveGroupsForAllWorkspaces({ project.apiWorkspaces.map(async (workspace) => { await cliContext.runTaskForWorkspace(workspace, async (context) => { if (sdkConfigV1 != null && (groupNames == null || groupNames.length === 0)) { - const resolved = resolveGroupsForSdkConfig({ workspace, sdkConfigV1, context }); - resolvedGroupNamesByWorkspace.set(workspace, resolved); + const sdkConfigGroup = workspace.generatorsConfiguration?.defaultGroup; + if (sdkConfigGroup == null) { + return context.failAndThrow("SDK Config workspace has no generation targets", undefined, { + code: CliError.Code.ConfigError + }); + } + resolvedGroupNamesByWorkspace.set(workspace, [sdkConfigGroup]); return; } const resolved = resolveGroupsForWorkspace({ @@ -291,127 +318,6 @@ async function resolveGroupsForAllWorkspaces({ return resolvedGroupNamesByWorkspace; } -export function resolveGroupsForSdkConfig({ - workspace, - sdkConfigV1, - context -}: { - workspace: AbstractAPIWorkspace; - sdkConfigV1: FernSdkConfigV1Payload; - context: Parameters[0]["context"]; -}): string[] { - const generatorsConfiguration = workspace.generatorsConfiguration; - if (generatorsConfiguration == null) { - return context.failAndThrow( - "SDK Config v1 generation requires generators.yml to load the Fern API workspace", - undefined, - { code: CliError.Code.ConfigError } - ); - } - - const requestedTargets = new Map(sdkConfigV1.targets.map((target) => [target.language, target])); - const requestedLanguages = new Set(requestedTargets.keys()); - const candidates = generatorsConfiguration.groups.flatMap((group) => { - const languages = group.generators.flatMap((generator) => { - const language = getFernSdkGenApiLanguage(generator.name); - if (language == null) { - return []; - } - const target = requestedTargets.get(language); - if (target == null || (target.generatorVersion != null && target.generatorVersion !== generator.version)) { - return []; - } - return [language]; - }); - if ( - languages.length === 0 || - languages.length !== group.generators.length || - languages.some((language) => !requestedLanguages.has(language)) - ) { - return []; - } - return [{ groupName: group.groupName, languages: new Set(languages) }]; - }); - const exactCovers = findMinimalExactGroupCovers(candidates, requestedLanguages); - if (exactCovers.length === 0) { - return context.failAndThrow( - `SDK Config v1 targets (${[...requestedLanguages].join(", ")}) cannot be matched exactly to generator groups in ${generatorsConfiguration.absolutePathToConfiguration}. Pass --group explicitly or regenerate sdk-config.yml from the current Fern configuration.`, - undefined, - { code: CliError.Code.ConfigError } - ); - } - const coversWithDefault = exactCovers.filter((cover) => - cover.some(({ groupName }) => groupName === generatorsConfiguration.defaultGroup) - ); - const preferredCovers = coversWithDefault.length > 0 ? coversWithDefault : exactCovers; - if (preferredCovers.length > 1) { - const descriptions = preferredCovers - .map((cover) => - cover - .map(({ groupName }) => groupName) - .sort() - .join(" + ") - ) - .sort() - .join("; "); - return context.failAndThrow( - `SDK Config v1 targets (${[...requestedLanguages].join(", ")}) match multiple generator group combinations: ${descriptions}. Pass --group explicitly.`, - undefined, - { code: CliError.Code.ConfigError } - ); - } - return (preferredCovers[0] ?? []).map(({ groupName }) => groupName).sort(); -} - -interface SdkConfigGroupCandidate { - groupName: string; - languages: Set; -} - -function findMinimalExactGroupCovers( - candidates: SdkConfigGroupCandidate[], - requestedLanguages: Set -): SdkConfigGroupCandidate[][] { - const sortedCandidates = [...candidates].sort((left, right) => left.groupName.localeCompare(right.groupName)); - let minimumGroupCount = Number.POSITIVE_INFINITY; - let covers: SdkConfigGroupCandidate[][] = []; - - const visit = (uncovered: Set, selected: SdkConfigGroupCandidate[]): void => { - if (uncovered.size === 0) { - if (selected.length < minimumGroupCount) { - minimumGroupCount = selected.length; - covers = [[...selected]]; - } else if (selected.length === minimumGroupCount) { - covers.push([...selected]); - } - return; - } - if (selected.length >= minimumGroupCount) { - return; - } - const nextLanguage = [...uncovered].sort()[0]; - if (nextLanguage == null) { - return; - } - for (const candidate of sortedCandidates) { - if ( - !candidate.languages.has(nextLanguage) || - [...candidate.languages].some((language) => !uncovered.has(language)) - ) { - continue; - } - const remaining = new Set(uncovered); - for (const language of candidate.languages) { - remaining.delete(language); - } - visit(remaining, [...selected, candidate]); - } - }; - - visit(new Set(requestedLanguages), []); - return covers; -} - /** * Walks the project's generators and prompts the user to confirm overwriting any local-file-system * output directories that already exist. Skips generators that wouldn't run anyway (per diff --git a/packages/cli/cli/src/commands/generate/loadSdkConfigV1.ts b/packages/cli/cli/src/commands/generate/loadSdkConfigV1.ts index 47fcee87de57..a2a785e72e74 100644 --- a/packages/cli/cli/src/commands/generate/loadSdkConfigV1.ts +++ b/packages/cli/cli/src/commands/generate/loadSdkConfigV1.ts @@ -1,12 +1,17 @@ import { readFile } from "node:fs/promises"; -import { resolve } from "node:path"; -import { cwd } from "@fern-api/fs-utils"; -import type { FernSdkConfigV1Payload } from "@fern-api/remote-workspace-runner"; -import { parseSdkConfigV1, validateSdkConfigV1 } from "@postman/sdk-config/sdk-config/v1"; +import { dirname, resolve } from "node:path"; +import { AbsoluteFilePath, cwd } from "@fern-api/fs-utils"; +import type { + FernSdkConfigV1Payload, + FernSdkGenApiPackageConfig, + FernSdkGenApiRequestedOutput +} from "@fern-api/remote-workspace-runner"; +import { parseSdkConfigV1, type SdkConfigV1, validateSdkConfigV1 } from "@postman/sdk-config/sdk-config/v1"; import YAML from "yaml"; export interface LoadedSdkConfigV1 { absolutePath: string; + config: SdkConfigV1; payload: FernSdkConfigV1Payload; } @@ -28,6 +33,7 @@ export async function loadSdkConfigV1(configPath: string): Promise ({ - language: target.language, - ...(target.generatorVersion != null ? { generatorVersion: target.generatorVersion } : {}), - ...(target.sdkName != null ? { sdkName: target.sdkName } : {}), - ...(target.sdkVersion != null ? { sdkVersion: target.sdkVersion } : {}), - ...(target.client?.pathParameterStyle != null - ? { clientPathParameterStyle: target.client.pathParameterStyle } - : {}) - })) + targets: parsed.targets.map((target) => { + const output = target.output ?? parsed.output; + return { + language: target.language, + ...(target.generatorVersion != null ? { generatorVersion: target.generatorVersion } : {}), + ...(target.sdkName != null ? { sdkName: target.sdkName } : {}), + ...(target.sdkVersion != null ? { sdkVersion: target.sdkVersion } : {}), + ...(target.client?.pathParameterStyle != null + ? { clientPathParameterStyle: target.client.pathParameterStyle } + : {}), + requestedOutput: toRequestedOutput(output), + ...(output?.delivery === "zip" + ? { + absolutePathToLocalOutputArchive: AbsoluteFilePath.of( + resolve( + dirname(absolutePath), + output.fileName ?? `generated/${target.language}.zip` + ) + ) + } + : {}), + package: { ...parsed.package, ...target.package } satisfies FernSdkGenApiPackageConfig + }; + }) } }; } catch (error) { @@ -57,3 +78,18 @@ export async function loadSdkConfigV1(configPath: string): Promise { }, targets: [ { - generatorVersion: "3.63.3", language: "typescript", output: { delivery: "zip" } } ] }); + expect(result.sdkConfig.targets[0]).not.toHaveProperty("generatorVersion"); expect(result.sdkConfig.sdkVersion).toBeUndefined(); expect(result.sdkConfig.apiVersion).toBeUndefined(); expect(result.sdkConfig.client).toBeUndefined(); diff --git a/packages/cli/cli/src/commands/sdk-migrate/mapFernGroupToSdkConfig.ts b/packages/cli/cli/src/commands/sdk-migrate/mapFernGroupToSdkConfig.ts index d32fa72ddeed..d5598d536dbb 100644 --- a/packages/cli/cli/src/commands/sdk-migrate/mapFernGroupToSdkConfig.ts +++ b/packages/cli/cli/src/commands/sdk-migrate/mapFernGroupToSdkConfig.ts @@ -267,6 +267,9 @@ function mapEnvironmentUrls( function normalizeGeneratorForMapping(generator: generatorsYml.GeneratorInvocation): FernResolvedGeneratorInput { return { ...generator, + // Migrated SDK Config targets intentionally float to the latest supported generator. + // Customers can add generatorVersion later when they want to pin an exact version. + version: undefined, outputMode: normalizeResolvedOutput(generator.outputMode) }; } diff --git a/packages/cli/ete-tests/src/tests/sdk-migrate/fixtures/sdk-config.yml b/packages/cli/ete-tests/src/tests/sdk-migrate/fixtures/sdk-config.yml index 31bd214b4c5b..be721a5d1828 100644 --- a/packages/cli/ete-tests/src/tests/sdk-migrate/fixtures/sdk-config.yml +++ b/packages/cli/ete-tests/src/tests/sdk-migrate/fixtures/sdk-config.yml @@ -12,9 +12,7 @@ targets: output: delivery: files path: ./generated/typescript - generatorVersion: 3.63.3 - language: python output: delivery: files path: ./generated/python - generatorVersion: 4.3.10 diff --git a/packages/cli/ete-tests/src/tests/sdk-migrate/sdk-migrate.test.ts b/packages/cli/ete-tests/src/tests/sdk-migrate/sdk-migrate.test.ts index 9959f0d28dc2..b61b0f598b2e 100644 --- a/packages/cli/ete-tests/src/tests/sdk-migrate/sdk-migrate.test.ts +++ b/packages/cli/ete-tests/src/tests/sdk-migrate/sdk-migrate.test.ts @@ -192,10 +192,10 @@ describe("fern sdk migrate", () => { } ); - expect((yaml.load(result.stdout) as { targets: unknown[] }).targets).toMatchObject([ - { language: "typescript", generatorVersion: "3.63.3" }, - { language: "python", generatorVersion: "4.3.10" } - ]); + const targets = (yaml.load(result.stdout) as { targets: Array> }).targets; + expect(targets).toMatchObject([{ language: "typescript" }, { language: "python" }]); + expect(targets).toHaveLength(2); + expect(targets.every((target) => !("generatorVersion" in target))).toBe(true); await temporaryDirectory.cleanup(); }); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteTaskHandler.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteTaskHandler.ts index fa91f02c018f..2709ae3387db 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteTaskHandler.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteTaskHandler.ts @@ -482,6 +482,30 @@ export async function downloadFilesForTask({ } } +/** Downloads the generated ZIP artifact without extracting it. */ +export async function downloadArchiveForTask({ + s3PreSignedReadUrl, + absolutePathToLocalOutput, + context +}: { + s3PreSignedReadUrl: string; + absolutePathToLocalOutput: AbsoluteFilePath; + context: InteractiveTaskContext; +}): Promise { + try { + const request = await axios.get(s3PreSignedReadUrl, { + responseType: "stream", + timeout: 60_000, + signal: AbortSignal.timeout(S3_DOWNLOAD_TIMEOUT_MS) + }); + await mkdir(path.dirname(absolutePathToLocalOutput), { recursive: true }); + await pipeline(request.data, createWriteStream(absolutePathToLocalOutput)); + context.logger.info(chalk.green(`Downloaded to ${absolutePathToLocalOutput}`)); + } catch (error) { + context.failAndThrow("Failed to download archive", error, { code: CliError.Code.NetworkError }); + } +} + /** Maximum time (ms) to wait for the S3 download to complete, including streaming. */ const S3_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1_000; diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/RemoteTaskHandler.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/RemoteTaskHandler.test.ts index 36c64738554f..fabb06f0662e 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/RemoteTaskHandler.test.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/RemoteTaskHandler.test.ts @@ -1,17 +1,46 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; import type { generatorsYml } from "@fern-api/configuration"; +import { AbsoluteFilePath } from "@fern-api/fs-utils"; import type { PipelineResult, ReplayStepResult } from "@fern-api/generator-cli/pipeline"; import { CONSOLE_LOGGER, type Logger } from "@fern-api/logger"; import type { InteractiveTaskContext, PosthogEvent, TaskResult } from "@fern-api/task-context"; import { FernFiddle } from "@fern-fern/fiddle-sdk"; +import axios from "axios"; import { describe, expect, it, vi } from "vitest"; import { + downloadArchiveForTask, extractVersionFromLogMessage, FIDDLE_PIPELINE_RESULT_LOG_REGEX, RemoteTaskHandler, tryParseReplayResult } from "../RemoteTaskHandler.js"; +describe("downloadArchiveForTask", () => { + it("persists the remote ZIP without extracting it", async () => { + const directory = await mkdtemp(join(tmpdir(), "fern-sdk-archive-")); + const destination = AbsoluteFilePath.of(join(directory, "nested", "sdk.zip")); + const info = vi.fn(); + vi.spyOn(axios, "get").mockResolvedValue({ data: Readable.from([Buffer.from("zip-bytes")]) } as never); + + try { + await downloadArchiveForTask({ + s3PreSignedReadUrl: "https://example.test/sdk.zip", + absolutePathToLocalOutput: destination, + context: { logger: { info } } as never + }); + + expect(await readFile(destination, "utf-8")).toBe("zip-bytes"); + expect(info).toHaveBeenCalledWith(expect.stringContaining(destination)); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); +}); + /** * Snapshot test for the Fiddle log-format coupling. If Fiddle ever changes the * exact prefix string in `GithubFiddleTask.java#974` from diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/fernSdkGenApi.test.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/fernSdkGenApi.test.ts index e734b453e454..331c26842d44 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/fernSdkGenApi.test.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/__test__/fernSdkGenApi.test.ts @@ -13,12 +13,14 @@ import { type FernSdkGenApiBuildParameters, type FernSdkGenApiPayload, FernSdkGenApiPreparationBatch, + type FernSdkGenApiRequestedOutput, getFernSdkGenApiLanguage, getFernSdkGenApiOrigin, isEligibleForFernSdkGenApi, isFernSdkGenApiEnabled, mapFernSdkGenApiOutput, preflightFernSdkGenApiBuild, + resolveSdkConfigRequestedOutput, runFernSdkGenApiBuild, selectFernSdkGenApiRoute } from "../fernSdkGenApi.js"; @@ -1773,6 +1775,45 @@ describe("fernapi/fern-mcp-server target", () => { expect(request.targets[0]?.package).toBeUndefined(); }); + it("uses SDK Config output metadata instead of the adapter invocation", () => { + const request = createFernSdkGenApiRequest({ + apiName: "Petstore", + organization: "acme", + cliVersion: "0.0.0", + generatorInvocation: mcpInvocation(), + sdkVersion: "0.0.1", + specsTarGzBuffer: Buffer.from("archive"), + payload: { ...sdkConfigPayload("{}"), package: { packageName: "@acme/sdk" } }, + requestedOutput: { + type: "github", + repository: "acme/sdk", + mode: "pull-request", + publish: { registry: "npm" } + } + }); + + expect(request.targets[0]).toMatchObject({ + package: { packageName: "@acme/sdk" }, + requestedOutput: { + type: "github", + repository: "acme/sdk", + mode: "pull-request", + publish: { registry: "npm" } + } + }); + }); + + it("forces SDK Config preview output to download instead of publishing", () => { + const githubOutput: FernSdkGenApiRequestedOutput = { + type: "github", + repository: "acme/sdk", + mode: "pull-request" + }; + + expect(resolveSdkConfigRequestedOutput(githubOutput, true)).toEqual({ type: "download" }); + expect(resolveSdkConfigRequestedOutput(githubOutput, false)).toBe(githubOutput); + }); + it("infers npm for legacy publish output with no explicit registry override", () => { const request = createFernSdkGenApiRequest({ apiName: "Petstore", diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/fernSdkGenApi.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/fernSdkGenApi.ts index d889b6eb2a54..522495e61fbe 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/fernSdkGenApi.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/fernSdkGenApi.ts @@ -11,7 +11,7 @@ import { createHash } from "crypto"; import FormData from "form-data"; import path from "path"; import { gunzipSync } from "zlib"; -import { downloadFilesForTask } from "./RemoteTaskHandler.js"; +import { downloadArchiveForTask, downloadFilesForTask } from "./RemoteTaskHandler.js"; import { type GenerationConfigKind, type GenerationConfigRoute, @@ -78,6 +78,13 @@ export type FernSdkGenApiRequestedOutput = } | { type: "publish"; publish: FernSdkGenApiPublishConfig }; +export function resolveSdkConfigRequestedOutput( + requestedOutput: FernSdkGenApiRequestedOutput | undefined, + isPreview: boolean +): FernSdkGenApiRequestedOutput | undefined { + return isPreview ? { type: "download" } : requestedOutput; +} + interface FernBuildStatus { buildId: string; status: "queued" | "running" | "succeeded" | "failed" | "partial_failure"; @@ -467,6 +474,10 @@ export interface FernSdkConfigV1Payload { sdkName?: string; sdkVersion?: string; clientPathParameterStyle?: "inline" | "wrapped" | "language-default"; + requestedOutput?: FernSdkGenApiRequestedOutput; + /** Local-only destination for a requested ZIP artifact; never serialized to sdk-gen-api. */ + absolutePathToLocalOutputArchive?: AbsoluteFilePath; + package?: FernSdkGenApiPackageConfig; }>; } @@ -481,6 +492,8 @@ export interface FernSdkGenApiBuildParameters { token: FernToken; specsTarGzBuffer: Buffer; payload: FernSdkGenApiPayload; + requestedOutput?: FernSdkGenApiRequestedOutput; + absolutePathToLocalOutputArchive?: AbsoluteFilePath; absolutePathToPreview: AbsoluteFilePath | undefined; context: InteractiveTaskContext; targetIdSeed?: string; @@ -753,7 +766,8 @@ function prepareFernSdkGenApiSubmission(participants: FernSdkGenApiBuildParamete targetIdSeed: participant.targetIdSeed, sourceSpecIndexes: participant.sourceSpecIndexes, audiences: participant.audiences, - payload: participant.payload + payload: participant.payload, + requestedOutput: participant.requestedOutput })) }); const serializedRequest = JSON.stringify(request); @@ -1129,17 +1143,26 @@ async function finishFernSdkGenApiTarget( code: CliError.Code.InternalError }); } - const outputPath = - participant.absolutePathToPreview != null - ? join( - participant.absolutePathToPreview, - RelativeFilePath.of(path.basename(participant.generatorInvocation.name)) - ) - : participant.generatorInvocation.absolutePathToLocalOutput; - if (outputPath != null) { + if (participant.absolutePathToPreview != null) { + await downloadFilesForTask({ + s3PreSignedReadUrl: target.result.artifactUrl, + absolutePathToLocalOutput: join( + participant.absolutePathToPreview, + RelativeFilePath.of(path.basename(participant.generatorInvocation.name)) + ), + context: participant.context, + skipFernignore: participant.skipFernignore + }); + } else if (participant.absolutePathToLocalOutputArchive != null) { + await downloadArchiveForTask({ + s3PreSignedReadUrl: target.result.artifactUrl, + absolutePathToLocalOutput: participant.absolutePathToLocalOutputArchive, + context: participant.context + }); + } else if (participant.generatorInvocation.absolutePathToLocalOutput != null) { await downloadFilesForTask({ s3PreSignedReadUrl: target.result.artifactUrl, - absolutePathToLocalOutput: outputPath, + absolutePathToLocalOutput: participant.generatorInvocation.absolutePathToLocalOutput, context: participant.context, skipFernignore: participant.skipFernignore }); @@ -1182,7 +1205,8 @@ export function createFernSdkGenApiRequest({ sdkVersion, apiVersion, specsTarGzBuffer, - payload + payload, + requestedOutput }: { apiName: string; organization: string; @@ -1193,13 +1217,14 @@ export function createFernSdkGenApiRequest({ apiVersion?: string; specsTarGzBuffer: Buffer; payload: FernSdkGenApiPayload; + requestedOutput?: FernSdkGenApiRequestedOutput; }): FernSdkGenApiRequest { return createFernSdkGenApiBatchRequest({ apiName, organization, cliVersion, specsTarGzBuffer, - targets: [{ generatorInvocation, sdkName, sdkVersion, apiVersion, payload }] + targets: [{ generatorInvocation, sdkName, sdkVersion, apiVersion, payload, requestedOutput }] }); } @@ -1223,6 +1248,7 @@ export function createFernSdkGenApiBatchRequest({ sourceSpecIndexes?: number[]; audiences?: string[]; payload: FernSdkGenApiPayload; + requestedOutput?: FernSdkGenApiRequestedOutput; }>; }): FernSdkGenApiRequest { if (targets.length === 0) { @@ -1241,7 +1267,10 @@ export function createFernSdkGenApiBatchRequest({ return id; }); const requestTargets = targets.map( - ({ generatorInvocation, sdkName, sdkVersion, apiVersion, targetIdSeed, audiences, payload }, index) => { + ( + { generatorInvocation, sdkName, sdkVersion, apiVersion, targetIdSeed, audiences, payload, requestedOutput }, + index + ) => { const language = getFernSdkGenApiLanguage(generatorInvocation.name); if (language == null) { throw new Error(`Unsupported Fern SDK generator: ${generatorInvocation.name}`); @@ -1292,7 +1321,7 @@ export function createFernSdkGenApiBatchRequest({ } : {}) }, - requestedOutput: output.requestedOutput + requestedOutput: requestedOutput ?? output.requestedOutput }; } ); diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/index.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/index.ts index 4e81a87edf85..572cef04a5c2 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/index.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/index.ts @@ -1,5 +1,9 @@ export { findGeneratorLineNumber, GeneratorOccurrenceTracker, getOutputRepoUrl } from "./automationMetadata.js"; -export type { FernSdkConfigV1Payload } from "./fernSdkGenApi.js"; +export type { + FernSdkConfigV1Payload, + FernSdkGenApiPackageConfig, + FernSdkGenApiRequestedOutput +} from "./fernSdkGenApi.js"; export { getFernSdkGenApiLanguage, isFernSdkGenApiEnabled } from "./fernSdkGenApi.js"; export type { FernSdkGenApiImportSettings, diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts index c4c1dc2731c1..b82a18b94e81 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts @@ -39,6 +39,7 @@ import { FernSdkGenApiPreparationBatch, getFernSdkGenApiLanguage, isEligibleForFernSdkGenApi, + resolveSdkConfigRequestedOutput, runFernSdkGenApiBuild } from "./fernSdkGenApi.js"; import type { FernSdkGenApiSourceArchive } from "./fernSdkGenApiSourceArchive.js"; @@ -324,7 +325,17 @@ export async function runRemoteGenerationForGenerator({ apiVersion: sdkConfigV1.apiVersion, token, specsTarGzBuffer: candidate.specsTarGzBuffer, - payload: { payloadKind: "sdk-config-v1", body: sdkConfigV1.body }, + payload: { + payloadKind: "sdk-config-v1", + body: sdkConfigV1.body, + package: sdkConfigTarget.package + }, + // Preview must never retain a publishing destination from SDK Config. + requestedOutput: resolveSdkConfigRequestedOutput( + sdkConfigTarget.requestedOutput, + absolutePathToPreview != null + ), + absolutePathToLocalOutputArchive: sdkConfigTarget.absolutePathToLocalOutputArchive, absolutePathToPreview, context: interactiveTaskContext, targetIdSeed: sdkGenApiTargetIdSeed, diff --git a/packages/cli/project-loader/src/__test__/loadProject.sdkConfig.test.ts b/packages/cli/project-loader/src/__test__/loadProject.sdkConfig.test.ts new file mode 100644 index 000000000000..95697b259402 --- /dev/null +++ b/packages/cli/project-loader/src/__test__/loadProject.sdkConfig.test.ts @@ -0,0 +1,36 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { AbsoluteFilePath } from "@fern-api/fs-utils"; +import { createMockTaskContext } from "@fern-api/task-context"; +import { afterEach, describe, expect, it } from "vitest"; + +import { loadProjectFromDirectory } from "../loadProject.js"; + +describe("loadProjectFromDirectory — caller-owned SDK Config workspace", () => { + const temporaryDirectories: string[] = []; + + afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true }))); + }); + + it("skips generators.yml discovery and allows the caller to provide the API workspace", async () => { + const fernDirectory = await mkdtemp(path.join(tmpdir(), "fern-sdk-config-project-")); + temporaryDirectories.push(fernDirectory); + await writeFile(path.join(fernDirectory, "fern.config.json"), '{"organization":"test","version":"*"}\n'); + await writeFile(path.join(fernDirectory, "generators.yml"), "this is intentionally invalid: [\n"); + + const project = await loadProjectFromDirectory({ + absolutePathToFernDirectory: AbsoluteFilePath.of(fernDirectory), + cliName: "fern", + cliVersion: "0.0.0", + commandLineApiWorkspace: undefined, + defaultToAllApiWorkspaces: false, + skipApiWorkspaces: true, + context: createMockTaskContext() + }); + + expect(project.apiWorkspaces).toEqual([]); + expect(project.config.organization).toBe("test"); + }); +}); diff --git a/packages/cli/project-loader/src/loadProject.ts b/packages/cli/project-loader/src/loadProject.ts index d51f16b4b920..d3315a3cc3a3 100644 --- a/packages/cli/project-loader/src/loadProject.ts +++ b/packages/cli/project-loader/src/loadProject.ts @@ -41,6 +41,8 @@ export declare namespace loadProject { nameOverride?: string; sdkLanguage?: generatorsYml.GenerationLanguage; preserveSchemaIds?: boolean; + /** Skip legacy API discovery when a caller-owned configuration loader provides the workspace. */ + skipApiWorkspaces?: boolean; } export interface LoadProjectFromDirectoryArgs extends Args { @@ -70,6 +72,7 @@ export async function loadProjectFromDirectory({ cliVersion, commandLineApiWorkspace, defaultToAllApiWorkspaces, + skipApiWorkspaces = false, context }: loadProject.LoadProjectFromDirectoryArgs): Promise { let apiWorkspaces: AbstractAPIWorkspace[] = []; @@ -85,7 +88,10 @@ export async function loadProjectFromDirectory({ doesPathExist(join(absolutePathToFernDirectory, RelativeFilePath.of(ASYNCAPI_DIRECTORY))) ]); - if (apisExists || defExists || genExists || genAltExists || openapiExists || asyncapiExists) { + if ( + !skipApiWorkspaces && + (apisExists || defExists || genExists || genAltExists || openapiExists || asyncapiExists) + ) { apiWorkspaces = await loadApis({ cliName, fernDirectory: absolutePathToFernDirectory, @@ -98,7 +104,7 @@ export async function loadProjectFromDirectory({ const docsWorkspaces = await loadDocsWorkspace({ fernDirectory: absolutePathToFernDirectory, context }); - if (apiWorkspaces.length === 0 && docsWorkspaces == null) { + if (apiWorkspaces.length === 0 && docsWorkspaces == null && !skipApiWorkspaces) { return context.failAndThrow( `No SDK specifications or docs specifications found. Please ensure one of the following .yml (not .yaml) files is present:\n` + ` › ${GENERATORS_CONFIGURATION_FILENAME}\n` + diff --git a/packages/cli/workspace/lazy-fern-workspace/src/__test__/bundleRemoteOpenAPI.test.ts b/packages/cli/workspace/lazy-fern-workspace/src/__test__/bundleRemoteOpenAPI.test.ts new file mode 100644 index 000000000000..c07a463ccbf4 --- /dev/null +++ b/packages/cli/workspace/lazy-fern-workspace/src/__test__/bundleRemoteOpenAPI.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it, vi } from "vitest"; + +import { bundleRemoteOpenAPI } from "../utils/bundleRemoteOpenAPI.js"; + +describe("bundleRemoteOpenAPI", () => { + it("resolves nested paths relative to the remote root URL", async () => { + const documents = new Map([ + [ + "https://specs.example.com/apis/openapi.yml", + `openapi: 3.0.0 +info: + title: Petstore + version: 1.0.0 +paths: + /pets: + get: + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: ./components/pet.yml#/Pet +` + ], + [ + "https://specs.example.com/apis/components/pet.yml", + `Pet: + type: object + required: [name] + properties: + name: + type: string + owner: + $ref: ./owner.yml#/Owner +` + ], + [ + "https://specs.example.com/apis/components/owner.yml", + `Owner: + type: object + properties: + id: + type: string +` + ] + ]); + const loadRemote = vi.fn(async (url: string) => { + const body = documents.get(url); + if (body == null) { + throw new Error(`Unexpected URL: ${url}`); + } + return { body, mimeType: "application/yaml" }; + }); + + const bundled = await bundleRemoteOpenAPI("https://specs.example.com/apis/openapi.yml", loadRemote); + + expect(loadRemote).toHaveBeenCalledWith("https://specs.example.com/apis/components/pet.yml"); + expect(loadRemote).toHaveBeenCalledWith("https://specs.example.com/apis/components/owner.yml"); + expect(JSON.stringify(bundled)).not.toContain("./components/pet.yml"); + expect(JSON.stringify(bundled)).not.toContain("./owner.yml"); + expect(bundled).toMatchObject({ + components: { + schemas: { + Pet: { + type: "object", + required: ["name"] + }, + Owner: { + type: "object" + } + } + } + }); + }); +}); diff --git a/packages/cli/workspace/lazy-fern-workspace/src/__test__/safeRemoteUrl.test.ts b/packages/cli/workspace/lazy-fern-workspace/src/__test__/safeRemoteUrl.test.ts new file mode 100644 index 000000000000..b2e948516e9d --- /dev/null +++ b/packages/cli/workspace/lazy-fern-workspace/src/__test__/safeRemoteUrl.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from "vitest"; + +import { isPublicIpAddress, safeFetchRemoteText, validateRemoteUrl } from "../utils/safeRemoteUrl.js"; + +describe("safe remote URL resolution", () => { + it.each([ + "0.0.0.0", + "10.0.0.1", + "100.64.0.1", + "127.0.0.1", + "169.254.169.254", + "172.16.0.1", + "192.168.0.1", + "::", + "::1", + "::ffff:127.0.0.1", + "::ffff:0:127.0.0.1", + "64:ff9b::127.0.0.1", + "64:ff9b:1::808:808", + "2002:7f00:1::", + "fc00::1", + "fe80::1" + ])("blocks non-public address %s", (address) => { + expect(isPublicIpAddress(address)).toBe(false); + }); + + it.each(["8.8.8.8", "1.1.1.1", "2606:4700:4700::1111"])("allows public address %s", (address) => { + expect(isPublicIpAddress(address)).toBe(true); + }); + + it("blocks hosts that resolve to cloud metadata or private addresses", async () => { + await expect( + validateRemoteUrl( + "https://metadata.example.com/openapi.yml", + vi.fn().mockResolvedValue([{ address: "169.254.169.254", family: 4 }]) as never + ) + ).rejects.toThrow("blocked address"); + }); + + it("blocks a host when any DNS answer is non-public", async () => { + await expect( + validateRemoteUrl( + "https://mixed.example.com/openapi.yml", + vi.fn().mockResolvedValue([ + { address: "8.8.8.8", family: 4 }, + { address: "10.0.0.1", family: 4 } + ]) as never + ) + ).rejects.toThrow("10.0.0.1"); + }); + + it("blocks URL credentials before DNS resolution", async () => { + const lookup = vi.fn(); + await expect( + validateRemoteUrl("https://user:password@example.com/openapi.yml", lookup as never) + ).rejects.toThrow("cannot contain credentials"); + expect(lookup).not.toHaveBeenCalled(); + }); + + it("revalidates redirect destinations before requesting them", async () => { + const lookup = vi.fn(async (hostname: string) => [ + { address: hostname === "public.example.com" ? "8.8.8.8" : "127.0.0.1", family: 4 } + ]); + const request = vi.fn().mockResolvedValue({ + statusCode: 302, + headers: { location: "http://internal.example.com/openapi.yml" }, + body: Buffer.alloc(0) + }); + + await expect( + safeFetchRemoteText("https://public.example.com/openapi.yml", { + lookup: lookup as never, + request + }) + ).rejects.toThrow("blocked address"); + expect(request).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/cli/workspace/lazy-fern-workspace/src/utils/bundleRemoteOpenAPI.ts b/packages/cli/workspace/lazy-fern-workspace/src/utils/bundleRemoteOpenAPI.ts new file mode 100644 index 000000000000..e69323432391 --- /dev/null +++ b/packages/cli/workspace/lazy-fern-workspace/src/utils/bundleRemoteOpenAPI.ts @@ -0,0 +1,37 @@ +import { DEFAULT_OPENAPI_BUNDLE_OPTIONS } from "@fern-api/api-workspace-commons"; +import { BaseResolver, bundle, Source } from "@redocly/openapi-core"; + +import { type RemoteTextResource, safeFetchRemoteText } from "./safeRemoteUrl.js"; + +type RemoteLoader = (url: string) => Promise; + +class SafeRemoteOpenAPIResolver extends BaseResolver { + constructor(private readonly loadRemote: RemoteLoader) { + super(); + } + + public override async loadExternalRef(absoluteRef: string): Promise { + if (!absoluteRef.startsWith("http://") && !absoluteRef.startsWith("https://")) { + return super.loadExternalRef(absoluteRef); + } + const resource = await this.loadRemote(absoluteRef); + return new Source(absoluteRef, resource.body, resource.mimeType); + } +} + +/** Bundles a remote OpenAPI document and all relative references into one in-memory document. */ +export async function bundleRemoteOpenAPI( + url: string, + loadRemote: RemoteLoader = safeFetchRemoteText +): Promise { + const result = await bundle({ + ...DEFAULT_OPENAPI_BUNDLE_OPTIONS, + ref: url, + externalRefResolver: new SafeRemoteOpenAPIResolver(loadRemote) + }); + const errors = result.problems.filter((problem) => problem.severity === "error"); + if (errors.length > 0) { + throw new Error(errors.map((problem) => problem.message).join("; ")); + } + return result.bundle.parsed; +} diff --git a/packages/cli/workspace/lazy-fern-workspace/src/utils/index.ts b/packages/cli/workspace/lazy-fern-workspace/src/utils/index.ts index e9d1c2983a9b..89f2bcfbf947 100644 --- a/packages/cli/workspace/lazy-fern-workspace/src/utils/index.ts +++ b/packages/cli/workspace/lazy-fern-workspace/src/utils/index.ts @@ -1,6 +1,8 @@ +export { bundleRemoteOpenAPI } from "./bundleRemoteOpenAPI.js"; export { convertOpenAPIV2ToV3 } from "./convertOpenAPIV2ToV3.js"; export { getAllOpenAPISpecs } from "./getAllOpenAPISpecs.js"; export { loadAsyncAPI } from "./loadAsyncAPI.js"; export { loadDependency } from "./loadDependency.js"; export { loadOpenAPI } from "./loadOpenAPI.js"; export { type WorkspaceLoader, WorkspaceLoaderFailureType } from "./Result.js"; +export { isPublicIpAddress, safeFetchRemoteText, validateRemoteUrl } from "./safeRemoteUrl.js"; diff --git a/packages/cli/workspace/lazy-fern-workspace/src/utils/safeRemoteUrl.ts b/packages/cli/workspace/lazy-fern-workspace/src/utils/safeRemoteUrl.ts new file mode 100644 index 000000000000..207c2c2e946d --- /dev/null +++ b/packages/cli/workspace/lazy-fern-workspace/src/utils/safeRemoteUrl.ts @@ -0,0 +1,266 @@ +import type { LookupAddress } from "node:dns"; +import { lookup as dnsLookup } from "node:dns/promises"; +import type { IncomingHttpHeaders } from "node:http"; +import { request as httpRequest } from "node:http"; +import { request as httpsRequest } from "node:https"; +import { isIP, type LookupFunction } from "node:net"; + +const MAX_REDIRECTS = 5; +const MAX_RESPONSE_BYTES = 25 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 30_000; + +export interface RemoteTextResource { + body: string; + mimeType?: string; +} + +interface RemoteResponse { + statusCode: number; + headers: IncomingHttpHeaders; + body: Buffer; +} + +interface SafeFetchOptions { + lookup?: typeof dnsLookup; + request?: (url: URL, addresses: LookupAddress[]) => Promise; +} + +/** Fetches a public HTTP(S) resource while validating every redirect and pinning DNS results. */ +export async function safeFetchRemoteText(url: string, options: SafeFetchOptions = {}): Promise { + const response = await fetchWithRedirects(url, options, 0); + const contentType = response.headers["content-type"]; + return { + body: response.body.toString("utf8"), + ...(typeof contentType === "string" ? { mimeType: contentType } : {}) + }; +} + +async function fetchWithRedirects( + value: string, + options: SafeFetchOptions, + redirectCount: number +): Promise { + const { url, addresses } = await validateRemoteUrl(value, options.lookup ?? dnsLookup); + const response = await (options.request ?? requestPinnedUrl)(url, addresses); + if (isRedirect(response.statusCode)) { + if (redirectCount >= MAX_REDIRECTS) { + throw new Error(`Remote API source exceeded ${MAX_REDIRECTS} redirects`); + } + const location = response.headers.location; + if (typeof location !== "string") { + throw new Error(`Remote API source returned redirect ${response.statusCode} without a Location header`); + } + return fetchWithRedirects(new URL(location, url).href, options, redirectCount + 1); + } + if (response.statusCode < 200 || response.statusCode >= 300) { + throw new Error(`Remote API source returned HTTP ${response.statusCode}`); + } + return response; +} + +export async function validateRemoteUrl( + value: string, + lookup: typeof dnsLookup = dnsLookup +): Promise<{ url: URL; addresses: LookupAddress[] }> { + const url = new URL(value); + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Remote API source must use HTTP or HTTPS: ${value}`); + } + if (url.username.length > 0 || url.password.length > 0) { + throw new Error("Remote API source URLs cannot contain credentials"); + } + const hostname = url.hostname.replace(/^\[|\]$/g, "").toLowerCase(); + if (hostname === "localhost" || hostname.endsWith(".localhost")) { + throw new Error(`Remote API source resolves to a blocked host: ${hostname}`); + } + const addresses = await lookup(hostname, { all: true, verbatim: true }); + if (addresses.length === 0) { + throw new Error(`Remote API source host did not resolve: ${hostname}`); + } + for (const address of addresses) { + if (!isPublicIpAddress(address.address)) { + throw new Error(`Remote API source resolves to a blocked address: ${address.address}`); + } + } + return { url, addresses }; +} + +export function isPublicIpAddress(address: string): boolean { + const version = isIP(address); + if (version === 4) { + return isPublicIpv4(address); + } + if (version === 6) { + return isPublicIpv6(address); + } + return false; +} + +function isPublicIpv4(address: string): boolean { + const octets = address.split(".").map(Number); + const [first, second, third] = octets; + if (octets.length !== 4 || octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) { + return false; + } + if (first == null || second == null || third == null) { + return false; + } + return !( + first === 0 || + first === 10 || + (first === 100 && second >= 64 && second <= 127) || + first === 127 || + (first === 169 && second === 254) || + (first === 172 && second >= 16 && second <= 31) || + (first === 192 && second === 0 && third === 0) || + (first === 192 && second === 0 && third === 2) || + (first === 192 && second === 168) || + (first === 198 && (second === 18 || second === 19)) || + (first === 198 && second === 51 && third === 100) || + (first === 203 && second === 0 && third === 113) || + first >= 224 + ); +} + +function isPublicIpv6(address: string): boolean { + const words = parseIpv6Words(address); + if (words == null) { + return false; + } + const first = words[0]; + if (first == null) { + return false; + } + const isUnspecifiedOrLoopback = words.slice(0, 7).every((word) => word === 0) && (words[7] === 0 || words[7] === 1); + const isUniqueLocal = (first & 0xfe00) === 0xfc00; + const isLinkLocal = (first & 0xffc0) === 0xfe80; + const isDeprecatedSiteLocal = (first & 0xffc0) === 0xfec0; + const isMulticast = (first & 0xff00) === 0xff00; + const isSpecialPurpose = first === 0x2001 && (words[1] ?? 0) <= 0x01ff; + const isSixToFour = first === 0x2002; + const isDiscardOnly = first === 0x0100 && words.slice(1, 4).every((word) => word === 0); + const isIpv4Mapped = words.slice(0, 5).every((word) => word === 0) && (words[5] === 0 || words[5] === 0xffff); + const isIpv4Translated = words.slice(0, 4).every((word) => word === 0) && words[4] === 0xffff && words[5] === 0; + const isNat64WellKnown = first === 0x0064 && words[1] === 0xff9b && words.slice(2, 6).every((word) => word === 0); + const isNat64Local = first === 0x0064 && words[1] === 0xff9b && words[2] === 1; + if (isIpv4Mapped || isIpv4Translated || isNat64WellKnown) { + const high = words[6]; + const low = words[7]; + if (high == null || low == null) { + return false; + } + return isPublicIpv4(`${high >> 8}.${high & 0xff}.${low >> 8}.${low & 0xff}`); + } + return !( + isUnspecifiedOrLoopback || + isUniqueLocal || + isLinkLocal || + isDeprecatedSiteLocal || + isMulticast || + isSpecialPurpose || + isSixToFour || + isDiscardOnly || + isNat64Local + ); +} + +function parseIpv6Words(address: string): number[] | undefined { + const withoutZone = address.toLowerCase().split("%")[0]; + if (withoutZone == null) { + return undefined; + } + let normalized = withoutZone; + const dottedTail = /(?:^|:)(\d+\.\d+\.\d+\.\d+)$/.exec(normalized)?.[1]; + if (dottedTail != null) { + const octets = dottedTail.split(".").map(Number); + if (octets.length !== 4 || octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) { + return undefined; + } + normalized = + normalized.slice(0, -dottedTail.length) + + `${(((octets[0] ?? 0) << 8) | (octets[1] ?? 0)).toString(16)}:${(((octets[2] ?? 0) << 8) | (octets[3] ?? 0)).toString(16)}`; + } + const halves = normalized.split("::"); + if (halves.length > 2) { + return undefined; + } + const left = halves[0]?.length ? halves[0].split(":") : []; + const right = halves[1]?.length ? halves[1].split(":") : []; + const missing = 8 - left.length - right.length; + if ((halves.length === 1 && missing !== 0) || (halves.length === 2 && missing < 1)) { + return undefined; + } + const groups = [...left, ...Array.from({ length: Math.max(0, missing) }, () => "0"), ...right]; + if (groups.length !== 8 || groups.some((group) => !/^[0-9a-f]{1,4}$/.test(group))) { + return undefined; + } + return groups.map((group) => Number.parseInt(group, 16)); +} + +function requestPinnedUrl(url: URL, addresses: LookupAddress[]): Promise { + return new Promise((resolve, reject) => { + const transport = url.protocol === "https:" ? httpsRequest : httpRequest; + const request = transport( + url, + { + method: "GET", + headers: { + accept: "application/json, application/yaml, text/yaml, */*", + "user-agent": "fern-cli" + }, + lookup: createPinnedLookup(addresses), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }, + (response) => { + const statusCode = response.statusCode ?? 0; + if (isRedirect(statusCode)) { + response.resume(); + resolve({ statusCode, headers: response.headers, body: Buffer.alloc(0) }); + return; + } + const contentLength = Number(response.headers["content-length"]); + if (Number.isFinite(contentLength) && contentLength > MAX_RESPONSE_BYTES) { + response.destroy(); + reject(new Error(`Remote API source exceeds the ${MAX_RESPONSE_BYTES} byte limit`)); + return; + } + const chunks: Buffer[] = []; + let bytes = 0; + response.on("data", (chunk: Buffer | string) => { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + bytes += buffer.length; + if (bytes > MAX_RESPONSE_BYTES) { + response.destroy(new Error(`Remote API source exceeds the ${MAX_RESPONSE_BYTES} byte limit`)); + return; + } + chunks.push(buffer); + }); + response.on("end", () => { + resolve({ statusCode, headers: response.headers, body: Buffer.concat(chunks) }); + }); + response.on("error", reject); + } + ); + request.on("error", reject); + request.end(); + }); +} + +function createPinnedLookup(addresses: LookupAddress[]): LookupFunction { + return (_hostname, options, callback) => { + if (options.all) { + callback(null, addresses); + return; + } + const address = addresses[0]; + if (address == null) { + callback(new Error("Remote API source has no validated address"), ""); + return; + } + callback(null, address.address, address.family); + }; +} + +function isRedirect(statusCode: number): boolean { + return statusCode === 301 || statusCode === 302 || statusCode === 303 || statusCode === 307 || statusCode === 308; +}