From 1b293d0caf5574e994eef5182b12ab6abec57696 Mon Sep 17 00:00:00 2001 From: Brent Rager Date: Tue, 2 Jun 2026 15:07:25 -0400 Subject: [PATCH] SMOODEV-1524: ESO manifest generator (@smooai/config/eso-manifests) Emit the two ESO resources that let a workload pull secrets from api.smoo.ai instead of Pulumi-baking them at SST deploy time: - buildClusterSecretStore(): ClusterSecretStore whose webhook points at the REAL config-values endpoint (GET /organizations/{org}/config/values/{key} ?environment={env} -> {value}, jsonPath $.value), org+env baked into the URL, bearer from the bootstrap Secret the eso-refresher (1523) keeps fresh. Never the hallucinated config.smoo.ai. - buildExternalSecret(): per-workload ExternalSecret mapping secret-tier config keys -> env-var names (UPPER_SNAKE_CASE default + explicit overrides like DASHSCOPE_API_KEY <- alibabaModelStudioApiKey), with duplicate-env-var guard and distinct target-Secret-name support for safe migration. Pure data (cdk8s/ArgoCD/kubectl all accept the objects). 12 unit tests. Epic SMOODEV-1522; consumed by smooai under SMOODEV-1525. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/eso-manifest-generator.md | 5 + package.json | 5 + src/eso-manifests/__tests__/index.test.ts | 110 ++++++++++++ src/eso-manifests/index.ts | 195 ++++++++++++++++++++++ tsup.config.ts | 1 + 5 files changed, 316 insertions(+) create mode 100644 .changeset/eso-manifest-generator.md create mode 100644 src/eso-manifests/__tests__/index.test.ts create mode 100644 src/eso-manifests/index.ts diff --git a/.changeset/eso-manifest-generator.md b/.changeset/eso-manifest-generator.md new file mode 100644 index 0000000..f12d86c --- /dev/null +++ b/.changeset/eso-manifest-generator.md @@ -0,0 +1,5 @@ +--- +'@smooai/config': minor +--- + +SMOODEV-1524: Add an ESO manifest generator (`@smooai/config/eso-manifests`). `buildClusterSecretStore()` emits a `ClusterSecretStore` whose webhook provider points at the real `api.smoo.ai` config-values endpoint (org + environment baked in, bearer from the bootstrap Secret the eso-refresher keeps fresh), and `buildExternalSecret()` emits a per-workload `ExternalSecret` mapping secret-tier config keys to env-var names (`UPPER_SNAKE_CASE` by default, with per-key overrides like `DASHSCOPE_API_KEY` ← `alibabaModelStudioApiKey`). Replaces the hand-maintained ESO YAML and makes ESO sync a first-class output of the config system. Epic SMOODEV-1522. diff --git a/package.json b/package.json index d0c8172..92f5602 100644 --- a/package.json +++ b/package.json @@ -176,6 +176,11 @@ "import": "./dist/eso-refresher/index.mjs", "require": "./dist/eso-refresher/index.js" }, + "./eso-manifests": { + "types": "./dist/eso-manifests/index.d.ts", + "import": "./dist/eso-manifests/index.mjs", + "require": "./dist/eso-manifests/index.js" + }, "./platform/client": { "types": "./dist/platform/client.d.ts", "browser": { diff --git a/src/eso-manifests/__tests__/index.test.ts b/src/eso-manifests/__tests__/index.test.ts new file mode 100644 index 0000000..c2e6d55 --- /dev/null +++ b/src/eso-manifests/__tests__/index.test.ts @@ -0,0 +1,110 @@ +/** + * SMOODEV-1524 — ESO manifest generator unit tests. + */ +import { describe, expect, it } from 'vitest'; +import { buildClusterSecretStore, buildExternalSecret, resolveSecretMapping } from '../index'; + +describe('buildClusterSecretStore (SMOODEV-1524)', () => { + const base = { apiUrl: 'https://api.smoo.ai', orgId: 'org-123', environment: 'production' }; + + it('bakes org + environment into the webhook URL and templates only the key', () => { + const store = buildClusterSecretStore(base) as any; + expect(store.kind).toBe('ClusterSecretStore'); + expect(store.spec.provider.webhook.url).toBe('https://api.smoo.ai/organizations/org-123/config/values/{{ .remoteRef.key }}?environment=production'); + expect(store.spec.provider.webhook.result.jsonPath).toBe('$.value'); + expect(store.spec.provider.webhook.headers.Authorization).toBe('Bearer {{ .auth.token }}'); + }); + + it('points at the real api.smoo.ai endpoint, never the hallucinated config.smoo.ai', () => { + const store = buildClusterSecretStore(base) as any; + expect(store.spec.provider.webhook.url).not.toContain('config.smoo.ai'); + expect(store.spec.provider.webhook.url).toContain('api.smoo.ai'); + }); + + it('defaults the bootstrap Secret ref and strips trailing slashes from apiUrl', () => { + const store = buildClusterSecretStore({ ...base, apiUrl: 'https://api.smoo.ai///' }) as any; + expect(store.spec.provider.webhook.url.startsWith('https://api.smoo.ai/organizations')).toBe(true); + const ref = store.spec.provider.webhook.secrets[0].secretRef; + expect(ref).toEqual({ name: 'smooai-config-bootstrap', namespace: 'external-secrets', key: 'bearer-token' }); + }); + + it('honors bootstrap Secret + name overrides and url-encodes the environment', () => { + const store = buildClusterSecretStore({ + ...base, + name: 'smooai-config-prod', + environment: 'pre prod', + bootstrapSecret: { name: 's', namespace: 'ns', key: 'k' }, + }) as any; + expect(store.metadata.name).toBe('smooai-config-prod'); + expect(store.spec.provider.webhook.url).toContain('environment=pre%20prod'); + expect(store.spec.provider.webhook.secrets[0].secretRef).toEqual({ name: 's', namespace: 'ns', key: 'k' }); + }); + + it('throws on missing required fields', () => { + expect(() => buildClusterSecretStore({ ...base, apiUrl: '' })).toThrow(/apiUrl/); + expect(() => buildClusterSecretStore({ ...base, orgId: '' })).toThrow(/orgId/); + expect(() => buildClusterSecretStore({ ...base, environment: '' })).toThrow(/environment/); + }); +}); + +describe('resolveSecretMapping (SMOODEV-1524)', () => { + it('snakecases the config key into an env-var name by default', () => { + expect(resolveSecretMapping('mimoApiKey')).toEqual({ configKey: 'mimoApiKey', envVar: 'MIMO_API_KEY' }); + }); + + it('honors an explicit env-var override (env name ≠ snakecase(key))', () => { + expect(resolveSecretMapping({ configKey: 'alibabaModelStudioApiKey', envVar: 'DASHSCOPE_API_KEY' })).toEqual({ + configKey: 'alibabaModelStudioApiKey', + envVar: 'DASHSCOPE_API_KEY', + }); + }); +}); + +describe('buildExternalSecret (SMOODEV-1524)', () => { + it('maps each config key to its env-var (snakecase default + explicit override)', () => { + const es = buildExternalSecret({ + name: 'litellm-config', + namespace: 'smooai-litellm', + secrets: ['mimoApiKey', { configKey: 'alibabaModelStudioApiKey', envVar: 'DASHSCOPE_API_KEY' }], + }) as any; + expect(es.kind).toBe('ExternalSecret'); + expect(es.spec.data).toEqual([ + { secretKey: 'MIMO_API_KEY', remoteRef: { key: 'mimoApiKey' } }, + { secretKey: 'DASHSCOPE_API_KEY', remoteRef: { key: 'alibabaModelStudioApiKey' } }, + ]); + }); + + it('defaults target Secret name to the resource name and store to smooai-config', () => { + const es = buildExternalSecret({ name: 'litellm-config', namespace: 'smooai-litellm', secrets: ['mimoApiKey'] }) as any; + expect(es.spec.target.name).toBe('litellm-config'); + expect(es.spec.target.creationPolicy).toBe('Owner'); + expect(es.spec.secretStoreRef).toEqual({ name: 'smooai-config', kind: 'ClusterSecretStore' }); + expect(es.spec.refreshInterval).toBe('1h'); + }); + + it('supports a distinct target Secret name (safe migration: sync to a new Secret first)', () => { + const es = buildExternalSecret({ + name: 'litellm-config-eso', + namespace: 'smooai-litellm', + targetSecretName: 'litellm-config-eso', + secrets: ['mimoApiKey'], + }) as any; + expect(es.spec.target.name).toBe('litellm-config-eso'); + }); + + it('rejects duplicate env-var names that would silently clobber', () => { + expect(() => + buildExternalSecret({ + name: 'x', + namespace: 'ns', + secrets: ['mimoApiKey', { configKey: 'somethingElse', envVar: 'MIMO_API_KEY' }], + }), + ).toThrow(/duplicate env-var/); + }); + + it('throws on missing required fields', () => { + expect(() => buildExternalSecret({ name: '', namespace: 'ns', secrets: ['k'] })).toThrow(/name/); + expect(() => buildExternalSecret({ name: 'n', namespace: '', secrets: ['k'] })).toThrow(/namespace/); + expect(() => buildExternalSecret({ name: 'n', namespace: 'ns', secrets: [] })).toThrow(/at least one/); + }); +}); diff --git a/src/eso-manifests/index.ts b/src/eso-manifests/index.ts new file mode 100644 index 0000000..e373d12 --- /dev/null +++ b/src/eso-manifests/index.ts @@ -0,0 +1,195 @@ +/** + * `@smooai/config/eso-manifests` — ExternalSecrets Operator (ESO) manifest + * generator (SMOODEV-1524, epic SMOODEV-1522). + * + * Emits the two ESO resources that let a Kubernetes workload pull its secrets + * from the @smooai/config HTTP API (`api.smoo.ai`) instead of having them + * Pulumi-baked at SST deploy time: + * + * 1. {@link buildClusterSecretStore} — a `ClusterSecretStore` whose `webhook` + * provider points at the REAL config-values endpoint, with org + env baked + * into the URL and the bearer sourced from the bootstrap Secret that the + * eso-refresher (SMOODEV-1523) keeps fresh. + * + * 2. {@link buildExternalSecret} — a per-workload `ExternalSecret` mapping the + * consumer's secret-tier config keys to the env-var names the workload + * reads (`UPPER_SNAKE_CASE(key)` by default, matching the SDK's env tier). + * + * Pure data — returns plain manifest objects (cdk8s / ArgoCD / `kubectl apply` + * all accept them). No cluster or network access. The smooai monorepo + * (SMOODEV-1525) consumes these to replace the stripped-out hand YAML. + * + * Endpoint contract (see packages/backend/src/routes/config/config-values.ts): + * GET {apiUrl}/organizations/{orgId}/config/values/{key}?environment={env} + * Authorization: Bearer + * 200 → { "value": } (jsonPath `$.value`) + */ +import { snakecase } from '@/utils'; + +/** A reference to the Kubernetes Secret + key holding the ESO bearer token. */ +export interface BootstrapSecretRef { + /** Secret name. Default `smooai-config-bootstrap`. */ + name?: string; + /** Secret namespace. Default `external-secrets`. */ + namespace?: string; + /** Data key holding the bearer. Default `bearer-token`. */ + key?: string; +} + +export interface ClusterSecretStoreOptions { + /** ClusterSecretStore name. Default `smooai-config`. */ + name?: string; + /** Config API base URL, no trailing slash. E.g. `https://api.smoo.ai`. */ + apiUrl: string; + /** Org id whose config this store reads. */ + orgId: string; + /** Environment name baked into the query string. E.g. `production`. */ + environment: string; + /** Bootstrap bearer Secret reference (kept fresh by the eso-refresher). */ + bootstrapSecret?: BootstrapSecretRef; +} + +export const ESO_DEFAULTS = { + clusterSecretStoreName: 'smooai-config', + bootstrapSecretName: 'smooai-config-bootstrap', + bootstrapSecretNamespace: 'external-secrets', + bootstrapSecretKey: 'bearer-token', + refreshInterval: '1h', + apiVersion: 'external-secrets.io/v1beta1', +} as const; + +function stripTrailingSlashes(s: string): string { + return s.replace(/\/+$/, ''); +} + +/** + * Build a `ClusterSecretStore` backed by the @smooai/config webhook provider. + * + * org + environment are baked into the URL because ESO's webhook only templates + * `{{ .remoteRef.key }}` per-secret — so a store is scoped to one (org, env) + * pair. Use one store per environment. + */ +export function buildClusterSecretStore(opts: ClusterSecretStoreOptions): Record { + if (!opts.apiUrl) throw new Error('buildClusterSecretStore: apiUrl is required'); + if (!opts.orgId) throw new Error('buildClusterSecretStore: orgId is required'); + if (!opts.environment) throw new Error('buildClusterSecretStore: environment is required'); + + const name = opts.name ?? ESO_DEFAULTS.clusterSecretStoreName; + const apiUrl = stripTrailingSlashes(opts.apiUrl); + const env = encodeURIComponent(opts.environment); + const secretName = opts.bootstrapSecret?.name ?? ESO_DEFAULTS.bootstrapSecretName; + const secretNamespace = opts.bootstrapSecret?.namespace ?? ESO_DEFAULTS.bootstrapSecretNamespace; + const secretKey = opts.bootstrapSecret?.key ?? ESO_DEFAULTS.bootstrapSecretKey; + + return { + apiVersion: ESO_DEFAULTS.apiVersion, + kind: 'ClusterSecretStore', + metadata: { name }, + spec: { + provider: { + webhook: { + // `{{ .remoteRef.key }}` is the only per-secret variable ESO + // substitutes; org + env are fixed for this store. + url: `${apiUrl}/organizations/${opts.orgId}/config/values/{{ .remoteRef.key }}?environment=${env}`, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer {{ .auth.token }}', + }, + result: { jsonPath: '$.value' }, + secrets: [ + { + name: 'auth', + secretRef: { + name: secretName, + namespace: secretNamespace, + key: secretKey, + }, + }, + ], + }, + }, + }, + }; +} + +/** One mapped secret: a config key → the env-var name the workload reads. */ +export interface SecretMapping { + /** The @smooai/config secret-tier key (camelCase), e.g. `mimoApiKey`. */ + configKey: string; + /** + * The env-var name the workload's Secret exposes. Defaults to + * `UPPER_SNAKE_CASE(configKey)` (matching the SDK env tier). Override when + * the workload reads a different name (e.g. `DASHSCOPE_API_KEY` ← + * `alibabaModelStudioApiKey`). + */ + envVar?: string; +} + +export interface ExternalSecretOptions { + /** ExternalSecret resource name (and default target Secret name). */ + name: string; + /** Namespace the ExternalSecret + target Secret live in. */ + namespace: string; + /** The secrets this workload needs — config keys (+ optional env-var override). */ + secrets: Array; + /** Target k8s Secret name the workload mounts via envFrom. Default = `name`. */ + targetSecretName?: string; + /** ClusterSecretStore to read from. Default `smooai-config`. */ + clusterSecretStoreName?: string; + /** ESO refresh interval. Default `1h`. */ + refreshInterval?: string; + /** Labels for the ExternalSecret resource. */ + labels?: Record; +} + +/** Normalize a mapping entry to `{ configKey, envVar }` with the snakecase default. */ +export function resolveSecretMapping(entry: SecretMapping | string): { configKey: string; envVar: string } { + const m: SecretMapping = typeof entry === 'string' ? { configKey: entry } : entry; + if (!m.configKey) throw new Error('resolveSecretMapping: configKey is required'); + return { configKey: m.configKey, envVar: m.envVar ?? snakecase(m.configKey) }; +} + +/** + * Build a per-workload `ExternalSecret`. Each entry becomes a `data` mapping of + * `secretKey` (the env-var name in the synced Secret) ← `remoteRef.key` (the + * @smooai/config key). The workload mounts the target Secret via `envFrom`. + */ +export function buildExternalSecret(opts: ExternalSecretOptions): Record { + if (!opts.name) throw new Error('buildExternalSecret: name is required'); + if (!opts.namespace) throw new Error('buildExternalSecret: namespace is required'); + if (!opts.secrets?.length) throw new Error('buildExternalSecret: at least one secret mapping is required'); + + const data = opts.secrets.map((entry) => { + const { configKey, envVar } = resolveSecretMapping(entry); + return { secretKey: envVar, remoteRef: { key: configKey } }; + }); + + // Guard against duplicate env-var names silently clobbering each other. + const envVars = data.map((d) => d.secretKey); + const dupes = envVars.filter((v, i) => envVars.indexOf(v) !== i); + if (dupes.length > 0) { + throw new Error(`buildExternalSecret: duplicate env-var names: ${[...new Set(dupes)].join(', ')}`); + } + + return { + apiVersion: ESO_DEFAULTS.apiVersion, + kind: 'ExternalSecret', + metadata: { + name: opts.name, + namespace: opts.namespace, + ...(opts.labels ? { labels: opts.labels } : {}), + }, + spec: { + refreshInterval: opts.refreshInterval ?? ESO_DEFAULTS.refreshInterval, + secretStoreRef: { + name: opts.clusterSecretStoreName ?? ESO_DEFAULTS.clusterSecretStoreName, + kind: 'ClusterSecretStore', + }, + target: { + name: opts.targetSecretName ?? opts.name, + creationPolicy: 'Owner', + }, + data, + }, + }; +} diff --git a/tsup.config.ts b/tsup.config.ts index fe461cf..582e347 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -23,6 +23,7 @@ const serverEntry = [ 'src/container/errors.ts', 'src/eso-refresher/index.ts', 'src/eso-refresher/run.ts', + 'src/eso-manifests/index.ts', 'src/platform/client.ts', 'src/platform/build.ts', 'src/nextjs/index.ts',