SMOODEV-1524: ESO manifest generator

.changeset/eso-manifest-generator.md

@@ -0,0 +1,5 @@
+---
+'@smooai/config': minor
+---
+
+SMOODEV-1524: Add an ESO manifest generator (`@smooai/config/eso-manifests`). `buildClusterSecretStore()` emits a `ClusterSecretStore` whose webhook provider points at the real `api.smoo.ai` config-values endpoint (org + environment baked in, bearer from the bootstrap Secret the eso-refresher keeps fresh), and `buildExternalSecret()` emits a per-workload `ExternalSecret` mapping secret-tier config keys to env-var names (`UPPER_SNAKE_CASE` by default, with per-key overrides like `DASHSCOPE_API_KEY` ← `alibabaModelStudioApiKey`). Replaces the hand-maintained ESO YAML and makes ESO sync a first-class output of the config system. Epic SMOODEV-1522.

package.json

@@ -176,6 +176,11 @@
             "import": "./dist/eso-refresher/index.mjs",
             "require": "./dist/eso-refresher/index.js"
         },
+        "./eso-manifests": {
+            "types": "./dist/eso-manifests/index.d.ts",
+            "import": "./dist/eso-manifests/index.mjs",
+            "require": "./dist/eso-manifests/index.js"
+        },
         "./platform/client": {
             "types": "./dist/platform/client.d.ts",
             "browser": {

src/eso-manifests/__tests__/index.test.ts

@@ -0,0 +1,110 @@
+/**
+ * SMOODEV-1524 — ESO manifest generator unit tests.
+ */
+import { describe, expect, it } from 'vitest';
+import { buildClusterSecretStore, buildExternalSecret, resolveSecretMapping } from '../index';
+
+describe('buildClusterSecretStore (SMOODEV-1524)', () => {
+    const base = { apiUrl: 'https://api.smoo.ai', orgId: 'org-123', environment: 'production' };
+
+    it('bakes org + environment into the webhook URL and templates only the key', () => {
+        const store = buildClusterSecretStore(base) as any;
+        expect(store.kind).toBe('ClusterSecretStore');
+        expect(store.spec.provider.webhook.url).toBe('https://api.smoo.ai/organizations/org-123/config/values/{{ .remoteRef.key }}?environment=production');
+        expect(store.spec.provider.webhook.result.jsonPath).toBe('$.value');
+        expect(store.spec.provider.webhook.headers.Authorization).toBe('Bearer {{ .auth.token }}');
+    });
+
+    it('points at the real api.smoo.ai endpoint, never the hallucinated config.smoo.ai', () => {
+        const store = buildClusterSecretStore(base) as any;
+        expect(store.spec.provider.webhook.url).not.toContain('config.smoo.ai');
+        expect(store.spec.provider.webhook.url).toContain('api.smoo.ai');
+    });
+
+    it('defaults the bootstrap Secret ref and strips trailing slashes from apiUrl', () => {
+        const store = buildClusterSecretStore({ ...base, apiUrl: 'https://api.smoo.ai///' }) as any;
+        expect(store.spec.provider.webhook.url.startsWith('https://api.smoo.ai/organizations')).toBe(true);
+        const ref = store.spec.provider.webhook.secrets[0].secretRef;
+        expect(ref).toEqual({ name: 'smooai-config-bootstrap', namespace: 'external-secrets', key: 'bearer-token' });
+    });
+
+    it('honors bootstrap Secret + name overrides and url-encodes the environment', () => {
+        const store = buildClusterSecretStore({
+            ...base,
+            name: 'smooai-config-prod',
+            environment: 'pre prod',
+            bootstrapSecret: { name: 's', namespace: 'ns', key: 'k' },
+        }) as any;
+        expect(store.metadata.name).toBe('smooai-config-prod');
+        expect(store.spec.provider.webhook.url).toContain('environment=pre%20prod');
+        expect(store.spec.provider.webhook.secrets[0].secretRef).toEqual({ name: 's', namespace: 'ns', key: 'k' });
+    });
+
+    it('throws on missing required fields', () => {
+        expect(() => buildClusterSecretStore({ ...base, apiUrl: '' })).toThrow(/apiUrl/);
+        expect(() => buildClusterSecretStore({ ...base, orgId: '' })).toThrow(/orgId/);
+        expect(() => buildClusterSecretStore({ ...base, environment: '' })).toThrow(/environment/);
+    });
+});
+
+describe('resolveSecretMapping (SMOODEV-1524)', () => {
+    it('snakecases the config key into an env-var name by default', () => {
+        expect(resolveSecretMapping('mimoApiKey')).toEqual({ configKey: 'mimoApiKey', envVar: 'MIMO_API_KEY' });
+    });
+
+    it('honors an explicit env-var override (env name ≠ snakecase(key))', () => {
+        expect(resolveSecretMapping({ configKey: 'alibabaModelStudioApiKey', envVar: 'DASHSCOPE_API_KEY' })).toEqual({
+            configKey: 'alibabaModelStudioApiKey',
+            envVar: 'DASHSCOPE_API_KEY',
+        });
+    });
+});
+
+describe('buildExternalSecret (SMOODEV-1524)', () => {
+    it('maps each config key to its env-var (snakecase default + explicit override)', () => {
+        const es = buildExternalSecret({
+            name: 'litellm-config',
+            namespace: 'smooai-litellm',
+            secrets: ['mimoApiKey', { configKey: 'alibabaModelStudioApiKey', envVar: 'DASHSCOPE_API_KEY' }],
+        }) as any;
+        expect(es.kind).toBe('ExternalSecret');
+        expect(es.spec.data).toEqual([
+            { secretKey: 'MIMO_API_KEY', remoteRef: { key: 'mimoApiKey' } },
+            { secretKey: 'DASHSCOPE_API_KEY', remoteRef: { key: 'alibabaModelStudioApiKey' } },
+        ]);
+    });
+
+    it('defaults target Secret name to the resource name and store to smooai-config', () => {
+        const es = buildExternalSecret({ name: 'litellm-config', namespace: 'smooai-litellm', secrets: ['mimoApiKey'] }) as any;
+        expect(es.spec.target.name).toBe('litellm-config');
+        expect(es.spec.target.creationPolicy).toBe('Owner');
+        expect(es.spec.secretStoreRef).toEqual({ name: 'smooai-config', kind: 'ClusterSecretStore' });
+        expect(es.spec.refreshInterval).toBe('1h');
+    });
+
+    it('supports a distinct target Secret name (safe migration: sync to a new Secret first)', () => {
+        const es = buildExternalSecret({
+            name: 'litellm-config-eso',
+            namespace: 'smooai-litellm',
+            targetSecretName: 'litellm-config-eso',
+            secrets: ['mimoApiKey'],
+        }) as any;
+        expect(es.spec.target.name).toBe('litellm-config-eso');
+    });
+
+    it('rejects duplicate env-var names that would silently clobber', () => {
+        expect(() =>
+            buildExternalSecret({
+                name: 'x',
+                namespace: 'ns',
+                secrets: ['mimoApiKey', { configKey: 'somethingElse', envVar: 'MIMO_API_KEY' }],
+            }),
+        ).toThrow(/duplicate env-var/);
+    });
+
+    it('throws on missing required fields', () => {
+        expect(() => buildExternalSecret({ name: '', namespace: 'ns', secrets: ['k'] })).toThrow(/name/);
+        expect(() => buildExternalSecret({ name: 'n', namespace: '', secrets: ['k'] })).toThrow(/namespace/);
+        expect(() => buildExternalSecret({ name: 'n', namespace: 'ns', secrets: [] })).toThrow(/at least one/);
+    });
+});

src/eso-manifests/index.ts

@@ -0,0 +1,195 @@
+/**
+ * `@smooai/config/eso-manifests` — ExternalSecrets Operator (ESO) manifest
+ * generator (SMOODEV-1524, epic SMOODEV-1522).
+ *
+ * Emits the two ESO resources that let a Kubernetes workload pull its secrets
+ * from the @smooai/config HTTP API (`api.smoo.ai`) instead of having them
+ * Pulumi-baked at SST deploy time:
+ *
+ *   1. {@link buildClusterSecretStore} — a `ClusterSecretStore` whose `webhook`
+ *      provider points at the REAL config-values endpoint, with org + env baked
+ *      into the URL and the bearer sourced from the bootstrap Secret that the
+ *      eso-refresher (SMOODEV-1523) keeps fresh.
+ *
+ *   2. {@link buildExternalSecret} — a per-workload `ExternalSecret` mapping the
+ *      consumer's secret-tier config keys to the env-var names the workload
+ *      reads (`UPPER_SNAKE_CASE(key)` by default, matching the SDK's env tier).
+ *
+ * Pure data — returns plain manifest objects (cdk8s / ArgoCD / `kubectl apply`
+ * all accept them). No cluster or network access. The smooai monorepo
+ * (SMOODEV-1525) consumes these to replace the stripped-out hand YAML.
+ *
+ * Endpoint contract (see packages/backend/src/routes/config/config-values.ts):
+ *   GET {apiUrl}/organizations/{orgId}/config/values/{key}?environment={env}
+ *   Authorization: Bearer <token>
+ *   200 → { "value": <any> }      (jsonPath `$.value`)
+ */
+import { snakecase } from '@/utils';
+
+/** A reference to the Kubernetes Secret + key holding the ESO bearer token. */
+export interface BootstrapSecretRef {
+    /** Secret name. Default `smooai-config-bootstrap`. */
+    name?: string;
+    /** Secret namespace. Default `external-secrets`. */
+    namespace?: string;
+    /** Data key holding the bearer. Default `bearer-token`. */
+    key?: string;
+}
+
+export interface ClusterSecretStoreOptions {
+    /** ClusterSecretStore name. Default `smooai-config`. */
+    name?: string;
+    /** Config API base URL, no trailing slash. E.g. `https://api.smoo.ai`. */
+    apiUrl: string;
+    /** Org id whose config this store reads. */
+    orgId: string;
+    /** Environment name baked into the query string. E.g. `production`. */
+    environment: string;
+    /** Bootstrap bearer Secret reference (kept fresh by the eso-refresher). */
+    bootstrapSecret?: BootstrapSecretRef;
+}
+
+export const ESO_DEFAULTS = {
+    clusterSecretStoreName: 'smooai-config',
+    bootstrapSecretName: 'smooai-config-bootstrap',
+    bootstrapSecretNamespace: 'external-secrets',
+    bootstrapSecretKey: 'bearer-token',
+    refreshInterval: '1h',
+    apiVersion: 'external-secrets.io/v1beta1',
+} as const;
+
+function stripTrailingSlashes(s: string): string {
+    return s.replace(/\/+$/, '');
+}
+
+/**
+ * Build a `ClusterSecretStore` backed by the @smooai/config webhook provider.
+ *
+ * org + environment are baked into the URL because ESO's webhook only templates
+ * `{{ .remoteRef.key }}` per-secret — so a store is scoped to one (org, env)
+ * pair. Use one store per environment.
+ */
+export function buildClusterSecretStore(opts: ClusterSecretStoreOptions): Record<string, unknown> {
+    if (!opts.apiUrl) throw new Error('buildClusterSecretStore: apiUrl is required');
+    if (!opts.orgId) throw new Error('buildClusterSecretStore: orgId is required');
+    if (!opts.environment) throw new Error('buildClusterSecretStore: environment is required');
+
+    const name = opts.name ?? ESO_DEFAULTS.clusterSecretStoreName;
+    const apiUrl = stripTrailingSlashes(opts.apiUrl);
+    const env = encodeURIComponent(opts.environment);
+    const secretName = opts.bootstrapSecret?.name ?? ESO_DEFAULTS.bootstrapSecretName;
+    const secretNamespace = opts.bootstrapSecret?.namespace ?? ESO_DEFAULTS.bootstrapSecretNamespace;
+    const secretKey = opts.bootstrapSecret?.key ?? ESO_DEFAULTS.bootstrapSecretKey;
+
+    return {
+        apiVersion: ESO_DEFAULTS.apiVersion,
+        kind: 'ClusterSecretStore',
+        metadata: { name },
+        spec: {
+            provider: {
+                webhook: {
+                    // `{{ .remoteRef.key }}` is the only per-secret variable ESO
+                    // substitutes; org + env are fixed for this store.
+                    url: `${apiUrl}/organizations/${opts.orgId}/config/values/{{ .remoteRef.key }}?environment=${env}`,
+                    headers: {
+                        'Content-Type': 'application/json',
+                        Authorization: 'Bearer {{ .auth.token }}',
+                    },
+                    result: { jsonPath: '$.value' },
+                    secrets: [
+                        {
+                            name: 'auth',
+                            secretRef: {
+                                name: secretName,
+                                namespace: secretNamespace,
+                                key: secretKey,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    };
+}
+
+/** One mapped secret: a config key → the env-var name the workload reads. */
+export interface SecretMapping {
+    /** The @smooai/config secret-tier key (camelCase), e.g. `mimoApiKey`. */
+    configKey: string;
+    /**
+     * The env-var name the workload's Secret exposes. Defaults to
+     * `UPPER_SNAKE_CASE(configKey)` (matching the SDK env tier). Override when
+     * the workload reads a different name (e.g. `DASHSCOPE_API_KEY` ←
+     * `alibabaModelStudioApiKey`).
+     */
+    envVar?: string;
+}
+
+export interface ExternalSecretOptions {
+    /** ExternalSecret resource name (and default target Secret name). */
+    name: string;
+    /** Namespace the ExternalSecret + target Secret live in. */
+    namespace: string;
+    /** The secrets this workload needs — config keys (+ optional env-var override). */
+    secrets: Array<SecretMapping | string>;
+    /** Target k8s Secret name the workload mounts via envFrom. Default = `name`. */
+    targetSecretName?: string;
+    /** ClusterSecretStore to read from. Default `smooai-config`. */
+    clusterSecretStoreName?: string;
+    /** ESO refresh interval. Default `1h`. */
+    refreshInterval?: string;
+    /** Labels for the ExternalSecret resource. */
+    labels?: Record<string, string>;
+}
+
+/** Normalize a mapping entry to `{ configKey, envVar }` with the snakecase default. */
+export function resolveSecretMapping(entry: SecretMapping | string): { configKey: string; envVar: string } {
+    const m: SecretMapping = typeof entry === 'string' ? { configKey: entry } : entry;
+    if (!m.configKey) throw new Error('resolveSecretMapping: configKey is required');
+    return { configKey: m.configKey, envVar: m.envVar ?? snakecase(m.configKey) };
+}
+
+/**
+ * Build a per-workload `ExternalSecret`. Each entry becomes a `data` mapping of
+ * `secretKey` (the env-var name in the synced Secret) ← `remoteRef.key` (the
+ * @smooai/config key). The workload mounts the target Secret via `envFrom`.
+ */
+export function buildExternalSecret(opts: ExternalSecretOptions): Record<string, unknown> {
+    if (!opts.name) throw new Error('buildExternalSecret: name is required');
+    if (!opts.namespace) throw new Error('buildExternalSecret: namespace is required');
+    if (!opts.secrets?.length) throw new Error('buildExternalSecret: at least one secret mapping is required');
+
+    const data = opts.secrets.map((entry) => {
+        const { configKey, envVar } = resolveSecretMapping(entry);
+        return { secretKey: envVar, remoteRef: { key: configKey } };
+    });
+
+    // Guard against duplicate env-var names silently clobbering each other.
+    const envVars = data.map((d) => d.secretKey);
+    const dupes = envVars.filter((v, i) => envVars.indexOf(v) !== i);
+    if (dupes.length > 0) {
+        throw new Error(`buildExternalSecret: duplicate env-var names: ${[...new Set(dupes)].join(', ')}`);
+    }
+
+    return {
+        apiVersion: ESO_DEFAULTS.apiVersion,
+        kind: 'ExternalSecret',
+        metadata: {
+            name: opts.name,
+            namespace: opts.namespace,
+            ...(opts.labels ? { labels: opts.labels } : {}),
+        },
+        spec: {
+            refreshInterval: opts.refreshInterval ?? ESO_DEFAULTS.refreshInterval,
+            secretStoreRef: {
+                name: opts.clusterSecretStoreName ?? ESO_DEFAULTS.clusterSecretStoreName,
+                kind: 'ClusterSecretStore',
+            },
+            target: {
+                name: opts.targetSecretName ?? opts.name,
+                creationPolicy: 'Owner',
+            },
+            data,
+        },
+    };
+}

tsup.config.ts

@@ -23,6 +23,7 @@ const serverEntry = [
     'src/container/errors.ts',
     'src/eso-refresher/index.ts',
     'src/eso-refresher/run.ts',
+    'src/eso-manifests/index.ts',
     'src/platform/client.ts',
     'src/platform/build.ts',
     'src/nextjs/index.ts',