From c9cb01a98741ec5164e63b403905f18d3f6daad6 Mon Sep 17 00:00:00 2001 From: Brent Rager Date: Tue, 2 Jun 2026 16:03:25 -0400 Subject: [PATCH 1/2] SMOODEV-1526: ESO manifest generator parity (Go/Python/Rust/C#) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port buildClusterSecretStore + buildExternalSecret from the TS reference to all four other SDK languages. Each emits the same ClusterSecretStore (webhook → real api.smoo.ai config-values endpoint, org+env baked, bearer from the bootstrap Secret) and per-workload ExternalSecret (secret-tier config keys → UPPER_SNAKE env vars via each language's native snakecase util, with env-var overrides + duplicate guard + distinct-target support). Tests: Go 8, Python 9, Rust 6, C# 7. Epic SMOODEV-1522. Refresher sidecar parity tracked separately. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/eso-manifests-parity.md | 5 + dotnet/src/SmooAI.Config/Eso/EsoManifests.cs | 243 ++++++++++++ .../SmooAI.Config.Tests/EsoManifestsTests.cs | 118 ++++++ go/config/eso_manifests.go | 239 ++++++++++++ go/config/eso_manifests_test.go | 120 ++++++ python/src/smooai_config/__init__.py | 8 + python/src/smooai_config/eso_manifests.py | 166 ++++++++ python/tests/test_eso_manifests.py | 101 +++++ rust/config/src/eso_manifests.rs | 358 ++++++++++++++++++ rust/config/src/lib.rs | 1 + 10 files changed, 1359 insertions(+) create mode 100644 .changeset/eso-manifests-parity.md create mode 100644 dotnet/src/SmooAI.Config/Eso/EsoManifests.cs create mode 100644 dotnet/tests/SmooAI.Config.Tests/EsoManifestsTests.cs create mode 100644 go/config/eso_manifests.go create mode 100644 go/config/eso_manifests_test.go create mode 100644 python/src/smooai_config/eso_manifests.py create mode 100644 python/tests/test_eso_manifests.py create mode 100644 rust/config/src/eso_manifests.rs diff --git a/.changeset/eso-manifests-parity.md b/.changeset/eso-manifests-parity.md new file mode 100644 index 0000000..c1cdc45 --- /dev/null +++ b/.changeset/eso-manifests-parity.md @@ -0,0 +1,5 @@ +--- +'@smooai/config': minor +--- + +SMOODEV-1526: Port the ESO manifest generator (`buildClusterSecretStore` + `buildExternalSecret`) to the Go, Python, Rust, and C# SDKs for language parity with the TypeScript reference. Each emits the same ClusterSecretStore (webhook → real api.smoo.ai config-values endpoint) and per-workload ExternalSecret (secret-tier config keys → UPPER_SNAKE_CASE env vars, with overrides + duplicate guard), using each language's native snakecase util. Epic SMOODEV-1522. diff --git a/dotnet/src/SmooAI.Config/Eso/EsoManifests.cs b/dotnet/src/SmooAI.Config/Eso/EsoManifests.cs new file mode 100644 index 0000000..8f9420e --- /dev/null +++ b/dotnet/src/SmooAI.Config/Eso/EsoManifests.cs @@ -0,0 +1,243 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SmooAI.Config.Eso; + +// ESO (ExternalSecrets Operator) manifest generator — C# parity port of the +// TypeScript src/eso-manifests (SMOODEV-1526, epic SMOODEV-1522). +// +// Emits the two ESO resources that let a Kubernetes workload pull its secrets +// from the @smooai/config HTTP API (api.smoo.ai) instead of having them baked +// at deploy time: +// 1. BuildClusterSecretStore — a ClusterSecretStore whose webhook provider +// points at the real config-values endpoint (org + env baked into the URL, +// bearer from the bootstrap Secret the eso-refresher keeps fresh). +// 2. BuildExternalSecret — a per-workload ExternalSecret mapping secret-tier +// config keys to env-var names (UPPER_SNAKE_CASE by default, overridable). +// +// Returns plain Dictionary structures (any YAML/JSON serializer accepts them). +// No cluster or network access. + +/// Reference to the k8s Secret + key holding the ESO bearer token. +public sealed class BootstrapSecretRef +{ + public string Name { get; init; } = EsoManifests.DefaultBootstrapSecretName; + public string Namespace { get; init; } = EsoManifests.DefaultBootstrapSecretNamespace; + public string Key { get; init; } = EsoManifests.DefaultBootstrapSecretKey; +} + +/// Options for . +public sealed class ClusterSecretStoreOptions +{ + /// ClusterSecretStore name; defaults to smooai-config. + public string? Name { get; init; } + + /// Config API base URL, e.g. https://api.smoo.ai (required). + public required string ApiUrl { get; init; } + + /// Org id whose config this store reads (required). + public required string OrgId { get; init; } + + /// Environment baked into the query string (required). + public required string Environment { get; init; } + + public BootstrapSecretRef? BootstrapSecret { get; init; } +} + +/// A config key → the env-var name the workload reads. EnvVar defaults +/// to UPPER_SNAKE_CASE(ConfigKey). +public sealed class SecretMapping +{ + public required string ConfigKey { get; init; } + public string? EnvVar { get; init; } + + public SecretMapping() { } + + [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] + public SecretMapping(string configKey, string? envVar = null) + { + ConfigKey = configKey; + EnvVar = envVar; + } +} + +/// Options for . +public sealed class ExternalSecretOptions +{ + public required string Name { get; init; } + public required string Namespace { get; init; } + public required IReadOnlyList Secrets { get; init; } + public string? TargetSecretName { get; init; } + public string? ClusterSecretStoreName { get; init; } + public string? RefreshInterval { get; init; } + public IReadOnlyDictionary? Labels { get; init; } +} + +public static class EsoManifests +{ + public const string DefaultClusterSecretStoreName = "smooai-config"; + public const string DefaultBootstrapSecretName = "smooai-config-bootstrap"; + public const string DefaultBootstrapSecretNamespace = "external-secrets"; + public const string DefaultBootstrapSecretKey = "bearer-token"; + public const string DefaultRefreshInterval = "1h"; + public const string ApiVersion = "external-secrets.io/v1beta1"; + + /// + /// Build a ClusterSecretStore backed by the @smooai/config webhook provider. + /// org + environment are baked into the URL because ESO's webhook only + /// templates {{ .remoteRef.key }} per-secret — so a store is scoped + /// to one (org, env) pair. + /// + public static Dictionary BuildClusterSecretStore(ClusterSecretStoreOptions opts) + { + if (string.IsNullOrEmpty(opts.ApiUrl)) + throw new ArgumentException("BuildClusterSecretStore: ApiUrl is required"); + if (string.IsNullOrEmpty(opts.OrgId)) + throw new ArgumentException("BuildClusterSecretStore: OrgId is required"); + if (string.IsNullOrEmpty(opts.Environment)) + throw new ArgumentException("BuildClusterSecretStore: Environment is required"); + + var name = string.IsNullOrEmpty(opts.Name) ? DefaultClusterSecretStoreName : opts.Name!; + var apiUrl = opts.ApiUrl.TrimEnd('/'); + var r = opts.BootstrapSecret ?? new BootstrapSecretRef(); + var url = $"{apiUrl}/organizations/{opts.OrgId}/config/values/{{{{ .remoteRef.key }}}}?environment={EncodeQueryComponent(opts.Environment)}"; + + return new Dictionary + { + ["apiVersion"] = ApiVersion, + ["kind"] = "ClusterSecretStore", + ["metadata"] = new Dictionary { ["name"] = name }, + ["spec"] = new Dictionary + { + ["provider"] = new Dictionary + { + ["webhook"] = new Dictionary + { + ["url"] = url, + ["headers"] = new Dictionary + { + ["Content-Type"] = "application/json", + ["Authorization"] = "Bearer {{ .auth.token }}", + }, + ["result"] = new Dictionary { ["jsonPath"] = "$.value" }, + ["secrets"] = new List + { + new Dictionary + { + ["name"] = "auth", + ["secretRef"] = new Dictionary + { + ["name"] = r.Name, + ["namespace"] = r.Namespace, + ["key"] = r.Key, + }, + }, + }, + }, + }, + }, + }; + } + + /// Normalize a mapping, defaulting EnvVar to the UPPER_SNAKE_CASE of ConfigKey. + public static (string ConfigKey, string EnvVar) ResolveSecretMapping(SecretMapping m) + { + if (string.IsNullOrEmpty(m.ConfigKey)) + throw new ArgumentException("ResolveSecretMapping: ConfigKey is required"); + var envVar = string.IsNullOrEmpty(m.EnvVar) ? CamelToUpperSnake(m.ConfigKey) : m.EnvVar!; + return (m.ConfigKey, envVar); + } + + /// + /// Build a per-workload ExternalSecret. Each entry becomes a data mapping of + /// secretKey (the env-var name in the synced Secret) ← remoteRef.key (the + /// @smooai/config key). + /// + public static Dictionary BuildExternalSecret(ExternalSecretOptions opts) + { + if (string.IsNullOrEmpty(opts.Name)) + throw new ArgumentException("BuildExternalSecret: Name is required"); + if (string.IsNullOrEmpty(opts.Namespace)) + throw new ArgumentException("BuildExternalSecret: Namespace is required"); + if (opts.Secrets == null || opts.Secrets.Count == 0) + throw new ArgumentException("BuildExternalSecret: at least one secret mapping is required"); + + var data = new List(opts.Secrets.Count); + var seen = new HashSet(); + foreach (var entry in opts.Secrets) + { + var (configKey, envVar) = ResolveSecretMapping(entry); + if (!seen.Add(envVar)) + throw new ArgumentException($"BuildExternalSecret: duplicate env-var name: {envVar}"); + data.Add(new Dictionary + { + ["secretKey"] = envVar, + ["remoteRef"] = new Dictionary { ["key"] = configKey }, + }); + } + + var metadata = new Dictionary + { + ["name"] = opts.Name, + ["namespace"] = opts.Namespace, + }; + if (opts.Labels is { Count: > 0 }) + metadata["labels"] = new Dictionary(opts.Labels); + + return new Dictionary + { + ["apiVersion"] = ApiVersion, + ["kind"] = "ExternalSecret", + ["metadata"] = metadata, + ["spec"] = new Dictionary + { + ["refreshInterval"] = string.IsNullOrEmpty(opts.RefreshInterval) ? DefaultRefreshInterval : opts.RefreshInterval!, + ["secretStoreRef"] = new Dictionary + { + ["name"] = string.IsNullOrEmpty(opts.ClusterSecretStoreName) ? DefaultClusterSecretStoreName : opts.ClusterSecretStoreName!, + ["kind"] = "ClusterSecretStore", + }, + ["target"] = new Dictionary + { + ["name"] = string.IsNullOrEmpty(opts.TargetSecretName) ? opts.Name : opts.TargetSecretName!, + ["creationPolicy"] = "Owner", + }, + ["data"] = data, + }, + }; + } + + // camelCase → UPPER_SNAKE_CASE, matching the env-tier mapping in + // Typed/EnvFileFallback.EnvVarNameFor (minus the prefix) so generated env + // var names match what the C# SDK reads from the env tier. + internal static string CamelToUpperSnake(string key) + { + var sb = new StringBuilder(key.Length + 8); + for (int i = 0; i < key.Length; i++) + { + var c = key[i]; + if (char.IsUpper(c) && i > 0) sb.Append('_'); + sb.Append(char.ToUpperInvariant(c)); + } + return sb.ToString(); + } + + // Percent-encode a query-string component (mirrors JS encodeURIComponent). + private static string EncodeQueryComponent(string s) + { + var sb = new StringBuilder(s.Length); + foreach (var b in Encoding.UTF8.GetBytes(s)) + { + var c = (char)b; + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') + || c == '-' || c == '_' || c == '.' || c == '~') + sb.Append(c); + else if (c == ' ') + sb.Append("%20"); + else + sb.Append('%').Append(b.ToString("X2")); + } + return sb.ToString(); + } +} diff --git a/dotnet/tests/SmooAI.Config.Tests/EsoManifestsTests.cs b/dotnet/tests/SmooAI.Config.Tests/EsoManifestsTests.cs new file mode 100644 index 0000000..36e2cc1 --- /dev/null +++ b/dotnet/tests/SmooAI.Config.Tests/EsoManifestsTests.cs @@ -0,0 +1,118 @@ +using System.Collections.Generic; +using SmooAI.Config.Eso; + +namespace SmooAI.Config.Tests; + +// SMOODEV-1526 — ESO manifest generator parity tests (C#). +public class EsoManifestsTests +{ + private static Dictionary Dict(object? o) => (Dictionary)o!; + private static List Arr(object? o) => (List)o!; + + private static object? Webhook(Dictionary store) => + Dict(Dict(Dict(store["spec"])["provider"])["webhook"]); + + [Fact] + public void ClusterSecretStore_BakesOrgAndEnv() + { + var store = EsoManifests.BuildClusterSecretStore(new ClusterSecretStoreOptions + { + ApiUrl = "https://api.smoo.ai", + OrgId = "org-123", + Environment = "production", + }); + var webhook = Dict(Webhook(store)); + var url = (string)webhook["url"]!; + Assert.Equal("https://api.smoo.ai/organizations/org-123/config/values/{{ .remoteRef.key }}?environment=production", url); + Assert.DoesNotContain("config.smoo.ai", url); + Assert.Equal("$.value", (string)Dict(webhook["result"])["jsonPath"]!); + } + + [Fact] + public void ClusterSecretStore_DefaultsAndEncoding() + { + var store = EsoManifests.BuildClusterSecretStore(new ClusterSecretStoreOptions + { + ApiUrl = "https://api.smoo.ai///", + OrgId = "o", + Environment = "pre prod", + }); + var webhook = Dict(Webhook(store)); + var url = (string)webhook["url"]!; + Assert.StartsWith("https://api.smoo.ai/organizations", url); + Assert.Contains("environment=pre%20prod", url); + var secretRef = Dict(Dict(Arr(webhook["secrets"])[0])["secretRef"]); + Assert.Equal("smooai-config-bootstrap", secretRef["name"]); + Assert.Equal("external-secrets", secretRef["namespace"]); + Assert.Equal("bearer-token", secretRef["key"]); + } + + [Fact] + public void ClusterSecretStore_RequiredFields() + { + Assert.Throws(() => + EsoManifests.BuildClusterSecretStore(new ClusterSecretStoreOptions { ApiUrl = "", OrgId = "o", Environment = "e" })); + Assert.Throws(() => + EsoManifests.BuildClusterSecretStore(new ClusterSecretStoreOptions { ApiUrl = "u", OrgId = "", Environment = "e" })); + Assert.Throws(() => + EsoManifests.BuildClusterSecretStore(new ClusterSecretStoreOptions { ApiUrl = "u", OrgId = "o", Environment = "" })); + } + + [Fact] + public void ResolveSecretMapping_DefaultsAndOverride() + { + Assert.Equal("MIMO_API_KEY", EsoManifests.ResolveSecretMapping(new SecretMapping("mimoApiKey")).EnvVar); + Assert.Equal("DASHSCOPE_API_KEY", EsoManifests.ResolveSecretMapping(new SecretMapping("alibabaModelStudioApiKey", "DASHSCOPE_API_KEY")).EnvVar); + } + + [Fact] + public void ExternalSecret_MapsKeys() + { + var es = EsoManifests.BuildExternalSecret(new ExternalSecretOptions + { + Name = "litellm-config", + Namespace = "smooai-litellm", + Secrets = new List + { + new("mimoApiKey"), + new("alibabaModelStudioApiKey", "DASHSCOPE_API_KEY"), + }, + }); + var spec = Dict(es["spec"]); + var data = Arr(spec["data"]); + var first = Dict(data[0]); + Assert.Equal("MIMO_API_KEY", first["secretKey"]); + Assert.Equal("mimoApiKey", Dict(first["remoteRef"])["key"]); + Assert.Equal("DASHSCOPE_API_KEY", Dict(data[1])["secretKey"]); + Assert.Equal("litellm-config", Dict(spec["target"])["name"]); + Assert.Equal("smooai-config", Dict(spec["secretStoreRef"])["name"]); + } + + [Fact] + public void ExternalSecret_DuplicateEnvVar() + { + var ex = Assert.Throws(() => + EsoManifests.BuildExternalSecret(new ExternalSecretOptions + { + Name = "x", + Namespace = "ns", + Secrets = new List + { + new("mimoApiKey"), + new("somethingElse", "MIMO_API_KEY"), + }, + })); + Assert.Contains("duplicate env-var", ex.Message); + } + + [Fact] + public void ExternalSecret_RequiredFields() + { + Assert.Throws(() => + EsoManifests.BuildExternalSecret(new ExternalSecretOptions { Name = "", Namespace = "ns", Secrets = new List { new("k") } })); + Assert.Throws(() => + EsoManifests.BuildExternalSecret(new ExternalSecretOptions { Name = "n", Namespace = "", Secrets = new List { new("k") } })); + Assert.Throws(() => + EsoManifests.BuildExternalSecret(new ExternalSecretOptions { Name = "n", Namespace = "ns", Secrets = new List() })); + } +} diff --git a/go/config/eso_manifests.go b/go/config/eso_manifests.go new file mode 100644 index 0000000..c708a14 --- /dev/null +++ b/go/config/eso_manifests.go @@ -0,0 +1,239 @@ +package config + +// ESO (ExternalSecrets Operator) manifest generator — Go parity port of the +// TypeScript `src/eso-manifests` (SMOODEV-1526, epic SMOODEV-1522). +// +// Emits the two ESO resources that let a Kubernetes workload pull its secrets +// from the @smooai/config HTTP API (api.smoo.ai) instead of having them baked +// at deploy time: +// +// 1. BuildClusterSecretStore — a ClusterSecretStore whose webhook provider +// points at the real config-values endpoint (org + env baked into the URL, +// bearer from the bootstrap Secret the eso-refresher keeps fresh). +// 2. BuildExternalSecret — a per-workload ExternalSecret mapping secret-tier +// config keys to env-var names (UPPER_SNAKE_CASE by default, overridable). +// +// Returns plain map structures (cdk8s / kubectl / YAML marshaling all accept +// them). No cluster or network access. + +import "strings" + +// Default values shared across the ESO manifests. +const ( + ESODefaultClusterSecretStoreName = "smooai-config" + ESODefaultBootstrapSecretName = "smooai-config-bootstrap" + ESODefaultBootstrapSecretNamespace = "external-secrets" + ESODefaultBootstrapSecretKey = "bearer-token" + ESODefaultRefreshInterval = "1h" + ESOAPIVersion = "external-secrets.io/v1beta1" +) + +// BootstrapSecretRef references the Kubernetes Secret + key holding the ESO +// bearer token. +type BootstrapSecretRef struct { + Name string // default "smooai-config-bootstrap" + Namespace string // default "external-secrets" + Key string // default "bearer-token" +} + +// ClusterSecretStoreOptions configures BuildClusterSecretStore. +type ClusterSecretStoreOptions struct { + Name string // ClusterSecretStore name; default "smooai-config" + APIURL string // config API base URL, e.g. "https://api.smoo.ai" (required) + OrgID string // org id whose config this store reads (required) + Environment string // environment baked into the query string (required) + BootstrapSecret *BootstrapSecretRef +} + +// BuildClusterSecretStore builds a ClusterSecretStore backed by the +// @smooai/config webhook provider. org + environment are baked into the URL +// because ESO's webhook only templates {{ .remoteRef.key }} per-secret — so a +// store is scoped to one (org, env) pair. Returns an error if required fields +// are missing. +func BuildClusterSecretStore(opts ClusterSecretStoreOptions) (map[string]any, error) { + if opts.APIURL == "" { + return nil, NewConfigError("BuildClusterSecretStore: APIURL is required") + } + if opts.OrgID == "" { + return nil, NewConfigError("BuildClusterSecretStore: OrgID is required") + } + if opts.Environment == "" { + return nil, NewConfigError("BuildClusterSecretStore: Environment is required") + } + + name := opts.Name + if name == "" { + name = ESODefaultClusterSecretStoreName + } + apiURL := strings.TrimRight(opts.APIURL, "/") + secretName := ESODefaultBootstrapSecretName + secretNamespace := ESODefaultBootstrapSecretNamespace + secretKey := ESODefaultBootstrapSecretKey + if opts.BootstrapSecret != nil { + if opts.BootstrapSecret.Name != "" { + secretName = opts.BootstrapSecret.Name + } + if opts.BootstrapSecret.Namespace != "" { + secretNamespace = opts.BootstrapSecret.Namespace + } + if opts.BootstrapSecret.Key != "" { + secretKey = opts.BootstrapSecret.Key + } + } + + url := apiURL + "/organizations/" + opts.OrgID + "/config/values/{{ .remoteRef.key }}?environment=" + urlQueryEscape(opts.Environment) + + return map[string]any{ + "apiVersion": ESOAPIVersion, + "kind": "ClusterSecretStore", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{ + "provider": map[string]any{ + "webhook": map[string]any{ + "url": url, + "headers": map[string]any{ + "Content-Type": "application/json", + "Authorization": "Bearer {{ .auth.token }}", + }, + "result": map[string]any{"jsonPath": "$.value"}, + "secrets": []any{ + map[string]any{ + "name": "auth", + "secretRef": map[string]any{ + "name": secretName, + "namespace": secretNamespace, + "key": secretKey, + }, + }, + }, + }, + }, + }, + }, nil +} + +// SecretMapping is one mapped secret: a config key → the env-var name the +// workload reads. EnvVar defaults to UPPER_SNAKE_CASE(ConfigKey). +type SecretMapping struct { + ConfigKey string + EnvVar string // optional; defaults to CamelToUpperSnake(ConfigKey) +} + +// ExternalSecretOptions configures BuildExternalSecret. +type ExternalSecretOptions struct { + Name string // ExternalSecret resource name (required) + Namespace string // namespace (required) + Secrets []SecretMapping // at least one (required) + TargetSecretName string // default = Name + ClusterSecretStoreName string // default "smooai-config" + RefreshInterval string // default "1h" + Labels map[string]string +} + +// ResolveSecretMapping normalizes a mapping, defaulting EnvVar to the +// UPPER_SNAKE_CASE form of ConfigKey. +func ResolveSecretMapping(m SecretMapping) (SecretMapping, error) { + if m.ConfigKey == "" { + return SecretMapping{}, NewConfigError("ResolveSecretMapping: ConfigKey is required") + } + envVar := m.EnvVar + if envVar == "" { + envVar = CamelToUpperSnake(m.ConfigKey) + } + return SecretMapping{ConfigKey: m.ConfigKey, EnvVar: envVar}, nil +} + +// BuildExternalSecret builds a per-workload ExternalSecret. Each entry becomes +// a data mapping of secretKey (the env-var name in the synced Secret) ← +// remoteRef.key (the @smooai/config key). Returns an error on missing required +// fields or duplicate env-var names. +func BuildExternalSecret(opts ExternalSecretOptions) (map[string]any, error) { + if opts.Name == "" { + return nil, NewConfigError("BuildExternalSecret: Name is required") + } + if opts.Namespace == "" { + return nil, NewConfigError("BuildExternalSecret: Namespace is required") + } + if len(opts.Secrets) == 0 { + return nil, NewConfigError("BuildExternalSecret: at least one secret mapping is required") + } + + data := make([]any, 0, len(opts.Secrets)) + seen := map[string]bool{} + for _, entry := range opts.Secrets { + resolved, err := ResolveSecretMapping(entry) + if err != nil { + return nil, err + } + if seen[resolved.EnvVar] { + return nil, NewConfigError("BuildExternalSecret: duplicate env-var name: " + resolved.EnvVar) + } + seen[resolved.EnvVar] = true + data = append(data, map[string]any{ + "secretKey": resolved.EnvVar, + "remoteRef": map[string]any{"key": resolved.ConfigKey}, + }) + } + + targetName := opts.TargetSecretName + if targetName == "" { + targetName = opts.Name + } + storeName := opts.ClusterSecretStoreName + if storeName == "" { + storeName = ESODefaultClusterSecretStoreName + } + refresh := opts.RefreshInterval + if refresh == "" { + refresh = ESODefaultRefreshInterval + } + + metadata := map[string]any{ + "name": opts.Name, + "namespace": opts.Namespace, + } + if len(opts.Labels) > 0 { + metadata["labels"] = opts.Labels + } + + return map[string]any{ + "apiVersion": ESOAPIVersion, + "kind": "ExternalSecret", + "metadata": metadata, + "spec": map[string]any{ + "refreshInterval": refresh, + "secretStoreRef": map[string]any{ + "name": storeName, + "kind": "ClusterSecretStore", + }, + "target": map[string]any{ + "name": targetName, + "creationPolicy": "Owner", + }, + "data": data, + }, + }, nil +} + +// urlQueryEscape percent-encodes a query-string value. Local helper to avoid a +// net/url import for a single value (mirrors the TS encodeURIComponent usage). +func urlQueryEscape(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', + r == '-', r == '_', r == '.', r == '~': + b.WriteRune(r) + case r == ' ': + b.WriteString("%20") + default: + for _, by := range []byte(string(r)) { + b.WriteString("%") + const hex = "0123456789ABCDEF" + b.WriteByte(hex[by>>4]) + b.WriteByte(hex[by&0x0F]) + } + } + } + return b.String() +} diff --git a/go/config/eso_manifests_test.go b/go/config/eso_manifests_test.go new file mode 100644 index 0000000..f0763d0 --- /dev/null +++ b/go/config/eso_manifests_test.go @@ -0,0 +1,120 @@ +package config + +import ( + "strings" + "testing" +) + +func TestBuildClusterSecretStore(t *testing.T) { + store, err := BuildClusterSecretStore(ClusterSecretStoreOptions{APIURL: "https://api.smoo.ai", OrgID: "org-123", Environment: "production"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + spec := store["spec"].(map[string]any) + webhook := spec["provider"].(map[string]any)["webhook"].(map[string]any) + url := webhook["url"].(string) + want := "https://api.smoo.ai/organizations/org-123/config/values/{{ .remoteRef.key }}?environment=production" + if url != want { + t.Errorf("url = %q, want %q", url, want) + } + if strings.Contains(url, "config.smoo.ai") { + t.Error("url must never reference the hallucinated config.smoo.ai") + } + if webhook["result"].(map[string]any)["jsonPath"].(string) != "$.value" { + t.Error("jsonPath should be $.value") + } +} + +func TestBuildClusterSecretStoreDefaultsAndOverrides(t *testing.T) { + store, _ := BuildClusterSecretStore(ClusterSecretStoreOptions{APIURL: "https://api.smoo.ai///", OrgID: "o", Environment: "pre prod"}) + webhook := store["spec"].(map[string]any)["provider"].(map[string]any)["webhook"].(map[string]any) + url := webhook["url"].(string) + if !strings.HasPrefix(url, "https://api.smoo.ai/organizations") { + t.Errorf("trailing slashes not stripped: %q", url) + } + if !strings.Contains(url, "environment=pre%20prod") { + t.Errorf("environment not url-encoded: %q", url) + } + ref := webhook["secrets"].([]any)[0].(map[string]any)["secretRef"].(map[string]any) + if ref["name"] != "smooai-config-bootstrap" || ref["namespace"] != "external-secrets" || ref["key"] != "bearer-token" { + t.Errorf("bootstrap secret ref defaults wrong: %v", ref) + } +} + +func TestBuildClusterSecretStoreRequiredFields(t *testing.T) { + if _, err := BuildClusterSecretStore(ClusterSecretStoreOptions{OrgID: "o", Environment: "e"}); err == nil { + t.Error("expected error for missing APIURL") + } + if _, err := BuildClusterSecretStore(ClusterSecretStoreOptions{APIURL: "u", Environment: "e"}); err == nil { + t.Error("expected error for missing OrgID") + } + if _, err := BuildClusterSecretStore(ClusterSecretStoreOptions{APIURL: "u", OrgID: "o"}); err == nil { + t.Error("expected error for missing Environment") + } +} + +func TestResolveSecretMapping(t *testing.T) { + m, _ := ResolveSecretMapping(SecretMapping{ConfigKey: "mimoApiKey"}) + if m.EnvVar != "MIMO_API_KEY" { + t.Errorf("default envVar = %q, want MIMO_API_KEY", m.EnvVar) + } + m2, _ := ResolveSecretMapping(SecretMapping{ConfigKey: "alibabaModelStudioApiKey", EnvVar: "DASHSCOPE_API_KEY"}) + if m2.EnvVar != "DASHSCOPE_API_KEY" { + t.Errorf("override envVar = %q, want DASHSCOPE_API_KEY", m2.EnvVar) + } +} + +func TestBuildExternalSecret(t *testing.T) { + es, err := BuildExternalSecret(ExternalSecretOptions{ + Name: "litellm-config", + Namespace: "smooai-litellm", + Secrets: []SecretMapping{ + {ConfigKey: "mimoApiKey"}, + {ConfigKey: "alibabaModelStudioApiKey", EnvVar: "DASHSCOPE_API_KEY"}, + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + spec := es["spec"].(map[string]any) + data := spec["data"].([]any) + if len(data) != 2 { + t.Fatalf("data len = %d, want 2", len(data)) + } + first := data[0].(map[string]any) + if first["secretKey"] != "MIMO_API_KEY" || first["remoteRef"].(map[string]any)["key"] != "mimoApiKey" { + t.Errorf("first mapping wrong: %v", first) + } + if spec["target"].(map[string]any)["name"] != "litellm-config" { + t.Error("target name should default to resource name") + } + if spec["secretStoreRef"].(map[string]any)["name"] != "smooai-config" { + t.Error("store should default to smooai-config") + } +} + +func TestBuildExternalSecretDuplicateEnvVar(t *testing.T) { + _, err := BuildExternalSecret(ExternalSecretOptions{ + Name: "x", + Namespace: "ns", + Secrets: []SecretMapping{ + {ConfigKey: "mimoApiKey"}, + {ConfigKey: "somethingElse", EnvVar: "MIMO_API_KEY"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "duplicate env-var") { + t.Errorf("expected duplicate env-var error, got %v", err) + } +} + +func TestBuildExternalSecretRequiredFields(t *testing.T) { + if _, err := BuildExternalSecret(ExternalSecretOptions{Namespace: "ns", Secrets: []SecretMapping{{ConfigKey: "k"}}}); err == nil { + t.Error("expected error for missing Name") + } + if _, err := BuildExternalSecret(ExternalSecretOptions{Name: "n", Secrets: []SecretMapping{{ConfigKey: "k"}}}); err == nil { + t.Error("expected error for missing Namespace") + } + if _, err := BuildExternalSecret(ExternalSecretOptions{Name: "n", Namespace: "ns"}); err == nil { + t.Error("expected error for empty Secrets") + } +} diff --git a/python/src/smooai_config/__init__.py b/python/src/smooai_config/__init__.py index 02ba93f..ed84324 100644 --- a/python/src/smooai_config/__init__.py +++ b/python/src/smooai_config/__init__.py @@ -22,6 +22,14 @@ init_container_config, select_mode, ) +from smooai_config.eso_manifests import ( + BootstrapSecretRef, + ExternalSecretOptions, + SecretMapping, + build_cluster_secret_store, + build_external_secret, + resolve_secret_mapping, +) from smooai_config.env_config import find_and_process_env_config from smooai_config.file_config import find_and_process_file_config, find_config_directory from smooai_config.local import LocalConfigManager diff --git a/python/src/smooai_config/eso_manifests.py b/python/src/smooai_config/eso_manifests.py new file mode 100644 index 0000000..96b43cd --- /dev/null +++ b/python/src/smooai_config/eso_manifests.py @@ -0,0 +1,166 @@ +"""ESO (ExternalSecrets Operator) manifest generator — Python parity port of the +TypeScript ``src/eso-manifests`` (SMOODEV-1526, epic SMOODEV-1522). + +Emits the two ESO resources that let a Kubernetes workload pull its secrets from +the @smooai/config HTTP API (api.smoo.ai) instead of having them baked at deploy +time: + +1. :func:`build_cluster_secret_store` — a ``ClusterSecretStore`` whose webhook + provider points at the real config-values endpoint (org + env baked into the + URL, bearer from the bootstrap Secret the eso-refresher keeps fresh). +2. :func:`build_external_secret` — a per-workload ``ExternalSecret`` mapping + secret-tier config keys to env-var names (UPPER_SNAKE_CASE by default, + overridable). + +Returns plain ``dict`` structures (cdk8s / kubectl / YAML all accept them). No +cluster or network access. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import quote + +from smooai_config.utils import SmooaiConfigError, camel_to_upper_snake + +ESO_DEFAULT_CLUSTER_SECRET_STORE_NAME = "smooai-config" +ESO_DEFAULT_BOOTSTRAP_SECRET_NAME = "smooai-config-bootstrap" +ESO_DEFAULT_BOOTSTRAP_SECRET_NAMESPACE = "external-secrets" +ESO_DEFAULT_BOOTSTRAP_SECRET_KEY = "bearer-token" +ESO_DEFAULT_REFRESH_INTERVAL = "1h" +ESO_API_VERSION = "external-secrets.io/v1beta1" + + +@dataclass +class BootstrapSecretRef: + """Reference to the k8s Secret + key holding the ESO bearer token.""" + + name: str = ESO_DEFAULT_BOOTSTRAP_SECRET_NAME + namespace: str = ESO_DEFAULT_BOOTSTRAP_SECRET_NAMESPACE + key: str = ESO_DEFAULT_BOOTSTRAP_SECRET_KEY + + +def build_cluster_secret_store( + *, + api_url: str, + org_id: str, + environment: str, + name: str = ESO_DEFAULT_CLUSTER_SECRET_STORE_NAME, + bootstrap_secret: BootstrapSecretRef | None = None, +) -> dict[str, Any]: + """Build a ``ClusterSecretStore`` backed by the @smooai/config webhook provider. + + ``org_id`` + ``environment`` are baked into the URL because ESO's webhook only + templates ``{{ .remoteRef.key }}`` per-secret — so a store is scoped to one + (org, env) pair. Raises :class:`ConfigError` on missing required fields. + """ + if not api_url: + raise SmooaiConfigError("build_cluster_secret_store: api_url is required") + if not org_id: + raise SmooaiConfigError("build_cluster_secret_store: org_id is required") + if not environment: + raise SmooaiConfigError("build_cluster_secret_store: environment is required") + + ref = bootstrap_secret or BootstrapSecretRef() + base = api_url.rstrip("/") + url = f"{base}/organizations/{org_id}/config/values/{{{{ .remoteRef.key }}}}?environment={quote(environment, safe='')}" + + return { + "apiVersion": ESO_API_VERSION, + "kind": "ClusterSecretStore", + "metadata": {"name": name}, + "spec": { + "provider": { + "webhook": { + "url": url, + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer {{ .auth.token }}", + }, + "result": {"jsonPath": "$.value"}, + "secrets": [ + { + "name": "auth", + "secretRef": { + "name": ref.name, + "namespace": ref.namespace, + "key": ref.key, + }, + } + ], + } + } + }, + } + + +@dataclass +class SecretMapping: + """A config key → the env-var name the workload reads. + + ``env_var`` defaults to ``UPPER_SNAKE_CASE(config_key)``. + """ + + config_key: str + env_var: str | None = None + + +def resolve_secret_mapping(mapping: SecretMapping | str) -> SecretMapping: + """Normalize a mapping, defaulting ``env_var`` to the snakecase of ``config_key``.""" + m = SecretMapping(config_key=mapping) if isinstance(mapping, str) else mapping + if not m.config_key: + raise SmooaiConfigError("resolve_secret_mapping: config_key is required") + return SecretMapping(config_key=m.config_key, env_var=m.env_var or camel_to_upper_snake(m.config_key)) + + +@dataclass +class ExternalSecretOptions: + name: str + namespace: str + secrets: list[SecretMapping | str] + target_secret_name: str | None = None + cluster_secret_store_name: str = ESO_DEFAULT_CLUSTER_SECRET_STORE_NAME + refresh_interval: str = ESO_DEFAULT_REFRESH_INTERVAL + labels: dict[str, str] = field(default_factory=dict) + + +def build_external_secret(opts: ExternalSecretOptions) -> dict[str, Any]: + """Build a per-workload ``ExternalSecret``. + + Each entry becomes a data mapping of ``secretKey`` (the env-var name in the + synced Secret) ← ``remoteRef.key`` (the @smooai/config key). Raises + :class:`ConfigError` on missing required fields or duplicate env-var names. + """ + if not opts.name: + raise SmooaiConfigError("build_external_secret: name is required") + if not opts.namespace: + raise SmooaiConfigError("build_external_secret: namespace is required") + if not opts.secrets: + raise SmooaiConfigError("build_external_secret: at least one secret mapping is required") + + data: list[dict[str, Any]] = [] + seen: set[str] = set() + for entry in opts.secrets: + resolved = resolve_secret_mapping(entry) + assert resolved.env_var is not None # resolve_secret_mapping always sets it + if resolved.env_var in seen: + raise SmooaiConfigError(f"build_external_secret: duplicate env-var name: {resolved.env_var}") + seen.add(resolved.env_var) + data.append({"secretKey": resolved.env_var, "remoteRef": {"key": resolved.config_key}}) + + metadata: dict[str, Any] = {"name": opts.name, "namespace": opts.namespace} + if opts.labels: + metadata["labels"] = opts.labels + + return { + "apiVersion": ESO_API_VERSION, + "kind": "ExternalSecret", + "metadata": metadata, + "spec": { + "refreshInterval": opts.refresh_interval, + "secretStoreRef": {"name": opts.cluster_secret_store_name, "kind": "ClusterSecretStore"}, + "target": {"name": opts.target_secret_name or opts.name, "creationPolicy": "Owner"}, + "data": data, + }, + } diff --git a/python/tests/test_eso_manifests.py b/python/tests/test_eso_manifests.py new file mode 100644 index 0000000..bd02451 --- /dev/null +++ b/python/tests/test_eso_manifests.py @@ -0,0 +1,101 @@ +"""SMOODEV-1526 — ESO manifest generator parity tests (Python).""" + +from __future__ import annotations + +import pytest +from smooai_config.eso_manifests import ( + BootstrapSecretRef, + ExternalSecretOptions, + SecretMapping, + build_cluster_secret_store, + build_external_secret, + resolve_secret_mapping, +) +from smooai_config.utils import SmooaiConfigError + + +def test_cluster_secret_store_bakes_org_and_env(): + store = build_cluster_secret_store(api_url="https://api.smoo.ai", org_id="org-123", environment="production") + url = store["spec"]["provider"]["webhook"]["url"] + assert url == "https://api.smoo.ai/organizations/org-123/config/values/{{ .remoteRef.key }}?environment=production" + assert "config.smoo.ai" not in url + assert store["spec"]["provider"]["webhook"]["result"]["jsonPath"] == "$.value" + + +def test_cluster_secret_store_defaults_and_encoding(): + store = build_cluster_secret_store(api_url="https://api.smoo.ai///", org_id="o", environment="pre prod") + url = store["spec"]["provider"]["webhook"]["url"] + assert url.startswith("https://api.smoo.ai/organizations") + assert "environment=pre%20prod" in url + ref = store["spec"]["provider"]["webhook"]["secrets"][0]["secretRef"] + assert ref == {"name": "smooai-config-bootstrap", "namespace": "external-secrets", "key": "bearer-token"} + + +def test_cluster_secret_store_overrides(): + store = build_cluster_secret_store( + api_url="https://api.smoo.ai", + org_id="o", + environment="production", + name="smooai-config-prod", + bootstrap_secret=BootstrapSecretRef(name="s", namespace="ns", key="k"), + ) + assert store["metadata"]["name"] == "smooai-config-prod" + assert store["spec"]["provider"]["webhook"]["secrets"][0]["secretRef"] == {"name": "s", "namespace": "ns", "key": "k"} + + +def test_cluster_secret_store_required_fields(): + with pytest.raises(SmooaiConfigError): + build_cluster_secret_store(api_url="", org_id="o", environment="e") + with pytest.raises(SmooaiConfigError): + build_cluster_secret_store(api_url="u", org_id="", environment="e") + with pytest.raises(SmooaiConfigError): + build_cluster_secret_store(api_url="u", org_id="o", environment="") + + +def test_resolve_secret_mapping(): + assert resolve_secret_mapping("mimoApiKey").env_var == "MIMO_API_KEY" + m = resolve_secret_mapping(SecretMapping(config_key="alibabaModelStudioApiKey", env_var="DASHSCOPE_API_KEY")) + assert m.env_var == "DASHSCOPE_API_KEY" + + +def test_build_external_secret_maps_keys(): + es = build_external_secret( + ExternalSecretOptions( + name="litellm-config", + namespace="smooai-litellm", + secrets=["mimoApiKey", SecretMapping(config_key="alibabaModelStudioApiKey", env_var="DASHSCOPE_API_KEY")], + ) + ) + assert es["spec"]["data"] == [ + {"secretKey": "MIMO_API_KEY", "remoteRef": {"key": "mimoApiKey"}}, + {"secretKey": "DASHSCOPE_API_KEY", "remoteRef": {"key": "alibabaModelStudioApiKey"}}, + ] + assert es["spec"]["target"]["name"] == "litellm-config" + assert es["spec"]["secretStoreRef"] == {"name": "smooai-config", "kind": "ClusterSecretStore"} + + +def test_build_external_secret_distinct_target(): + es = build_external_secret( + ExternalSecretOptions(name="litellm-config-eso", namespace="smooai-litellm", secrets=["mimoApiKey"], target_secret_name="litellm-config-eso") + ) + assert es["spec"]["target"]["name"] == "litellm-config-eso" + + +def test_build_external_secret_duplicate_env_var(): + with pytest.raises(SmooaiConfigError, match="duplicate env-var"): + build_external_secret( + ExternalSecretOptions( + name="x", + namespace="ns", + secrets=["mimoApiKey", SecretMapping(config_key="somethingElse", env_var="MIMO_API_KEY")], + ) + ) + + +def test_build_external_secret_required_fields(): + with pytest.raises(SmooaiConfigError): + build_external_secret(ExternalSecretOptions(name="", namespace="ns", secrets=["k"])) + with pytest.raises(SmooaiConfigError): + build_external_secret(ExternalSecretOptions(name="n", namespace="", secrets=["k"])) + with pytest.raises(SmooaiConfigError): + build_external_secret(ExternalSecretOptions(name="n", namespace="ns", secrets=[])) diff --git a/rust/config/src/eso_manifests.rs b/rust/config/src/eso_manifests.rs new file mode 100644 index 0000000..3caa641 --- /dev/null +++ b/rust/config/src/eso_manifests.rs @@ -0,0 +1,358 @@ +//! ESO (ExternalSecrets Operator) manifest generator — Rust parity port of the +//! TypeScript `src/eso-manifests` (SMOODEV-1526, epic SMOODEV-1522). +//! +//! Emits the two ESO resources that let a Kubernetes workload pull its secrets +//! from the @smooai/config HTTP API (api.smoo.ai) instead of having them baked +//! at deploy time: +//! +//! 1. [`build_cluster_secret_store`] — a `ClusterSecretStore` whose webhook +//! provider points at the real config-values endpoint (org + env baked into +//! the URL, bearer from the bootstrap Secret the eso-refresher keeps fresh). +//! 2. [`build_external_secret`] — a per-workload `ExternalSecret` mapping +//! secret-tier config keys to env-var names (UPPER_SNAKE_CASE by default, +//! overridable). +//! +//! Returns `serde_json::Value` (cdk8s / kubectl / YAML all accept it). No +//! cluster or network access. + +use serde_json::{json, Value}; +use std::collections::HashSet; + +use crate::utils::{camel_to_upper_snake, SmooaiConfigError}; + +pub const ESO_DEFAULT_CLUSTER_SECRET_STORE_NAME: &str = "smooai-config"; +pub const ESO_DEFAULT_BOOTSTRAP_SECRET_NAME: &str = "smooai-config-bootstrap"; +pub const ESO_DEFAULT_BOOTSTRAP_SECRET_NAMESPACE: &str = "external-secrets"; +pub const ESO_DEFAULT_BOOTSTRAP_SECRET_KEY: &str = "bearer-token"; +pub const ESO_DEFAULT_REFRESH_INTERVAL: &str = "1h"; +pub const ESO_API_VERSION: &str = "external-secrets.io/v1beta1"; + +/// Reference to the k8s Secret + key holding the ESO bearer token. +#[derive(Debug, Clone)] +pub struct BootstrapSecretRef { + pub name: String, + pub namespace: String, + pub key: String, +} + +impl Default for BootstrapSecretRef { + fn default() -> Self { + Self { + name: ESO_DEFAULT_BOOTSTRAP_SECRET_NAME.to_string(), + namespace: ESO_DEFAULT_BOOTSTRAP_SECRET_NAMESPACE.to_string(), + key: ESO_DEFAULT_BOOTSTRAP_SECRET_KEY.to_string(), + } + } +} + +/// Options for [`build_cluster_secret_store`]. +#[derive(Debug, Clone)] +pub struct ClusterSecretStoreOptions { + /// ClusterSecretStore name; defaults to `smooai-config`. + pub name: Option, + /// Config API base URL, e.g. `https://api.smoo.ai` (required). + pub api_url: String, + /// Org id whose config this store reads (required). + pub org_id: String, + /// Environment baked into the query string (required). + pub environment: String, + pub bootstrap_secret: Option, +} + +/// Build a `ClusterSecretStore` backed by the @smooai/config webhook provider. +/// +/// org + environment are baked into the URL because ESO's webhook only templates +/// `{{ .remoteRef.key }}` per-secret — so a store is scoped to one (org, env). +pub fn build_cluster_secret_store( + opts: &ClusterSecretStoreOptions, +) -> Result { + if opts.api_url.is_empty() { + return Err(SmooaiConfigError::new( + "build_cluster_secret_store: api_url is required", + )); + } + if opts.org_id.is_empty() { + return Err(SmooaiConfigError::new( + "build_cluster_secret_store: org_id is required", + )); + } + if opts.environment.is_empty() { + return Err(SmooaiConfigError::new( + "build_cluster_secret_store: environment is required", + )); + } + + let name = opts + .name + .clone() + .unwrap_or_else(|| ESO_DEFAULT_CLUSTER_SECRET_STORE_NAME.to_string()); + let api_url = opts.api_url.trim_end_matches('/'); + let r = opts.bootstrap_secret.clone().unwrap_or_default(); + + let url = format!( + "{}/organizations/{}/config/values/{{{{ .remoteRef.key }}}}?environment={}", + api_url, + opts.org_id, + encode_query_component(&opts.environment) + ); + + Ok(json!({ + "apiVersion": ESO_API_VERSION, + "kind": "ClusterSecretStore", + "metadata": { "name": name }, + "spec": { + "provider": { + "webhook": { + "url": url, + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer {{ .auth.token }}" + }, + "result": { "jsonPath": "$.value" }, + "secrets": [ + { + "name": "auth", + "secretRef": { + "name": r.name, + "namespace": r.namespace, + "key": r.key + } + } + ] + } + } + } + })) +} + +/// A config key → the env-var name the workload reads. `env_var` defaults to +/// `UPPER_SNAKE_CASE(config_key)`. +#[derive(Debug, Clone)] +pub struct SecretMapping { + pub config_key: String, + pub env_var: Option, +} + +impl SecretMapping { + pub fn new(config_key: impl Into) -> Self { + Self { + config_key: config_key.into(), + env_var: None, + } + } + + pub fn with_env_var(config_key: impl Into, env_var: impl Into) -> Self { + Self { + config_key: config_key.into(), + env_var: Some(env_var.into()), + } + } +} + +/// Normalize a mapping, defaulting `env_var` to the snakecase of `config_key`. +/// Returns `(config_key, env_var)`. +pub fn resolve_secret_mapping(m: &SecretMapping) -> Result<(String, String), SmooaiConfigError> { + if m.config_key.is_empty() { + return Err(SmooaiConfigError::new( + "resolve_secret_mapping: config_key is required", + )); + } + let env_var = m + .env_var + .clone() + .unwrap_or_else(|| camel_to_upper_snake(&m.config_key)); + Ok((m.config_key.clone(), env_var)) +} + +/// Options for [`build_external_secret`]. +#[derive(Debug, Clone)] +pub struct ExternalSecretOptions { + pub name: String, + pub namespace: String, + pub secrets: Vec, + pub target_secret_name: Option, + pub cluster_secret_store_name: Option, + pub refresh_interval: Option, + pub labels: Option>, +} + +/// Build a per-workload `ExternalSecret`. Each entry becomes a data mapping of +/// `secretKey` (the env-var name in the synced Secret) ← `remoteRef.key` (the +/// @smooai/config key). +pub fn build_external_secret(opts: &ExternalSecretOptions) -> Result { + if opts.name.is_empty() { + return Err(SmooaiConfigError::new( + "build_external_secret: name is required", + )); + } + if opts.namespace.is_empty() { + return Err(SmooaiConfigError::new( + "build_external_secret: namespace is required", + )); + } + if opts.secrets.is_empty() { + return Err(SmooaiConfigError::new( + "build_external_secret: at least one secret mapping is required", + )); + } + + let mut data: Vec = Vec::with_capacity(opts.secrets.len()); + let mut seen: HashSet = HashSet::new(); + for entry in &opts.secrets { + let (config_key, env_var) = resolve_secret_mapping(entry)?; + if !seen.insert(env_var.clone()) { + return Err(SmooaiConfigError::new(&format!( + "build_external_secret: duplicate env-var name: {env_var}" + ))); + } + data.push(json!({ "secretKey": env_var, "remoteRef": { "key": config_key } })); + } + + let target_name = opts + .target_secret_name + .clone() + .unwrap_or_else(|| opts.name.clone()); + let store_name = opts + .cluster_secret_store_name + .clone() + .unwrap_or_else(|| ESO_DEFAULT_CLUSTER_SECRET_STORE_NAME.to_string()); + let refresh = opts + .refresh_interval + .clone() + .unwrap_or_else(|| ESO_DEFAULT_REFRESH_INTERVAL.to_string()); + + let mut metadata = json!({ "name": opts.name, "namespace": opts.namespace }); + if let Some(labels) = &opts.labels { + if !labels.is_empty() { + metadata["labels"] = json!(labels); + } + } + + Ok(json!({ + "apiVersion": ESO_API_VERSION, + "kind": "ExternalSecret", + "metadata": metadata, + "spec": { + "refreshInterval": refresh, + "secretStoreRef": { "name": store_name, "kind": "ClusterSecretStore" }, + "target": { "name": target_name, "creationPolicy": "Owner" }, + "data": data + } + })) +} + +/// Percent-encode a query-string component (mirrors JS `encodeURIComponent`). +fn encode_query_component(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + b' ' => out.push_str("%20"), + _ => { + out.push('%'); + out.push_str(&format!("{b:02X}")); + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn store_opts() -> ClusterSecretStoreOptions { + ClusterSecretStoreOptions { + name: None, + api_url: "https://api.smoo.ai".to_string(), + org_id: "org-123".to_string(), + environment: "production".to_string(), + bootstrap_secret: None, + } + } + + #[test] + fn cluster_store_bakes_org_and_env() { + let s = build_cluster_secret_store(&store_opts()).unwrap(); + let url = s["spec"]["provider"]["webhook"]["url"].as_str().unwrap(); + assert_eq!( + url, + "https://api.smoo.ai/organizations/org-123/config/values/{{ .remoteRef.key }}?environment=production" + ); + assert!(!url.contains("config.smoo.ai")); + assert_eq!(s["spec"]["provider"]["webhook"]["result"]["jsonPath"], "$.value"); + } + + #[test] + fn cluster_store_defaults_and_encoding() { + let mut o = store_opts(); + o.api_url = "https://api.smoo.ai///".to_string(); + o.environment = "pre prod".to_string(); + let s = build_cluster_secret_store(&o).unwrap(); + let url = s["spec"]["provider"]["webhook"]["url"].as_str().unwrap(); + assert!(url.starts_with("https://api.smoo.ai/organizations")); + assert!(url.contains("environment=pre%20prod")); + let r = &s["spec"]["provider"]["webhook"]["secrets"][0]["secretRef"]; + assert_eq!(r["name"], "smooai-config-bootstrap"); + assert_eq!(r["namespace"], "external-secrets"); + assert_eq!(r["key"], "bearer-token"); + } + + #[test] + fn cluster_store_required_fields() { + let mut o = store_opts(); + o.api_url = String::new(); + assert!(build_cluster_secret_store(&o).is_err()); + } + + #[test] + fn resolve_mapping_defaults_and_override() { + let (_, env) = resolve_secret_mapping(&SecretMapping::new("mimoApiKey")).unwrap(); + assert_eq!(env, "MIMO_API_KEY"); + let (_, env2) = + resolve_secret_mapping(&SecretMapping::with_env_var("alibabaModelStudioApiKey", "DASHSCOPE_API_KEY")).unwrap(); + assert_eq!(env2, "DASHSCOPE_API_KEY"); + } + + #[test] + fn external_secret_maps_keys() { + let es = build_external_secret(&ExternalSecretOptions { + name: "litellm-config".to_string(), + namespace: "smooai-litellm".to_string(), + secrets: vec![ + SecretMapping::new("mimoApiKey"), + SecretMapping::with_env_var("alibabaModelStudioApiKey", "DASHSCOPE_API_KEY"), + ], + target_secret_name: None, + cluster_secret_store_name: None, + refresh_interval: None, + labels: None, + }) + .unwrap(); + assert_eq!(es["spec"]["data"][0]["secretKey"], "MIMO_API_KEY"); + assert_eq!(es["spec"]["data"][0]["remoteRef"]["key"], "mimoApiKey"); + assert_eq!(es["spec"]["data"][1]["secretKey"], "DASHSCOPE_API_KEY"); + assert_eq!(es["spec"]["target"]["name"], "litellm-config"); + assert_eq!(es["spec"]["secretStoreRef"]["name"], "smooai-config"); + } + + #[test] + fn external_secret_duplicate_env_var() { + let err = build_external_secret(&ExternalSecretOptions { + name: "x".to_string(), + namespace: "ns".to_string(), + secrets: vec![ + SecretMapping::new("mimoApiKey"), + SecretMapping::with_env_var("somethingElse", "MIMO_API_KEY"), + ], + target_secret_name: None, + cluster_secret_store_name: None, + refresh_interval: None, + labels: None, + }); + assert!(err.is_err()); + assert!(err.unwrap_err().to_string().contains("duplicate env-var")); + } +} diff --git a/rust/config/src/lib.rs b/rust/config/src/lib.rs index f92615e..10d09f7 100644 --- a/rust/config/src/lib.rs +++ b/rust/config/src/lib.rs @@ -11,6 +11,7 @@ pub mod config_manager; pub mod container; pub mod deferred; pub mod env_config; +pub mod eso_manifests; pub mod file_config; pub mod local; pub mod merge; From ee27bfbfd7282d2cf6c3b0151536209043d26b21 Mon Sep 17 00:00:00 2001 From: Brent Rager Date: Tue, 2 Jun 2026 16:09:32 -0400 Subject: [PATCH 2/2] SMOODEV-1526: ESO refresher core parity (Go/Python/Rust/C#) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the refresh algorithm + SecretWriter abstraction from the TS reference to all four other SDKs. Each mirrors: invalidate-then-mint each cycle (so the bootstrap Secret always holds a near-full-TTL token), fail-loud initial write, non-fatal loop-tick retries — driven by the language's own TokenProvider and unit-tested with a fake writer (no live cluster). The k8s-backed writer is intentionally an optional adapter (NOT in core) so base SDK consumers don't pull a heavy k8s client (client-go/kube/kubernetes/ KubernetesClient are all absent from these SDKs today); the TS sidecar remains the canonical deployable. Tests: Go 5, Python 7, Rust 4, C# 5. Epic SMOODEV-1522. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/eso-refresher-parity.md | 5 + dotnet/src/SmooAI.Config/Eso/EsoRefresher.cs | 94 +++++++++ .../SmooAI.Config.Tests/EsoRefresherTests.cs | 95 +++++++++ go/config/eso_refresher.go | 120 +++++++++++ go/config/eso_refresher_test.go | 133 ++++++++++++ python/src/smooai_config/__init__.py | 6 + python/src/smooai_config/eso_refresher.py | 116 +++++++++++ python/tests/test_eso_refresher.py | 119 +++++++++++ rust/config/src/eso_refresher.rs | 190 ++++++++++++++++++ rust/config/src/lib.rs | 1 + 10 files changed, 879 insertions(+) create mode 100644 .changeset/eso-refresher-parity.md create mode 100644 dotnet/src/SmooAI.Config/Eso/EsoRefresher.cs create mode 100644 dotnet/tests/SmooAI.Config.Tests/EsoRefresherTests.cs create mode 100644 go/config/eso_refresher.go create mode 100644 go/config/eso_refresher_test.go create mode 100644 python/src/smooai_config/eso_refresher.py create mode 100644 python/tests/test_eso_refresher.py create mode 100644 rust/config/src/eso_refresher.rs diff --git a/.changeset/eso-refresher-parity.md b/.changeset/eso-refresher-parity.md new file mode 100644 index 0000000..571c8b7 --- /dev/null +++ b/.changeset/eso-refresher-parity.md @@ -0,0 +1,5 @@ +--- +'@smooai/config': minor +--- + +SMOODEV-1526: Port the ESO bearer-token refresher core (the refresh algorithm + `SecretWriter` abstraction) to the Go, Python, Rust, and C# SDKs for parity with the TypeScript reference. Each mirrors the same behavior — invalidate-then-mint each cycle so the bootstrap Secret always holds a near-full-TTL token, fail-loud initial write, non-fatal loop-tick retries — driven by the language's own TokenProvider and unit-tested with a fake writer (no live cluster). The k8s-backed writer is intentionally an optional adapter so base SDK consumers don't pull a heavy k8s client; the TypeScript sidecar remains the canonical deployable. Epic SMOODEV-1522. diff --git a/dotnet/src/SmooAI.Config/Eso/EsoRefresher.cs b/dotnet/src/SmooAI.Config/Eso/EsoRefresher.cs new file mode 100644 index 0000000..41f93ad --- /dev/null +++ b/dotnet/src/SmooAI.Config/Eso/EsoRefresher.cs @@ -0,0 +1,94 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace SmooAI.Config.Eso; + +// ESO bearer-token refresher core — C# parity port of the TypeScript +// src/eso-refresher (SMOODEV-1526, epic SMOODEV-1522). +// +// ESO's webhook provider reads a STATIC bearer from a k8s Secret, but the config +// API issues short-lived client_credentials JWTs (~1h) — so a static token goes +// stale and ESO sync silently 401s. This refresher re-mints the token on a short +// interval via the same TokenProvider the SDK uses and writes it into the +// bootstrap Secret, so ESO always reads a fresh bearer. +// +// The k8s write is abstracted behind ISecretWriter so the loop is unit-testable +// with a fake (no live cluster). A native KubernetesClient-backed writer is an +// optional adapter (kept out of this core so base SDK consumers do not pull a +// heavy k8s client) — the TypeScript sidecar remains the canonical deployable; +// this gives the refresh ALGORITHM parity in C#. + +/// Writes the freshly-minted bearer token into the target Secret. +public interface ISecretWriter +{ + Task PatchBearerTokenAsync(string token, CancellationToken cancellationToken = default); +} + +/// The slice of TokenProvider the refresher needs. The real +/// TokenProvider satisfies it; tests inject a fake. +public interface ITokenSource +{ + Task GetAccessTokenAsync(CancellationToken cancellationToken = default); + void Invalidate(); +} + +/// Drives the ESO bearer refresh: re-mints a fresh token and writes it +/// to the target Secret on each cycle. +public sealed class EsoRefresher +{ + public const int DefaultIntervalSeconds = 900; + + private readonly ITokenSource _tokenSource; + private readonly ISecretWriter _secretWriter; + + /// The configured re-mint interval. + public TimeSpan Interval { get; } + + public EsoRefresher(ITokenSource tokenSource, ISecretWriter secretWriter, TimeSpan interval = default) + { + _tokenSource = tokenSource ?? throw new ArgumentException("EsoRefresher: tokenSource is required", nameof(tokenSource)); + _secretWriter = secretWriter ?? throw new ArgumentException("EsoRefresher: secretWriter is required", nameof(secretWriter)); + Interval = interval <= TimeSpan.Zero ? TimeSpan.FromSeconds(DefaultIntervalSeconds) : interval; + } + + /// + /// Force a brand-new token mint + write. Invalidates first so the Secret + /// always holds a token with (close to) a full TTL ahead — ESO must never + /// read a token about to expire. + /// + public async Task RefreshOnceAsync(CancellationToken cancellationToken = default) + { + _tokenSource.Invalidate(); + var token = await _tokenSource.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false); + await _secretWriter.PatchBearerTokenAsync(token, cancellationToken).ConfigureAwait(false); + } + + /// + /// Run the refresher: an initial fail-loud mint+write, then loop on the + /// interval until cancellation. Loop failures are swallowed (the current + /// Secret token is still valid for the rest of its TTL) and retried next tick. + /// + public async Task RunAsync(CancellationToken cancellationToken) + { + // Initial mint+write — fail-loud (exceptions propagate out of RunAsync). + await RefreshOnceAsync(cancellationToken).ConfigureAwait(false); + + using var timer = new PeriodicTimer(Interval); + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + try + { + await RefreshOnceAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch + { + // Non-fatal: retry on the next tick. + } + } + } +} diff --git a/dotnet/tests/SmooAI.Config.Tests/EsoRefresherTests.cs b/dotnet/tests/SmooAI.Config.Tests/EsoRefresherTests.cs new file mode 100644 index 0000000..726e46b --- /dev/null +++ b/dotnet/tests/SmooAI.Config.Tests/EsoRefresherTests.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using SmooAI.Config.Eso; + +namespace SmooAI.Config.Tests; + +// SMOODEV-1526 — ESO refresher core parity tests (C#). +public class EsoRefresherTests +{ + private sealed class FakeTokenSource : ITokenSource + { + private readonly string[] _tokens; + private int _idx; + public int Calls; + public int Invalidations; + + public FakeTokenSource(params string[] tokens) => _tokens = tokens; + + public Task GetAccessTokenAsync(CancellationToken cancellationToken = default) + { + Calls++; + var t = _tokens[Math.Min(_idx, _tokens.Length - 1)]; + _idx++; + return Task.FromResult(t); + } + + public void Invalidate() => Invalidations++; + } + + private sealed class RecordingWriter : ISecretWriter + { + private readonly int _failOnCall; + private int _call; + public readonly List Written = new(); + + public RecordingWriter(int failOnCall = -1) => _failOnCall = failOnCall; + + public Task PatchBearerTokenAsync(string token, CancellationToken cancellationToken = default) + { + _call++; + if (_call == _failOnCall) throw new InvalidOperationException("simulated k8s patch failure"); + Written.Add(token); + return Task.CompletedTask; + } + } + + [Fact] + public async Task RefreshOnce_WritesFreshToken() + { + var ts = new FakeTokenSource("tok-1"); + var w = new RecordingWriter(); + var r = new EsoRefresher(ts, w); + await r.RefreshOnceAsync(); + Assert.Equal(1, ts.Invalidations); + Assert.Equal(new List { "tok-1" }, w.Written); + } + + [Fact] + public async Task ForcesFreshEachCycle() + { + var ts = new FakeTokenSource("tok-1", "tok-2"); + var w = new RecordingWriter(); + var r = new EsoRefresher(ts, w); + await r.RefreshOnceAsync(); + await r.RefreshOnceAsync(); + Assert.Equal(2, ts.Calls); + Assert.Equal(2, ts.Invalidations); + Assert.Equal(new List { "tok-1", "tok-2" }, w.Written); + } + + [Fact] + public async Task RefreshOnce_PropagatesWriteFailure() + { + var ts = new FakeTokenSource("tok-1"); + var w = new RecordingWriter(failOnCall: 1); + var r = new EsoRefresher(ts, w); + await Assert.ThrowsAsync(() => r.RefreshOnceAsync()); + } + + [Fact] + public void RequiredFields() + { + Assert.Throws(() => new EsoRefresher(null!, new RecordingWriter())); + Assert.Throws(() => new EsoRefresher(new FakeTokenSource("t"), null!)); + } + + [Fact] + public void DefaultsIntervalWhenZero() + { + var r = new EsoRefresher(new FakeTokenSource("t"), new RecordingWriter()); + Assert.Equal(TimeSpan.FromSeconds(EsoRefresher.DefaultIntervalSeconds), r.Interval); + } +} diff --git a/go/config/eso_refresher.go b/go/config/eso_refresher.go new file mode 100644 index 0000000..372cd99 --- /dev/null +++ b/go/config/eso_refresher.go @@ -0,0 +1,120 @@ +package config + +// ESO bearer-token refresher core — Go parity port of the TypeScript +// src/eso-refresher (SMOODEV-1526, epic SMOODEV-1522). +// +// ESO's webhook provider reads a STATIC bearer from a k8s Secret, but the +// config API issues short-lived client_credentials JWTs (~1h) — so a static +// token goes stale and ESO sync silently 401s. This refresher re-mints the +// token on a short interval via the same TokenProvider the SDK uses and writes +// it into the bootstrap Secret, so ESO always reads a fresh bearer. +// +// The k8s write is abstracted behind SecretWriter so the loop is unit-testable +// with a fake (no live cluster). A native client-go-backed SecretWriter is an +// optional adapter (kept out of this core package so base SDK consumers do not +// pull a heavy k8s client) — the TypeScript sidecar remains the canonical +// deployable; this gives the refresh ALGORITHM parity in Go. + +import ( + "context" + "time" +) + +// ESO refresher defaults. +const ( + ESORefresherDefaultIntervalSeconds = 900 +) + +// SecretWriter writes the freshly-minted bearer token into the target store. +// Abstracted so the refresh loop can be unit-tested without a live cluster. +type SecretWriter interface { + // PatchBearerToken writes token into the configured Secret/key (the impl + // base64-encodes for k8s). + PatchBearerToken(token string) error +} + +// tokenSource is the slice of TokenProvider the refresher needs. *TokenProvider +// satisfies it; tests inject a fake. +type tokenSource interface { + GetAccessToken(ctx context.Context) (string, error) + Invalidate() +} + +// EsoRefresherOptions configures RunEsoRefresher. +type EsoRefresherOptions struct { + // TokenSource mints/refreshes the bearer. Required. + TokenSource tokenSource + // SecretWriter writes the bearer into the target Secret. Required. + SecretWriter SecretWriter + // Interval between re-mints. Defaults to 900s. + Interval time.Duration +} + +// EsoRefresherHandle controls a running refresher. +type EsoRefresherHandle struct { + // RefreshNow forces an immediate re-mint + write. + RefreshNow func() error + // Stop halts the refresh loop. Idempotent. + Stop func() +} + +// RunEsoRefresher performs an initial mint+write synchronously (fail-loud on +// misconfiguration), then starts a background loop re-minting on Interval. +// Returns a handle to force a refresh or stop the loop. +func RunEsoRefresher(opts EsoRefresherOptions) (*EsoRefresherHandle, error) { + if opts.TokenSource == nil { + return nil, NewConfigError("RunEsoRefresher: TokenSource is required") + } + if opts.SecretWriter == nil { + return nil, NewConfigError("RunEsoRefresher: SecretWriter is required") + } + interval := opts.Interval + if interval <= 0 { + interval = ESORefresherDefaultIntervalSeconds * time.Second + } + + refreshNow := func() error { + // Force a brand-new token each cycle so the Secret always holds one with + // (close to) a full TTL ahead — ESO must never read a token about to expire. + opts.TokenSource.Invalidate() + token, err := opts.TokenSource.GetAccessToken(context.Background()) + if err != nil { + return err + } + return opts.SecretWriter.PatchBearerToken(token) + } + + // Initial mint+write — fail-loud (caller exits non-zero on error). + if err := refreshNow(); err != nil { + return nil, err + } + + stop := make(chan struct{}) + stopped := false + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + // Loop failures are non-fatal: the current Secret token is still + // valid for the rest of its TTL. The caller's logger (if any) can + // observe via a wrapping SecretWriter; here we simply retry next tick. + _ = refreshNow() + } + } + }() + + return &EsoRefresherHandle{ + RefreshNow: refreshNow, + Stop: func() { + if stopped { + return + } + stopped = true + close(stop) + }, + }, nil +} diff --git a/go/config/eso_refresher_test.go b/go/config/eso_refresher_test.go new file mode 100644 index 0000000..6c34f45 --- /dev/null +++ b/go/config/eso_refresher_test.go @@ -0,0 +1,133 @@ +package config + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +type fakeTokenSource struct { + mu sync.Mutex + tokens []string + idx int + invalidations int + calls int +} + +func (f *fakeTokenSource) GetAccessToken(_ context.Context) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + t := f.tokens[minIdx(f.idx, len(f.tokens)-1)] + f.idx++ + return t, nil +} + +func (f *fakeTokenSource) Invalidate() { + f.mu.Lock() + defer f.mu.Unlock() + f.invalidations++ +} + +func minIdx(a, b int) int { + if a < b { + return a + } + return b +} + +type recordingWriter struct { + mu sync.Mutex + written []string + failOnCall int + call int +} + +func (w *recordingWriter) PatchBearerToken(token string) error { + w.mu.Lock() + defer w.mu.Unlock() + w.call++ + if w.call == w.failOnCall { + return errors.New("simulated k8s patch failure") + } + w.written = append(w.written, token) + return nil +} + +func (w *recordingWriter) snapshot() []string { + w.mu.Lock() + defer w.mu.Unlock() + return append([]string(nil), w.written...) +} + +func TestRunEsoRefresherInitialWrite(t *testing.T) { + ts := &fakeTokenSource{tokens: []string{"tok-1"}} + w := &recordingWriter{} + h, err := RunEsoRefresher(EsoRefresherOptions{TokenSource: ts, SecretWriter: w, Interval: time.Hour}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer h.Stop() + if got := w.snapshot(); len(got) != 1 || got[0] != "tok-1" { + t.Errorf("initial write = %v, want [tok-1]", got) + } +} + +func TestRunEsoRefresherForcesFreshEachCall(t *testing.T) { + ts := &fakeTokenSource{tokens: []string{"tok-1", "tok-2", "tok-3"}} + w := &recordingWriter{} + h, err := RunEsoRefresher(EsoRefresherOptions{TokenSource: ts, SecretWriter: w, Interval: time.Hour}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer h.Stop() + if err := h.RefreshNow(); err != nil { + t.Fatalf("RefreshNow: %v", err) + } + // Startup + one forced refresh = two mints, each preceded by an invalidate. + if ts.calls != 2 || ts.invalidations != 2 { + t.Errorf("calls=%d invalidations=%d, want 2/2", ts.calls, ts.invalidations) + } + if got := w.snapshot(); len(got) != 2 || got[0] != "tok-1" || got[1] != "tok-2" { + t.Errorf("written = %v, want [tok-1 tok-2]", got) + } +} + +func TestRunEsoRefresherFailLoudInitial(t *testing.T) { + ts := &fakeTokenSource{tokens: []string{"tok-1"}} + w := &recordingWriter{failOnCall: 1} // first (initial) write fails + _, err := RunEsoRefresher(EsoRefresherOptions{TokenSource: ts, SecretWriter: w, Interval: time.Hour}) + if err == nil { + t.Error("expected RunEsoRefresher to fail loud when the initial write fails") + } +} + +func TestRunEsoRefresherRequiredFields(t *testing.T) { + if _, err := RunEsoRefresher(EsoRefresherOptions{SecretWriter: &recordingWriter{}}); err == nil { + t.Error("expected error for missing TokenSource") + } + if _, err := RunEsoRefresher(EsoRefresherOptions{TokenSource: &fakeTokenSource{tokens: []string{"t"}}}); err == nil { + t.Error("expected error for missing SecretWriter") + } +} + +func TestRunEsoRefresherLoopTicks(t *testing.T) { + ts := &fakeTokenSource{tokens: []string{"t1", "t2", "t3", "t4", "t5"}} + w := &recordingWriter{} + h, err := RunEsoRefresher(EsoRefresherOptions{TokenSource: ts, SecretWriter: w, Interval: 10 * time.Millisecond}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer h.Stop() + // Wait for the loop to tick at least once past the initial write. + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if len(w.snapshot()) >= 2 { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Errorf("loop did not tick; writes=%v", w.snapshot()) +} diff --git a/python/src/smooai_config/__init__.py b/python/src/smooai_config/__init__.py index ed84324..4279693 100644 --- a/python/src/smooai_config/__init__.py +++ b/python/src/smooai_config/__init__.py @@ -30,6 +30,12 @@ build_external_secret, resolve_secret_mapping, ) +from smooai_config.eso_refresher import ( + EsoRefresherHandle, + SecretWriter, + TokenSource, + run_eso_refresher, +) from smooai_config.env_config import find_and_process_env_config from smooai_config.file_config import find_and_process_file_config, find_config_directory from smooai_config.local import LocalConfigManager diff --git a/python/src/smooai_config/eso_refresher.py b/python/src/smooai_config/eso_refresher.py new file mode 100644 index 0000000..398f537 --- /dev/null +++ b/python/src/smooai_config/eso_refresher.py @@ -0,0 +1,116 @@ +"""ESO bearer-token refresher core — Python parity port of the TypeScript +``src/eso-refresher`` (SMOODEV-1526, epic SMOODEV-1522). + +ESO's webhook provider reads a STATIC bearer from a k8s Secret, but the config +API issues short-lived ``client_credentials`` JWTs (~1h) — so a static token goes +stale and ESO sync silently 401s. This refresher re-mints the token on a short +interval via the same TokenProvider the SDK uses and writes it into the bootstrap +Secret, so ESO always reads a fresh bearer. + +The k8s write is abstracted behind :class:`SecretWriter` so the loop is +unit-testable with a fake (no live cluster). A native ``kubernetes``-backed +writer is an optional adapter (kept out of this core so base SDK consumers do not +pull a heavy k8s client) — the TypeScript sidecar remains the canonical +deployable; this gives the refresh ALGORITHM parity in Python. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass +from typing import Callable, Protocol + +from smooai_config.utils import SmooaiConfigError + +ESO_REFRESHER_DEFAULT_INTERVAL_SECONDS = 900.0 + + +class SecretWriter(Protocol): + """Writes the freshly-minted bearer token into the target Secret.""" + + def patch_bearer_token(self, token: str) -> None: ... + + +class TokenSource(Protocol): + """The slice of TokenProvider the refresher needs. The real ``TokenProvider`` + satisfies it; tests inject a fake.""" + + def get_access_token(self) -> str: ... + + def invalidate(self) -> None: ... + + +# A scheduler starts a repeating callback every ``interval`` seconds and returns +# a cancel callable. Injectable so tests drive ticks deterministically. +Scheduler = Callable[[Callable[[], None], float], Callable[[], None]] + + +def _default_scheduler(fn: Callable[[], None], interval: float) -> Callable[[], None]: + stop = threading.Event() + + def loop() -> None: + while not stop.wait(interval): + fn() + + thread = threading.Thread(target=loop, daemon=True) + thread.start() + return stop.set + + +@dataclass +class EsoRefresherHandle: + """Controls a running refresher.""" + + refresh_now: Callable[[], None] + stop: Callable[[], None] + + +def run_eso_refresher( + *, + token_source: TokenSource, + secret_writer: SecretWriter, + interval_seconds: float = ESO_REFRESHER_DEFAULT_INTERVAL_SECONDS, + scheduler: Scheduler | None = None, +) -> EsoRefresherHandle: + """Start the refresher. + + Performs an initial mint+write synchronously (fail-loud on misconfiguration), + then schedules periodic refreshes. Returns a handle to force a refresh or + stop the loop. Loop failures are swallowed (the current Secret token is still + valid for the rest of its TTL) and retried on the next tick. + """ + if token_source is None: + raise SmooaiConfigError("run_eso_refresher: token_source is required") + if secret_writer is None: + raise SmooaiConfigError("run_eso_refresher: secret_writer is required") + if interval_seconds <= 0: + interval_seconds = ESO_REFRESHER_DEFAULT_INTERVAL_SECONDS + + def refresh_now() -> None: + # Force a brand-new token each cycle so the Secret always holds one with + # (close to) a full TTL ahead — ESO must never read a token about to expire. + token_source.invalidate() + token = token_source.get_access_token() + secret_writer.patch_bearer_token(token) + + # Initial mint+write — fail-loud. + refresh_now() + + def tick() -> None: + try: + refresh_now() + except Exception: # noqa: BLE001 — loop failures are non-fatal, retried next tick + pass + + sched = scheduler or _default_scheduler + cancel = sched(tick, interval_seconds) + + stopped = {"v": False} + + def stop() -> None: + if stopped["v"]: + return + stopped["v"] = True + cancel() + + return EsoRefresherHandle(refresh_now=refresh_now, stop=stop) diff --git a/python/tests/test_eso_refresher.py b/python/tests/test_eso_refresher.py new file mode 100644 index 0000000..134bba6 --- /dev/null +++ b/python/tests/test_eso_refresher.py @@ -0,0 +1,119 @@ +"""SMOODEV-1526 — ESO refresher core parity tests (Python).""" + +from __future__ import annotations + +from typing import Callable + +import pytest +from smooai_config.eso_refresher import run_eso_refresher +from smooai_config.utils import SmooaiConfigError + + +class FakeTokenSource: + def __init__(self, tokens: list[str]) -> None: + self._tokens = tokens + self._idx = 0 + self.calls = 0 + self.invalidations = 0 + + def get_access_token(self) -> str: + self.calls += 1 + t = self._tokens[min(self._idx, len(self._tokens) - 1)] + self._idx += 1 + return t + + def invalidate(self) -> None: + self.invalidations += 1 + + +class RecordingWriter: + def __init__(self, fail_on_call: int = -1) -> None: + self.written: list[str] = [] + self._fail_on_call = fail_on_call + self._call = 0 + + def patch_bearer_token(self, token: str) -> None: + self._call += 1 + if self._call == self._fail_on_call: + raise RuntimeError("simulated k8s patch failure") + self.written.append(token) + + +class ManualScheduler: + """Captures the tick fn so tests drive it deterministically.""" + + def __init__(self) -> None: + self.fn: Callable[[], None] | None = None + self.interval: float = 0 + self.cancelled = False + + def __call__(self, fn: Callable[[], None], interval: float) -> Callable[[], None]: + self.fn = fn + self.interval = interval + + def cancel() -> None: + self.cancelled = True + + return cancel + + def tick(self) -> None: + assert self.fn is not None + self.fn() + + +def test_initial_write(): + ts = FakeTokenSource(["tok-1"]) + w = RecordingWriter() + run_eso_refresher(token_source=ts, secret_writer=w, scheduler=ManualScheduler()) + assert w.written == ["tok-1"] + + +def test_forces_fresh_each_cycle(): + ts = FakeTokenSource(["tok-1", "tok-2", "tok-3"]) + w = RecordingWriter() + sched = ManualScheduler() + run_eso_refresher(token_source=ts, secret_writer=w, scheduler=sched) + sched.tick() + assert ts.calls == 2 + assert ts.invalidations == 2 + assert w.written == ["tok-1", "tok-2"] + + +def test_survives_tick_failure(): + ts = FakeTokenSource(["tok-1", "tok-2", "tok-3"]) + w = RecordingWriter(fail_on_call=2) # first scheduled tick fails + sched = ManualScheduler() + run_eso_refresher(token_source=ts, secret_writer=w, scheduler=sched) + sched.tick() # fails internally, must not raise + sched.tick() # recovers + assert w.written == ["tok-1", "tok-3"] + + +def test_fail_loud_initial(): + ts = FakeTokenSource(["tok-1"]) + w = RecordingWriter(fail_on_call=1) # initial write fails + with pytest.raises(RuntimeError): + run_eso_refresher(token_source=ts, secret_writer=w, scheduler=ManualScheduler()) + + +def test_stop_cancels_loop(): + ts = FakeTokenSource(["tok-1"]) + w = RecordingWriter() + sched = ManualScheduler() + handle = run_eso_refresher(token_source=ts, secret_writer=w, scheduler=sched) + assert sched.cancelled is False + handle.stop() + assert sched.cancelled is True + + +def test_required_fields(): + with pytest.raises(SmooaiConfigError): + run_eso_refresher(token_source=None, secret_writer=RecordingWriter(), scheduler=ManualScheduler()) # type: ignore[arg-type] + with pytest.raises(SmooaiConfigError): + run_eso_refresher(token_source=FakeTokenSource(["t"]), secret_writer=None, scheduler=ManualScheduler()) # type: ignore[arg-type] + + +def test_honors_interval_override(): + sched = ManualScheduler() + run_eso_refresher(token_source=FakeTokenSource(["t"]), secret_writer=RecordingWriter(), interval_seconds=123.0, scheduler=sched) + assert sched.interval == 123.0 diff --git a/rust/config/src/eso_refresher.rs b/rust/config/src/eso_refresher.rs new file mode 100644 index 0000000..5040d71 --- /dev/null +++ b/rust/config/src/eso_refresher.rs @@ -0,0 +1,190 @@ +//! ESO bearer-token refresher core — Rust parity port of the TypeScript +//! `src/eso-refresher` (SMOODEV-1526, epic SMOODEV-1522). +//! +//! ESO's webhook provider reads a STATIC bearer from a k8s Secret, but the +//! config API issues short-lived `client_credentials` JWTs (~1h) — so a static +//! token goes stale and ESO sync silently 401s. This refresher re-mints the +//! token on a short interval via the same `TokenProvider` the SDK uses and +//! writes it into the bootstrap Secret, so ESO always reads a fresh bearer. +//! +//! The k8s write is abstracted behind [`SecretWriter`] so the loop is +//! unit-testable with a fake (no live cluster). A native `kube`-backed writer is +//! an optional adapter (kept out of this core so base SDK consumers do not pull +//! a heavy k8s client) — the TypeScript sidecar remains the canonical +//! deployable; this gives the refresh ALGORITHM parity in Rust. + +use std::future::Future; +use std::time::Duration; + +use crate::utils::SmooaiConfigError; + +pub const ESO_REFRESHER_DEFAULT_INTERVAL_SECONDS: u64 = 900; + +/// Writes the freshly-minted bearer token into the target Secret. Abstracted so +/// the refresh loop is unit-testable without a live cluster. +pub trait SecretWriter { + fn patch_bearer_token( + &self, + token: &str, + ) -> impl Future>; +} + +/// The slice of `TokenProvider` the refresher needs. The real `TokenProvider` +/// satisfies it; tests inject a fake. +pub trait TokenSource { + fn get_access_token(&self) -> impl Future>; + fn invalidate(&self) -> impl Future; +} + +/// Drives the ESO bearer refresh: re-mints a fresh token and writes it to the +/// target Secret on each cycle. +pub struct EsoRefresher { + token_source: T, + secret_writer: W, + interval: Duration, +} + +impl EsoRefresher { + /// Build a refresher. `interval` defaults to 900s when zero. + pub fn new(token_source: T, secret_writer: W, interval: Duration) -> Self { + let interval = if interval.is_zero() { + Duration::from_secs(ESO_REFRESHER_DEFAULT_INTERVAL_SECONDS) + } else { + interval + }; + Self { + token_source, + secret_writer, + interval, + } + } + + /// The configured re-mint interval. + pub fn interval(&self) -> Duration { + self.interval + } + + /// Force a brand-new token mint + write. Invalidates first so the Secret + /// always holds a token with (close to) a full TTL ahead — ESO must never + /// read a token about to expire. + pub async fn refresh_once(&self) -> Result<(), SmooaiConfigError> { + self.token_source.invalidate().await; + let token = self.token_source.get_access_token().await?; + self.secret_writer.patch_bearer_token(&token).await + } + + /// Run the refresher: an initial fail-loud mint+write, then loop on the + /// interval until `stop` resolves. Loop failures are swallowed (the current + /// Secret token is still valid for the rest of its TTL) and retried next + /// tick. + pub async fn run(&self, stop: impl Future) -> Result<(), SmooaiConfigError> { + // Initial mint+write — fail-loud. + self.refresh_once().await?; + + let mut ticker = tokio::time::interval(self.interval); + ticker.tick().await; // consume the immediate first tick + tokio::pin!(stop); + loop { + tokio::select! { + _ = &mut stop => return Ok(()), + _ = ticker.tick() => { + let _ = self.refresh_once().await; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + struct FakeTokenSource { + tokens: Vec, + idx: RefCell, + calls: RefCell, + invalidations: RefCell, + } + + impl FakeTokenSource { + fn new(tokens: &[&str]) -> Self { + Self { + tokens: tokens.iter().map(|s| s.to_string()).collect(), + idx: RefCell::new(0), + calls: RefCell::new(0), + invalidations: RefCell::new(0), + } + } + } + + impl TokenSource for FakeTokenSource { + async fn get_access_token(&self) -> Result { + *self.calls.borrow_mut() += 1; + let i = *self.idx.borrow(); + let t = self.tokens[i.min(self.tokens.len() - 1)].clone(); + *self.idx.borrow_mut() += 1; + Ok(t) + } + async fn invalidate(&self) { + *self.invalidations.borrow_mut() += 1; + } + } + + struct RecordingWriter { + written: RefCell>, + fail_on_call: usize, + call: RefCell, + } + + impl RecordingWriter { + fn new(fail_on_call: usize) -> Self { + Self { + written: RefCell::new(Vec::new()), + fail_on_call, + call: RefCell::new(0), + } + } + } + + impl SecretWriter for RecordingWriter { + async fn patch_bearer_token(&self, token: &str) -> Result<(), SmooaiConfigError> { + *self.call.borrow_mut() += 1; + if *self.call.borrow() == self.fail_on_call { + return Err(SmooaiConfigError::new("simulated k8s patch failure")); + } + self.written.borrow_mut().push(token.to_string()); + Ok(()) + } + } + + #[tokio::test] + async fn refresh_once_writes_fresh_token() { + let r = EsoRefresher::new(FakeTokenSource::new(&["tok-1"]), RecordingWriter::new(0), Duration::ZERO); + r.refresh_once().await.unwrap(); + assert_eq!(*r.token_source.invalidations.borrow(), 1); + assert_eq!(r.secret_writer.written.borrow().clone(), vec!["tok-1".to_string()]); + } + + #[tokio::test] + async fn forces_fresh_each_cycle() { + let r = EsoRefresher::new(FakeTokenSource::new(&["tok-1", "tok-2"]), RecordingWriter::new(0), Duration::ZERO); + r.refresh_once().await.unwrap(); + r.refresh_once().await.unwrap(); + assert_eq!(*r.token_source.calls.borrow(), 2); + assert_eq!(*r.token_source.invalidations.borrow(), 2); + assert_eq!(r.secret_writer.written.borrow().clone(), vec!["tok-1".to_string(), "tok-2".to_string()]); + } + + #[tokio::test] + async fn refresh_once_propagates_write_failure() { + let r = EsoRefresher::new(FakeTokenSource::new(&["tok-1"]), RecordingWriter::new(1), Duration::ZERO); + assert!(r.refresh_once().await.is_err()); + } + + #[tokio::test] + async fn defaults_interval_when_zero() { + let r = EsoRefresher::new(FakeTokenSource::new(&["t"]), RecordingWriter::new(0), Duration::ZERO); + assert_eq!(r.interval(), Duration::from_secs(ESO_REFRESHER_DEFAULT_INTERVAL_SECONDS)); + } +} diff --git a/rust/config/src/lib.rs b/rust/config/src/lib.rs index 10d09f7..44101a2 100644 --- a/rust/config/src/lib.rs +++ b/rust/config/src/lib.rs @@ -12,6 +12,7 @@ pub mod container; pub mod deferred; pub mod env_config; pub mod eso_manifests; +pub mod eso_refresher; pub mod file_config; pub mod local; pub mod merge;