From c04a10190043a6e325797f80713ef7d06be0d97c Mon Sep 17 00:00:00 2001 From: Krstan Vjestica Date: Thu, 23 Jul 2026 14:32:08 +0200 Subject: [PATCH] fix: cache Secrets in the operator namespace instead of cluster-wide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The credential reconciler watches Secrets, and the manager cached them at cluster scope. Once the Secret grant became a namespaced Role, that informer's list was refused, the shared cache never reported synced, and the manager exited with "failed to wait for ... caches to sync" — a crash loop that took all nine controllers down, not just Credential. Scope the Secret cache to the operator's own namespace so the informer asks only for what the Role grants. The namespace comes from a POD_NAMESPACE downward-API env (added to both the Helm deployment and the kustomize manager manifest), falling back to the ServiceAccount namespace file so an older manifest still scopes correctly. Out-of-cluster runs have neither and keep the unscoped cache, which matches the cluster-admin kubeconfig they use. Verified in the 7-node dev cluster: 0 restarts, all nine controllers start, a Credential CR reconciles its Secret through the scoped cache, and the operator still cannot read Secrets outside its namespace. --- CHANGELOG.md | 12 +++++++++++ README.md | 12 +++++++++++ cmd/main.go | 38 ++++++++++++++++++++++++++++++++++ config/manager/manager.yaml | 7 +++++++ helm/templates/deployment.yaml | 7 +++++++ 5 files changed, 76 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 083a9a9..eefff36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,18 @@ and the project uses SemVer. now a namespaced `Role`/`RoleBinding` in the operator's own namespace (kustomize + Helm), driven by a namespaced `+kubebuilder:rbac` marker. +### Fixed +- **Operator no longer crash-loops under the scoped Secret role.** The credential + reconciler watches Secrets, and the manager cached them at *cluster* scope, so + the informer's list was refused by the namespaced role, the shared cache never + reported synced, and the manager exited with + `failed to wait for ... caches to sync` — taking all nine controllers down, not + just Credential. The manager now caches Secrets only in its own namespace, + resolved from the `POD_NAMESPACE` downward-API env (added to the Helm + deployment and the kustomize manager manifest) and falling back to the mounted + ServiceAccount namespace file. Out-of-cluster runs, where neither is present, + keep the unscoped cache and log why. + ## [2026.06.1] - 2026-06-20 ### Changed diff --git a/README.md b/README.md index 6fc9e63..37ab8ea 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,18 @@ helm install forail-operator ./helm -n forail-operator --create-namespace \ --set forail.token=$TOKEN ``` +Two namespace notes: + +- **Secrets referenced by `Credential` CRs must live in the operator's own + namespace.** The operator holds a namespaced Secret `Role` (not a ClusterRole) + and caches Secrets only there, so a `spec.inputsFrom` pointing at a Secret in + another namespace will not resolve. +- **`forail.url` must be a host Forail accepts.** The chart's + `forail.allowedHosts` covers the `forail-web` Service DNS names, so the URL + above works as-is. Reaching Forail under any other name — an ingress host, an + external load balancer — means passing `--set forail.hostHeader=`, + otherwise Django rejects every call with `400`. + ### Via OLM (recommended for OpenShift / OperatorHub) ```bash diff --git a/cmd/main.go b/cmd/main.go index a3ccf57..095ebaa 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -3,11 +3,15 @@ package main import ( "flag" "os" + "strings" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -54,12 +58,31 @@ func main() { setupLog.Info("no default Forail backend configured; CRs without spec.forailInstance will not reconcile until --forail-url and --forail-token (or FORAIL_URL / FORAIL_TOKEN env) are set") } + // needtofix M12 follow-up: the credential reconciler watches Secrets, and the + // manager's default cache watches them at CLUSTER scope. Since M12 narrowed the + // Secret grant to a namespaced Role, that informer can never list — it fails + // permanently, the shared cache never reports synced, and the manager exits + // ("failed to wait for ... caches to sync"), taking every controller down with + // it. Scope the Secret cache to the operator's own namespace so the informer + // asks only for what the Role actually grants. + cacheOpts := cache.Options{} + if ns := operatorNamespace(); ns != "" { + cacheOpts.ByObject = map[client.Object]cache.ByObject{ + &corev1.Secret{}: {Namespaces: map[string]cache.Config{ns: {}}}, + } + } else { + // Out-of-cluster runs (`make run`) use a kubeconfig that is normally + // cluster-admin, so an unscoped Secret cache is correct there. + setupLog.Info("operator namespace unknown (no POD_NAMESPACE env, no service-account namespace file); caching Secrets cluster-wide") + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: metricsAddr}, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "forail-operator-leader.forail.forail-platform.io", + Cache: cacheOpts, }) if err != nil { setupLog.Error(err, "unable to start manager") @@ -169,3 +192,18 @@ func main() { os.Exit(1) } } + +// operatorNamespace resolves the namespace the operator runs in: the +// POD_NAMESPACE downward-API env the chart sets, falling back to the namespace +// file every in-cluster ServiceAccount is mounted with (so a manifest that +// predates the env var still scopes correctly). Empty means out-of-cluster. +func operatorNamespace() string { + if ns := strings.TrimSpace(os.Getenv("POD_NAMESPACE")); ns != "" { + return ns + } + data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d8e1569..0caec1d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -42,6 +42,13 @@ spec: - --metrics-bind-address=:8080 - --health-probe-bind-address=:8081 env: + # Scopes the manager's Secret cache to this namespace, matching the + # namespaced Secret Role (needtofix M12). Without it the cache would + # watch Secrets cluster-wide, get 403, and never sync. + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace # In-cluster service URL — bypasses the Ingress, plain HTTP on # the pod network. No Host header needed because forail-web's # internal nginx accepts any Host. diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index f09b830..e19a045 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -31,6 +31,13 @@ spec: - --leader-elect=true {{- end }} env: + # Scopes the manager's Secret cache to this namespace, matching the + # namespaced Secret Role (needtofix M12). Without it the cache would + # watch Secrets cluster-wide, get 403, and never sync. + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: FORAIL_URL value: {{ .Values.forail.url | quote }} {{- if .Values.forail.hostHeader }}