OCPBUGS-58102: Expose ExternalDNS operand metrics via kube-rbac-proxy sidecar

bundle/manifests/external-dns-operator.clusterserviceversion.yaml

@@ -472,6 +472,7 @@ spec:
                 - --operator-namespace=$(OPERATOR_NAMESPACE)
                 - --operand-namespace=$(OPERATOR_NAMESPACE)
                 - --externaldns-image=$(RELATED_IMAGE_EXTERNAL_DNS)
+                - --kube-rbac-proxy-image=$(RELATED_IMAGE_KUBE_RBAC_PROXY)
                 - --trusted-ca-configmap=$(TRUSTED_CA_CONFIGMAP_NAME)
                 - --leader-elect
                 - --webhook-disable-http2
@@ -482,6 +483,8 @@ spec:
                       fieldPath: metadata.namespace
                 - name: RELATED_IMAGE_EXTERNAL_DNS
                   value: quay.io/external-dns-operator/external-dns:latest
+                - name: RELATED_IMAGE_KUBE_RBAC_PROXY
+                  value: quay.io/openshift/origin-kube-rbac-proxy:latest
                 - name: TRUSTED_CA_CONFIGMAP_NAME
                 image: quay.io/openshift/origin-external-dns-operator:latest
                 name: external-dns-operator
@@ -559,6 +562,7 @@ spec:
           - configmaps
           - secrets
           - serviceaccounts
+          - services
           verbs:
           - create
           - delete
@@ -587,6 +591,18 @@ spec:
           - patch
           - update
           - watch
+        - apiGroups:
+          - monitoring.coreos.com
+          resources:
+          - servicemonitors
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - patch
+          - update
+          - watch
         - apiGroups:
           - coordination.k8s.io
           resources:

bundle/manifests/external-dns_rbac.authorization.k8s.io_v1_clusterrole.yaml

@@ -31,3 +31,15 @@ rules:
   - get
   - watch
   - list
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create

config/manager/manager.yaml

@@ -36,6 +36,7 @@ spec:
         - --operator-namespace=$(OPERATOR_NAMESPACE)
         - --operand-namespace=$(OPERATOR_NAMESPACE)
         - --externaldns-image=$(RELATED_IMAGE_EXTERNAL_DNS)
+        - --kube-rbac-proxy-image=$(RELATED_IMAGE_KUBE_RBAC_PROXY)
         - --trusted-ca-configmap=$(TRUSTED_CA_CONFIGMAP_NAME)
         - --leader-elect
         - --webhook-disable-http2
@@ -49,6 +50,8 @@ spec:
             # Use "latest" floating tag to avoid problems with the prunning of older mirorred images.
             # Ref: https://issues.redhat.com/browse/OCPBUGS-57339.
           value: quay.io/external-dns-operator/external-dns:latest
+        - name: RELATED_IMAGE_KUBE_RBAC_PROXY
+          value: quay.io/openshift/origin-kube-rbac-proxy:latest
         - name: TRUSTED_CA_CONFIGMAP_NAME
         securityContext:
           capabilities:

config/rbac/operand_role.yaml

@@ -31,3 +31,15 @@ rules:
       - get
       - watch
       - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create

config/rbac/role.yaml

@@ -73,6 +73,7 @@ rules:
   - configmaps
   - secrets
   - serviceaccounts
+  - services
   verbs:
   - create
   - delete
@@ -101,3 +102,15 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

docs/openshift.md

@@ -10,3 +10,18 @@ Use the following convenience script to secure communication between the API and
 ```bash
 $ ./hack/add-serving-cert.sh --namespace external-dns-operator --service webhook-service --webhook validating-webhook-configuration --secret webhook-server-cert
 ```
+
+## Operand metrics
+
+Each ExternalDNS operand deployment includes a [kube-rbac-proxy](https://github.com/brancz/kube-rbac-proxy) sidecar per zone container to expose metrics over HTTPS. A `Service` and `ServiceMonitor` are created per `ExternalDNS` CR for Prometheus discovery.
+
+### Multi-zone metric differentiation
+
+When an `ExternalDNS` instance manages multiple zones, one ExternalDNS container runs per zone — each exposing the same metric names. Metrics are kept separate (not combined or deduplicated) through the following mechanism:
+
+- Each zone container binds its metrics to a distinct localhost port (`:7979`, `:7980`, etc.).
+- Each kube-rbac-proxy sidecar proxies one of those ports on a distinct secure port (`:9091`, `:9092`, etc.).
+- The `ServiceMonitor` creates one endpoint entry per port.
+- Prometheus assigns a unique `instance` label (`pod_ip:port`) to each scrape target.
+
+This means metrics from different zones within the same `ExternalDNS` instance are differentiated by port via the `instance` label. Metrics from different `ExternalDNS` instances are differentiated by pod or deployment name.

pkg/operator/controller/externaldns/controller.go

@@ -25,6 +25,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -102,6 +103,16 @@ func New(mgr manager.Manager, cfg Config) (controller.Controller, error) {
 		return nil, err
 	}
 
+	if err := c.Watch(source.Kind[client.Object](operatorCache, &corev1.Service{}, handler.EnqueueRequestForOwner(operatorScheme, operatorRESTMapper, &operatorv1beta1.ExternalDNS{}, handler.OnlyControllerOwner()))); err != nil {
+		return nil, err
+	}
+
+	smInformer := &unstructured.Unstructured{}
+	smInformer.SetGroupVersionKind(serviceMonitorGVK)
+	if err := c.Watch(source.Kind[client.Object](operatorCache, smInformer, handler.EnqueueRequestForOwner(operatorScheme, operatorRESTMapper, &operatorv1beta1.ExternalDNS{}, handler.OnlyControllerOwner()))); err != nil {
+		return nil, err
+	}
+
 	// secret replicated by the credentials controller
 	// needs to trigger the reconciliation of the corresponding ExternalDNS
 	// because of the annotation with the secret's hash in the operand deployment
@@ -212,6 +223,13 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return reconcile.Result{}, fmt.Errorf("failed to ensure externalDNS deployment: %w", err)
 	}
 
+	if err := r.ensureExternalDNSMetricsService(ctx, r.config.Namespace, externalDNS); err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to ensure externalDNS metrics service: %w", err)
+	}
+	if err := r.ensureExternalDNSServiceMonitor(ctx, r.config.Namespace, externalDNS); err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to ensure externalDNS service monitor: %w", err)
+	}
+
 	if err := r.updateExternalDNSStatus(ctx, externalDNS, currentDeployment, true); err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to update externalDNS custom resource %s: %w", externalDNS.Name, err)
 	}

pkg/operator/controller/externaldns/deployment.go

@@ -109,16 +109,16 @@ func (r *reconciler) ensureExternalDNSDeployment(ctx context.Context, namespace,
 	}
 
 	desired, err := desiredExternalDNSDeployment(&deploymentConfig{
-		namespace,
-		image,
-		serviceAccount,
-		externalDNS,
-		r.config.IsOpenShift,
-		r.config.PlatformStatus,
-		credSecret.Name,
-		credSecretHash,
-		trustCAConfigMapName,
-		trustCAConfigMapHash,
+		namespace:              namespace,
+		image:                  image,
+		serviceAccount:         serviceAccount,
+		externalDNS:            externalDNS,
+		isOpenShift:            r.config.IsOpenShift,
+		platformStatus:         r.config.PlatformStatus,
+		secret:                 credSecret.Name,
+		secretHash:             credSecretHash,
+		trustedCAConfigMapName: trustCAConfigMapName,
+		trustedCAConfigMapHash: trustCAConfigMapHash,
 	})
 	if err != nil {
 		return false, nil, fmt.Errorf("failed to build externalDNS deployment: %w", err)
@@ -296,6 +296,9 @@ func desiredExternalDNSDeployment(cfg *deploymentConfig) (*appsv1.Deployment, er
 			depl.Spec.Template.Spec.Containers = append(depl.Spec.Template.Spec.Containers, *container)
 		}
 	}
+	certVolume := metricsCertVolume(controller.ExternalDNSMetricsSecretName(cfg.externalDNS))
+	depl.Spec.Template.Spec.Volumes = append(depl.Spec.Template.Spec.Volumes, certVolume)
+
 	return depl, nil
 }
 
@@ -418,6 +421,10 @@ func externalDNSContainersChanged(current, expected, updated *appsv1.Deployment)
 				updated.Spec.Template.Spec.Containers[currCont.Index].SecurityContext = updatedContext
 				changed = true
 			}
+			if !equalContainerPorts(currCont.Ports, expCont.Ports) {
+				updated.Spec.Template.Spec.Containers[currCont.Index].Ports = expCont.Ports
+				changed = true
+			}
 		} else {
 			// expected container is not present - add it
 			updated.Spec.Template.Spec.Containers = append(updated.Spec.Template.Spec.Containers, expCont.Container)
@@ -701,6 +708,27 @@ func securityContextChanged(current, updated, desired *corev1.SecurityContext) (
 	return changed, updated
 }
 
+// equalContainerPorts returns true if 2 container port slices have the same content.
+func equalContainerPorts(current, expected []corev1.ContainerPort) bool {
+	if len(current) != len(expected) {
+		return false
+	}
+	currentMap := map[string]corev1.ContainerPort{}
+	for _, p := range current {
+		currentMap[p.Name] = p
+	}
+	for _, ep := range expected {
+		cp, found := currentMap[ep.Name]
+		if !found {
+			return false
+		}
+		if cp.ContainerPort != ep.ContainerPort || cp.Protocol != ep.Protocol {
+			return false
+		}
+	}
+	return true
+}
+
 func equalBoolPtr(current, desired *bool) bool {
 	if desired == nil {
 		return true