remove observedGeneration from postgresDatabase

api/v4/postgresdatabase_types.go

@@ -65,9 +65,6 @@ type PostgresDatabaseStatus struct {
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 	// +optional
 	Databases []DatabaseInfo `json:"databases,omitempty"`
-	// ObservedGeneration represents the .metadata.generation that the status was set based upon.
-	// +optional
-	ObservedGeneration *int64 `json:"observedGeneration,omitempty"`
 }
 
 // +kubebuilder:object:root=true

internal/controller/postgresdatabase_controller.go

@@ -18,14 +18,14 @@ package controller
 
 import (
 	"context"
-	"reflect"
 
 	cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
 	enterprisev4 "github.com/splunk/splunk-operator/api/v4"
 	dbadapter "github.com/splunk/splunk-operator/pkg/postgresql/database/adapter"
 	dbcore "github.com/splunk/splunk-operator/pkg/postgresql/database/core"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -95,31 +95,45 @@ func (r *PostgresDatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		return err
 	}
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&enterprisev4.PostgresDatabase{}, builder.WithPredicates(
-			predicate.Or(
-				predicate.GenerationChangedPredicate{},
-				predicate.Funcs{
-					UpdateFunc: func(e event.UpdateEvent) bool {
-						return !reflect.DeepEqual(
-							e.ObjectOld.GetFinalizers(),
-							e.ObjectNew.GetFinalizers(),
-						)
-					},
-				},
-			),
-		)).
-		Owns(&cnpgv1.Database{}, builder.WithPredicates(predicate.Funcs{
-			CreateFunc: func(event.CreateEvent) bool { return false },
-		})).
-		Owns(&corev1.Secret{}, builder.WithPredicates(predicate.Funcs{
-			CreateFunc: func(event.CreateEvent) bool { return false },
-		})).
-		Owns(&corev1.ConfigMap{}, builder.WithPredicates(predicate.Funcs{
-			CreateFunc: func(event.CreateEvent) bool { return false },
-		})).
+		WithEventFilter(predicate.Funcs{GenericFunc: func(event.GenericEvent) bool { return false }}).
+		For(&enterprisev4.PostgresDatabase{}, builder.WithPredicates(postgresDatabasePredicator())).
+		Owns(&cnpgv1.Database{}, builder.WithPredicates(postgresDatabaseCNPGDatabasePredicator())).
+		Owns(&corev1.Secret{}, builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})).
+		Owns(&corev1.ConfigMap{}, builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})).
 		Named("postgresdatabase").
 		WithOptions(controller.Options{
 			MaxConcurrentReconciles: DatabaseTotalWorker,
 		}).
 		Complete(r)
 }
+
+func postgresDatabasePredicator() predicate.Predicate {
+	return predicate.Or(
+		predicate.GenerationChangedPredicate{},
+		predicate.Funcs{
+			UpdateFunc: func(e event.UpdateEvent) bool {
+				if !equality.Semantic.DeepEqual(e.ObjectOld.GetDeletionTimestamp(), e.ObjectNew.GetDeletionTimestamp()) {
+					return true
+				}
+				return !equality.Semantic.DeepEqual(e.ObjectOld.GetFinalizers(), e.ObjectNew.GetFinalizers())
+			},
+		},
+	)
+}
+
+func postgresDatabaseCNPGDatabasePredicator() predicate.Predicate {
+	return predicate.Or(
+		predicate.GenerationChangedPredicate{},
+		predicate.Funcs{
+			UpdateFunc: func(e event.UpdateEvent) bool {
+				oldObj, okOld := e.ObjectOld.(*cnpgv1.Database)
+				newObj, okNew := e.ObjectNew.(*cnpgv1.Database)
+				if !okOld || !okNew {
+					return true
+				}
+				return !equality.Semantic.DeepEqual(oldObj.Status.Applied, newObj.Status.Applied) ||
+					ownerReferencesChanged(oldObj, newObj)
+			},
+		},
+	)
+}

internal/controller/postgresdatabase_controller_test.go

@@ -35,6 +35,8 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
@@ -332,10 +334,8 @@ func expectStatusCondition(current *enterprisev4.PostgresDatabase, conditionType
 	Expect(condition.Reason).To(Equal(expectedReason), "unexpected reason for %s", conditionType)
 }
 
-func expectReadyStatus(current *enterprisev4.PostgresDatabase, generation int64, expectedDatabase enterprisev4.DatabaseInfo) {
+func expectReadyStatus(current *enterprisev4.PostgresDatabase, expectedDatabase enterprisev4.DatabaseInfo) {
 	expectStatusPhase(current, "Ready")
-	Expect(current.Status.ObservedGeneration).NotTo(BeNil())
-	Expect(*current.Status.ObservedGeneration).To(Equal(generation))
 	Expect(current.Status.Databases).To(HaveLen(1))
 	Expect(current.Status.Databases[0].Name).To(Equal(expectedDatabase.Name))
 	Expect(current.Status.Databases[0].Ready).To(Equal(expectedDatabase.Ready))
@@ -344,6 +344,33 @@ func expectReadyStatus(current *enterprisev4.PostgresDatabase, generation int64,
 	Expect(current.Status.Databases[0].ConfigMapRef).NotTo(BeNil())
 }
 
+func reconcilePostgresDatabaseToReady(ctx context.Context, scenario readyClusterScenario, poolerEnabled bool) *enterprisev4.PostgresDatabase {
+	seedReadyClusterScenario(ctx, scenario, poolerEnabled)
+
+	result, err := reconcilePostgresDatabase(ctx, scenario.requestName)
+	expectEmptyReconcileResult(result, err)
+
+	current := expectFinalizerAdded(ctx, scenario.requestName)
+	seedExistingDatabaseStatus(ctx, current, scenario.dbName)
+
+	result, err = reconcilePostgresDatabase(ctx, scenario.requestName)
+	expectReconcileResult(result, err, 15*time.Second)
+	expectProvisionedArtifacts(ctx, scenario, current)
+	expectManagedRolesPatched(ctx, scenario)
+
+	result, err = reconcilePostgresDatabase(ctx, scenario.requestName)
+	expectReconcileResult(result, err, 15*time.Second)
+	cnpgDatabase := expectCNPGDatabaseCreated(ctx, scenario, current)
+	markCNPGDatabaseApplied(ctx, cnpgDatabase)
+
+	result, err = reconcilePostgresDatabase(ctx, scenario.requestName)
+	expectEmptyReconcileResult(result, err)
+
+	current = fetchPostgresDatabase(ctx, scenario.requestName)
+	expectReadyStatus(current, enterprisev4.DatabaseInfo{Name: scenario.dbName, Ready: true})
+	return current
+}
+
 var _ = Describe("PostgresDatabase Controller", func() {
 	var (
 		ctx       context.Context
@@ -386,8 +413,6 @@ var _ = Describe("PostgresDatabase Controller", func() {
 				current := fetchPostgresDatabase(ctx, requestName)
 				expectStatusPhase(current, "Pending")
 				expectStatusCondition(current, "ClusterReady", metav1.ConditionFalse, "ClusterNotFound")
-				clusterReady := meta.FindStatusCondition(current.Status.Conditions, "ClusterReady")
-				Expect(clusterReady.ObservedGeneration).To(Equal(current.Generation))
 			})
 		})
 	})
@@ -418,7 +443,7 @@ var _ = Describe("PostgresDatabase Controller", func() {
 				expectEmptyReconcileResult(result, err)
 
 				current = fetchPostgresDatabase(ctx, scenario.requestName)
-				expectReadyStatus(current, current.Generation, enterprisev4.DatabaseInfo{Name: scenario.dbName, Ready: true})
+				expectReadyStatus(current, enterprisev4.DatabaseInfo{Name: scenario.dbName, Ready: true})
 				expectStatusCondition(current, "ClusterReady", metav1.ConditionTrue, "ClusterAvailable")
 				expectStatusCondition(current, "SecretsReady", metav1.ConditionTrue, "SecretsCreated")
 				expectStatusCondition(current, "ConfigMapsReady", metav1.ConditionTrue, "ConfigMapsCreated")
@@ -446,6 +471,164 @@ var _ = Describe("PostgresDatabase Controller", func() {
 		})
 	})
 
+	When("owned resource drift occurs after the PostgresDatabase is ready", func() {
+		It("repairs configmap content drift", func() {
+			scenario := newReadyClusterScenario(namespace, "configmap-drift", "tenant-cluster", "tenant-cnpg", "appdb")
+			owner := reconcilePostgresDatabaseToReady(ctx, scenario, false)
+
+			configMap := &corev1.ConfigMap{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-%s-config", scenario.resourceName, scenario.dbName), Namespace: scenario.namespace}, configMap)).To(Succeed())
+			configMap.Data["rw-host"] = "unexpected.example"
+			Expect(k8sClient.Update(ctx, configMap)).To(Succeed())
+
+			result, err := reconcilePostgresDatabase(ctx, scenario.requestName)
+			expectEmptyReconcileResult(result, err)
+
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: configMap.Name, Namespace: configMap.Namespace}, configMap)).To(Succeed())
+			Expect(configMap.Data).To(HaveKeyWithValue("rw-host", "tenant-rw."+scenario.namespace+".svc.cluster.local"))
+
+			current := fetchPostgresDatabase(ctx, scenario.requestName)
+			expectReadyStatus(current, enterprisev4.DatabaseInfo{Name: scenario.dbName, Ready: true})
+			Expect(metav1.IsControlledBy(configMap, owner)).To(BeTrue())
+		})
+
+		It("recreates a deleted configmap", func() {
+			scenario := newReadyClusterScenario(namespace, "configmap-delete", "tenant-cluster", "tenant-cnpg", "appdb")
+			reconcilePostgresDatabaseToReady(ctx, scenario, false)
+
+			configMapName := fmt.Sprintf("%s-%s-config", scenario.resourceName, scenario.dbName)
+			Expect(k8sClient.Delete(ctx, &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{Name: configMapName, Namespace: scenario.namespace},
+			})).To(Succeed())
+
+			result, err := reconcilePostgresDatabase(ctx, scenario.requestName)
+			expectEmptyReconcileResult(result, err)
+
+			configMap := &corev1.ConfigMap{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: scenario.namespace}, configMap)).To(Succeed())
+			Expect(configMap.Data).To(HaveKeyWithValue("rw-host", "tenant-rw."+scenario.namespace+".svc.cluster.local"))
+		})
+
+		It("does not recreate a deleted managed user secret", func() {
+			scenario := newReadyClusterScenario(namespace, "secret-delete", "tenant-cluster", "tenant-cnpg", "appdb")
+			reconcilePostgresDatabaseToReady(ctx, scenario, false)
+
+			secretName := fmt.Sprintf("%s-%s-admin", scenario.resourceName, scenario.dbName)
+			Expect(k8sClient.Delete(ctx, &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: scenario.namespace},
+			})).To(Succeed())
+
+			result, err := reconcilePostgresDatabase(ctx, scenario.requestName)
+			expectReconcileResult(result, err, 15*time.Second)
+
+			current := fetchPostgresDatabase(ctx, scenario.requestName)
+			expectStatusPhase(current, "Provisioning")
+			expectStatusCondition(current, "SecretsReady", metav1.ConditionFalse, "SecretsDriftDetected")
+
+			missing := &corev1.Secret{}
+			err = k8sClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: scenario.namespace}, missing)
+			Expect(apierrors.IsNotFound(err)).To(BeTrue())
+		})
+
+		It("re-attaches ownership when a managed user secret loses its owner reference", func() {
+			scenario := newReadyClusterScenario(namespace, "secret-adopt", "tenant-cluster", "tenant-cnpg", "appdb")
+			owner := reconcilePostgresDatabaseToReady(ctx, scenario, false)
+
+			secret := &corev1.Secret{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-%s-admin", scenario.resourceName, scenario.dbName), Namespace: scenario.namespace}, secret)).To(Succeed())
+			secret.OwnerReferences = nil
+			Expect(k8sClient.Update(ctx, secret)).To(Succeed())
+
+			result, err := reconcilePostgresDatabase(ctx, scenario.requestName)
+			expectEmptyReconcileResult(result, err)
+
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}, secret)).To(Succeed())
+			Expect(metav1.IsControlledBy(secret, owner)).To(BeTrue())
+
+			current := fetchPostgresDatabase(ctx, scenario.requestName)
+			expectReadyStatus(current, enterprisev4.DatabaseInfo{Name: scenario.dbName, Ready: true})
+		})
+
+		It("creates secrets and configmaps for a newly added database while preserving existing ones", func() {
+			scenario := newReadyClusterScenario(namespace, "new-database", "tenant-cluster", "tenant-cnpg", "appdb")
+			current := reconcilePostgresDatabaseToReady(ctx, scenario, false)
+
+			current.Spec.Databases = append(current.Spec.Databases, enterprisev4.DatabaseDefinition{Name: "analytics"})
+			Expect(k8sClient.Update(ctx, current)).To(Succeed())
+
+			result, err := reconcilePostgresDatabase(ctx, scenario.requestName)
+			expectReconcileResult(result, err, 15*time.Second)
+
+			for _, secretName := range []string{
+				fmt.Sprintf("%s-analytics-admin", scenario.resourceName),
+				fmt.Sprintf("%s-analytics-rw", scenario.resourceName),
+			} {
+				secret := &corev1.Secret{}
+				Expect(k8sClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: scenario.namespace}, secret)).To(Succeed())
+			}
+
+			configMap := &corev1.ConfigMap{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-analytics-config", scenario.resourceName), Namespace: scenario.namespace}, configMap)).To(Succeed())
+			Expect(configMap.Data).To(HaveKeyWithValue("dbname", "analytics"))
+
+			existingSecret := &corev1.Secret{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-%s-admin", scenario.resourceName, scenario.dbName), Namespace: scenario.namespace}, existingSecret)).To(Succeed())
+		})
+	})
+
+	When("postgresdatabase secondary-resource predicates run", func() {
+		It("treats cnpg database applied-state, create, and delete changes as drift triggers", func() {
+			pred := postgresDatabaseCNPGDatabasePredicator()
+
+			oldApplied := true
+			newApplied := false
+			Expect(pred.Create(event.CreateEvent{})).To(BeTrue())
+			Expect(pred.Update(event.UpdateEvent{
+				ObjectOld: &cnpgv1.Database{Status: cnpgv1.DatabaseStatus{Applied: &oldApplied}},
+				ObjectNew: &cnpgv1.Database{Status: cnpgv1.DatabaseStatus{Applied: &newApplied}},
+			})).To(BeTrue())
+			Expect(pred.Delete(event.DeleteEvent{})).To(BeTrue())
+		})
+
+		It("ignores cnpg database updates that do not change readiness or ownership", func() {
+			pred := postgresDatabaseCNPGDatabasePredicator()
+
+			applied := true
+			Expect(pred.Update(event.UpdateEvent{
+				ObjectOld: &cnpgv1.Database{
+					ObjectMeta: metav1.ObjectMeta{Name: "db", Namespace: "test"},
+					Status:     cnpgv1.DatabaseStatus{Applied: &applied},
+				},
+				ObjectNew: &cnpgv1.Database{
+					ObjectMeta: metav1.ObjectMeta{Name: "db", Namespace: "test"},
+					Status:     cnpgv1.DatabaseStatus{Applied: &applied},
+				},
+			})).To(BeFalse())
+		})
+
+		It("treats secret create, update, and delete events as drift triggers", func() {
+			pred := predicate.ResourceVersionChangedPredicate{}
+
+			Expect(pred.Create(event.CreateEvent{})).To(BeTrue())
+			Expect(pred.Update(event.UpdateEvent{
+				ObjectOld: &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret", Namespace: "test", ResourceVersion: "1"}},
+				ObjectNew: &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret", Namespace: "test", ResourceVersion: "2"}},
+			})).To(BeTrue())
+			Expect(pred.Delete(event.DeleteEvent{})).To(BeTrue())
+		})
+
+		It("treats configmap create, update, and delete events as drift triggers", func() {
+			pred := predicate.ResourceVersionChangedPredicate{}
+
+			Expect(pred.Create(event.CreateEvent{})).To(BeTrue())
+			Expect(pred.Update(event.UpdateEvent{
+				ObjectOld: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "config", Namespace: "test", ResourceVersion: "1"}},
+				ObjectNew: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "config", Namespace: "test", ResourceVersion: "2"}},
+			})).To(BeTrue())
+			Expect(pred.Delete(event.DeleteEvent{})).To(BeTrue())
+		})
+	})
+
 	When("role ownership conflicts exist", func() {
 		It("marks the resource failed and stops provisioning dependent resources", func() {
 			resourceName := "conflict-cluster"

pkg/postgresql/cluster/core/cluster.go

@@ -451,11 +451,11 @@ func PostgresClusterService(ctx context.Context, rc *ReconcileContext, req ctrl.
 		oldPhase = *postgresCluster.Status.Phase
 	}
 	if err := syncStatus(ctx, c, postgresCluster, cnpgCluster); err != nil {
-		logger.Error(err, "Failed to sync status")
 		if apierrors.IsConflict(err) {
 			logger.Info("Conflict during status update, will requeue")
 			return ctrl.Result{Requeue: true}, nil
 		}
+		logger.Error(err, "Failed to sync status")
 		return ctrl.Result{}, fmt.Errorf("failed to sync status: %w", err)
 	}
 	var newPhase string