[DPE-10203] fix(async-replication): recover async replication after a dead datacenter

src/backups.py

@@ -186,7 +186,7 @@ def _can_unit_perform_backup(self) -> tuple[bool, str | None]:
             return False, "Unit cannot perform backups as the database seems to be offline"
 
         # Only enable backups on primary if there are replicas but TLS is not enabled.
-        if is_primary and self.charm.app.planned_units() > 1:
+        if is_primary and self.charm._planned_units > 1:
             return False, "Unit cannot perform backups as it is the cluster primary"
 
         if not self.charm.patroni_manager.member_started:
@@ -1313,7 +1313,7 @@ def _pre_restore_checks(self, event: ActionEvent) -> bool:
             return False
 
         logger.info("Checking that the cluster does not have more than one unit")
-        if self.charm.app.planned_units() > 1:
+        if self.charm._planned_units > 1:
             error_message = (
                 "Unit cannot restore backup as there are more than one unit in the cluster"
             )

src/charm.py

@@ -1565,6 +1565,21 @@ def _hosts(self) -> set[str]:
                 hosts.append(unit.name.replace("/", "-"))
         return set(hosts)
 
+    @property
+    def _planned_units(self) -> int:
+        """Number of planned units, resilient to a transient goal-state failure.
+
+        ops implements ``Application.planned_units()`` via ``goal-state``, which fails
+        ("saas application ... not found") while a cross-model SAAS force-removed during a
+        dead-DC teardown still lingers in goal-state. Fall back to the count of currently known
+        units so the hook reconciles instead of crashing the ``_patroni`` property and every
+        hook that touches it (DPE-10203).
+        """
+        try:
+            return self.app.planned_units()
+        except ModelError:
+            return len(self._hosts)
+
     @cached_property
     def _patroni(self) -> Patroni:
         """Returns an instance of the Patroni object."""
@@ -2375,6 +2390,11 @@ def _on_update_status(self, _) -> None:
         # Update the sync-standby endpoint in the async replication data.
         self.async_replication.update_async_replication_data()
 
+        # Clear a promoted-cluster-counter orphaned by a dead-DC teardown whose relation-broken
+        # never fired (Juju CMR limitation); otherwise a newly-formed async relation re-counts it
+        # and create-replication wrongly reports "There is already a replication set up.".
+        self.async_replication.clear_stale_promotion()
+
         self.backup.coordinate_stanza_fields()
 
         # self.logical_replication.retry_validations()
@@ -2540,10 +2560,7 @@ def _set_primary_status_message(self) -> None:
                 danger_state = ""
                 if not self._patroni.has_raft_quorum():
                     danger_state = " (read-only)"
-                elif (
-                    len(self.patroni_manager.get_running_cluster_members())
-                    < self.app.planned_units()
-                ):
+                elif len(self.patroni_manager.get_running_cluster_members()) < self._planned_units:
                     danger_state = " (degraded)"
                 unit_status = "Standby" if self.is_standby_leader else "Primary"
                 self.set_unit_status(ActiveStatus(f"{unit_status}{danger_state}"))
@@ -2771,12 +2788,13 @@ def update_config(
         """Updates Patroni config file based on the existence of the TLS files."""
         if refresh is None:
             refresh = self.refresh
-        return self.config_manager.update_config(
+        primary_cluster_endpoint = self.async_replication.get_primary_cluster_endpoint()
+        result = self.config_manager.update_config(
             self.postgresql,
             is_creating_backup=is_creating_backup,
             relations_user_databases_map=self.relations_user_databases_map,
             ldap_parameters=self.get_ldap_parameters(),
-            async_primary_cluster_endpoint=self.async_replication.get_primary_cluster_endpoint(),
+            async_primary_cluster_endpoint=primary_cluster_endpoint,
             async_partner_addresses=self.async_replication.get_partner_addresses(),
             async_standby_endpoints=self.async_replication.get_standby_endpoints(),
             watcher_raft_address=self.watcher_offer.watcher_raft_address
@@ -2785,6 +2803,21 @@ def update_config(
             no_peers=no_peers,
             refresh=refresh,
         )
+        # The lib's apply_api_config only SETS the DCS standby_cluster (when another
+        # cluster is primary) and never CLEARS it. A force-promote bumps the
+        # promoted-cluster-counter but — while the dead-DC relation still lingers — does
+        # not call promote_standby_cluster(), so without this the reconciler never clears
+        # the stale standby and the cluster stays a read-only standby leader (DPE-10203).
+        if (
+            result
+            and not no_peers
+            and self.patroni_manager.member_started
+            and primary_cluster_endpoint is None
+        ):
+            self.patroni_manager.bulk_update_parameters_controller_by_patroni(
+                {}, {"standby_cluster": None}
+            )
+        return result
 
     def _validate_config_options(self) -> None:
         """Validates specific config options that need access to the database or to the TLS status."""