feat: pgbackrest test

tests/robot/Lib/pgsLibrary.py

@@ -781,6 +781,135 @@ def check_last_backup_id(self):
         last_backup_id = health_json["storage"]["lastSuccessful"]["id"]
         return last_backup_id
 
+    def pgbackrest_backup_exists(self, backup_id):
+        try:
+            backups = self.get_pgbackrest_backup_list()
+            if backup_id in backups:
+                logging.info("PgBackRest backup %s found through backup daemon /list", backup_id)
+                return True
+        except Exception as e:
+            logging.info("Cannot check backup daemon list for PgBackRest backup %s: %s", backup_id, e)
+
+        status = self.get_pgbackrest_sidecar_backup_status(backup_id)
+        if self._pgbackrest_backup_matches(backup_id, status):
+            logging.info("PgBackRest backup %s found through backrest /status", backup_id)
+            return True
+
+        backups = self.get_pgbackrest_sidecar_backup_list()
+        if self._pgbackrest_backup_found_in_list(backup_id, backups):
+            logging.info("PgBackRest backup %s found through backrest /list", backup_id)
+            return True
+
+        return False
+
+    def get_pgbackrest_backup_list(self):
+        response = requests.get(f"{self._scheme}://postgres-backup-daemon:8081/list", verify=False, timeout=10)
+        response.raise_for_status()
+        backups = response.json()
+        logging.info("Backup daemon backup list: {}".format(backups))
+        return backups
+
+    def get_pgbackrest_sidecar_backup_status(self, backup_id):
+        for service in ["backrest", "backrest-headless"]:
+            try:
+                response = requests.get(
+                    "http://{}:3000/status?timestamp={}".format(service, backup_id),
+                    timeout=10
+                )
+                response.raise_for_status()
+                status = response.json()
+                logging.info("PgBackRest status from %s for %s: %s", service, backup_id, status)
+                if self._pgbackrest_backup_matches(backup_id, status):
+                    return status
+            except Exception as e:
+                logging.info("Cannot get PgBackRest status from %s for %s: %s", service, backup_id, e)
+        return {}
+
+    def get_pgbackrest_sidecar_backup_list(self):
+        errors = {}
+        for service in ["backrest", "backrest-headless"]:
+            try:
+                response = requests.get("http://{}:3000/list".format(service), timeout=10)
+                response.raise_for_status()
+                backups = response.json()
+                logging.info("PgBackRest list from %s: %s", service, backups)
+                return backups
+            except Exception as e:
+                errors[service] = str(e)
+                logging.info("Cannot get PgBackRest list from %s: %s", service, e)
+        logging.info("Cannot get PgBackRest list from sidecar services: %s", errors)
+        return []
+
+    def _pgbackrest_backup_matches(self, backup_id, backup):
+        if not isinstance(backup, dict):
+            return False
+        annotation = backup.get("annotation") or {}
+        return annotation.get("timestamp") == backup_id and not backup.get("error", False)
+
+    def _pgbackrest_backup_found_in_list(self, backup_id, backups):
+        if isinstance(backups, dict):
+            if backup_id in backups:
+                return True
+            backups = backups.values()
+        if not isinstance(backups, list):
+            return False
+        for backup in backups:
+            if self._pgbackrest_backup_matches(backup_id, backup):
+                return True
+        return False
+
+    def restore_pgbackrest_backup(self, backup_id):
+        pod = self.get_pod(label='app:postgres-backup-daemon', status='Running')
+        command = "cd /maintenance/recovery && SET={} python3 pg_back_rest_recovery.py".format(backup_id)
+        logging.info("Start PgBackRest restore from backup daemon pod {} with backup id {}".format(
+            pod.metadata.name, backup_id))
+        output, errors = self.execute_in_pod(pod.metadata.name, command)
+        logging.info("PgBackRest restore output: {}".format(output))
+        if errors:
+            logging.info("PgBackRest restore stderr: {}".format(errors))
+        return output
+
+    def get_backup_daemon_restart_count(self):
+        pod = self.get_pod(label='app:postgres-backup-daemon', status='Running')
+        restart_count = 0
+        for container_status in pod.status.container_statuses or []:
+            restart_count += container_status.restart_count
+        logging.info("Backup daemon pod {} restart count: {}".format(pod.metadata.name, restart_count))
+        return restart_count
+
+    def get_pgbackrest_prerequisite_status(self):
+        status = {
+            "storage_type": None,
+            "backup_daemon_pod": None,
+            "pgbackrest_configmap_exists": False,
+            "pgbackrest_sidecar_pods": [],
+            "missing": []
+        }
+
+        backup_daemon = self.get_pod(label='app:postgres-backup-daemon', status='Running')
+        status["backup_daemon_pod"] = backup_daemon.metadata.name
+        status["storage_type"] = self.get_env_for_pod(backup_daemon, "STORAGE_TYPE")
+        if status["storage_type"] != "pgbackrest":
+            status["missing"].append("backup daemon STORAGE_TYPE is not pgbackrest")
+
+        try:
+            self.pl_lib.get_config_map("pgbackrest-conf", self._namespace)
+            status["pgbackrest_configmap_exists"] = True
+        except Exception:
+            status["missing"].append("pgbackrest-conf config map is absent")
+
+        pg_cluster_name = os.getenv("PG_CLUSTER_NAME", "patroni")
+        for pod in self.get_pods(label="pgcluster:{}".format(pg_cluster_name), status="Running"):
+            for container in pod.spec.containers:
+                if container.name == "pgbackrest-sidecar":
+                    status["pgbackrest_sidecar_pods"].append(pod.metadata.name)
+                    break
+        if not status["pgbackrest_sidecar_pods"]:
+            status["missing"].append("pgbackrest-sidecar container is absent in running patroni pods")
+
+        logging.info("PgBackRest prerequisite status: {}".format(status))
+        return status
+
     def schedule_evict(self, last_backup_id):
         health_json = requests.delete(f"{self._scheme}://postgres-backup-daemon:8081/evict?id={last_backup_id}", verify=False).json()
         return health_json

tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot

@@ -0,0 +1,82 @@
+*** Settings ***
+Documentation     Check positive full restore cycle with PgBackRest storage
+Library           Collections
+Library           OperatingSystem
+Library           String
+Resource          ../Lib/lib.robot
+
+*** Variables ***
+${OPERATION_RETRY_COUNT}       60
+${OPERATION_RETRY_INTERVAL}    5s
+
+*** Test Cases ***
+Check PgBackRest Full Backup Restore
+    [Tags]  pgbackrest  pgbackrest_restore
+    [Documentation]
+    ...  Positive PgBackRest cycle:
+    ...  1. Verify backup daemon uses PgBackRest storage.
+    ...  2. Create database and seed data.
+    ...  3. Create full backup through backup daemon.
+    ...  4. Add data after backup.
+    ...  5. Restore Patroni cluster from the created PgBackRest backup.
+    ...  6. Verify cluster is healthy and data state matches the backup.
+    ${pg_cluster_name}=  Get Environment Variable  PG_CLUSTER_NAME  default=patroni
+    ${postfix}=  Generate Random String  5  [LOWER]
+    ${db_name}=  Set Variable  pgbackrest_restore_${postfix}
+    Set Test Variable  \${db_name}  ${db_name}
+    Log To Console  \n[pgbackrest] cluster=${pg_cluster_name}, database=${db_name}
+    Skip Test If PgBackRest Is Not Configured
+    Create Database  ${db_name}
+    Wait Until Keyword Succeeds  ${OPERATION_RETRY_COUNT}  ${OPERATION_RETRY_INTERVAL}
+    ...  Check Database Exists  ${pg_cluster_name}  ${db_name}
+    ${rid_before}  ${expected_before}=  Insert Test Record  database=${db_name}
+    ${restart_count_before}=  Get Backup Daemon Restart Count
+    ${backup_id}=  Create PgBackRest Full Backup
+    Log To Console  [pgbackrest] backup_id=${backup_id}, restart_count_before=${restart_count_before}
+    ${rid_after}  ${expected_after}=  Insert Test Record  database=${db_name}
+    ${restore_output}=  Restore Pgbackrest Backup  ${backup_id}
+    Log  ${restore_output}
+    Log To Console  [pgbackrest] restore started for backup_id=${backup_id}
+    Wait Until Keyword Succeeds  20 min  10 sec  Patroni Ready
+    Check Test Record  pg-${pg_cluster_name}  ${rid_before}  ${expected_before}  ${db_name}
+    Check Test Record Is Absent  pg-${pg_cluster_name}  ${rid_after}  ${expected_after}  ${db_name}
+    ${restart_count_after}=  Get Backup Daemon Restart Count
+    Log To Console  [pgbackrest] restart_count_after=${restart_count_after}
+    Should Be Equal As Integers  ${restart_count_after}  ${restart_count_before}
+    [Teardown]  Delete Database  ${db_name}
+
+*** Keywords ***
+Skip Test If PgBackRest Is Not Configured
+    ${status}=  Get Pgbackrest Prerequisite Status
+    Log  PgBackRest prerequisites: ${status}
+    ${missing}=  Get From Dictionary  ${status}  missing
+    ${missing_count}=  Get Length  ${missing}
+    Run Keyword If  ${missing_count} > 0  Pass Execution  PgBackRest is not configured for this environment: ${missing}
+
+Check Database Exists
+    [Arguments]  ${pg_cluster_name}  ${db_name}
+    ${databases}=  Execute Query  pg-${pg_cluster_name}  SELECT datname FROM pg_database
+    Should Contain  str(${databases})  ${db_name}
+
+Create PgBackRest Full Backup
+    ${pod}=  Get Pod  label=app:postgres-backup-daemon  status=Running
+    ${dump_count}=  Get Backup Count
+    ${schedule_response}=  Schedule Backup
+    Log  PgBackRest backup schedule response: ${schedule_response}
+    Dictionary Should Contain Key  ${schedule_response}  backup_id
+    ${backup_id}=  Get From Dictionary  ${schedule_response}  backup_id
+    Log To Console  [pgbackrest] waiting backup_id=${backup_id}
+    Wait Until Keyword Succeeds  30 min  15 sec  Check PgBackRest Backup Exists  ${backup_id}
+    ${dump_count_after}=  Get Backup Count
+    Log  PgBackRest backup ${backup_id} is listed, dump_count_before=${dump_count}, dump_count_after=${dump_count_after}
+    RETURN  ${backup_id}
+
+Check PgBackRest Backup Exists
+    [Arguments]  ${backup_id}
+    ${exists}=  Pgbackrest Backup Exists  ${backup_id}
+    Should Be True  ${exists}  msg=PgBackRest backup ${backup_id} was not found in backrest list
+
+Check Test Record Is Absent
+    [Arguments]  ${pod_name}  ${rid}  ${expected}  ${database}
+    ${res}=  Execute Query  ${pod_name}  select * from test_insert_robot where id=${rid}  dbname=${database}
+    Should Not Be True  """${expected}""" in """${res}"""  msg=Record added after backup is still present after restore: ${res}