From 4b0001e6d7538ba1f9828b681b36a8337351db5a Mon Sep 17 00:00:00 2001 From: Tvion Date: Mon, 20 Oct 2025 18:09:13 +0500 Subject: [PATCH 1/4] Refactor backup functions to support optional blob_path parameter in backups.py and granular.py --- docker/granular/backups.py | 75 +++++++++++++++++--------------- docker/granular/granular.py | 82 +++++++++++++++++++---------------- docker/granular/pg_backup.py | 24 +++++----- docker/granular/pg_restore.py | 40 +++++++++-------- 4 files changed, 120 insertions(+), 101 deletions(-) diff --git a/docker/granular/backups.py b/docker/granular/backups.py index 35c55b2..6ecbcb4 100644 --- a/docker/granular/backups.py +++ b/docker/granular/backups.py @@ -77,73 +77,76 @@ def is_valid_namespace(namespace): return re.match("^[a-zA-z0-9_]+$", namespace) is not None -def backup_exists(backup_id, namespace=configs.default_namespace(), external_backup_storage=None): - return os.path.exists(build_backup_path(backup_id, namespace, external_backup_storage)) +def backup_exists(backup_id, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): + if blob_path is not None: + return os.path.exists(build_backup_path(backup_id, blob_path=blob_path)) + else: + return os.path.exists(build_backup_path(backup_id, namespace, external_backup_storage)) -def database_backup_exists(backup_id, database, namespace=configs.default_namespace(), external_backup_storage=None): - return os.path.exists(build_database_backup_path(backup_id, database, namespace, external_backup_storage)) +def database_backup_exists(backup_id, database, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): + return os.path.exists(build_database_backup_path(backup_id, database, namespace, external_backup_storage, blob_path)) def build_namespace_path(namespace=configs.default_namespace()): return '%s/%s' % (configs.backups_storage(), namespace) -def build_database_backup_path(backup_id, database, - namespace=configs.default_namespace(), external_backup_storage=None): - if configs.get_encryption(): - return '%s/%s_enc.dump' % ( - build_backup_path(backup_id, namespace, external_backup_storage), database) - else: - return '%s/%s.dump' % ( - build_backup_path(backup_id, namespace, external_backup_storage), database) - +def build_database_backup_path(backup_id, database, + namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): + ext = '_enc.dump' if configs.get_encryption() else '.dump' + return '%s/%s%s' % ( + build_backup_path(backup_id, namespace, external_backup_storage, blob_path=blob_path), database, ext) def build_roles_backup_path(backup_id, database, - namespace=configs.default_namespace(), external_backup_storage=None): - if configs.get_encryption(): - return "%s/%s.roles_enc.sql" % ( - build_backup_path(backup_id, namespace, external_backup_storage), database) - else: - return "%s/%s.roles.sql" % ( - build_backup_path(backup_id, namespace, external_backup_storage), database) - + namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): + ext = 'roles_enc.sql' if configs.get_encryption() else 'roles.sql' + return "%s/%s.%s" % ( + build_backup_path(backup_id, namespace, external_backup_storage, blob_path=blob_path), database, ext) def build_database_backup_full_path(backup_id, database, storage_root, namespace=configs.default_namespace(), - ): - if configs.get_encryption(): - return '%s/%s/%s/%s_enc.dump' % ( - storage_root, namespace, backup_id, database) + blob_path=None): + ext = '_enc.dump' if configs.get_encryption() else '.dump' + if blob_path is not None: + return '%s/backups/%s/logical/dbaas/%s%s' % (blob_path, backup_id, database, ext) else: - return '%s/%s/%s/%s.dump' % ( - storage_root, namespace, backup_id, database) + return '%s/%s/%s/%s%s' % (storage_root, namespace, backup_id, database, ext) def build_database_restore_report_path(backup_id, database, restore_tracking_id, namespace=configs.default_namespace()): return '%s/%s.%s.report' % (build_backup_path(backup_id, namespace), database, restore_tracking_id) -def build_backup_path(backup_id, namespace=configs.default_namespace(), external_backup_storage=None): - return '%s/%s/%s' % (configs.backups_storage() if external_backup_storage is None else external_backup_storage, - namespace, backup_id) +def build_backup_path(backup_id, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): + if blob_path is not None: + return '%s/backups/%s/logical/dbaas' % (blob_path, backup_id) + else: + return '%s/%s/%s' % (configs.backups_storage() if external_backup_storage is None else external_backup_storage, + namespace, backup_id) def build_external_backup_root(external_backup_path): return '%s/%s' % (os.getenv("EXTERNAL_STORAGE_ROOT"), external_backup_path) -def build_backup_status_file_path(backup_id, namespace=configs.default_namespace(), external_backup_storage=None): - return '%s/status.json' % build_backup_path(backup_id, namespace, external_backup_storage) +def build_backup_status_file_path(backup_id, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): + if blob_path is not None: + return '%s/status.json' % build_backup_path(backup_id, blob_path=blob_path) + else: + return '%s/status.json' % build_backup_path(backup_id, namespace, external_backup_storage) def build_restore_status_file_path(backup_id, tracking_id, namespace=configs.default_namespace(), - external_backup_storage=None): - return '%s/%s.json' % (build_backup_path(backup_id, namespace, external_backup_storage), tracking_id) + external_backup_storage=None, blob_path=None): + if blob_path is not None: + return '%s/%s.json' % (build_backup_path(backup_id, blob_path=blob_path), tracking_id) + else: + return '%s/%s.json' % (build_backup_path(backup_id, namespace, external_backup_storage), tracking_id) -def get_key_name_by_backup_id(backup_id, namespace, external_backup_storage=None): - status_path = build_backup_status_file_path(backup_id, namespace, external_backup_storage) +def get_key_name_by_backup_id(backup_id, namespace, external_backup_storage=None, blob_path=None): + status_path = build_backup_status_file_path(backup_id, namespace, external_backup_storage, blob_path) with open(status_path) as f: data = json.load(f) return data.get("key_name") diff --git a/docker/granular/granular.py b/docker/granular/granular.py index 24aba1c..b8e41b8 100644 --- a/docker/granular/granular.py +++ b/docker/granular/granular.py @@ -286,7 +286,8 @@ def __init__(self): 'keep', 'compressionLevel', 'externalBackupPath', - 'storageName'] + 'storageName', + 'blobPath'] def perform_granular_backup(self, backup_request): # # for gke full backup @@ -333,7 +334,7 @@ def perform_granular_backup(self, backup_request): backup_id = backups.generate_backup_id() backup_request['backupId'] = backup_id - worker = pg_backup.PostgreSQLDumpWorker(databases, backup_request) + worker = pg_backup.PostgreSQLDumpWorker(databases, backup_request, backup_request.get('blobPath')) worker.start() @@ -1080,10 +1081,12 @@ def post(self): if databases and not isinstance(databases, (list, tuple)): return {"message": "databases must be an array"}, http.client.BAD_REQUEST + blob_path = normalize_blobPath(blob_path) + # Reuse old logic directly backup_request = { "databases": list(databases), - "externalBackupPath": blob_path, + "blobPath": blob_path, "storageName": storage_name } resp = GranularBackupRequestEndpoint().perform_granular_backup(backup_request) @@ -1094,6 +1097,8 @@ def post(self): elif isinstance(resp, dict): body, code = resp, http.client.ACCEPTED else: + if code == http.client.BAD_REQUEST: + return resp, http.client.NOT_FOUND return resp try: @@ -1146,10 +1151,9 @@ def get(self, backup_id): if not backups.is_valid_namespace(namespace): return "Invalid namespace name: %s." % namespace.encode("utf-8"), http.client.BAD_REQUEST - external_backup_path = request.args.get("blobPath") - external_backup_root = backups.build_external_backup_root(external_backup_path) if external_backup_path else None - - status_path = backups.build_backup_status_file_path(backup_id, namespace, external_backup_root) + blob_path = request.args.get("blobPath") + blob_path = normalize_blobPath(blob_path) + status_path = backups.build_backup_status_file_path(backup_id, blob_path=blob_path) if self.s3: try: @@ -1162,8 +1166,8 @@ def get(self, backup_id): with open(status_path) as f: raw = json.load(f) - if external_backup_path and not (raw.get("blobPath") or raw.get("externalBackupPath")): - raw["blobPath"] = external_backup_path + if blob_path and not (raw.get("blobPath") or raw.get("externalBackupPath")): + raw["blobPath"] = blob_path return backups.transform_backup_status_v1(raw), http.client.OK @@ -1174,12 +1178,12 @@ def delete(self, backup_id): return {"backupId": backup_id, "message": "Backup ID is not specified", "status": "Failed"}, http.client.BAD_REQUEST req_ns = request.args.get("namespace") - external_backup_path = request.args.get("blobPath") or request.args.get("externalBackupPath") - if not external_backup_path: + blob_path = request.args.get("blobPath") + if not blob_path: return {"backupId": backup_id, "message": "blobPath query parameter is required (e.g. ?blobPath=tmp/a/b/c).", "status": "Failed"}, http.client.BAD_REQUEST - external_backup_root = backups.build_external_backup_root(external_backup_path) + blob_path = normalize_blobPath(blob_path) def _exists(p: str) -> bool: if self.s3: @@ -1199,14 +1203,14 @@ def _exists(p: str) -> bool: if "default" not in candidates: candidates.append("default") for cand in candidates: - status_try = backups.build_backup_status_file_path(backup_id, cand, external_backup_root) + status_try = backups.build_backup_status_file_path(backup_id, cand, blob_path=blob_path) if _exists(status_try): namespace = cand break if not namespace: namespace = configs.default_namespace() - status_path = backups.build_backup_status_file_path(backup_id, namespace, external_backup_root) + status_path = backups.build_backup_status_file_path(backup_id, namespace, blob_path=blob_path) existed_before = _exists(status_path) resp = TerminateBackupEndpoint().post(backup_id) @@ -1236,13 +1240,13 @@ def _exists(p: str) -> bool: self.log.info("Terminate response for %s: code=%s body=%s", backup_id, term_code, term_body) try: - target_dir = backups.build_backup_path(backup_id, namespace, external_backup_root) + target_dir = backups.build_backup_path(backup_id, blob_path=blob_path) if self.s3: self.s3.delete_objects(target_dir) else: if os.path.isdir(target_dir): shutil.rmtree(target_dir, ignore_errors=False) - if external_backup_root is None: + if blob_path is None: ns_dir = backups.build_namespace_path(namespace) if os.path.isdir(ns_dir) and not os.listdir(ns_dir) and namespace != "default": shutil.rmtree(ns_dir, ignore_errors=True) @@ -1293,6 +1297,7 @@ def post(self, backup_id): return {"message": "blobPath is required"}, http.client.BAD_REQUEST if not isinstance(pairs, (list, tuple)): return {"message": "databases must be an array of objects"}, http.client.BAD_REQUEST + blob_path = normalize_blobPath(blob_path) databases = [] databases_mapping = {} @@ -1310,8 +1315,7 @@ def post(self, backup_id): return "Invalid namespace name: %s." % namespace.encode("utf-8"), http.client.BAD_REQUEST external_backup_path = blob_path - external_backup_root = backups.build_external_backup_root(external_backup_path) if external_backup_path else None - backup_details_file = backups.build_backup_status_file_path(backup_id, namespace, external_backup_root) + backup_details_file = backups.build_backup_status_file_path(backup_id, blob_path=blob_path) if self.s3: try: @@ -1331,9 +1335,9 @@ def post(self, backup_id): if self.s3: for database in list(backup_details.get("databases", {}).keys()): - if not self.s3.is_file_exists(backups.build_database_backup_path(backup_id, database, namespace, external_backup_root)): + if not self.s3.is_file_exists(backups.build_database_backup_path(backup_id, database, blob_path=blob_path)): return "Backup in bucket is not found.", http.client.NOT_FOUND - elif not backups.backup_exists(backup_id, namespace, external_backup_root): + elif not backups.backup_exists(backup_id, blob_path=blob_path): return "Backup is not found.", http.client.NOT_FOUND ghost_databases = [] @@ -1371,7 +1375,7 @@ def post(self, backup_id): worker = pg_restore.PostgreSQLRestoreWorker( requested, force, {"backupId": backup_id, "namespace": namespace, "externalBackupPath": external_backup_path, "trackingId": tracking_id}, - databases_mapping, owners_mapping, restore_roles, single_transaction, body.get("dbaasClone") + databases_mapping, owners_mapping, restore_roles, single_transaction, body.get("dbaasClone"), blob_path ) worker.start() @@ -1421,11 +1425,7 @@ def post(self, backup_id): "externalBackupPath": external_backup_path or "", "sourceBackupId": backup_id } - try: - status_path = backups.build_restore_status_file_path(backup_id, tracking_id, namespace, - backups.build_external_backup_root(external_backup_path) if external_backup_path else None) - except TypeError: - status_path = backups.build_restore_status_file_path(backup_id, tracking_id, namespace) + status_path = backups.build_restore_status_file_path(backup_id, tracking_id, blob_path=blob_path) try: if hasattr(utils, "write_in_json"): @@ -1468,10 +1468,10 @@ def get(self, restore_id): if not backups.is_valid_namespace(namespace): return "Invalid namespace name: %s." % namespace.encode("utf-8"), http.client.BAD_REQUEST - external_backup_path = request.args.get("blobPath") - external_backup_root = backups.build_external_backup_root(external_backup_path) if external_backup_path else None + blob_path = request.args.get("blobPath") + blob_path = normalize_blobPath(blob_path) storage_name = request.args.get("storageName") or os.environ.get("STORAGE_NAME") - status_path = backups.build_restore_status_file_path(backup_id, restore_id, namespace, external_backup_root) + status_path = backups.build_restore_status_file_path(backup_id, restore_id, blob_path=blob_path) if self.s3: try: @@ -1484,8 +1484,8 @@ def get(self, restore_id): with open(status_path) as f: raw = json.load(f) - if external_backup_path and not (raw.get("blobPath") or raw.get("externalBackupPath")): - raw["blobPath"] = external_backup_path + if blob_path and not (raw.get("blobPath") or raw.get("externalBackupPath")): + raw["blobPath"] = blob_path if storage_name and not raw.get("storageName"): raw["storageName"] = storage_name @@ -1528,17 +1528,17 @@ def delete(self, restore_id): "termination": {"code": term_code, "body": term_body} }, http.client.OK - external_backup_path = request.args.get("blobPath") or request.args.get("externalBackupPath") - if not external_backup_path: + blob_path = request.args.get("blobPath") or request.args.get("externalBackupPath") + if not blob_path: return { "restoreId": restore_id, "message": "blobPath query parameter is required for cleanup (e.g. ?blobPath=tmp/a/b/c).", "status": "Failed" }, http.client.BAD_REQUEST + blob_path = normalize_blobPath(blob_path) - external_backup_root = backups.build_external_backup_root(external_backup_path) - status_path = backups.build_restore_status_file_path(backup_id, restore_id, namespace, external_backup_root) - backup_base = backups.build_backup_path(backup_id, namespace, external_backup_root) + status_path = backups.build_restore_status_file_path(backup_id, restore_id, blob_path=blob_path) + backup_base = backups.build_backup_path(backup_id, blob_path=blob_path) pattern_name = f"{restore_id}" pattern_glob = os.path.join(backup_base, pattern_name + "*") @@ -1632,7 +1632,15 @@ def _prefix_exists() -> bool: return {"restoreId": restore_id, "message": msg, "status": "Successful", "termination": {"code": term_code, "body": term_body}}, http.client.OK - + +def normalize_blobPath(blobpath): + # Normalize blob_path by removing a single leading and trailing slash + if isinstance(blob_path, str): + if blob_path.startswith("/"): + blob_path = blob_path[1:] + if blob_path.endswith("/") and blob_path != "/": + blob_path = blob_path[:-1] + def get_pgbackrest_service(): if os.getenv("BACKUP_FROM_STANDBY") == "true": try: diff --git a/docker/granular/pg_backup.py b/docker/granular/pg_backup.py index 0c169b0..0bbfcfd 100644 --- a/docker/granular/pg_backup.py +++ b/docker/granular/pg_backup.py @@ -32,7 +32,7 @@ class PostgreSQLDumpWorker(Thread): - def __init__(self, databases, backup_request): + def __init__(self, databases, backup_request, blob_path=None): Thread.__init__(self) self.log = logging.getLogger("PostgreSQLDumpWorker") @@ -50,7 +50,11 @@ def __init__(self, databases, backup_request): self.bin_path = configs.get_pgsql_bin_path(self.postgres_version) self.parallel_jobs = configs.get_parallel_jobs() self.databases = databases if databases else [] - self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) + if blob_path is not None: + self.blob_path = blob_path + self.backup_dir = backups.build_backup_path(self.backup_id, blob_path=blob_path) + else: + self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) self.create_backup_dir() self.s3 = storage_s3.AwsS3Vault() if os.environ['STORAGE_TYPE'] == "s3" else None self._cancel_event = Event() @@ -111,7 +115,7 @@ def update_status(self, key, value, database=None, flush=False): self.flush_status(self.external_backup_root) def flush_status(self, external_backup_storage=None): - path = backups.build_backup_status_file_path(self.backup_id, self.namespace, external_backup_storage) + path = backups.build_backup_status_file_path(self.backup_id, self.namespace, external_backup_storage, self.blob_path) utils.write_in_json(path, self.status) if self.s3: try: @@ -270,7 +274,7 @@ def backup_single_database(self, database): command.extend(['-T','cron.*']) database_backup_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, blob_path=self.blob_path) with open(database_backup_path, 'w+') as dump, \ open(self.stderr_file(database), "w+") as stderr: @@ -350,9 +354,9 @@ def fetch_roles(self, database): self.log.debug(self.log_msg("Roles {} have been fetched for backup ".format(rolesList))) roles_backup_path = backups.build_roles_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) database_backup_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) path_for_parallel_flag_backup = os.path.join(pg_dump_backup_path, database) @@ -401,7 +405,7 @@ def fetch_roles(self, database): def dump_roles_for_db(self, roles, database): roles_backup_path = backups.build_roles_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) with open(roles_backup_path, 'w+') as dump, \ open(self.stderr_file(database), "w+") as stderr: @@ -446,9 +450,9 @@ def cleanup(self, database): os.remove(self.stderr_file(database)) def on_success(self, database): - database_backup_path = backups.build_database_backup_path(self.backup_id, database, self.namespace, self.external_backup_root) + database_backup_path = backups.build_database_backup_path(self.backup_id, database, self.namespace, self.external_backup_root, self.blob_path) - pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) + pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) path_for_parallel_flag_backup = os.path.join(pg_dump_backup_path, database) if self.s3: @@ -463,7 +467,7 @@ def on_success(self, database): size_bytes = os.path.getsize(database_backup_path) self.update_status('path', backups. build_database_backup_full_path( - self.backup_id, database, self.location, self.namespace), database) + self.backup_id, database, self.location, self.namespace, blob_path=self.blob_path), database) self.update_status('sizeBytes', size_bytes, database) self.update_status('size', backups.sizeof_fmt(size_bytes), database) if self.encryption: diff --git a/docker/granular/pg_restore.py b/docker/granular/pg_restore.py index ee65c61..b2cb09e 100644 --- a/docker/granular/pg_restore.py +++ b/docker/granular/pg_restore.py @@ -35,7 +35,7 @@ class PostgreSQLRestoreWorker(Thread): - def __init__(self, databases, force, restore_request, databases_mapping, owners_mapping, restore_roles=True, single_transaction=False, dbaas_clone=False): + def __init__(self, databases, force, restore_request, databases_mapping, owners_mapping, restore_roles=True, single_transaction=False, dbaas_clone=False, blobPath=None): Thread.__init__(self) self.log = logging.getLogger("PostgreSQLRestoreWorker") @@ -57,7 +57,7 @@ def __init__(self, databases, force, restore_request, databases_mapping, owners_ self.restore_roles = restore_roles self.postgres_version = utils.get_version_of_pgsql_server() self.location = configs.backups_storage(self.postgres_version) if self.is_standard_storage \ - else backups.build_external_backup_root(restore_request.get('externalBackupPath')) + else backups.build_external_backup_root(restore_request.get('externalBackupPath')) if restore_request.get('externalBackupPath') is not None else blobPath self.external_backup_root = None if self.is_standard_storage else self.location self.databases_mapping = databases_mapping self.owners_mapping = owners_mapping @@ -65,12 +65,13 @@ def __init__(self, databases, force, restore_request, databases_mapping, owners_ self.parallel_jobs = configs.get_parallel_jobs() self.s3 = storage_s3.AwsS3Vault() if os.environ['STORAGE_TYPE'] == "s3" else None if self.s3: - self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) + self.blob_path = blobPath + self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) self.create_backup_dir(self.backup_dir) if configs.get_encryption(): self.encryption = True self.key_name = backups.get_key_name_by_backup_id(self.backup_id, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) #TODO: check if this is correct else: self.encryption = False if databases_mapping: @@ -94,8 +95,11 @@ def log_msg(self, msg): return "[trackingId=%s] %s" % (self.tracking_id, msg) def flush_status(self, external_backup_storage=None): - path = backups.build_restore_status_file_path(self.backup_id, self.tracking_id, self.namespace, - external_backup_storage) + if self.blob_path is not None: + path = backups.build_restore_status_file_path(self.backup_id, self.tracking_id, blob_path=self.blob_path) + else: + path = backups.build_restore_status_file_path(self.backup_id, self.tracking_id, self.namespace, + external_backup_storage) utils.write_in_json(path, self.status) if self.s3: try: @@ -169,21 +173,21 @@ def restore_single_database(self, database): self.update_status('status', backups.BackupStatus.IN_PROGRESS, database) self.update_status('source', backups.build_database_backup_full_path( self.backup_id, database, self.location, - self.namespace), database, flush=True) + self.namespace, blob_path=self.blob_path), database, flush=True) if int(self.parallel_jobs) > 1: pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) dump_path = os.path.join(pg_dump_backup_path, database) else: dump_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) roles_backup_path = backups.build_roles_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) stderr_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) stderr_path = stderr_path + '.stderr' stdout_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) sql_script_path = stdout_path + '.sql' stdout_path = stdout_path + '.stdout' if self.s3: @@ -195,7 +199,7 @@ def restore_single_database(self, database): raise e self.update_status('path', backups.build_database_backup_full_path( - self.backup_id, database, self.location, self.namespace), + self.backup_id, database, self.location, self.namespace, blob_path=self.blob_path), database=database, flush=True) new_bd_name = self.databases_mapping.get(database) or database @@ -521,7 +525,7 @@ def process_restore_request(self): self.update_status('started', str(datetime.datetime.fromtimestamp(start_timestamp).isoformat()), flush=True) backup_status_file = backups.build_backup_status_file_path(self.backup_id, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) if self.s3: try: status = self.s3.read_object(backup_status_file) @@ -550,15 +554,15 @@ def process_restore_request(self): for database in self.databases: if self.s3: is_backup_exist = self.s3.is_file_exists(backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root)) + self.namespace, self.external_backup_root, self.blob_path)) else: if int(self.parallel_jobs) > 1: - pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) + pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) path_for_parallel_flag_backup = os.path.join(pg_dump_backup_path, database) is_backup_exist = os.path.exists(path_for_parallel_flag_backup) else: is_backup_exist = backups.database_backup_exists(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path) if not is_backup_exist: raise backups.BackupNotFoundException(self.backup_id, database, self.namespace) @@ -576,11 +580,11 @@ def process_restore_request(self): finally: try: if int(self.parallel_jobs) > 1: - pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) + pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) backup_path = os.path.join(pg_dump_backup_path, database) else: backup_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root) + self.namespace, self.external_backup_root, self.blob_path ) os.remove(backup_path + '.stderr') except OSError as ex: self.log.exception(self.log_msg("Unable to remove stderr log due to: %s " % str(ex))) From e1838c2df97a77da76b9f31bec627258472e8e36 Mon Sep 17 00:00:00 2001 From: Tvion Date: Thu, 23 Oct 2025 16:05:28 +0500 Subject: [PATCH 2/4] Refactor backup path handling in backups.py, pg_backup.py, and pg_restore.py to streamline blob_path usage and improve code readability. --- docker/granular/backups.py | 23 +++++++---------------- docker/granular/pg_backup.py | 7 ++----- docker/granular/pg_restore.py | 7 +++---- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/docker/granular/backups.py b/docker/granular/backups.py index 6ecbcb4..a973b41 100644 --- a/docker/granular/backups.py +++ b/docker/granular/backups.py @@ -78,10 +78,7 @@ def is_valid_namespace(namespace): def backup_exists(backup_id, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): - if blob_path is not None: - return os.path.exists(build_backup_path(backup_id, blob_path=blob_path)) - else: - return os.path.exists(build_backup_path(backup_id, namespace, external_backup_storage)) + return os.path.exists(build_backup_path(backup_id, namespace, external_backup_storage, blob_path)) def database_backup_exists(backup_id, database, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): @@ -96,20 +93,20 @@ def build_database_backup_path(backup_id, database, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): ext = '_enc.dump' if configs.get_encryption() else '.dump' return '%s/%s%s' % ( - build_backup_path(backup_id, namespace, external_backup_storage, blob_path=blob_path), database, ext) + build_backup_path(backup_id, namespace, external_backup_storage, blob_path), database, ext) def build_roles_backup_path(backup_id, database, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): ext = 'roles_enc.sql' if configs.get_encryption() else 'roles.sql' return "%s/%s.%s" % ( - build_backup_path(backup_id, namespace, external_backup_storage, blob_path=blob_path), database, ext) + build_backup_path(backup_id, namespace, external_backup_storage, blob_path), database, ext) def build_database_backup_full_path(backup_id, database, storage_root, namespace=configs.default_namespace(), blob_path=None): ext = '_enc.dump' if configs.get_encryption() else '.dump' if blob_path is not None: - return '%s/backups/%s/logical/dbaas/%s%s' % (blob_path, backup_id, database, ext) + return '%s/%s/%s%s' % (blob_path, backup_id, database, ext) else: return '%s/%s/%s/%s%s' % (storage_root, namespace, backup_id, database, ext) @@ -120,7 +117,7 @@ def build_database_restore_report_path(backup_id, database, restore_tracking_id, def build_backup_path(backup_id, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): if blob_path is not None: - return '%s/backups/%s/logical/dbaas' % (blob_path, backup_id) + return '%s/%s' % (blob_path, backup_id) else: return '%s/%s/%s' % (configs.backups_storage() if external_backup_storage is None else external_backup_storage, namespace, backup_id) @@ -131,18 +128,12 @@ def build_external_backup_root(external_backup_path): def build_backup_status_file_path(backup_id, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): - if blob_path is not None: - return '%s/status.json' % build_backup_path(backup_id, blob_path=blob_path) - else: - return '%s/status.json' % build_backup_path(backup_id, namespace, external_backup_storage) + return '%s/status.json' % build_backup_path(backup_id, namespace, external_backup_storage, blob_path) def build_restore_status_file_path(backup_id, tracking_id, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): - if blob_path is not None: - return '%s/%s.json' % (build_backup_path(backup_id, blob_path=blob_path), tracking_id) - else: - return '%s/%s.json' % (build_backup_path(backup_id, namespace, external_backup_storage), tracking_id) + return '%s/%s.json' % (build_backup_path(backup_id, namespace, external_backup_storage, blob_path), tracking_id) def get_key_name_by_backup_id(backup_id, namespace, external_backup_storage=None, blob_path=None): diff --git a/docker/granular/pg_backup.py b/docker/granular/pg_backup.py index 0bbfcfd..4707b7f 100644 --- a/docker/granular/pg_backup.py +++ b/docker/granular/pg_backup.py @@ -50,11 +50,8 @@ def __init__(self, databases, backup_request, blob_path=None): self.bin_path = configs.get_pgsql_bin_path(self.postgres_version) self.parallel_jobs = configs.get_parallel_jobs() self.databases = databases if databases else [] - if blob_path is not None: - self.blob_path = blob_path - self.backup_dir = backups.build_backup_path(self.backup_id, blob_path=blob_path) - else: - self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) + self.blob_path = blob_path + self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) self.create_backup_dir() self.s3 = storage_s3.AwsS3Vault() if os.environ['STORAGE_TYPE'] == "s3" else None self._cancel_event = Event() diff --git a/docker/granular/pg_restore.py b/docker/granular/pg_restore.py index b2cb09e..74390f1 100644 --- a/docker/granular/pg_restore.py +++ b/docker/granular/pg_restore.py @@ -64,10 +64,9 @@ def __init__(self, databases, force, restore_request, databases_mapping, owners_ self.bin_path = configs.get_pgsql_bin_path(self.postgres_version) self.parallel_jobs = configs.get_parallel_jobs() self.s3 = storage_s3.AwsS3Vault() if os.environ['STORAGE_TYPE'] == "s3" else None - if self.s3: - self.blob_path = blobPath - self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) - self.create_backup_dir(self.backup_dir) + self.blob_path = blobPath + self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) + self.create_backup_dir(self.backup_dir) if configs.get_encryption(): self.encryption = True self.key_name = backups.get_key_name_by_backup_id(self.backup_id, From 42ffab50ecf680d21c18cad1badade73c14fb0b3 Mon Sep 17 00:00:00 2001 From: Tvion Date: Fri, 24 Oct 2025 18:32:27 +0500 Subject: [PATCH 3/4] Fix: use blob_path only for files in S3 --- docker/granular/backups.py | 8 +++---- docker/granular/granular.py | 41 ++++++++++------------------------- docker/granular/pg_backup.py | 20 ++++++++--------- docker/granular/pg_restore.py | 26 +++++++++++----------- docker/granular/storage_s3.py | 22 ++++++++++++++----- 5 files changed, 55 insertions(+), 62 deletions(-) diff --git a/docker/granular/backups.py b/docker/granular/backups.py index e79b1ac..e20778a 100644 --- a/docker/granular/backups.py +++ b/docker/granular/backups.py @@ -77,12 +77,12 @@ def is_valid_namespace(namespace): return re.match("^[a-zA-z0-9_]+$", namespace) is not None -def backup_exists(backup_id, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): - return os.path.exists(build_backup_path(backup_id, namespace, external_backup_storage, blob_path)) +def backup_exists(backup_id, namespace=configs.default_namespace(), external_backup_storage=None): + return os.path.exists(build_backup_path(backup_id, namespace, external_backup_storage)) -def database_backup_exists(backup_id, database, namespace=configs.default_namespace(), external_backup_storage=None, blob_path=None): - return os.path.exists(build_database_backup_path(backup_id, database, namespace, external_backup_storage, blob_path)) +def database_backup_exists(backup_id, database, namespace=configs.default_namespace(), external_backup_storage=None): + return os.path.exists(build_database_backup_path(backup_id, database, namespace, external_backup_storage)) def build_namespace_path(namespace=configs.default_namespace()): diff --git a/docker/granular/granular.py b/docker/granular/granular.py index 364acda..e7d1f61 100644 --- a/docker/granular/granular.py +++ b/docker/granular/granular.py @@ -1087,8 +1087,6 @@ def post(self): blob_path = normalize_blobPath(blob_path) - blob_path = normalize_blobPath(blob_path) - # Reuse old logic directly backup_request = { "databases": list(databases), @@ -1162,8 +1160,7 @@ def get(self, backup_id): if not backups.is_valid_namespace(namespace): return "Invalid namespace name: %s." % namespace.encode("utf-8"), http.client.BAD_REQUEST - blob_path = request.args.get("blobPath") - blob_path = normalize_blobPath(blob_path) + blob_path = normalize_blobPath(request.args.get("blobPath")) status_path = backups.build_backup_status_file_path(backup_id, blob_path=blob_path) @@ -1172,7 +1169,7 @@ def get(self, backup_id): except Exception: return "Backup in bucket is not found.", http.client.NOT_FOUND - if blob_path and not (raw.get("blobPath") or raw.get("externalBackupPath")): + if blob_path and not raw.get("blobPath"): raw["blobPath"] = blob_path return backups.transform_backup_status_v1(raw), http.client.OK @@ -1301,7 +1298,6 @@ def post(self, backup_id): if not isinstance(pairs, (list, tuple)): return {"message": "databases must be an array of objects"}, http.client.BAD_REQUEST blob_path = normalize_blobPath(blob_path) - blob_path = normalize_blobPath(blob_path) databases = [] databases_mapping = {} @@ -1318,7 +1314,6 @@ def post(self, backup_id): if not backups.is_valid_namespace(namespace): return "Invalid namespace name: %s." % namespace.encode("utf-8"), http.client.BAD_REQUEST - external_backup_path = blob_path backup_details_file = backups.build_backup_status_file_path(backup_id, blob_path=blob_path) try: @@ -1370,7 +1365,7 @@ def post(self, backup_id): if not dry_run: worker = pg_restore.PostgreSQLRestoreWorker( requested, force, - {"backupId": backup_id, "namespace": namespace, "externalBackupPath": external_backup_path, "trackingId": tracking_id}, + {"backupId": backup_id, "namespace": namespace, "trackingId": tracking_id}, databases_mapping, owners_mapping, restore_roles, single_transaction, body.get("dbaasClone"), blob_path ) worker.start() @@ -1382,7 +1377,6 @@ def post(self, backup_id): created_iso = "" storage_name = body.get("storageName") or "" - blob_path = blob_path or external_backup_path or "" dbs_out = [] for prev in (requested or []): @@ -1418,19 +1412,13 @@ def post(self, backup_id): "creationTime": created_iso} for prev in (requested or []) }, "storageName": storage_name, "blobPath": blob_path, - "externalBackupPath": external_backup_path or "", "sourceBackupId": backup_id } if dry_run: return enriched, http.client.OK - try: - status_path = backups.build_restore_status_file_path(backup_id, tracking_id, namespace, - backups.build_external_backup_root(external_backup_path) if external_backup_path else None) - except TypeError: - status_path = backups.build_restore_status_file_path(backup_id, tracking_id, namespace) - status_path = backups.build_restore_status_file_path(backup_id, tracking_id, blob_path=blob_path) + status_path = backups.build_restore_status_file_path(backup_id, tracking_id, namespace) try: if hasattr(utils, "write_in_json"): @@ -1476,8 +1464,7 @@ def get(self, restore_id): if not backups.is_valid_namespace(namespace): return "Invalid namespace name: %s." % namespace.encode("utf-8"), http.client.BAD_REQUEST - blob_path = request.args.get("blobPath") - blob_path = normalize_blobPath(blob_path) + blob_path = normalize_blobPath(request.args.get("blobPath")) storage_name = request.args.get("storageName") or os.environ.get("STORAGE_NAME") status_path = backups.build_restore_status_file_path(backup_id, restore_id, blob_path=blob_path) @@ -1486,7 +1473,7 @@ def get(self, restore_id): except Exception: return "Backup in bucket is not found.", http.client.NOT_FOUND - if blob_path and not (raw.get("blobPath") or raw.get("externalBackupPath")): + if blob_path and not raw.get("blobPath"): raw["blobPath"] = blob_path if storage_name and not raw.get("storageName"): raw["storageName"] = storage_name @@ -1533,7 +1520,7 @@ def delete(self, restore_id): "termination": {"code": term_code, "body": term_body} }, http.client.OK - blob_path = request.args.get("blobPath") or request.args.get("externalBackupPath") + blob_path = request.args.get("blobPath") if not blob_path: return { "restoreId": restore_id, @@ -1606,18 +1593,12 @@ def _prefix_exists() -> bool: return { "restoreId": restore_id, "message": f"Termination attempted; cleanup encountered an error: {e}", - "status": "Successful", + "status": "Failed", "termination": {"code": term_code, "body": term_body} - }, http.client.OK - - if term_code == 404: - return {"restoreId": restore_id, "message": "Restore is not found.", "status": "Failed", - "termination": {"code": term_code, "body": term_body}}, http.client.NOT_FOUND + }, http.client.INTERNAL_SERVER_ERROR if term_code and 200 <= term_code < 300: msg = "Restore terminated successfully. Cleanup completed." - elif term_code == 404: - msg = "No active restore (already finished or not found). Cleanup completed." else: msg = "Termination attempted. Cleanup completed." @@ -1627,8 +1608,8 @@ def _prefix_exists() -> bool: def normalize_blobPath(blob_path): # Normalize blob_path by removing a single leading and trailing slash if isinstance(blob_path, str): - if not blob_path.startswith("/"): - blob_path = f'/{blob_path}' + if blob_path.startswith("/"): + blob_path = blob_path[1:] if blob_path.endswith("/"): blob_path = blob_path[:-1] return blob_path diff --git a/docker/granular/pg_backup.py b/docker/granular/pg_backup.py index 4707b7f..d5ccb79 100644 --- a/docker/granular/pg_backup.py +++ b/docker/granular/pg_backup.py @@ -51,7 +51,7 @@ def __init__(self, databases, backup_request, blob_path=None): self.parallel_jobs = configs.get_parallel_jobs() self.databases = databases if databases else [] self.blob_path = blob_path - self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) + self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) self.create_backup_dir() self.s3 = storage_s3.AwsS3Vault() if os.environ['STORAGE_TYPE'] == "s3" else None self._cancel_event = Event() @@ -112,12 +112,12 @@ def update_status(self, key, value, database=None, flush=False): self.flush_status(self.external_backup_root) def flush_status(self, external_backup_storage=None): - path = backups.build_backup_status_file_path(self.backup_id, self.namespace, external_backup_storage, self.blob_path) + path = backups.build_backup_status_file_path(self.backup_id, self.namespace, external_backup_storage) utils.write_in_json(path, self.status) if self.s3: try: # upload dumpfile - self.s3.upload_file(path) + self.s3.upload_file(path, self.blob_path, self.backup_id) except Exception as e: raise e @@ -271,7 +271,7 @@ def backup_single_database(self, database): command.extend(['-T','cron.*']) database_backup_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root, blob_path=self.blob_path) + self.namespace, self.external_backup_root) with open(database_backup_path, 'w+') as dump, \ open(self.stderr_file(database), "w+") as stderr: @@ -297,9 +297,9 @@ def backup_single_database(self, database): if self.s3: try: # upload dumpfile - self.s3.upload_file(database_backup_path) + self.s3.upload_file(database_backup_path, self.blob_path, self.backup_id) # upload errorfile - self.s3.upload_file(self.stderr_file(database)) + self.s3.upload_file(self.stderr_file(database), self.blob_path, self.backup_id) except Exception as e: raise e @@ -351,9 +351,9 @@ def fetch_roles(self, database): self.log.debug(self.log_msg("Roles {} have been fetched for backup ".format(rolesList))) roles_backup_path = backups.build_roles_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root, self.blob_path) + self.namespace, self.external_backup_root) database_backup_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root, self.blob_path) + self.namespace, self.external_backup_root) pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) path_for_parallel_flag_backup = os.path.join(pg_dump_backup_path, database) @@ -402,7 +402,7 @@ def fetch_roles(self, database): def dump_roles_for_db(self, roles, database): roles_backup_path = backups.build_roles_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root, self.blob_path) + self.namespace, self.external_backup_root) with open(roles_backup_path, 'w+') as dump, \ open(self.stderr_file(database), "w+") as stderr: @@ -434,7 +434,7 @@ def dump_roles_for_db(self, roles, database): if self.s3: try: logging.info("Streaming {} roles to AWS".format(database)) - self.s3.upload_file(roles_backup_path) + self.s3.upload_file(roles_backup_path, self.blob_path, self.backup_id) except Exception as e: raise e finally: diff --git a/docker/granular/pg_restore.py b/docker/granular/pg_restore.py index feabee0..9743bce 100644 --- a/docker/granular/pg_restore.py +++ b/docker/granular/pg_restore.py @@ -65,12 +65,12 @@ def __init__(self, databases, force, restore_request, databases_mapping, owners_ self.parallel_jobs = configs.get_parallel_jobs() self.s3 = storage_s3.AwsS3Vault() if os.environ['STORAGE_TYPE'] == "s3" else None self.blob_path = blobPath - self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) + self.backup_dir = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) self.create_backup_dir(self.backup_dir) if configs.get_encryption(): self.encryption = True self.key_name = backups.get_key_name_by_backup_id(self.backup_id, - self.namespace, self.external_backup_root, self.blob_path) #TODO: check if this is correct + self.namespace, self.external_backup_root) else: self.encryption = False if databases_mapping: @@ -95,12 +95,12 @@ def log_msg(self, msg): def flush_status(self, external_backup_storage=None): path = backups.build_restore_status_file_path(self.backup_id, self.tracking_id, self.namespace, - external_backup_storage, self.blob_path) + external_backup_storage) utils.write_in_json(path, self.status) if self.s3: try: # upload status file - self.s3.upload_file(path) + self.s3.upload_file(path, self.blob_path, self.backup_id) except Exception as e: raise e @@ -178,21 +178,21 @@ def restore_single_database(self, database): dump_path = os.path.join(pg_dump_backup_path, database) else: dump_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root, self.blob_path) + self.namespace, self.external_backup_root) roles_backup_path = backups.build_roles_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root, self.blob_path) + self.namespace, self.external_backup_root) stderr_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root, self.blob_path) + self.namespace, self.external_backup_root) stderr_path = stderr_path + '.stderr' stdout_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root, self.blob_path) + self.namespace, self.external_backup_root) sql_script_path = stdout_path + '.sql' stdout_path = stdout_path + '.stdout' if self.s3: try: - self.s3.download_file(dump_path) + self.s3.download_file(dump_path, self.backup_id, self.blob_path) if self.restore_roles: - self.s3.download_file(roles_backup_path) + self.s3.download_file(roles_backup_path, self.backup_id, self.blob_path) except Exception as e: raise e self.update_status('path', @@ -560,7 +560,7 @@ def process_restore_request(self): is_backup_exist = os.path.exists(path_for_parallel_flag_backup) else: is_backup_exist = backups.database_backup_exists(self.backup_id, database, - self.namespace, self.external_backup_root, self.blob_path) + self.namespace, self.external_backup_root) if not is_backup_exist: raise backups.BackupNotFoundException(self.backup_id, database, self.namespace) @@ -578,11 +578,11 @@ def process_restore_request(self): finally: try: if int(self.parallel_jobs) > 1: - pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root, self.blob_path) + pg_dump_backup_path = backups.build_backup_path(self.backup_id, self.namespace, self.external_backup_root) backup_path = os.path.join(pg_dump_backup_path, database) else: backup_path = backups.build_database_backup_path(self.backup_id, database, - self.namespace, self.external_backup_root, self.blob_path ) + self.namespace, self.external_backup_root) os.remove(backup_path + '.stderr') except OSError as ex: self.log.exception(self.log_msg("Unable to remove stderr log due to: %s " % str(ex))) diff --git a/docker/granular/storage_s3.py b/docker/granular/storage_s3.py index 7bee12b..e273b68 100644 --- a/docker/granular/storage_s3.py +++ b/docker/granular/storage_s3.py @@ -61,8 +61,13 @@ def get_s3_client(self): verify=(False if os.getenv("AWS_S3_UNTRUSTED_CERT", "false").lower() == "true" else None)) @retry(stop_max_attempt_number=RETRY_COUNT, wait_fixed=RETRY_WAIT) - def upload_file(self, file_path): - return self.get_s3_client().upload_file(file_path, self.bucket, self.aws_prefix + file_path) + def upload_file(self, file_path, blob_path=None, backup_id=None): + if blob_path: + file_name = file_path.rsplit('/',1)[1] + s3FilePath = self.aws_prefix + f'{blob_path}/{backup_id}/{file_name}' + return self.get_s3_client().upload_file(file_path, self.bucket, s3FilePath) + else: + return self.get_s3_client().upload_file(file_path, self.bucket, self.aws_prefix + file_path) @retry(stop_max_attempt_number=RETRY_COUNT, wait_fixed=RETRY_WAIT) def delete_file(self, filename): @@ -91,10 +96,17 @@ def get_file_size(self, file_path): self.__log.info("Requested {} file not found".format(file_path)) @retry(stop_max_attempt_number=RETRY_COUNT, wait_fixed=RETRY_WAIT) - def download_file(self, filename): - logging.info("Downloading file {}" .format(self.aws_prefix + filename)) + def download_file(self, filename, backup_id, blob_path=None): try: - self.get_s3_client().download_file(self.bucket, self.aws_prefix + filename, filename) + if blob_path: + only_file_name = filename.rsplit('/',1)[1] + file_path = f'{blob_path}/{backup_id}/{only_file_name}' + logging.info(f'Downloading file {file_path} to {filename}') + + self.get_s3_client().download_file(self.bucket, self.aws_prefix + file_path, filename) + else: + logging.info("Downloading file {}" .format(self.aws_prefix + filename)) + self.get_s3_client().download_file(self.bucket, self.aws_prefix + filename, filename) except Exception as e: raise e return From 0687b99c13c58d13f55ad870566163619538c33f Mon Sep 17 00:00:00 2001 From: Tvion Date: Tue, 28 Oct 2025 15:35:37 +0500 Subject: [PATCH 4/4] Fix: return statements added for S3 configuration checks in backup and restore resources --- docker/granular/granular.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docker/granular/granular.py b/docker/granular/granular.py index e7d1f61..b01cbc5 100644 --- a/docker/granular/granular.py +++ b/docker/granular/granular.py @@ -1073,7 +1073,7 @@ def get_endpoints(): @auth.login_required def post(self): if not self.s3: - "S3 is not configured for backup daemon", http.client.FORBIDDEN + return "S3 is not configured for backup daemon", http.client.FORBIDDEN body = request.get_json(silent=True) or {} storage_name = body.get("storageName") @@ -1151,7 +1151,7 @@ def get_endpoints(): @auth.login_required def get(self, backup_id): if not self.s3: - "S3 is not configured for backup daemon", http.client.FORBIDDEN + return "S3 is not configured for backup daemon", http.client.FORBIDDEN if not backup_id: return "Backup ID is not specified.", http.client.BAD_REQUEST @@ -1178,7 +1178,7 @@ def get(self, backup_id): @superuser_authorization def delete(self, backup_id): if not self.s3: - "S3 is not configured for backup daemon", http.client.FORBIDDEN + return "S3 is not configured for backup daemon", http.client.FORBIDDEN if not backup_id: return {"backupId": backup_id, "message": "Backup ID is not specified", "status": "Failed"}, http.client.BAD_REQUEST @@ -1283,7 +1283,7 @@ def get_endpoints(): @superuser_authorization def post(self, backup_id): if not self.s3: - "S3 is not configured for backup daemon", http.client.FORBIDDEN + return "S3 is not configured for backup daemon", http.client.FORBIDDEN body = request.get_json(silent=True) or {} blob_path = body.get("blobPath") @@ -1450,7 +1450,7 @@ def get_endpoints(): @superuser_authorization def get(self, restore_id): if not self.s3: - "S3 is not configured for backup daemon", http.client.FORBIDDEN + return "S3 is not configured for backup daemon", http.client.FORBIDDEN if not restore_id: return "Restore tracking ID is not specified.", http.client.BAD_REQUEST @@ -1484,7 +1484,7 @@ def get(self, restore_id): @superuser_authorization def delete(self, restore_id): if not self.s3: - "S3 is not configured for backup daemon", http.client.FORBIDDEN + return "S3 is not configured for backup daemon", http.client.FORBIDDEN if not restore_id: return {"restoreId": restore_id, "message": "Restore ID is not specified", "status": "Failed"}, http.client.BAD_REQUEST @@ -1609,7 +1609,11 @@ def normalize_blobPath(blob_path): # Normalize blob_path by removing a single leading and trailing slash if isinstance(blob_path, str): if blob_path.startswith("/"): - blob_path = blob_path[1:] + if not os.getenv("AWS_S3_PREFIX", ""): + blob_path = blob_path[1:] + else: + if os.getenv("AWS_S3_PREFIX", ""): + blob_path = "/" + blob_path if blob_path.endswith("/"): blob_path = blob_path[:-1] return blob_path