diff --git a/.gitignore b/.gitignore index 8b6f0c9..b799a68 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,5 @@ dmypy.json #Other .DS_Store + +playwright/.auth diff --git a/figshare/Article.py b/figshare/Article.py index 378e1a7..b2407df 100644 --- a/figshare/Article.py +++ b/figshare/Article.py @@ -524,18 +524,25 @@ def __download_files(self, files, version_data, folder_name): if (len(files) > 0): version_no = format_version(version_data["version"]) - article_folder = folder_name + "/" + version_no + article_folder = os.path.join(folder_name, version_no) file_no = 0 for file in files: if (file['is_link_only'] is False): - article_files_folder = article_folder + "/DATA" + article_files_folder = os.path.join(article_folder, "DATA") ingest_staging_storage = self.ingest_staging_storage - article_folder_path = ingest_staging_storage + article_files_folder + article_folder_path = os.path.join(ingest_staging_storage, article_files_folder) article_files_path_exists = os.path.exists(article_folder_path) if (article_files_path_exists is False): os.makedirs(article_folder_path, exist_ok=True) - file_name_with_path = article_folder_path + "/" + str(file['id']) + "_" + file['name'] + folder_for_file = '' + # if an item version has no folders, folders dict will be empty + if len(version_data['folder_structure'].keys()) > 0 and str(file['id']) in version_data['folder_structure'].keys(): + folder_for_file = version_data['folder_structure'][str(file['id'])] + filepath = os.path.join(article_folder_path, folder_for_file) + if not os.path.exists(filepath): + os.makedirs(filepath, exist_ok=True) + file_name_with_path = os.path.join(filepath, str(file['id']) + "_" + file['name']) self.logs.write_log_in_file("info", f"Downloading file {file['id']} for article {version_data['id']} - " + f"version {version_data['version']}", True) @@ -733,10 +740,10 @@ def check_required_space(self, required_space): """ def __check_file_hash(self, files, version_data, folder_path): version_no = format_version(version_data["version"]) - article_version_folder = folder_path + "/" + version_no - article_files_folder = article_version_folder + "/DATA" + article_version_folder = os.path.join(folder_path, version_no) + article_files_folder = os.path.join(article_version_folder, "DATA") ingest_staging_storage = self.ingest_staging_storage - article_folder_path = ingest_staging_storage + article_files_folder + article_folder_path = os.path.join(ingest_staging_storage, article_files_folder) # check if preservation dir is reachable self.check_access_of_directories(ingest_staging_storage, "preservation") @@ -750,7 +757,12 @@ def __check_file_hash(self, files, version_data, folder_path): if (len(get_files) > 0): self.logs.write_log_in_file('info', "Comparing Figshare file hashes against existing local files.", True) for file in files: - file_path = article_folder_path + "/" + str(file['id']) + "_" + file['name'] + # if an item version has no folders, folders dict will be empty + if len(version_data['folder_structure'].keys()) > 0 and str(file['id']) in version_data['folder_structure'].keys(): + folder_for_file = version_data['folder_structure'][str(file['id'])] + else: + folder_for_file = '' + file_path = os.path.join(article_folder_path, folder_for_file, str(file['id']) + "_" + file['name']) file_exists = os.path.exists(file_path) compare_hash = file['supplied_md5'] if (compare_hash == ""): @@ -1116,7 +1128,7 @@ def process_articles(self, articles): if (self.matched_curation_folder_list): curation_folder_size = 0 for folder in self.matched_curation_folder_list: - path = curation_storage_location + folder + path = os.path.join(curation_storage_location, folder) curation_folder_size += self.get_file_size_of_given_path(path, "UAL_RDM") elif len(self.matched_curation_folder_list) == 0 and len(article_data) != 0: curation_folder_size = 0 @@ -1175,7 +1187,7 @@ def process_articles(self, articles): self.logs.write_log_in_file("info", "Pre-processing script finished successfully.", True) # check main folder exists in preservation storage. ingest_staging_storage = self.ingest_staging_storage - check_dir = ingest_staging_storage + folder_name + check_dir = os.path.join(ingest_staging_storage, folder_name) check_files = True copy_files = True self.logs.write_log_in_file("info", f"Checking if {check_dir} exists.", True) diff --git a/figshare/Utils.py b/figshare/Utils.py index c9b9e28..e59b956 100644 --- a/figshare/Utils.py +++ b/figshare/Utils.py @@ -414,7 +414,7 @@ def check_local_path(article_id: int, version_no: Any, path="") -> list: if path == "": config = configparser.ConfigParser() - config.read('bagger/config/default.toml') + config.read(os.path.join('bagger', 'config', 'default.toml')) default_config = config['Defaults'] path = default_config['archival_staging_storage'] @@ -603,9 +603,9 @@ def get_article_id_and_version_from_path(path: str) -> tuple: version_no = '' article_id = '' if path: - path_elements = path.split('/') - version_no = path_elements[-2] - article_id = path_elements[-3].split('_')[-1] + path_elements = os.path.split(os.path.split(path)[0]) + version_no = path_elements[1] + article_id = os.path.split(path_elements[0])[1].split('_')[-1] return article_id, version_no