Feat: extract folder structure for datasets (Issue #125)

.gitignore

@@ -142,3 +142,5 @@ dmypy.json
 
 #Other
 .DS_Store
+
+playwright/.auth

figshare/Article.py

@@ -524,18 +524,25 @@ def __download_files(self, files, version_data, folder_name):
 
         if (len(files) > 0):
             version_no = format_version(version_data["version"])
-            article_folder = folder_name + "/" + version_no
+            article_folder = os.path.join(folder_name, version_no)
             file_no = 0
             for file in files:
                 if (file['is_link_only'] is False):
-                    article_files_folder = article_folder + "/DATA"
+                    article_files_folder = os.path.join(article_folder, "DATA")
                     ingest_staging_storage = self.ingest_staging_storage
-                    article_folder_path = ingest_staging_storage + article_files_folder
+                    article_folder_path = os.path.join(ingest_staging_storage, article_files_folder)
                     article_files_path_exists = os.path.exists(article_folder_path)
                     if (article_files_path_exists is False):
                         os.makedirs(article_folder_path, exist_ok=True)
 
-                    file_name_with_path = article_folder_path + "/" + str(file['id']) + "_" + file['name']
+                    folder_for_file = ''
+                    # if an item version has no folders, folders dict will be empty
+                    if len(version_data['folder_structure'].keys()) > 0 and str(file['id']) in version_data['folder_structure'].keys():
+                        folder_for_file = version_data['folder_structure'][str(file['id'])]
+                    filepath = os.path.join(article_folder_path, folder_for_file)
+                    if not os.path.exists(filepath):
+                        os.makedirs(filepath, exist_ok=True)
+                    file_name_with_path = os.path.join(filepath, str(file['id']) + "_" + file['name'])
                     self.logs.write_log_in_file("info",
                                                 f"Downloading file {file['id']} for article {version_data['id']} - "
                                                 + f"version {version_data['version']}", True)
@@ -733,10 +740,10 @@ def check_required_space(self, required_space):
     """
     def __check_file_hash(self, files, version_data, folder_path):
         version_no = format_version(version_data["version"])
-        article_version_folder = folder_path + "/" + version_no
-        article_files_folder = article_version_folder + "/DATA"
+        article_version_folder = os.path.join(folder_path, version_no)
+        article_files_folder = os.path.join(article_version_folder, "DATA")
         ingest_staging_storage = self.ingest_staging_storage
-        article_folder_path = ingest_staging_storage + article_files_folder
+        article_folder_path = os.path.join(ingest_staging_storage, article_files_folder)
 
         # check if preservation dir is reachable
         self.check_access_of_directories(ingest_staging_storage, "preservation")
@@ -750,7 +757,12 @@ def __check_file_hash(self, files, version_data, folder_path):
             if (len(get_files) > 0):
                 self.logs.write_log_in_file('info', "Comparing Figshare file hashes against existing local files.", True)
                 for file in files:
-                    file_path = article_folder_path + "/" + str(file['id']) + "_" + file['name']
+                    # if an item version has no folders, folders dict will be empty
+                    if len(version_data['folder_structure'].keys()) > 0 and str(file['id']) in version_data['folder_structure'].keys():
+                        folder_for_file = version_data['folder_structure'][str(file['id'])]
+                    else:
+                        folder_for_file = ''
+                    file_path = os.path.join(article_folder_path, folder_for_file, str(file['id']) + "_" + file['name'])
                     file_exists = os.path.exists(file_path)
                     compare_hash = file['supplied_md5']
                     if (compare_hash == ""):
@@ -1116,7 +1128,7 @@ def process_articles(self, articles):
         if (self.matched_curation_folder_list):
             curation_folder_size = 0
             for folder in self.matched_curation_folder_list:
-                path = curation_storage_location + folder
+                path = os.path.join(curation_storage_location, folder)
                 curation_folder_size += self.get_file_size_of_given_path(path, "UAL_RDM")
         elif len(self.matched_curation_folder_list) == 0 and len(article_data) != 0:
             curation_folder_size = 0
@@ -1175,7 +1187,7 @@ def process_articles(self, articles):
                             self.logs.write_log_in_file("info", "Pre-processing script finished successfully.", True)
                             # check main folder exists in preservation storage.
                             ingest_staging_storage = self.ingest_staging_storage
-                            check_dir = ingest_staging_storage + folder_name
+                            check_dir = os.path.join(ingest_staging_storage, folder_name)
                             check_files = True
                             copy_files = True
                             self.logs.write_log_in_file("info", f"Checking if {check_dir} exists.", True)

figshare/Utils.py

@@ -414,7 +414,7 @@ def check_local_path(article_id: int, version_no: Any, path="") -> list:
 
     if path == "":
         config = configparser.ConfigParser()
-        config.read('bagger/config/default.toml')
+        config.read(os.path.join('bagger', 'config', 'default.toml'))
         default_config = config['Defaults']
         path = default_config['archival_staging_storage']
 
@@ -603,9 +603,9 @@ def get_article_id_and_version_from_path(path: str) -> tuple:
     version_no = ''
     article_id = ''
     if path:
-        path_elements = path.split('/')
-        version_no = path_elements[-2]
-        article_id = path_elements[-3].split('_')[-1]
+        path_elements = os.path.split(os.path.split(path)[0])
+        version_no = path_elements[1]
+        article_id = os.path.split(path_elements[0])[1].split('_')[-1]
 
     return article_id, version_no