Feat: Inspect dart-runner executable

app.py

@@ -2,7 +2,7 @@
 import argparse
 from version import __version__, __commit__
 from Log import Log
-from figshare.Utils import upload_to_remote, get_archival_staging_storage
+from figshare.Utils import inspect_dart, upload_to_remote, get_archival_staging_storage
 from figshare.Article import Article
 from datetime import datetime
 from Config import Config
@@ -82,6 +82,7 @@ def main():
     figshare_config = config_obj.figshare_config()
     system_config = config_obj.system_config()
     figshare_api_url = figshare_config["url"]
+    check_dart = inspect_dart()
     log = Log(env_file)
     log_location = system_config["logs_location"]
     ingest_staging_storage = system_config["ingest_staging_storage"]
@@ -144,6 +145,14 @@ def main():
                                   + " not be reached or read.",
                                   True, False)
 
+    if check_dart is None:
+        log.write_log_in_file('warning',
+                              "dart-runner executable not found. Bagging will not occur.", True)
+    elif check_dart is False:
+        log.write_log_in_file('warning',
+                              "dart-runner is not executable or version is lower than 1.0. Bagging will not occur.",
+                              True)
+
     return config_obj, log
 
 

figshare/Article.py

@@ -10,7 +10,7 @@
 from figshare.Integration import Integration
 from figshare.Utils import standardize_api_result, sorter_api_result, get_preserved_version_hash_and_size, metadata_to_hash, check_local_path
 from figshare.Utils import compare_hash, check_wasabi, calculate_payload_size, get_article_id_and_version_from_path, stringify_metadata
-from figshare.Utils import format_version, get_folder_name_in_local_storage, upload_to_remote
+from figshare.Utils import format_version, get_folder_name_in_local_storage, upload_to_remote, inspect_dart
 from slugify import slugify
 from requests.adapters import HTTPAdapter, Retry
 
@@ -64,6 +64,7 @@ def __init__(self, config, log, ids):
                                           'articles_locally_preserved': 0, 'articles_versions_with_processing_error': 0
                                           }
         self.skipped_article_versions = {}
+        self.check_dart = inspect_dart()
         self.processor = Integration(self.config_obj, self.logs)
 
     """
@@ -1048,21 +1049,31 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
                 self.logs.write_log_in_file("info", "Saving json in metadata folder for each version.", True)
                 success = success & self.__save_json_in_metadata(version_data, folder_name)
 
-                # only run the postprocessor if all above steps succeeded
-                if success:
-                    value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process)
-                    if (value_post_process != 0):
-                        self.logs.write_log_in_file("error",
-                                                    f"{version_data['id']} version {version_data['version']} - Post-processing script failed.",
+                # only run the postprocessor if all above steps succeeded and dart-runner is available
+                if self.check_dart:
+                    if success:
+                        value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process)
+                        if (value_post_process != 0):
+                            self.logs.write_log_in_file("error",
+                                                        f"{version_data['id']} version {version_data['version']} - Post-processing script failed.",
+                                                        True)
+                            success = False
+                        else:
+                            success = True
+                    else:
+                        self.logs.write_log_in_file("info",
+                                                    f"No further processing for {version_data['id']} version "
+                                                    + f"{version_data['version']} due to errors.",
                                                     True)
                         success = False
-                    else:
-                        success = True
                 else:
-                    self.logs.write_log_in_file("info",
-                                                f"No further processing for {version_data['id']} version {version_data['version']} due to errors.",
-                                                True)
                     success = False
+                    self.logs.write_log_in_file("error",
+                                                f"dart-runner not available. No bagging for {version_data['id']} version {version_data['version']}",
+                                                True)
+                    if self.system_config['continue-on-error'] == "False":
+                        self.logs.write_log_in_file("info", "Aborting execution.", True, True)
+
             else:
                 # if download process has any errors then delete complete folder
                 self.logs.write_log_in_file("info", "Download process had an error so complete folder is being deleted.", True)
@@ -1076,23 +1087,32 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
                 success = False
         else:
             if check_files or copy_files:
-                if success:
-                    # call post process script function for each matched item.
-                    value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process)
-                    if (value_post_process != 0):
-                        self.logs.write_log_in_file("error",
-                                                    f"{version_data['id']} version {version_data['version']} - Post-processing script failed.",
+                if self.check_dart:
+                    if success:
+                        # call post process script function for each matched item.
+                        value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process)
+                        if (value_post_process != 0):
+                            self.logs.write_log_in_file("error",
+                                                        f"{version_data['id']} version {version_data['version']} - Post-processing script failed.",
+                                                        True)
+                            success = False
+                        else:
+                            success = True
+                    else:
+                        self.logs.write_log_in_file("info",
+                                                    f"No further processing for {version_data['id']} version "
+                                                    + f"{version_data['version']} due to errors.",
                                                     True)
                         success = False
-                    else:
-                        success = True
                 else:
-                    self.logs.write_log_in_file("info",
-                                                f"No further processing for {version_data['id']} version {version_data['version']} due to errors.",
-                                                True)
                     success = False
+                    self.logs.write_log_in_file("error",
+                                                f"dart-runner not available. No bagging for {version_data['id']} version {version_data['version']}",
+                                                True)
+                    if self.system_config['continue-on-error'] == "False":
+                        self.logs.write_log_in_file("info", "Aborting execution.", True, True)
             else:
-                self.logs.write_log_in_file("error", "Unexpected condidion in final processing. No further actions taken.", True)
+                self.logs.write_log_in_file("error", "Unexpected condition in final processing. No further actions taken.", True)
                 success = False
         return success
 
@@ -1237,10 +1257,18 @@ def process_articles(self, articles):
                         else:
                             self.logs.write_log_in_file("error", "Pre-processing script failed. Running post-processing script.", True)
                             # call post process script function for each matched item.
-                            value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process)
-                            if (value_post_process != 0):
-                                self.logs.write_log_in_file("error", f"{version_data['id']} version {version_data['version']} - "
-                                                            + "Post-processing script failed.", True)
+                            if self.check_dart:
+                                value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process)
+                                if (value_post_process != 0):
+                                    self.logs.write_log_in_file("error", f"{version_data['id']} version {version_data['version']} - "
+                                                                + "Post-processing script failed.", True)
+                            else:
+                                self.logs.write_log_in_file("error",
+                                                            f"dart-runner not available. No bagging for {version_data['id']} "
+                                                            + f"version {version_data['version']}",
+                                                            True)
+                                if self.system_config['continue-on-error'] == "False":
+                                    self.logs.write_log_in_file("info", "Aborting execution.", True, True)
         return processed_count, self.skipped_items_counts_dict['ap_trust_preserved_versions'], \
             self.skipped_items_counts_dict['wasabi_preserved_versions'], len(self.skipped_items_counts_dict['articles_with_processing_error']), \
             self.skipped_items_counts_dict['articles_versions_with_processing_error']

figshare/Collection.py

@@ -8,6 +8,7 @@
 from figshare.Integration import Integration
 from figshare.Utils import standardize_api_result, sorter_api_result, get_preserved_version_hash_and_size, format_version, metadata_to_hash
 from figshare.Utils import compare_hash, check_wasabi, check_local_path, get_folder_name_in_local_storage, upload_to_remote, stringify_metadata
+from figshare.Utils import inspect_dart
 
 
 class Collection:
@@ -42,6 +43,7 @@ def __init__(self, config, log, ids):
                                               'already_preserved_versions': 0, 'wasabi_preserved_versions': 0,
                                               'ap_trust_preserved_versions': 0
                                               }
+        self.check_dart = inspect_dart()
         self.processor = Integration(self.config_obj, self.logs)
 
     """
@@ -370,11 +372,18 @@ def process_collections(self, collections):
                     self.__save_json_in_metadata(collection, version, folder_name)
                     collection_preservation_path = self.ingest_staging_storage + \
                         os.path.basename(os.path.dirname(os.path.dirname(folder_name)))
-                    value_post_process = self.processor.post_process_script_function("Collection", collection_preservation_path)
-                    if (value_post_process != 0):
-                        self.logs.write_log_in_file("error", f"collection {collection} - post-processing script failed.", True)
+                    if self.check_dart:
+                        value_post_process = self.processor.post_process_script_function("Collection", collection_preservation_path)
+                        if (value_post_process != 0):
+                            self.logs.write_log_in_file("error", f"collection {collection} - post-processing script failed.", True)
+                        else:
+                            processed_count += 1
                     else:
-                        processed_count += 1
+                        self.logs.write_log_in_file("error",
+                                                    f"dart-runner not available. No bagging for collection {collection}",
+                                                    True)
+                        if self.system_config['continue-on-error'] == "False":
+                            self.logs.write_log_in_file("info", "Aborting execution.", True, True)
                 else:
                     self.logs.write_log_in_file("info", "*Dry Run* File download and post-processing with "
                                                 + f"{self.system_config['post_process_script_command']} skipped.", True)

figshare/Utils.py

@@ -2,11 +2,46 @@
 import json
 import os
 import re
+import tempfile
+import configparser
 from typing import Any
 from time import sleep
-import tempfile
+from subprocess import Popen, PIPE
 from bagger.wasabi import Wasabi
-import configparser
+
+
+def inspect_dart() -> Any:
+    """
+    Checks if dart-runner executable exists and it's usable
+
+    :return: Return True if dart-runner executable exists and it's usable
+    :rtype: Any
+    """
+    config = configparser.ConfigParser()
+    config.read('bagger/config/default.toml')
+    default_config = config['Defaults']
+    dart_cmd = default_config['dart_command'].replace('\"', '')
+
+    try:
+        dart_cmd = f'{dart_cmd} --version'
+        dart = Popen(dart_cmd,
+                     shell=True,
+                     stdin=PIPE,
+                     stdout=PIPE,
+                     stderr=PIPE,
+                     text=True,
+                     close_fds=True
+                     )
+        dart_version, _ = dart.communicate()
+        dart_version_re = re.compile("v[0-9]*.")
+        dart_major_version = re.findall(dart_version_re, dart_version)[0].replace('v', '').replace('.', '')
+        if int(dart_major_version) < 1:
+            return False
+        return True
+    except FileNotFoundError:
+        return None
+    except PermissionError:
+        return False
 
 
 def standardize_api_result(api_result) -> dict: