diff --git a/app.py b/app.py index 9202787..7424685 100644 --- a/app.py +++ b/app.py @@ -2,7 +2,7 @@ import argparse from version import __version__, __commit__ from Log import Log -from figshare.Utils import upload_to_remote, get_archival_staging_storage +from figshare.Utils import inspect_dart, upload_to_remote, get_archival_staging_storage from figshare.Article import Article from datetime import datetime from Config import Config @@ -82,6 +82,7 @@ def main(): figshare_config = config_obj.figshare_config() system_config = config_obj.system_config() figshare_api_url = figshare_config["url"] + check_dart = inspect_dart() log = Log(env_file) log_location = system_config["logs_location"] ingest_staging_storage = system_config["ingest_staging_storage"] @@ -144,6 +145,14 @@ def main(): + " not be reached or read.", True, False) + if check_dart is None: + log.write_log_in_file('warning', + "dart-runner executable not found. Bagging will not occur.", True) + elif check_dart is False: + log.write_log_in_file('warning', + "dart-runner is not executable or version is lower than 1.0. Bagging will not occur.", + True) + return config_obj, log diff --git a/figshare/Article.py b/figshare/Article.py index e7bb33a..b1896a7 100644 --- a/figshare/Article.py +++ b/figshare/Article.py @@ -10,7 +10,7 @@ from figshare.Integration import Integration from figshare.Utils import standardize_api_result, sorter_api_result, get_preserved_version_hash_and_size, metadata_to_hash, check_local_path from figshare.Utils import compare_hash, check_wasabi, calculate_payload_size, get_article_id_and_version_from_path, stringify_metadata -from figshare.Utils import format_version, get_folder_name_in_local_storage, upload_to_remote +from figshare.Utils import format_version, get_folder_name_in_local_storage, upload_to_remote, inspect_dart from slugify import slugify from requests.adapters import HTTPAdapter, Retry @@ -64,6 +64,7 @@ def __init__(self, config, log, ids): 'articles_locally_preserved': 0, 'articles_versions_with_processing_error': 0 } self.skipped_article_versions = {} + self.check_dart = inspect_dart() self.processor = Integration(self.config_obj, self.logs) """ @@ -1048,21 +1049,31 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold self.logs.write_log_in_file("info", "Saving json in metadata folder for each version.", True) success = success & self.__save_json_in_metadata(version_data, folder_name) - # only run the postprocessor if all above steps succeeded - if success: - value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process) - if (value_post_process != 0): - self.logs.write_log_in_file("error", - f"{version_data['id']} version {version_data['version']} - Post-processing script failed.", + # only run the postprocessor if all above steps succeeded and dart-runner is available + if self.check_dart: + if success: + value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process) + if (value_post_process != 0): + self.logs.write_log_in_file("error", + f"{version_data['id']} version {version_data['version']} - Post-processing script failed.", + True) + success = False + else: + success = True + else: + self.logs.write_log_in_file("info", + f"No further processing for {version_data['id']} version " + + f"{version_data['version']} due to errors.", True) success = False - else: - success = True else: - self.logs.write_log_in_file("info", - f"No further processing for {version_data['id']} version {version_data['version']} due to errors.", - True) success = False + self.logs.write_log_in_file("error", + f"dart-runner not available. No bagging for {version_data['id']} version {version_data['version']}", + True) + if self.system_config['continue-on-error'] == "False": + self.logs.write_log_in_file("info", "Aborting execution.", True, True) + else: # if download process has any errors then delete complete folder self.logs.write_log_in_file("info", "Download process had an error so complete folder is being deleted.", True) @@ -1076,23 +1087,32 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold success = False else: if check_files or copy_files: - if success: - # call post process script function for each matched item. - value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process) - if (value_post_process != 0): - self.logs.write_log_in_file("error", - f"{version_data['id']} version {version_data['version']} - Post-processing script failed.", + if self.check_dart: + if success: + # call post process script function for each matched item. + value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process) + if (value_post_process != 0): + self.logs.write_log_in_file("error", + f"{version_data['id']} version {version_data['version']} - Post-processing script failed.", + True) + success = False + else: + success = True + else: + self.logs.write_log_in_file("info", + f"No further processing for {version_data['id']} version " + + f"{version_data['version']} due to errors.", True) success = False - else: - success = True else: - self.logs.write_log_in_file("info", - f"No further processing for {version_data['id']} version {version_data['version']} due to errors.", - True) success = False + self.logs.write_log_in_file("error", + f"dart-runner not available. No bagging for {version_data['id']} version {version_data['version']}", + True) + if self.system_config['continue-on-error'] == "False": + self.logs.write_log_in_file("info", "Aborting execution.", True, True) else: - self.logs.write_log_in_file("error", "Unexpected condidion in final processing. No further actions taken.", True) + self.logs.write_log_in_file("error", "Unexpected condition in final processing. No further actions taken.", True) success = False return success @@ -1237,10 +1257,18 @@ def process_articles(self, articles): else: self.logs.write_log_in_file("error", "Pre-processing script failed. Running post-processing script.", True) # call post process script function for each matched item. - value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process) - if (value_post_process != 0): - self.logs.write_log_in_file("error", f"{version_data['id']} version {version_data['version']} - " - + "Post-processing script failed.", True) + if self.check_dart: + value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process) + if (value_post_process != 0): + self.logs.write_log_in_file("error", f"{version_data['id']} version {version_data['version']} - " + + "Post-processing script failed.", True) + else: + self.logs.write_log_in_file("error", + f"dart-runner not available. No bagging for {version_data['id']} " + + f"version {version_data['version']}", + True) + if self.system_config['continue-on-error'] == "False": + self.logs.write_log_in_file("info", "Aborting execution.", True, True) return processed_count, self.skipped_items_counts_dict['ap_trust_preserved_versions'], \ self.skipped_items_counts_dict['wasabi_preserved_versions'], len(self.skipped_items_counts_dict['articles_with_processing_error']), \ self.skipped_items_counts_dict['articles_versions_with_processing_error'] diff --git a/figshare/Collection.py b/figshare/Collection.py index fc4dc44..100244a 100644 --- a/figshare/Collection.py +++ b/figshare/Collection.py @@ -8,6 +8,7 @@ from figshare.Integration import Integration from figshare.Utils import standardize_api_result, sorter_api_result, get_preserved_version_hash_and_size, format_version, metadata_to_hash from figshare.Utils import compare_hash, check_wasabi, check_local_path, get_folder_name_in_local_storage, upload_to_remote, stringify_metadata +from figshare.Utils import inspect_dart class Collection: @@ -42,6 +43,7 @@ def __init__(self, config, log, ids): 'already_preserved_versions': 0, 'wasabi_preserved_versions': 0, 'ap_trust_preserved_versions': 0 } + self.check_dart = inspect_dart() self.processor = Integration(self.config_obj, self.logs) """ @@ -370,11 +372,18 @@ def process_collections(self, collections): self.__save_json_in_metadata(collection, version, folder_name) collection_preservation_path = self.ingest_staging_storage + \ os.path.basename(os.path.dirname(os.path.dirname(folder_name))) - value_post_process = self.processor.post_process_script_function("Collection", collection_preservation_path) - if (value_post_process != 0): - self.logs.write_log_in_file("error", f"collection {collection} - post-processing script failed.", True) + if self.check_dart: + value_post_process = self.processor.post_process_script_function("Collection", collection_preservation_path) + if (value_post_process != 0): + self.logs.write_log_in_file("error", f"collection {collection} - post-processing script failed.", True) + else: + processed_count += 1 else: - processed_count += 1 + self.logs.write_log_in_file("error", + f"dart-runner not available. No bagging for collection {collection}", + True) + if self.system_config['continue-on-error'] == "False": + self.logs.write_log_in_file("info", "Aborting execution.", True, True) else: self.logs.write_log_in_file("info", "*Dry Run* File download and post-processing with " + f"{self.system_config['post_process_script_command']} skipped.", True) diff --git a/figshare/Utils.py b/figshare/Utils.py index c2dad8b..29cbe86 100644 --- a/figshare/Utils.py +++ b/figshare/Utils.py @@ -2,11 +2,46 @@ import json import os import re +import tempfile +import configparser from typing import Any from time import sleep -import tempfile +from subprocess import Popen, PIPE from bagger.wasabi import Wasabi -import configparser + + +def inspect_dart() -> Any: + """ + Checks if dart-runner executable exists and it's usable + + :return: Return True if dart-runner executable exists and it's usable + :rtype: Any + """ + config = configparser.ConfigParser() + config.read('bagger/config/default.toml') + default_config = config['Defaults'] + dart_cmd = default_config['dart_command'].replace('\"', '') + + try: + dart_cmd = f'{dart_cmd} --version' + dart = Popen(dart_cmd, + shell=True, + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + text=True, + close_fds=True + ) + dart_version, _ = dart.communicate() + dart_version_re = re.compile("v[0-9]*.") + dart_major_version = re.findall(dart_version_re, dart_version)[0].replace('v', '').replace('.', '') + if int(dart_major_version) < 1: + return False + return True + except FileNotFoundError: + return None + except PermissionError: + return False def standardize_api_result(api_result) -> dict: