From 60581fe8d1c6916e6e02f556f92714e63f39c255 Mon Sep 17 00:00:00 2001 From: kcho Date: Thu, 24 Aug 2023 16:24:27 +0000 Subject: [PATCH 01/17] feat: skip downloading mp3 or mp4 files for non-interview datatypes --- lochness/box/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lochness/box/__init__.py b/lochness/box/__init__.py index 49820ec6..2175bc59 100644 --- a/lochness/box/__init__.py +++ b/lochness/box/__init__.py @@ -559,6 +559,15 @@ def sync_module(Lochness: 'lochness.config', if not product: continue + # ignore mp3 or mp4 files in non-interviews datatypes + if datatype != 'interviews': + if box_file_object.name.endswith('mp3') or \ + box_file_object.name.endswith('mp4'): + logger.warning('mp3 or mp4 detected in ' + f'non-interviews datatype in {root}') + continue + + protect = product.get('protect', True) # GENERAL / STUDY or PROTECTED / STUDY From 32f05c5a6ac57b74de368ea6ed0e0eff2710b953 Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 1 Sep 2023 15:43:11 -0400 Subject: [PATCH 02/17] test: test init for lochness.xnat.new_sync --- tests/lochness_test/xnat/test_new_sync.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/lochness_test/xnat/test_new_sync.py diff --git a/tests/lochness_test/xnat/test_new_sync.py b/tests/lochness_test/xnat/test_new_sync.py new file mode 100644 index 00000000..346c2ae9 --- /dev/null +++ b/tests/lochness_test/xnat/test_new_sync.py @@ -0,0 +1,18 @@ +from lochness.xnat import sync_new + + +class Subject(): + def __init__(self): + self.xnat = {'xnat.PronetSL': 'SL00005'} + self.protected_folder = 'test_protected_folder' + + +def test_sync_new(): + Lochness = {'keyring': { + 'URL': 'XXXX', + 'USERNAME': 'XXXX', + 'PASSWORD': 'XXXX'} + } + subject = Subject() + dry = False + sync_new(Lochness, subject, dry=dry) From 31498a37392a2a24610ec9ff443ff86ace95408a Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 1 Sep 2023 15:44:05 -0400 Subject: [PATCH 03/17] feat: a new sync_new using XNATpy added --- lochness/xnat/__init__.py | 75 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/lochness/xnat/__init__.py b/lochness/xnat/__init__.py index 7783db94..90bf9edf 100644 --- a/lochness/xnat/__init__.py +++ b/lochness/xnat/__init__.py @@ -1,6 +1,7 @@ import os import yaml import uuid +import xnat import yaxil import lochness import logging @@ -17,6 +18,7 @@ logger = logging.getLogger(__name__) + @net.retry(max_attempts=5) def sync_old(Lochness, subject, dry=False): logger.debug('exploring {0}/{1}'.format(subject.study, subject.id)) @@ -30,7 +32,7 @@ def sync_old(Lochness, subject, dry=False): upper case IDs if the data for one ID do not exist, experiments(auth, xnat_uid) returns nothing preventing the execution of inner loop ''' - _xnat_uids= xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids] + _xnat_uids = xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids] for xnat_uid in _xnat_uids: for experiment in experiments(auth, xnat_uid): logger.info(experiment) @@ -57,9 +59,9 @@ def sync_old(Lochness, subject, dry=False): 'it does not match the data on XNAT.' \ 'Please check MRI data saved in {dst} and ' \ 'compared to the XNAT data.' - + lochness.notify(Lochness, message, study=subject.study) - #lochness.backup(dst) + # lochness.backup(dst) continue message = 'downloading {PROJECT}/{LABEL} to {FOLDER}' @@ -80,7 +82,6 @@ def sync_old(Lochness, subject, dry=False): os.rename(tmpdir, dst) - @net.retry(max_attempts=5) def sync(Lochness, subject, dry=False): logger.debug('exploring {0}/{1}'.format(subject.study, subject.id)) @@ -94,7 +95,7 @@ def sync(Lochness, subject, dry=False): upper case IDs if the data for one ID do not exist, experiments(auth, xnat_uid) returns nothing preventing the execution of inner loop ''' - _xnat_uids= xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids] + _xnat_uids = xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids] for xnat_uid in _xnat_uids: for experiment in experiments(auth, xnat_uid): logger.info(experiment) @@ -136,6 +137,70 @@ def sync(Lochness, subject, dry=False): fp.write(archieved_date) +class NoMatchingSubjectXNAT(Exception): + pass + + +@net.retry(max_attempts=5) +def sync_new(Lochness, subject, dry=False): + """A new sync function with XNATpy""" + logger.debug('exploring {0}/{1}'.format(subject.study, subject.id)) + + for alias, xnat_uids in iter(subject.xnat.items()): + Keyring = Lochness['keyring'][alias] + session = xnat.connect(Keyring['URL'], + username=Keyring['USERNAME'], + password=Keyring['PASSWORD']) + ''' + pull XNAT data agnostic to the case of subject IDs loop over lower and + upper case IDs if the data for one ID do not exist, experiments(auth, + xnat_uid) returns nothing preventing the execution of inner loop + ''' + _xnat_uids = xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids] + site = xnat_uids[:2] + for ses in session.projects: + if 'pronet' in ses.lower(): + if site in ses: + break + + project = session.projects[ses] + xnat_subject = '' + for xnat_uid in _xnat_uids: + try: + xnat_subject = project.subjects[xnat_uid] + except KeyError: + continue + + if xnat_subject == '': + logger.warn('There is no matching subject in XNAT database: ' + f'{xnat_uid}') + raise NoMatchingSubjectXNAT('There is no matching subject in XNAT') + + for exp_id, experiment in xnat_subject.experiments.items(): + dirname = tree.get('mri', + subject.protected_folder, + processed=False, + BIDS=Lochness['BIDS']) + dst = os.path.join(dirname, f'{experiment.label.upper()}.zip') + + logger.info(experiment) + + # do not re-download already transferred & removed data + if is_transferred_and_removed(Lochness, dst): + continue + + if os.path.exists(dst): + continue + + message = 'downloading {PROJECT}/{LABEL} to {FOLDER}' + logger.debug(message.format(PROJECT=experiment.project, + LABEL=experiment.label, + FOLDER=dst)) + + if not dry: + experiment.download(experiment) + + def check_consistency(d, experiment): '''check that local data still matches data in xnat''' experiment_file = os.path.join(d, '.experiment') From 965bb3027531a93b457666431a6699cff5a89a20 Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 1 Sep 2023 22:21:45 +0000 Subject: [PATCH 04/17] test: working version --- tests/lochness_test/xnat/test_new_sync.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/lochness_test/xnat/test_new_sync.py b/tests/lochness_test/xnat/test_new_sync.py index 346c2ae9..e748a743 100644 --- a/tests/lochness_test/xnat/test_new_sync.py +++ b/tests/lochness_test/xnat/test_new_sync.py @@ -1,18 +1,22 @@ +import os from lochness.xnat import sync_new +from lochness.config import load class Subject(): - def __init__(self): - self.xnat = {'xnat.PronetSL': 'SL00005'} - self.protected_folder = 'test_protected_folder' + def __init__(self, subject): + self.xnat = {f'xnat.Pronet{subject[:2]}': subject} + self.protected_folder = subject + self.study ='study' + self.id ='id' def test_sync_new(): - Lochness = {'keyring': { - 'URL': 'XXXX', - 'USERNAME': 'XXXX', - 'PASSWORD': 'XXXX'} - } - subject = Subject() + Lochness = load('/mnt/ProNET/Lochness/config.yml') + subject = Subject('SL00005') dry = False sync_new(Lochness, subject, dry=dry) + + print(os.popen('tree').read()) + print(os.popen('rm -rf processed raw')) + From 8855e7d06b60ff1690131e0a003c5d70b9b1d86b Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 1 Sep 2023 22:22:22 +0000 Subject: [PATCH 05/17] feat: sync_new working version - tested --- lochness/xnat/__init__.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lochness/xnat/__init__.py b/lochness/xnat/__init__.py index 90bf9edf..0fb0257b 100644 --- a/lochness/xnat/__init__.py +++ b/lochness/xnat/__init__.py @@ -3,8 +3,9 @@ import uuid import xnat import yaxil -import lochness +import shutil import logging +import lochness import tempfile as tf import collections as col from pathlib import Path @@ -149,14 +150,14 @@ def sync_new(Lochness, subject, dry=False): for alias, xnat_uids in iter(subject.xnat.items()): Keyring = Lochness['keyring'][alias] session = xnat.connect(Keyring['URL'], - username=Keyring['USERNAME'], - password=Keyring['PASSWORD']) + Keyring['USERNAME'], + Keyring['PASSWORD']) ''' pull XNAT data agnostic to the case of subject IDs loop over lower and upper case IDs if the data for one ID do not exist, experiments(auth, xnat_uid) returns nothing preventing the execution of inner loop ''' - _xnat_uids = xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids] + _xnat_uids = [xnat_uids.upper(), xnat_uids.lower()] site = xnat_uids[:2] for ses in session.projects: if 'pronet' in ses.lower(): @@ -184,11 +185,6 @@ def sync_new(Lochness, subject, dry=False): dst = os.path.join(dirname, f'{experiment.label.upper()}.zip') logger.info(experiment) - - # do not re-download already transferred & removed data - if is_transferred_and_removed(Lochness, dst): - continue - if os.path.exists(dst): continue @@ -198,7 +194,10 @@ def sync_new(Lochness, subject, dry=False): FOLDER=dst)) if not dry: - experiment.download(experiment) + with tf.NamedTemporaryFile(delete=False) as tmpfilename: + experiment.download(tmpfilename.name) + shutil.move(tmpfilename.name, dst) + os.chmod(dst, 0o0755) def check_consistency(d, experiment): From 1475c9f7e14973a0ef30102a2cdd04e343f21631 Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 1 Sep 2023 22:28:17 +0000 Subject: [PATCH 06/17] minor: check IDs in small letters first --- lochness/xnat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lochness/xnat/__init__.py b/lochness/xnat/__init__.py index 0fb0257b..a08c528c 100644 --- a/lochness/xnat/__init__.py +++ b/lochness/xnat/__init__.py @@ -157,7 +157,7 @@ def sync_new(Lochness, subject, dry=False): upper case IDs if the data for one ID do not exist, experiments(auth, xnat_uid) returns nothing preventing the execution of inner loop ''' - _xnat_uids = [xnat_uids.upper(), xnat_uids.lower()] + _xnat_uids = [xnat_uids.lower(), xnat_uids.upper()] site = xnat_uids[:2] for ses in session.projects: if 'pronet' in ses.lower(): From a34f68a8d365764e7b1e530f07f69c61f041929b Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 1 Sep 2023 22:28:39 +0000 Subject: [PATCH 07/17] minor: break the loop once matched / PEP8 --- lochness/xnat/__init__.py | 7 ++++--- tests/lochness_test/xnat/test_new_sync.py | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lochness/xnat/__init__.py b/lochness/xnat/__init__.py index a08c528c..d55586e1 100644 --- a/lochness/xnat/__init__.py +++ b/lochness/xnat/__init__.py @@ -169,13 +169,14 @@ def sync_new(Lochness, subject, dry=False): for xnat_uid in _xnat_uids: try: xnat_subject = project.subjects[xnat_uid] + break except KeyError: continue if xnat_subject == '': - logger.warn('There is no matching subject in XNAT database: ' - f'{xnat_uid}') - raise NoMatchingSubjectXNAT('There is no matching subject in XNAT') + msg = f'There is no matching subject in XNAT database: {xnat_uid}' + logger.warn(msg) + raise NoMatchingSubjectXNAT('No matching subject in XNAT') for exp_id, experiment in xnat_subject.experiments.items(): dirname = tree.get('mri', diff --git a/tests/lochness_test/xnat/test_new_sync.py b/tests/lochness_test/xnat/test_new_sync.py index e748a743..a94d24b1 100644 --- a/tests/lochness_test/xnat/test_new_sync.py +++ b/tests/lochness_test/xnat/test_new_sync.py @@ -20,3 +20,12 @@ def test_sync_new(): print(os.popen('tree').read()) print(os.popen('rm -rf processed raw')) + +def test_wrong_id(): + Lochness = load('/mnt/ProNET/Lochness/config.yml') + subject = Subject('SL00000') + dry = False + sync_new(Lochness, subject, dry=dry) + + print(os.popen('tree').read()) + print(os.popen('rm -rf processed raw')) From fcadea3a6ea352a67f5b78945205322c9764e802 Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 1 Sep 2023 22:34:37 +0000 Subject: [PATCH 08/17] feat: name change to sync_new to sync_xnatpy --- lochness/xnat/__init__.py | 10 +++++----- tests/lochness_test/xnat/test_new_sync.py | 13 ++++++------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/lochness/xnat/__init__.py b/lochness/xnat/__init__.py index d55586e1..cdc9b940 100644 --- a/lochness/xnat/__init__.py +++ b/lochness/xnat/__init__.py @@ -143,15 +143,15 @@ class NoMatchingSubjectXNAT(Exception): @net.retry(max_attempts=5) -def sync_new(Lochness, subject, dry=False): +def sync_xnatpy(Lochness, subject, dry=False): """A new sync function with XNATpy""" logger.debug('exploring {0}/{1}'.format(subject.study, subject.id)) for alias, xnat_uids in iter(subject.xnat.items()): - Keyring = Lochness['keyring'][alias] - session = xnat.connect(Keyring['URL'], - Keyring['USERNAME'], - Keyring['PASSWORD']) + keyring = Lochness['keyring'][alias] + session = xnat.connect(keyring['URL'], + keyring['USERNAME'], + keyring['PASSWORD']) ''' pull XNAT data agnostic to the case of subject IDs loop over lower and upper case IDs if the data for one ID do not exist, experiments(auth, diff --git a/tests/lochness_test/xnat/test_new_sync.py b/tests/lochness_test/xnat/test_new_sync.py index a94d24b1..ea715914 100644 --- a/tests/lochness_test/xnat/test_new_sync.py +++ b/tests/lochness_test/xnat/test_new_sync.py @@ -1,6 +1,7 @@ import os -from lochness.xnat import sync_new +from lochness.xnat import sync_xnatpy from lochness.config import load +import pytest class Subject(): @@ -11,11 +12,11 @@ def __init__(self, subject): self.id ='id' -def test_sync_new(): +def test_sync_xnatpy(): Lochness = load('/mnt/ProNET/Lochness/config.yml') subject = Subject('SL00005') dry = False - sync_new(Lochness, subject, dry=dry) + sync_xnatpy(Lochness, subject, dry=dry) print(os.popen('tree').read()) print(os.popen('rm -rf processed raw')) @@ -25,7 +26,5 @@ def test_wrong_id(): Lochness = load('/mnt/ProNET/Lochness/config.yml') subject = Subject('SL00000') dry = False - sync_new(Lochness, subject, dry=dry) - - print(os.popen('tree').read()) - print(os.popen('rm -rf processed raw')) + with pytest.raises(Exception): + sync_xnatpy(Lochness, subject, dry=dry) From 0c47d8111f9dbf50f54d38e80796750b4b5d7af1 Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 1 Sep 2023 22:36:53 +0000 Subject: [PATCH 09/17] feat: replace sync.py to use xnat.sync_xnatpy instead of xnat.sync --- scripts/sync.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/sync.py b/scripts/sync.py index a8eb9a0c..1e178cbb 100755 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -241,8 +241,13 @@ def do(args, Lochness): for Module in args.hdd: lochness.attempt(Module.sync, Lochness, subject, dry=args.dry) else: - for Module in args.source: - lochness.attempt(Module.sync, Lochness, subject, dry=args.dry) + for source, Module in zip(args.input_sources, args.source): + if source == 'xnat': + lochness.attempt(Module.sync_xnatpy, Lochness, + subject, dry=args.dry) + else: + lochness.attempt(Module.sync, Lochness, + subject, dry=args.dry) n += 1 # anonymize PII From e3f2969cc819c28fc4c61c82a30216c7135c9ecc Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 13 Sep 2023 19:22:23 +0000 Subject: [PATCH 10/17] feat: exception added when 'removed_df_loc' doesn't exist in config --- lochness/cleaner/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lochness/cleaner/__init__.py b/lochness/cleaner/__init__.py index b4a4ff8f..a9ec5e82 100644 --- a/lochness/cleaner/__init__.py +++ b/lochness/cleaner/__init__.py @@ -167,7 +167,12 @@ def is_transferred_and_removed(Lochness, bool: True if removed previously ''' if removed_df_loc is None: - removed_df_loc = Path(Lochness['removed_df_loc']) + try: + removed_df_loc = Path(Lochness['removed_df_loc']) + except KeyError: + removed_df_loc = 'not_defined' + + if Path(removed_df_loc).is_file(): df_removed = pd.read_csv(removed_df_loc, index_col=0) From aa2bd20869fdb4e120daf968c2e25201653196fe Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 13 Sep 2023 19:24:09 +0000 Subject: [PATCH 11/17] fix: get correct AMP-SCZ IDs in small letters --- lochness/xnat/__init__.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lochness/xnat/__init__.py b/lochness/xnat/__init__.py index cdc9b940..071d450d 100644 --- a/lochness/xnat/__init__.py +++ b/lochness/xnat/__init__.py @@ -157,8 +157,10 @@ def sync_xnatpy(Lochness, subject, dry=False): upper case IDs if the data for one ID do not exist, experiments(auth, xnat_uid) returns nothing preventing the execution of inner loop ''' - _xnat_uids = [xnat_uids.lower(), xnat_uids.upper()] - site = xnat_uids[:2] + site = xnat_uids[0][1][:2] + _xnat_uids = [(x[0], x[1].upper()) for x in xnat_uids] + \ + [(x[0], x[1].lower()) for x in xnat_uids] + for ses in session.projects: if 'pronet' in ses.lower(): if site in ses: @@ -168,15 +170,16 @@ def sync_xnatpy(Lochness, subject, dry=False): xnat_subject = '' for xnat_uid in _xnat_uids: try: - xnat_subject = project.subjects[xnat_uid] + xnat_subject = project.subjects[xnat_uid[1]] break - except KeyError: + except KeyError as e: continue if xnat_subject == '': - msg = f'There is no matching subject in XNAT database: {xnat_uid}' - logger.warn(msg) - raise NoMatchingSubjectXNAT('No matching subject in XNAT') + msg = 'There is no matching subject in XNAT database: ' + msg += f"{' / '.join([x[1] for x in _xnat_uids])}" + logger.debug(msg) + continue for exp_id, experiment in xnat_subject.experiments.items(): dirname = tree.get('mri', @@ -185,7 +188,6 @@ def sync_xnatpy(Lochness, subject, dry=False): BIDS=Lochness['BIDS']) dst = os.path.join(dirname, f'{experiment.label.upper()}.zip') - logger.info(experiment) if os.path.exists(dst): continue From 96e247c43131954ea6d1a0cc2810a827ca24626e Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 20 Sep 2023 20:40:11 +0000 Subject: [PATCH 12/17] minor: messages to box module --- lochness/box/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lochness/box/__init__.py b/lochness/box/__init__.py index 1e4cf1d5..a3b820b5 100644 --- a/lochness/box/__init__.py +++ b/lochness/box/__init__.py @@ -468,8 +468,10 @@ def sync_module(Lochness: 'lochness.config', enterprise_id) # error get_access_token may occur when your box app is not authorized except TokenAccessError as err: + logger.debug('Refreshing token') refresh_token_path = Path(Lochness['keyring_file']).parent / \ '.refresh_token' + logger.debug(refresh_token_path) api_token = refresh_access_token(refresh_token_path, client_id, client_secret) @@ -549,8 +551,8 @@ def sync_module(Lochness: 'lochness.config', for root, dirs, files in walk_from_folder_object( bx_head, datatype_obj): for box_file_object in files: - logger.info(f'Found a file matching the pattern: ' - f'{box_file_object}') + logger.debug(f'Found a file matching the pattern: ' + f'{box_file_object}') bx_tail = join(basename(root), box_file_object.name) product = _find_product(bx_tail, product, From 917a71d66d72afa2200cbe992ba923075b80b0bd Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 20 Sep 2023 20:40:46 +0000 Subject: [PATCH 13/17] feat: tmpdir to be loaded from config.yml --- lochness/xnat/__init__.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/lochness/xnat/__init__.py b/lochness/xnat/__init__.py index 071d450d..e125ef6f 100644 --- a/lochness/xnat/__init__.py +++ b/lochness/xnat/__init__.py @@ -142,11 +142,31 @@ class NoMatchingSubjectXNAT(Exception): pass +def set_TMPDIR(Lochness): + try: + tmp_dir = Lochness['tmp_dir'] + except KeyError: + tmp_dir = os.environ.get('TMPDIR') + if tmp_dir is None: + tmp_dir = tf.gettempdir() + + # Set the TMPDIR environment variable to a default value + os.environ['TMPDIR'] = tmp_dir + + return tmp_dir + + @net.retry(max_attempts=5) def sync_xnatpy(Lochness, subject, dry=False): """A new sync function with XNATpy""" logger.debug('exploring {0}/{1}'.format(subject.study, subject.id)) + tmp_dir = set_TMPDIR(Lochness) + + # remove xnatpy tmp files + for tmp_file in Path(tmp_dir).glob('*generated_xnat.py'): + os.remove(tmp_file) + for alias, xnat_uids in iter(subject.xnat.items()): keyring = Lochness['keyring'][alias] session = xnat.connect(keyring['URL'], @@ -197,12 +217,15 @@ def sync_xnatpy(Lochness, subject, dry=False): FOLDER=dst)) if not dry: - with tf.NamedTemporaryFile(delete=False) as tmpfilename: + with tf.NamedTemporaryFile(dir=tmp_dir, + delete=False) as tmpfilename: experiment.download(tmpfilename.name) shutil.move(tmpfilename.name, dst) os.chmod(dst, 0o0755) + + def check_consistency(d, experiment): '''check that local data still matches data in xnat''' experiment_file = os.path.join(d, '.experiment') From 66cfc45d2021947026a35ff301edd75aa943f946 Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 20 Sep 2023 20:41:20 +0000 Subject: [PATCH 14/17] minor: minor updates --- scripts/lochness_create_template.py | 2 +- scripts/sync.py | 2 +- tests/lochness_test/box/test_box.py | 30 +++++++++++++++++++++++++++++ tests/test_lochness.py | 1 + 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/scripts/lochness_create_template.py b/scripts/lochness_create_template.py index 6a430947..e6df3b7f 100755 --- a/scripts/lochness_create_template.py +++ b/scripts/lochness_create_template.py @@ -438,7 +438,7 @@ def create_config_template(config_loc: Path, args: object) -> None: - product: transcripts data_dir: {study}_Interviews/transcripts/Approved out_dir: transcripts - pattern: '*.txt + pattern: '*.txt' ''' config_example += line_to_add diff --git a/scripts/sync.py b/scripts/sync.py index 1e178cbb..ac156e63 100755 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -123,7 +123,7 @@ def main(): # replace args.source with corresponding lochness modules if args.source: args.input_sources = args.source - args.source = list(set([SOURCES[x] for x in args.source])) + args.source = [SOURCES[x] for x in args.source] else: args.input_sources = [] diff --git a/tests/lochness_test/box/test_box.py b/tests/lochness_test/box/test_box.py index cf53ef44..007fa8fa 100644 --- a/tests/lochness_test/box/test_box.py +++ b/tests/lochness_test/box/test_box.py @@ -532,3 +532,33 @@ def test_box_sync_Phil(args_and_Lochness): sync(Lochness, subject, dry=False) # show_tree_then_delete('tmp_lochness') + + +def test_box_sync_issue_Aug(): + '''Test pulling data keeping the structure of the subdirectories in box''' + args = Args('tmp_lochness') + args.studies = ['PronetKC', 'PronetKC', 'PronetPA'] + args.sources = ['box'] + create_lochness_template(args) + keyring = KeyringAndEncrypt(args.outdir) + # {'subject_id': 'PV06517', + # 'source_id': 'PV06517'}, + information_to_add_to_metadata = {'box': + [ + {'subject_id': 'PA14348', + 'source_id': 'PA14348'} + ] + } + + for study in args.studies: + keyring.update_for_box(study) + + # update box metadata + initialize_metadata_test('tmp_lochness/PHOENIX', study, + information_to_add_to_metadata) + + Lochness = config_load_test('tmp_lochness/config.yml', '') + + for subject in lochness.read_phoenix_metadata(Lochness): + sync(Lochness, subject, dry=False) + diff --git a/tests/test_lochness.py b/tests/test_lochness.py index fb1a0bbd..a17a619d 100644 --- a/tests/test_lochness.py +++ b/tests/test_lochness.py @@ -203,6 +203,7 @@ def config_load_test(f: 'location', archive_base=None): # box file pattern strings from the config to string template # regardless of the selected study in the args if 'box' in Lochness: + print(Lochness['box']) for _, study_dict in Lochness['box'].items(): for _, modality_values in study_dict['file_patterns'].items(): for modality_dict in modality_values: From 886bb9c9a59e96eb5fb94937bc2c75a523b11e77 Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 20 Sep 2023 20:47:04 +0000 Subject: [PATCH 15/17] feat: control xnat tempfile --- lochness/xnat/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lochness/xnat/__init__.py b/lochness/xnat/__init__.py index e125ef6f..bfc183f9 100644 --- a/lochness/xnat/__init__.py +++ b/lochness/xnat/__init__.py @@ -167,6 +167,9 @@ def sync_xnatpy(Lochness, subject, dry=False): for tmp_file in Path(tmp_dir).glob('*generated_xnat.py'): os.remove(tmp_file) + for tmp_file in Path(tmp_dir).glob('tmp_xnat*'): + os.remove(tmp_file) + for alias, xnat_uids in iter(subject.xnat.items()): keyring = Lochness['keyring'][alias] session = xnat.connect(keyring['URL'], @@ -218,6 +221,7 @@ def sync_xnatpy(Lochness, subject, dry=False): if not dry: with tf.NamedTemporaryFile(dir=tmp_dir, + prefix='tmp_xnat_', delete=False) as tmpfilename: experiment.download(tmpfilename.name) shutil.move(tmpfilename.name, dst) From 5afbe13242f7d572a37858a070c5ddc5e30af822 Mon Sep 17 00:00:00 2001 From: kcho Date: Thu, 12 Oct 2023 10:28:22 +0000 Subject: [PATCH 16/17] feat: bypassing exception for REDCap issue for initializing metadata --- lochness/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lochness/__init__.py b/lochness/__init__.py index 2ed4b3b7..1f828ab6 100644 --- a/lochness/__init__.py +++ b/lochness/__init__.py @@ -100,10 +100,15 @@ def initialize_metadata(Lochness, args, elif 'redcap' in args.input_sources: id_fieldname = Lochness['redcap_id_colname'] consent_fieldname = Lochness['redcap_consent_colname'] - REDCap.initialize_metadata( - Lochness, study_name, id_fieldname, consent_fieldname, - multiple_site_in_a_repo, upenn_redcap) - + try: + REDCap.initialize_metadata( + Lochness, study_name, id_fieldname, consent_fieldname, + multiple_site_in_a_repo, upenn_redcap) + except lochness.redcap.REDCapError as e: + logger.debug('REDCap not accessible') + logger.debug(e) + print('REDCap not accessible') + continue else: pass From a4c03dc3d7513eda900c404be8ce4f8791cb334e Mon Sep 17 00:00:00 2001 From: kcho Date: Thu, 12 Oct 2023 10:28:49 +0000 Subject: [PATCH 17/17] fix: continue to next day when there is an error on a specific date --- lochness/mindlamp/__init__.py | 11 ++++++++--- scripts/sync.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lochness/mindlamp/__init__.py b/lochness/mindlamp/__init__.py index 329db9fd..10a269cd 100644 --- a/lochness/mindlamp/__init__.py +++ b/lochness/mindlamp/__init__.py @@ -181,9 +181,14 @@ def sync(Lochness: 'lochness.config', # pull data from mindlamp begin = time.time() - data_dict = function_to_execute(LAMP, subject_id, - from_ts=time_utc_00_ts, - to_ts=time_utc_24_ts) + try: + data_dict = function_to_execute(LAMP, subject_id, + from_ts=time_utc_00_ts, + to_ts=time_utc_24_ts) + except Exception as e: + print(e) + continue + end = time.time() logger.debug( f'Mindlamp {subject_id} {date_str} {data_name} data pull' diff --git a/scripts/sync.py b/scripts/sync.py index ac156e63..7cc29d1f 100755 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -201,7 +201,7 @@ def main(): # email if args.daily_summary: - check_source(Lochness) + send_out_daily_updates(Lochness) if args.check_source: check_source(Lochness)