lochness.xnat to use XNATpy module instead of Yaxil

lochness/__init__.py

@@ -100,10 +100,15 @@ def initialize_metadata(Lochness, args,
         elif 'redcap' in args.input_sources:
             id_fieldname = Lochness['redcap_id_colname']
             consent_fieldname = Lochness['redcap_consent_colname']
-            REDCap.initialize_metadata(
-                    Lochness, study_name, id_fieldname, consent_fieldname,
-                    multiple_site_in_a_repo, upenn_redcap)
-
+            try:
+                REDCap.initialize_metadata(
+                        Lochness, study_name, id_fieldname, consent_fieldname,
+                        multiple_site_in_a_repo, upenn_redcap)
+            except lochness.redcap.REDCapError as e:
+                logger.debug('REDCap not accessible')
+                logger.debug(e)
+                print('REDCap not accessible')
+                continue
         else:
             pass
 

lochness/box/__init__.py

@@ -468,8 +468,10 @@ def sync_module(Lochness: 'lochness.config',
                                          enterprise_id)
         # error get_access_token may occur when your box app is not authorized
         except TokenAccessError as err:
+            logger.debug('Refreshing token')
             refresh_token_path = Path(Lochness['keyring_file']).parent / \
                     '.refresh_token'
+            logger.debug(refresh_token_path)
             api_token = refresh_access_token(refresh_token_path,
                                              client_id,
                                              client_secret)
@@ -549,8 +551,8 @@ def sync_module(Lochness: 'lochness.config',
                 for root, dirs, files in walk_from_folder_object(
                         bx_head, datatype_obj):
                     for box_file_object in files:
-                        logger.info(f'Found a file matching the pattern: '
-                                    f'{box_file_object}')
+                        logger.debug(f'Found a file matching the pattern: '
+                                     f'{box_file_object}')
                         bx_tail = join(basename(root), box_file_object.name)
                         product = _find_product(bx_tail,
                                                 product,
@@ -559,6 +561,15 @@ def sync_module(Lochness: 'lochness.config',
                         if not product:
                             continue
 
+                        # ignore mp3 or mp4 files in non-interviews datatypes
+                        if datatype != 'interviews':
+                            if box_file_object.name.endswith('mp3') or \
+                                    box_file_object.name.endswith('mp4'):
+                                logger.warning('mp3 or mp4 detected in '
+                                    f'non-interviews datatype in {root}')
+                                continue
+
+
                         protect = product.get('protect', True)
 
                         # GENERAL / STUDY or PROTECTED / STUDY

lochness/cleaner/__init__.py

@@ -167,7 +167,12 @@ def is_transferred_and_removed(Lochness,
         bool: True if removed previously
     '''
     if removed_df_loc is None:
-        removed_df_loc = Path(Lochness['removed_df_loc'])
+        try:
+            removed_df_loc = Path(Lochness['removed_df_loc'])
+        except KeyError:
+            removed_df_loc = 'not_defined'
+
+
 
     if Path(removed_df_loc).is_file():
         df_removed = pd.read_csv(removed_df_loc, index_col=0)

lochness/mindlamp/__init__.py

@@ -181,9 +181,14 @@ def sync(Lochness: 'lochness.config',
 
             # pull data from mindlamp
             begin = time.time()
-            data_dict = function_to_execute(LAMP, subject_id,
-                                            from_ts=time_utc_00_ts,
-                                            to_ts=time_utc_24_ts)
+            try:
+                data_dict = function_to_execute(LAMP, subject_id,
+                                                from_ts=time_utc_00_ts,
+                                                to_ts=time_utc_24_ts)
+            except Exception as e:
+                print(e)
+                continue
+
             end = time.time()
             logger.debug(
                 f'Mindlamp {subject_id} {date_str} {data_name} data pull'

lochness/xnat/__init__.py

@@ -1,9 +1,11 @@
 import os
 import yaml
 import uuid
+import xnat
 import yaxil
-import lochness
+import shutil
 import logging
+import lochness
 import tempfile as tf
 import collections as col
 from pathlib import Path
@@ -17,6 +19,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 @net.retry(max_attempts=5)
 def sync_old(Lochness, subject, dry=False):
     logger.debug('exploring {0}/{1}'.format(subject.study, subject.id))
@@ -30,7 +33,7 @@ def sync_old(Lochness, subject, dry=False):
         upper case IDs if the data for one ID do not exist, experiments(auth,
         xnat_uid) returns nothing preventing the execution of inner loop
         '''
-        _xnat_uids= xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids]
+        _xnat_uids = xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids]
         for xnat_uid in _xnat_uids:
             for experiment in experiments(auth, xnat_uid):
                 logger.info(experiment)
@@ -57,9 +60,9 @@ def sync_old(Lochness, subject, dry=False):
                                   'it does not match the data on XNAT.' \
                                   'Please check MRI data saved in {dst} and ' \
                                   'compared to the XNAT data.'
-                            
+
                         lochness.notify(Lochness, message, study=subject.study)
-                        #lochness.backup(dst)
+                        # lochness.backup(dst)
                         continue
 
                 message = 'downloading {PROJECT}/{LABEL} to {FOLDER}'
@@ -80,7 +83,6 @@ def sync_old(Lochness, subject, dry=False):
                     os.rename(tmpdir, dst)
 
 
-
 @net.retry(max_attempts=5)
 def sync(Lochness, subject, dry=False):
     logger.debug('exploring {0}/{1}'.format(subject.study, subject.id))
@@ -94,7 +96,7 @@ def sync(Lochness, subject, dry=False):
         upper case IDs if the data for one ID do not exist, experiments(auth,
         xnat_uid) returns nothing preventing the execution of inner loop
         '''
-        _xnat_uids= xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids]
+        _xnat_uids = xnat_uids + [(x[0], x[1].lower()) for x in xnat_uids]
         for xnat_uid in _xnat_uids:
             for experiment in experiments(auth, xnat_uid):
                 logger.info(experiment)
@@ -136,6 +138,98 @@ def sync(Lochness, subject, dry=False):
                         fp.write(archieved_date)
 
 
+class NoMatchingSubjectXNAT(Exception):
+    pass
+
+
+def set_TMPDIR(Lochness):
+    try:
+        tmp_dir = Lochness['tmp_dir']
+    except KeyError:
+        tmp_dir = os.environ.get('TMPDIR')
+        if tmp_dir is None:
+            tmp_dir = tf.gettempdir()
+
+    # Set the TMPDIR environment variable to a default value
+    os.environ['TMPDIR'] = tmp_dir
+
+    return tmp_dir
+
+
+@net.retry(max_attempts=5)
+def sync_xnatpy(Lochness, subject, dry=False):
+    """A new sync function with XNATpy"""
+    logger.debug('exploring {0}/{1}'.format(subject.study, subject.id))
+
+    tmp_dir = set_TMPDIR(Lochness)
+
+    # remove xnatpy tmp files
+    for tmp_file in Path(tmp_dir).glob('*generated_xnat.py'):
+        os.remove(tmp_file)
+
+    for tmp_file in Path(tmp_dir).glob('tmp_xnat*'):
+        os.remove(tmp_file)
+
+    for alias, xnat_uids in iter(subject.xnat.items()):
+        keyring = Lochness['keyring'][alias]
+        session = xnat.connect(keyring['URL'],
+                               keyring['USERNAME'],
+                               keyring['PASSWORD'])
+        '''
+        pull XNAT data agnostic to the case of subject IDs loop over lower and
+        upper case IDs if the data for one ID do not exist, experiments(auth,
+        xnat_uid) returns nothing preventing the execution of inner loop
+        '''
+        site = xnat_uids[0][1][:2]
+        _xnat_uids = [(x[0], x[1].upper()) for x in xnat_uids] + \
+                     [(x[0], x[1].lower()) for x in xnat_uids]
+
+        for ses in session.projects:
+            if 'pronet' in ses.lower():
+                if site in ses:
+                    break
+
+        project = session.projects[ses]
+        xnat_subject = ''
+        for xnat_uid in _xnat_uids:
+            try:
+                xnat_subject = project.subjects[xnat_uid[1]]
+                break
+            except KeyError as e:
+                continue
+
+        if xnat_subject == '':
+            msg = 'There is no matching subject in XNAT database: '
+            msg += f"{' / '.join([x[1] for x in _xnat_uids])}"
+            logger.debug(msg)
+            continue
+
+        for exp_id, experiment in xnat_subject.experiments.items():
+            dirname = tree.get('mri',
+                               subject.protected_folder,
+                               processed=False,
+                               BIDS=Lochness['BIDS'])
+            dst = os.path.join(dirname, f'{experiment.label.upper()}.zip')
+
+            if os.path.exists(dst):
+                continue
+
+            message = 'downloading {PROJECT}/{LABEL} to {FOLDER}'
+            logger.debug(message.format(PROJECT=experiment.project,
+                                        LABEL=experiment.label,
+                                        FOLDER=dst))
+
+            if not dry:
+                with tf.NamedTemporaryFile(dir=tmp_dir,
+                                           prefix='tmp_xnat_',
+                                           delete=False) as tmpfilename:
+                    experiment.download(tmpfilename.name)
+                    shutil.move(tmpfilename.name, dst)
+                    os.chmod(dst, 0o0755)
+
+
+
+
 def check_consistency(d, experiment):
     '''check that local data still matches data in xnat'''
     experiment_file = os.path.join(d, '.experiment')

scripts/lochness_create_template.py

@@ -438,7 +438,7 @@ def create_config_template(config_loc: Path, args: object) -> None:
                    - product: transcripts
                      data_dir: {study}_Interviews/transcripts/Approved
                      out_dir: transcripts
-                     pattern: '*.txt
+                     pattern: '*.txt'
              '''
 
             config_example += line_to_add

scripts/sync.py

@@ -123,7 +123,7 @@ def main():
     # replace args.source with corresponding lochness modules
     if args.source:
         args.input_sources = args.source
-        args.source = list(set([SOURCES[x] for x in args.source]))
+        args.source = [SOURCES[x] for x in args.source]
     else:
         args.input_sources = []
 
@@ -201,7 +201,7 @@ def main():
 
         # email
         if args.daily_summary:
-            check_source(Lochness)
+            send_out_daily_updates(Lochness)
             if args.check_source:
                 check_source(Lochness)
 
@@ -241,8 +241,13 @@ def do(args, Lochness):
             for Module in args.hdd:
                 lochness.attempt(Module.sync, Lochness, subject, dry=args.dry)
         else:
-            for Module in args.source:
-                lochness.attempt(Module.sync, Lochness, subject, dry=args.dry)
+            for source, Module in zip(args.input_sources, args.source):
+                if source == 'xnat':
+                    lochness.attempt(Module.sync_xnatpy, Lochness,
+                                     subject, dry=args.dry)
+                else:
+                    lochness.attempt(Module.sync, Lochness,
+                                     subject, dry=args.dry)
         n += 1
 
     # anonymize PII

tests/lochness_test/box/test_box.py

@@ -532,3 +532,33 @@ def test_box_sync_Phil(args_and_Lochness):
         sync(Lochness, subject, dry=False)
 
     # show_tree_then_delete('tmp_lochness')
+
+
+def test_box_sync_issue_Aug():
+    '''Test pulling data keeping the structure of the subdirectories in box'''
+    args = Args('tmp_lochness')
+    args.studies = ['PronetKC', 'PronetKC', 'PronetPA']
+    args.sources = ['box']
+    create_lochness_template(args)
+    keyring = KeyringAndEncrypt(args.outdir)
+        # {'subject_id': 'PV06517',
+        # 'source_id': 'PV06517'},
+    information_to_add_to_metadata = {'box':
+            [
+        {'subject_id': 'PA14348',
+        'source_id': 'PA14348'}
+        ]
+        }
+
+    for study in args.studies:
+        keyring.update_for_box(study)
+
+        # update box metadata
+        initialize_metadata_test('tmp_lochness/PHOENIX', study,
+                                 information_to_add_to_metadata)
+
+    Lochness = config_load_test('tmp_lochness/config.yml', '')
+
+    for subject in lochness.read_phoenix_metadata(Lochness):
+        sync(Lochness, subject, dry=False)
+

tests/lochness_test/xnat/test_new_sync.py

@@ -0,0 +1,30 @@
+import os
+from lochness.xnat import sync_xnatpy
+from lochness.config import load
+import pytest
+
+
+class Subject():
+    def __init__(self, subject):
+        self.xnat = {f'xnat.Pronet{subject[:2]}': subject}
+        self.protected_folder = subject
+        self.study ='study'
+        self.id ='id'
+
+
+def test_sync_xnatpy():
+    Lochness = load('/mnt/ProNET/Lochness/config.yml')
+    subject = Subject('SL00005')
+    dry = False
+    sync_xnatpy(Lochness, subject, dry=dry)
+
+    print(os.popen('tree').read())
+    print(os.popen('rm -rf processed raw'))
+
+
+def test_wrong_id():
+    Lochness = load('/mnt/ProNET/Lochness/config.yml')
+    subject = Subject('SL00000')
+    dry = False
+    with pytest.raises(Exception):
+        sync_xnatpy(Lochness, subject, dry=dry)

tests/test_lochness.py

@@ -203,6 +203,7 @@ def config_load_test(f: 'location', archive_base=None):
     # box file pattern strings from the config to string template
     # regardless of the selected study in the args
     if 'box' in Lochness:
+        print(Lochness['box'])
         for _, study_dict in Lochness['box'].items():
             for _, modality_values in study_dict['file_patterns'].items():
                 for modality_dict in modality_values: