Update from Prescient server to master

README.md

@@ -103,14 +103,14 @@ Running one of the commands above will create the structure below
 ├── 1_encrypt_command.sh
 ├── 2_sync_command.sh
 ├── PHOENIX
-│   ├── GENERAL
+│   ├── PROTECTED
 │   │   ├── PronetLA
 │   │   │   └── PronetLA_metadata.csv
 │   │   ├── PronetSL
 │   │   │   └── PronetSL_metadata.csv
 │   │   └── PronetWU
 │   │       └── PronetWU_metadata.csv
-│   └── PROTECTED
+│   └── GENERAL
 │       ├── PronetLA
 │       ├── PronetSL
 │       └── PronetWU
@@ -131,7 +131,7 @@ has all information, it should be encrypted. (See step 3 below)
 non-sensitive information about the server, lochness instance and different
 data sources.
 
-2. Either manually update the `PHOENIX/GENERAL/*/*_metadata.csv` or
+2. Either manually update the `PHOENIX/PROTECTED/*/*_metadata.csv` or
    amend the field names in REDCap / RPMS sources correctly for lochness to
    automatically update the metadata files.
 

lochness/__init__.py

@@ -121,17 +121,18 @@ def read_phoenix_metadata(Lochness, studies=None):
         studies = lochness.listdir(Lochness, general_folder)
     # iterate over studies
     for study_name in studies:
-        f = os.path.join(general_folder,
-                         study_name,
-                         f'{study_name}_metadata.csv')
-        if not os.path.exists(f):
-            logger.error('metadata file does not exist {0}'.format(f))
+        metadata_loc = os.path.join(protected_folder,
+                                    study_name,
+                                    f'{study_name}_metadata.csv')
+        if not os.path.exists(metadata_loc):
+            logger.error('metadata file does not exist {0}'.format
+                    (metadata_loc))
             continue
-        logger.debug('reading metadata file {0}'.format(f))
+        logger.debug('reading metadata file {0}'.format(metadata_loc))
         try:
             # iterate over rows in metadata file
             for subject in _subjects(Lochness, study_name, general_folder,
-                                     protected_folder, f):
+                                     protected_folder, metadata_loc):
                 yield subject
         except StudyMetadataError as e:
             logger.error(e)

lochness/email/__init__.py

@@ -1,3 +1,4 @@
+import re
 import os
 from pathlib import Path
 import string
@@ -119,7 +120,8 @@ def send_detail(Lochness,
 
 
 def send_out_daily_updates(Lochness, days: int = 1,
-                           test: bool = False, mailx: bool = True):
+                           test: bool = False, mailx: bool = True,
+                           shorten: bool = True):
     '''Send daily updates from Lochness'''
 
     s3_log = Path(Lochness['phoenix_root']) / 's3_log.csv'
@@ -142,13 +144,16 @@ def send_out_daily_updates(Lochness, days: int = 1,
 
         s3_df_selected['date'] = s3_df_selected['timestamp'].apply(
                 lambda x: x.date())
+
+        # remove logfile
+        s3_df_selected = s3_df_selected[s3_df_selected.filename != '.log']
         count_df = s3_df_selected.groupby([
             'date', 'protected', 'study',
-            'processed', 'subject', 'datatypes']).count()[['filename']]
+            'processed', 'datatypes']).count()[['filename']]
         count_df.columns = ['file count']
         count_df = count_df.reset_index()
         count_df.columns = ['Transfer date', 'Protected vs General', 'Site',
-                'Raw vs Processed', 'Subject', 'Data type',
+                'Raw vs Processed', 'Data type',
                 'Number of files transferred']
         s3_df_selected.drop('date', axis=1, inplace=True)
 
@@ -200,6 +205,40 @@ def send_out_daily_updates(Lochness, days: int = 1,
 
         in_mail_footer = 'Only the files transferred to NDA are shown.'
 
+        if shorten:
+            s3_df_tmp = s3_df_selected.copy()
+            for index, row in s3_df_tmp.iterrows():
+                s3_df_tmp.loc[index, 'File name'] = re.sub(
+                        row['Subject'], 'SUBJECT', row['File name'])
+
+                mindlamp_act_patt = r'_activity_(\d{4}_\d{2}_\d{2}.+)$'
+                if re.search(mindlamp_act_patt, row['File name']):
+                    rest_str = re.search(mindlamp_act_patt,
+                                         row['File name']).group(1)
+                    s3_df_tmp.loc[index, 'File name'] = \
+                            f'SUBJECT_activity_{rest_str}'
+
+                for patt in ['-diaryTranscriptQC-day', '_audioJournal_day',
+                        '-diaryAudioQC-day']:
+                    if re.search(patt,
+                                 row['File name']):
+                        s3_df_tmp.loc[index, 'File name'] = \
+                                re.search(r'([a-zA-Z]+)[-_]day', patt).group(1)
+
+            gb = s3_df_tmp.groupby('File name')
+            for unique_filename, table in gb:
+                s3_df_tmp.loc[table.index, 'Subject'] = \
+                    f'{len(table)} subjects'
+                s3_df_tmp.loc[table.index, 'Transfer time (UTC)'] = \
+                        table['Transfer time (UTC)'].iloc[0]
+                s3_df_tmp.loc[table.index, 'Download time (UTC)'] = \
+                        table['Download time (UTC)'].iloc[0]
+            s3_df_selected = s3_df_tmp.drop_duplicates()
+            rename_Subject = lambda x: 'Number of subjects' if x == 'Subject' \
+                    else x
+            s3_df_selected.columns = [rename_Subject(x) for x in
+                                      s3_df_selected.columns]
+
         send_detail(
             Lochness,
             Lochness['sender'],

lochness/email/bootdey_template.html

@@ -1,7 +1,7 @@
 <html>
 <link rel=”stylesheet” href=”https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css”>
 <script src=”https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js”></script>
-<!--<script src=”https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.16.0/umd/popper.min.js”></script>-->
+<script src=”https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.16.0/umd/popper.min.js”></script>
 <script src=”https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js”></script>
 <table class="body-wrap" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6">
     <tbody>
@@ -16,7 +16,7 @@
                             <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
                                 <td class="" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #38414a; margin: 0; padding: 20px;"
                                     align="center" bgcolor="#71b6f9" valign="top">
-                                    <img src="https://github.com/AMP-SCZ/lochness/blob/kcho/readthedocs/docs/source/images/AMP%20SCZ%20Logo%20small.png?raw=true"><br>
+                                    <img src="https://raw.githubusercontent.com/AMP-SCZ/lochness/master/docs/source/images/AMP%20SCZ%20Logo%20small.png"><br>
                                     <a href="https://github.com/AMP-SCZ/lochness" style="font-size:32px;color:#fff;">{{ title }}</a> <br>
                                     <span style="margin-top: 10px;display: block;">{{ subtitle }}</span>
                                 </td>
@@ -75,4 +75,5 @@
     </tbody>
 </table>
 
+
 </html>

lochness/email/template.html

@@ -45,7 +45,6 @@
        <tbody>
         <tr>
          <td id="logo-box">
-          <img height="60" src="https://harvard-nrg.github.io/img/shield.png" alt="shield" />
          </td>
         </tr>
         <tr>

lochness/mediaflux/__init__.py

@@ -94,7 +94,7 @@ def sync_module(Lochness: 'lochness.config',
 
                     # obtain mediaflux remote paths
                     with tf.TemporaryDirectory() as tmpdir:
-                        diff_path= pjoin(tmpdir,'diff.csv')
+                        diff_path = pjoin(tmpdir,'diff.csv')
                         cmd = (' ').join(['unimelb-mf-check',
                                           '--mf.config', mflux_cfg,
                                           '--nb-retries 5',
@@ -112,10 +112,21 @@ def sync_module(Lochness: 'lochness.config',
                             continue
 
                         df = pd.read_csv(diff_path)
-                        for remote in df['SRC_PATH'].values:
-                            if pd.isnull(remote):
+                        for _, row in df[~df['SRC_PATH'].isnull()].iterrows():
+                            remote = row.SRC_PATH
+                            checksum = row.SRC_CHECKSUM
+
+                            if pd.isnull(checksum):
                                 continue
 
+                            # ignore mp3 or mp4 in non-interviews
+                            if datatype != 'interviews':
+                                if remote.endswith('mp3') or \
+                                        remote.endswith('mp4'):
+                                    logger.warning('mp3 or mp4 file detected '
+                                        f'in non-interviews datatype: {root}')
+                                    continue
+
                             if not re.search(
                                     patt.replace('*', '(.+?)').lower(),
                                     remote.lower()):
@@ -155,6 +166,21 @@ def sync_module(Lochness: 'lochness.config',
                                     if 'out_dir' in prod else mf_local) / \
                                         subpath.parent
 
+                            # do not re-download already downloaded data
+                            prev_checksum_file = Path(mf_local) / \
+                                    f'.checksum_{subpath.name}'
+
+
+                            if prev_checksum_file.is_file():
+                                with open(prev_checksum_file, 'r') as fp:
+                                    prev_checksum = fp.read().strip()
+
+                                if prev_checksum == checksum:
+                                    continue
+                                # else:
+                                    # pass
+                                    # os.remove(Path(mf_local) / subpath.name)
+
                             # do not re-download already transferred &
                             # removed data
                             if is_transferred_and_removed(
@@ -170,24 +196,32 @@ def sync_module(Lochness: 'lochness.config',
                                               '--mf.config', mflux_cfg,
                                               '-o', f'"{mf_local}"',
                                               '--nb-retries 5',
+                                              '--overwrite',
                                               f'\"{remote}\"'])
 
                             p = Popen(cmd, shell=True,
                                       stdout=DEVNULL, stderr=STDOUT)
                             p.wait()
 
-                            # verify checksum after download completes if
-                            # checksum does not match, data will be downloaded
-                            # again ENH should we verify checksum 5 times?
-                            cmd += ' --csum-check'
-                            p = Popen(cmd, shell=True,
-                                      stdout=DEVNULL, stderr=STDOUT)
-                            p.wait()
+                            # write checksum to local
+                            with open(prev_checksum_file, 'w') as fp:
+                                fp.write(checksum)
 
                             # for A/V related files, force permission to 770
                             # so A/V pipeline can work
                             if 'interviews' in str(mf_local):
-                                for root, dirs, files in os.walk(mf_local):
+                                interviews_root = tree.get(
+                                        datatype,
+                                        subj_dir,
+                                        processed=processed,
+                                        BIDS=Lochness['BIDS'],
+                                        makedirs=True)
+                                for root, dirs, files in os.walk(
+                                        interviews_root):
+                                    dir_path = Path(root)
+                                    perm = oct(dir_path.stat().st_mode)[-3:]
+                                    if perm != '770':
+                                        os.chmod(dir_path, 0o0770)
                                     for file in files:
                                         file_p = Path(root) / file
                                         perm = oct(file_p.stat().st_mode)[-3:]

lochness/mindlamp/__init__.py

@@ -144,7 +144,8 @@ def sync(Lochness: 'lochness.config',
         logger.debug(f'Mindlamp {subject_id} {date_str} data pull - start')
 
         # store both data types
-        for data_name in ['activity', 'sensor']:
+        # for data_name in ['activity', 'sensor']:
+        for data_name in ['activity']:
             dst = Path(dst_folder) / \
                 f'{subject_id}_{subject.study}_{data_name}_{date_str}.json'
 
@@ -169,9 +170,14 @@ def sync(Lochness: 'lochness.config',
 
             # pull data from mindlamp
             begin = time.time()
-            data_dict = function_to_execute(
-                    LAMP, subject_id,
-                    from_ts=time_utc_00_ts, to_ts=time_utc_24_ts)
+            try:
+                data_dict = function_to_execute(
+                        LAMP, subject_id,
+                        from_ts=time_utc_00_ts, to_ts=time_utc_24_ts)
+            except Exception as e:
+                print(e)
+                continue
+
             end = time.time()
             logger.debug(
                 f'Mindlamp {subject_id} {date_str} {data_name} data pull'

lochness/redcap/__init__.py

@@ -81,8 +81,8 @@ def initialize_metadata(Lochness: 'Lochness object',
     project_name = study_name.split(site_code_study)[0]  # 'Pronet'
 
     # metadata study location
-    general_path = Path(Lochness['phoenix_root']) / 'GENERAL'
-    metadata_study = general_path / study_name / f"{study_name}_metadata.csv"
+    protected_path = Path(Lochness['phoenix_root']) / 'PROTECTED'
+    metadata_study = protected_path / study_name / f"{study_name}_metadata.csv"
 
     # use redcap_project function to load the redcap keyrings for the project
     _, api_url, api_key = next(redcap_projects(
@@ -237,8 +237,8 @@ def initialize_metadata_rm(Lochness: 'Lochness object',
     project_name = study_name.split(site_code_study)[0]  # 'Pronet'
 
     # metadata study location
-    general_path = Path(Lochness['phoenix_root']) / 'GENERAL'
-    metadata_study = general_path / study_name / f"{study_name}_metadata.csv"
+    protected_path = Path(Lochness['phoenix_root']) / 'PROTECTED'
+    metadata_study = protected_path / study_name / f"{study_name}_metadata.csv"
 
     # use redcap_project function to load the redcap keyrings for the project
     _, api_url, api_key = next(redcap_projects(
@@ -682,13 +682,28 @@ def sync(Lochness, subject, dry=False):
                 # UPENN REDCap is set up with its own record_id, but have added
                 # "session_subid" field to note AMP-SCZ ID
                 redcap_subject_sl = redcap_subject.lower()
+                digits = [1, 2, 3, 4, 5, 6, 7, 8, 9]
+                digits_str = [str(x) for x in digits]
+                contains_logic = []
+                for subject_id in [redcap_subject, redcap_subject_sl]:
+                    contains_logic += [
+                            f"contains([session_subid], '{subject_id}_{x}')"
+                            for x in digits_str]
+                    contains_logic += [
+                            f"contains([session_subid], '{subject_id}={x}')"
+                            for x in digits_str]
+
+
                 record_query = {
                     'token': api_key,
                     'content': 'record',
                     'format': 'json',
                     'filterLogic': f"[session_subid] = '{redcap_subject}' or "
-                                   f"[session_subid] = '{redcap_subject_sl}'"
+                                   f"[session_subid] = '{redcap_subject_sl}' or "
+                                   f"{' or '.join(contains_logic)}"
                 }
+                                   # f"[session_subid] = '{redcap_subject_sl}' or "
+                                   # f"{' or '.join(contains_logic)}"
 
             else:
                 id_field = Lochness['redcap_id_colname']