From 74c262534f05a67d698ebe421105de8e08829065 Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 12 May 2023 08:26:03 +1000 Subject: [PATCH 01/29] feat: allow = pattern --- lochness/redcap/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lochness/redcap/__init__.py b/lochness/redcap/__init__.py index 8a7089a7..3dfa0ddc 100644 --- a/lochness/redcap/__init__.py +++ b/lochness/redcap/__init__.py @@ -689,9 +689,9 @@ def sync(Lochness, subject, dry=False): contains_logic += [ f"contains([session_subid], '{subject_id}_{x}')" for x in digits_str] - # contains_logic += [ - # f"contains([session_subid], '{subject_id}={x}')" - # for x in digits_str] + contains_logic += [ + f"contains([session_subid], '{subject_id}={x}')" + for x in digits_str] record_query = { From 4418b199948ede642e6afab40cf8d819814e2a9e Mon Sep 17 00:00:00 2001 From: Tashrif Billah <35086881+tashrifbillah@users.noreply.github.com> Date: Sat, 27 May 2023 17:14:46 -0400 Subject: [PATCH 02/29] do not read RPMS CSV files within 7:55-8:10 pm they are exported by RPMS and modified by Tashrif's programs during this window --- lochness/rpms/__init__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lochness/rpms/__init__.py b/lochness/rpms/__init__.py index 5d0b0915..9f892691 100644 --- a/lochness/rpms/__init__.py +++ b/lochness/rpms/__init__.py @@ -12,6 +12,7 @@ from typing import List, Dict, Union import pandas as pd from datetime import datetime +from time import sleep import re from lochness.redcap.process_piis import process_and_copy_db pd.set_option('mode.chained_assignment', None) @@ -22,6 +23,24 @@ logger = logging.getLogger(__name__) +def _wait(): + + # calculate _sleep in minutes + # it is between 15 (7:55 pm) and 0 (8:10 pm) + _sleep=0 + now=datetime.now() + if now.hour==19: + if now.minute>=55: + _sleep=15-(now.minute-55) + + elif now.hour==20: + if now.minute<=10: + _sleep=10-now.minute + + # argument of sleep() is in seconds + sleep(_sleep*60) + + def get_rpms_database(rpms_root_path: str) -> Dict[str, pd.DataFrame]: '''Return dictionary of RPMS database in pandas dataframes @@ -65,6 +84,10 @@ def get_rpms_database(rpms_root_path: str) -> Dict[str, pd.DataFrame]: 'measure_file_date', ascending=False).iterrows(): if n == 0: try: + # do not read CSV files within 7:55-8:10 pm + # they are exported by RPMS and modified by Tashrif's programs + # during this window + _wait() df_tmp = pd.read_csv(row.measure_file, dtype=str) except pd.errors.EmptyDataError: # ignore csv is empty shutil.move(row.measure_file, From 1aaeead2f022fab88b6979ea265d52a510b1cdb7 Mon Sep 17 00:00:00 2001 From: kcho Date: Sat, 29 Jul 2023 00:45:14 +1000 Subject: [PATCH 03/29] fix: PennCNB source check function to take account of AB00000_ and AB00000= PennCNB ID patterns --- lochness/utils/source_check.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lochness/utils/source_check.py b/lochness/utils/source_check.py index 96d7328a..9e38b765 100644 --- a/lochness/utils/source_check.py +++ b/lochness/utils/source_check.py @@ -96,6 +96,16 @@ def check_list_all_penn_cnb_subjects( df['site_orig'].str.split('_').str[1].str.upper() df['modality'] = 'PENN_CNB' + + # PennCNB accepts subject IDs with '_\d' suffix that denotes different + # timepoint of the data submission + + # get only the first XX\d\d\d\d\d pattern + df['subject_orig'] = df['subject'].copy() + pattern = r'([A-Z]{2}\d{5}(?=_|=|$))' + df['subject'] = df['subject'].str.extract(pattern) + df['subject'] = df['subject'].fillna(df['subject_orig']) + df['subject_check'] = df['subject'].apply(ampscz_penn_validate) df['exist_in_db'] = df['subject'].str[:7].str.upper().isin( subject_list).fillna(False) From 095780293f7d8305031b66f52b84f04d4b4cdfeb Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 25 Aug 2023 02:19:53 +1000 Subject: [PATCH 04/29] minor: removing js scripts didn't help the lag in reading emails --- lochness/email/bootdey_template.html | 4 ++-- lochness/email/template.html | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lochness/email/bootdey_template.html b/lochness/email/bootdey_template.html index a48b8433..0c08f3f1 100644 --- a/lochness/email/bootdey_template.html +++ b/lochness/email/bootdey_template.html @@ -1,7 +1,7 @@ - + @@ -16,7 +16,7 @@ diff --git a/lochness/email/template.html b/lochness/email/template.html index 8fd0ac5c..19f91b97 100644 --- a/lochness/email/template.html +++ b/lochness/email/template.html @@ -45,7 +45,6 @@ From 24d3d045ce0a9c7f1352cff3f6c9682d87e58892 Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 25 Aug 2023 02:22:04 +1000 Subject: [PATCH 05/29] feat: skip uploadgin mp3 or mp4 files from non-interview datatypes --- lochness/email/bootdey_template.html | 1 + lochness/mediaflux/__init__.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lochness/email/bootdey_template.html b/lochness/email/bootdey_template.html index 0c08f3f1..d0348827 100644 --- a/lochness/email/bootdey_template.html +++ b/lochness/email/bootdey_template.html @@ -75,4 +75,5 @@
-
+
{{ title }}
{{ subtitle }}
- shield
+ diff --git a/lochness/mediaflux/__init__.py b/lochness/mediaflux/__init__.py index 0eecfacc..a32a7cbd 100644 --- a/lochness/mediaflux/__init__.py +++ b/lochness/mediaflux/__init__.py @@ -94,7 +94,7 @@ def sync_module(Lochness: 'lochness.config', # obtain mediaflux remote paths with tf.TemporaryDirectory() as tmpdir: - diff_path= pjoin(tmpdir,'diff.csv') + diff_path = pjoin(tmpdir,'diff.csv') cmd = (' ').join(['unimelb-mf-check', '--mf.config', mflux_cfg, '--nb-retries 5', @@ -116,6 +116,14 @@ def sync_module(Lochness: 'lochness.config', if pd.isnull(remote): continue + # ignore mp3 or mp4 in non-interviews + if datatype != 'interviews': + if remote.endswith('mp3') or \ + remote.endswith('mp4'): + logger.warning('mp3 or mp4 file detected ' + f'in non-interviews datatype: {root}') + continue + if not re.search( patt.replace('*', '(.+?)').lower(), remote.lower()): From 522ddae00a9be12ca5f81a16bf9214a360e53871 Mon Sep 17 00:00:00 2001 From: kcho Date: Sun, 27 Aug 2023 20:48:42 +1000 Subject: [PATCH 06/29] fix: remove duplicated check_source --- scripts/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync.py b/scripts/sync.py index a8eb9a0c..64942c8b 100755 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -201,7 +201,7 @@ def main(): # email if args.daily_summary: - check_source(Lochness) + # check_source(Lochness) if args.check_source: check_source(Lochness) From 986a676a7e8a8f5dcbfdf98d1e517652147445dc Mon Sep 17 00:00:00 2001 From: kcho Date: Thu, 7 Sep 2023 08:07:04 +1000 Subject: [PATCH 07/29] fix: mediaflux checksum --- lochness/mediaflux/__init__.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/lochness/mediaflux/__init__.py b/lochness/mediaflux/__init__.py index a32a7cbd..9960552f 100644 --- a/lochness/mediaflux/__init__.py +++ b/lochness/mediaflux/__init__.py @@ -112,9 +112,9 @@ def sync_module(Lochness: 'lochness.config', continue df = pd.read_csv(diff_path) - for remote in df['SRC_PATH'].values: - if pd.isnull(remote): - continue + for row in df[~df['SRC_PATH'].isnull()]: + remote = row.SRC_PATH + checksum = row.SRC_CHECKSUM # ignore mp3 or mp4 in non-interviews if datatype != 'interviews': @@ -163,6 +163,18 @@ def sync_module(Lochness: 'lochness.config', if 'out_dir' in prod else mf_local) / \ subpath.parent + # do not re-download already downloaded data + prev_checksum_file = Path(mf_local).parent / \ + f'.checksum_{Path(mf_local).name}' + if prev_checksum_file.is_file(): + with open(prev_checksum_file, 'r') as fp: + prev_checksum = fp.read().strip() + + if prev_checksum == checksum: + continue + else: + shutil.remove(mf_local) + # do not re-download already transferred & # removed data if is_transferred_and_removed( @@ -184,13 +196,9 @@ def sync_module(Lochness: 'lochness.config', stdout=DEVNULL, stderr=STDOUT) p.wait() - # verify checksum after download completes if - # checksum does not match, data will be downloaded - # again ENH should we verify checksum 5 times? - cmd += ' --csum-check' - p = Popen(cmd, shell=True, - stdout=DEVNULL, stderr=STDOUT) - p.wait() + # write checksum to local + with open(prev_checksum_file, 'w') as fp: + fp.write(checksum) # for A/V related files, force permission to 770 # so A/V pipeline can work From 6dc465a7481f1683684f950acc37b5db549f5707 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com> Date: Sat, 9 Sep 2023 14:08:43 -0400 Subject: [PATCH 08/29] one idea: make use of RPMS_DUMP_HOUR env var caveat: will require us to change it during AEST/AEDT change --- lochness/rpms/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lochness/rpms/__init__.py b/lochness/rpms/__init__.py index 9f892691..7d860c22 100644 --- a/lochness/rpms/__init__.py +++ b/lochness/rpms/__init__.py @@ -25,15 +25,18 @@ def _wait(): + # RPMS dumps data at 20:00 (AEDT) or at 19:00 (AEST) + RPMS_DUMP_HOUR=os.getenv('RPMS_DUMP_HOUR',20) + # calculate _sleep in minutes # it is between 15 (7:55 pm) and 0 (8:10 pm) _sleep=0 now=datetime.now() - if now.hour==19: + if now.hour==RPMS_DUMP_HOUR-1: if now.minute>=55: _sleep=15-(now.minute-55) - elif now.hour==20: + elif now.hour==RPMS_DUMP_HOUR: if now.minute<=10: _sleep=10-now.minute From 4040cd6f1bcce4d5523c5742fdf7d9265959b82c Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com> Date: Sat, 9 Sep 2023 14:19:25 -0400 Subject: [PATCH 09/29] second idea: obtain TIME_ZONE env var that is set at system level disclosure: Tashrif set this in /etc/profile.d/timezone.sh --- lochness/rpms/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lochness/rpms/__init__.py b/lochness/rpms/__init__.py index 7d860c22..aeb2054d 100644 --- a/lochness/rpms/__init__.py +++ b/lochness/rpms/__init__.py @@ -26,7 +26,8 @@ def _wait(): # RPMS dumps data at 20:00 (AEDT) or at 19:00 (AEST) - RPMS_DUMP_HOUR=os.getenv('RPMS_DUMP_HOUR',20) + tz=os.getenv('TIME_ZONE','AEDT') + RPMS_DUMP_HOUR=20 if tz=='AEDT' else 19 # calculate _sleep in minutes # it is between 15 (7:55 pm) and 0 (8:10 pm) From 38982c6c25f40795c4c91c2c16d8b5762d8030da Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 11 Oct 2023 22:16:17 +1100 Subject: [PATCH 10/29] feat: proper checksum to mediaflux --- lochness/mediaflux/__init__.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lochness/mediaflux/__init__.py b/lochness/mediaflux/__init__.py index 9960552f..1f112281 100644 --- a/lochness/mediaflux/__init__.py +++ b/lochness/mediaflux/__init__.py @@ -112,10 +112,13 @@ def sync_module(Lochness: 'lochness.config', continue df = pd.read_csv(diff_path) - for row in df[~df['SRC_PATH'].isnull()]: + for _, row in df[~df['SRC_PATH'].isnull()].iterrows(): remote = row.SRC_PATH checksum = row.SRC_CHECKSUM + if pd.isnull(checksum): + continue + # ignore mp3 or mp4 in non-interviews if datatype != 'interviews': if remote.endswith('mp3') or \ @@ -164,16 +167,19 @@ def sync_module(Lochness: 'lochness.config', subpath.parent # do not re-download already downloaded data - prev_checksum_file = Path(mf_local).parent / \ - f'.checksum_{Path(mf_local).name}' + prev_checksum_file = Path(mf_local) / \ + f'.checksum_{subpath.name}' + + if prev_checksum_file.is_file(): with open(prev_checksum_file, 'r') as fp: prev_checksum = fp.read().strip() if prev_checksum == checksum: continue - else: - shutil.remove(mf_local) + # else: + # pass + # os.remove(Path(mf_local) / subpath.name) # do not re-download already transferred & # removed data @@ -190,6 +196,7 @@ def sync_module(Lochness: 'lochness.config', '--mf.config', mflux_cfg, '-o', f'"{mf_local}"', '--nb-retries 5', + '--overwrite', f'\"{remote}\"']) p = Popen(cmd, shell=True, From 9fe36b60b1e8a00c76dc10003a8c9e61a69742cc Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 11 Oct 2023 22:16:54 +1100 Subject: [PATCH 11/29] feat: continue to the next day even when there is an error on a date --- lochness/mindlamp/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lochness/mindlamp/__init__.py b/lochness/mindlamp/__init__.py index 5c20e593..be73a275 100644 --- a/lochness/mindlamp/__init__.py +++ b/lochness/mindlamp/__init__.py @@ -169,9 +169,14 @@ def sync(Lochness: 'lochness.config', # pull data from mindlamp begin = time.time() - data_dict = function_to_execute( - LAMP, subject_id, - from_ts=time_utc_00_ts, to_ts=time_utc_24_ts) + try: + data_dict = function_to_execute( + LAMP, subject_id, + from_ts=time_utc_00_ts, to_ts=time_utc_24_ts) + except Exception as e: + print(e) + continue + end = time.time() logger.debug( f'Mindlamp {subject_id} {date_str} {data_name} data pull' From 93708e0462507eaa5a00395d2187008a0d34c784 Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 11 Oct 2023 22:17:20 +1100 Subject: [PATCH 12/29] feat: ignore .check_sum --- lochness/transfer/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lochness/transfer/__init__.py b/lochness/transfer/__init__.py index d230296c..944409e1 100644 --- a/lochness/transfer/__init__.py +++ b/lochness/transfer/__init__.py @@ -299,7 +299,7 @@ def send_file_to_s3_phoenix(Lochness, source_file: Path) -> None: command = f"aws s3 cp \ {source_file} s3://{s3_bucket_name}/{target_path} \ - --exclude '*.mp3' --exclude '.checksum*'" + --exclude '*.mp3' --exclude '.checksum*' --exclude '.check_sum*'" command_out = os.popen(command).read() # update s3 log @@ -561,7 +561,8 @@ def lochness_to_lochness_transfer_s3_protected(Lochness, command = f"aws s3 sync \ {source_directory}/ \ s3://{s3_bucket_name}/{s3_phoenix_root_dtype} \ - --exclude '*.mp3' --exclude '.check_sum*'" + --exclude '*.mp3' --exclude '.check_sum*' \ + --exclude '.checksum*' " # logger.debug(re.sub(r'\s+', r' ', command)) From ea81b26995c226fcea07724ebc7b181545cbd2af Mon Sep 17 00:00:00 2001 From: Dheshan Mohandass Date: Mon, 11 Dec 2023 08:13:52 -0500 Subject: [PATCH 13/29] feat.: update AV QC requirements #130 --- lochness/utils/path_checker.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/lochness/utils/path_checker.py b/lochness/utils/path_checker.py index 28ac0da4..66202713 100644 --- a/lochness/utils/path_checker.py +++ b/lochness/utils/path_checker.py @@ -235,6 +235,32 @@ def update_by_removing_unused_files(df: pd.DataFrame) -> None: ds_store_index = df[df.file_name == '.DS_Store'].index df.drop(ds_store_index, inplace=True) + # recording.conf files + recording_conf_index = df[df.file_name == 'recording.conf'].index + df.drop(recording_conf_index, inplace=True) + + # chat.txt files + chat_txt_index = df[df.file_name == 'chat.txt'].index + df.drop(chat_txt_index, inplace=True) + + +def update_skipped_av_files(df: pd.DataFrame) -> pd.DataFrame: + """ + Update the 'file_check' column in the DataFrame based on the 'parent_dir' column. + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + pd.DataFrame: The updated DataFrame. + """ + for idx, row in df.iterrows(): + if "Extraneous files" in row["parent_dir"]: + df.loc[idx, "file_check"] = True + elif "Additional interview files" in row["parent_dir"]: + df.loc[idx, "file_check"] = True + return df + def update_by_removing_genetics_and_fluids(df: pd.DataFrame) -> None: '''Remove files under GeneticsAndFluids directory''' @@ -288,13 +314,14 @@ def check_file_path_df(df: pd.DataFrame, update_interviews_check(df) update_interviews_transcript_check(df) update_interviews_teams_data_check(df) + update_skipped_av_files(df) update_interviews_video_check(df) update_interviews_audio_check(df) # ignore genetics and fluids update_by_removing_genetics_and_fluids(df) - # ignore .DS_Store files + # ignore .DS_Store, recording.conf, chat.txt update_by_removing_unused_files(df) # check if the subject exist in metadata From 3ab72ce1fc97f9609ee57cefa65d2b1aadef3ee3 Mon Sep 17 00:00:00 2001 From: Dheshan Mohandass Date: Sat, 16 Dec 2023 03:28:36 +1100 Subject: [PATCH 14/29] fix: AVQC changes being overwritten by QC Checks - change Skip assertions to use 'file_path' instead of 'parent_dir' --- lochness/utils/path_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lochness/utils/path_checker.py b/lochness/utils/path_checker.py index 66202713..8635ee93 100644 --- a/lochness/utils/path_checker.py +++ b/lochness/utils/path_checker.py @@ -255,9 +255,9 @@ def update_skipped_av_files(df: pd.DataFrame) -> pd.DataFrame: pd.DataFrame: The updated DataFrame. """ for idx, row in df.iterrows(): - if "Extraneous files" in row["parent_dir"]: + if "Extraneous files" in row["file_path"]: df.loc[idx, "file_check"] = True - elif "Additional interview files" in row["parent_dir"]: + elif "Additional interview files" in row["file_path"]: df.loc[idx, "file_check"] = True return df @@ -314,9 +314,9 @@ def check_file_path_df(df: pd.DataFrame, update_interviews_check(df) update_interviews_transcript_check(df) update_interviews_teams_data_check(df) - update_skipped_av_files(df) update_interviews_video_check(df) update_interviews_audio_check(df) + update_skipped_av_files(df) # ignore genetics and fluids update_by_removing_genetics_and_fluids(df) From 4130bde6e3cfe5a578fbd75f51a57ce664089663 Mon Sep 17 00:00:00 2001 From: kcho Date: Sat, 27 Jan 2024 03:12:17 +1100 Subject: [PATCH 15/29] fix: permission on interviews root --- lochness/mediaflux/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lochness/mediaflux/__init__.py b/lochness/mediaflux/__init__.py index 1f112281..0fbcba70 100644 --- a/lochness/mediaflux/__init__.py +++ b/lochness/mediaflux/__init__.py @@ -210,7 +210,18 @@ def sync_module(Lochness: 'lochness.config', # for A/V related files, force permission to 770 # so A/V pipeline can work if 'interviews' in str(mf_local): - for root, dirs, files in os.walk(mf_local): + interviews_root = tree.get( + datatype, + subj_dir, + processed=processed, + BIDS=Lochness['BIDS'], + makedirs=True) + for root, dirs, files in os.walk( + interviews_root): + dir_path = Path(root) + perm = oct(dir_path.stat().st_mode)[-3:] + if perm != '770': + os.chmod(dir_path, 0o0770) for file in files: file_p = Path(root) / file perm = oct(file_p.stat().st_mode)[-3:] From 4793cc29c93ea1b969b8272db6f512425bb06496 Mon Sep 17 00:00:00 2001 From: kcho Date: Sat, 27 Jan 2024 03:14:21 +1100 Subject: [PATCH 16/29] tmp: no longer pulling sensor data from mindlamp for prescidnt --- lochness/mindlamp/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lochness/mindlamp/__init__.py b/lochness/mindlamp/__init__.py index be73a275..7e48642a 100644 --- a/lochness/mindlamp/__init__.py +++ b/lochness/mindlamp/__init__.py @@ -144,7 +144,8 @@ def sync(Lochness: 'lochness.config', logger.debug(f'Mindlamp {subject_id} {date_str} data pull - start') # store both data types - for data_name in ['activity', 'sensor']: + # for data_name in ['activity', 'sensor']: + for data_name in ['activity']: dst = Path(dst_folder) / \ f'{subject_id}_{subject.study}_{data_name}_{date_str}.json' From 123bdba1743677579bac4a472bf3d8a5d2f49ee4 Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 7 Feb 2024 08:47:43 +1100 Subject: [PATCH 17/29] test: init --- .../lochness_test/utils/test_path_checker.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/lochness_test/utils/test_path_checker.py diff --git a/tests/lochness_test/utils/test_path_checker.py b/tests/lochness_test/utils/test_path_checker.py new file mode 100644 index 00000000..cd8d4530 --- /dev/null +++ b/tests/lochness_test/utils/test_path_checker.py @@ -0,0 +1,51 @@ +import lochness +from pathlib import Path +import sys +import pandas as pd +lochness_root = Path(lochness.__path__[0]).parent +scripts_dir = lochness_root / 'scripts' +test_dir = lochness_root / 'tests' +sys.path.append(str(scripts_dir)) +sys.path.append(str(test_dir)) + +from lochness.utils.path_checker import nth_item_from_path, \ + ampscz_id_validate, update_interviews_transcript_check, \ + check_file_path_df, update_by_removing_unused_files, \ + update_eeg_check +from lochness.utils.source_check import collect_mediaflux_files_info, \ + get_subject_list_from_metadata + + +def test_update_interviews_transcript_check(): + from lochness.config import load + config_loc = '/mnt/prescient/Prescient_production/config.yml' + Lochness = load(config_loc) + + subject_id_list = get_subject_list_from_metadata(Lochness) + ignore_id_list = [] + + if Lochness.get('id_list_csv', False): + # Prescient network read full subject list from RPMS + check_only_subject_id_list = True + unique_subjects = [] + for csv in Path(Lochness['RPMS_PATH']).glob('*csv'): + try: + [unique_subjects.append(x) for x in + pd.read_csv(csv)[Lochness['RPMS_id_colname']] + if x not in unique_subjects] + except: + pass + ignore_id_list += [x for x in unique_subjects + if x not in subject_id_list] + ignore_id_list = [x for x in ignore_id_list if type(x) == str] + + if Path('mediaflux_df_test.csv').is_file(): + mediaflux_df = pd.read_csv('mediaflux_df_test.csv', index_col=0) + else: + mediaflux_df = collect_mediaflux_files_info(Lochness, ignore_id_list) + mediaflux_df.to_csv('mediaflux_df_test.csv') + + df = mediaflux_df.copy() + all_df = check_file_path_df(df, subject_id_list) + + df_tmp.to_csv('test.csv') From c78fc7f78cba28545630dfb13080e3a979a3921d Mon Sep 17 00:00:00 2001 From: kcho Date: Wed, 7 Feb 2024 08:48:17 +1100 Subject: [PATCH 18/29] merged --- lochness/utils/path_checker.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lochness/utils/path_checker.py b/lochness/utils/path_checker.py index 8635ee93..8d12fef5 100644 --- a/lochness/utils/path_checker.py +++ b/lochness/utils/path_checker.py @@ -243,6 +243,10 @@ def update_by_removing_unused_files(df: pd.DataFrame) -> None: chat_txt_index = df[df.file_name == 'chat.txt'].index df.drop(chat_txt_index, inplace=True) + # Ignore For review + for_review_index = df[df.file_path.str.contains( + 'TRANSCRIPTS/For review')].index + df.drop(for_review_index, inplace=True) def update_skipped_av_files(df: pd.DataFrame) -> pd.DataFrame: """ @@ -275,7 +279,7 @@ def update_by_checking_against_subject_list( '''pass''' df['exist_in_db'] = df['subject'].isin(subject_id_list).fillna(False) - df.loc[df[df['exist_in_db']].index, + df.loc[~df[df['exist_in_db']].index, 'notes'] = 'Subject missing from database' From 61c9502442c4ec606451388848d26cdb408d0260 Mon Sep 17 00:00:00 2001 From: Dheshan Mohandass Date: Fri, 9 Feb 2024 09:45:21 -0500 Subject: [PATCH 19/29] fix.: Use new WAV file template from Speech SOP --- lochness/utils/path_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lochness/utils/path_checker.py b/lochness/utils/path_checker.py index 8d12fef5..6ce0622f 100644 --- a/lochness/utils/path_checker.py +++ b/lochness/utils/path_checker.py @@ -183,7 +183,7 @@ def update_interviews_teams_data_check(df: pd.DataFrame) -> pd.DataFrame: # file name pattern check video_int_df['file_pattern_check'] = video_int_df['file_name'].str.match( - r'\d{4}\d{2}\d{2}\d{6}_[A-Z]{2}\d{5}_(OPEN|PSYSCS).(wav|WAV)') + r'\d{4}\d{2}\d{2}\d{6}(wav|WAV)') video_int_df['file_check'] = video_int_df['directory_check'] & \ video_int_df['file_name'] df.loc[video_int_index] = video_int_df From a5722515091c0257cd91169b605ec46b57e51e74 Mon Sep 17 00:00:00 2001 From: Dheshan Mohandass Date: Fri, 9 Feb 2024 10:00:56 -0500 Subject: [PATCH 20/29] chore.: Ignore unwanted files (.log, Zoomver.tag and recording.conf) files - Improve documentation --- lochness/utils/path_checker.py | 96 +++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 12 deletions(-) diff --git a/lochness/utils/path_checker.py b/lochness/utils/path_checker.py index 6ce0622f..1b9cf114 100644 --- a/lochness/utils/path_checker.py +++ b/lochness/utils/path_checker.py @@ -1,3 +1,11 @@ +""" +Checks Mediaflux file paths for deviations from the standard operating +procedures (SOP). + +Note: This module checks files at source (Mediaflux) and not at the +destination (local storage). So this will check files that will be +pulled by Lochness and those that will not be pulled by Lochness. +""" import pandas as pd import re from pathlib import Path @@ -168,7 +176,20 @@ def update_interviews_video_check(df: pd.DataFrame) -> pd.DataFrame: def update_interviews_teams_data_check(df: pd.DataFrame) -> pd.DataFrame: - '''Check logics in rows for Interviews video''' + ''' + Check logics in rows for Interviews with WAV audio files. + + Validate the following: + - The directory structure + - The file name pattern + - The file name should be in the format of 'YYYYMMDDHHMMSS.wav' + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + pd.DataFrame: The updated DataFrame. + ''' # interviews transcript video_int_index = df[ (df.modality=='Interviews') & @@ -183,21 +204,41 @@ def update_interviews_teams_data_check(df: pd.DataFrame) -> pd.DataFrame: # file name pattern check video_int_df['file_pattern_check'] = video_int_df['file_name'].str.match( - r'\d{4}\d{2}\d{2}\d{6}(wav|WAV)') + r'\d{4}\d{2}\d{2}\d{6}(wav|WAV)' + ) video_int_df['file_check'] = video_int_df['directory_check'] & \ video_int_df['file_name'] df.loc[video_int_index] = video_int_df def update_interviews_audio_check(df: pd.DataFrame) -> pd.DataFrame: - '''Check logics in rows for Interviews audio''' + ''' + Check logics in rows for Interviews audio + + Skip the following files: + - 'playback.m3u' files + - 'Audio Record' directory + - 'Zoomver.tag' files + - files ending with '.log' + + Skip here means to set the 'file_check' column to True, essentially + ignoring these files. + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + pd.DataFrame: The updated DataFrame. + ''' # interviews audio - audio_int_index = df[(df.modality=='Interviews') & - (df.file_name.str.endswith('.m4a'))].index + audio_int_index = df[ + (df.modality == 'Interviews') & (df.file_name.str.endswith('.m4a')) + ].index audio_int_df = df.loc[audio_int_index] audio_int_df['zoom_name'] = audio_int_df['parent_dir'].str.extract( - '\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2} (.+)') + '\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2} (.+)' + ) # audio files need to be under either the zoom or Audio Record folder audio_int_df['file_check_1'] = ~audio_int_df['zoom_name'].isnull() @@ -209,19 +250,36 @@ def update_interviews_audio_check(df: pd.DataFrame) -> pd.DataFrame: df.loc[audio_int_index] = audio_int_df # ignore playback.m3u audio files - audio_int_index = df[(df.modality=='Interviews') & - (df.file_name == 'playback.m3u')].index + audio_int_index = df[ + (df.modality == 'Interviews') & (df.file_name == 'playback.m3u') + ].index audio_int_df = df.loc[audio_int_index] audio_int_df['file_check'] = True df.loc[audio_int_index] = audio_int_df # ignore 'Audio Record' folder - rec_index = df[(df.modality=='Interviews') & - (df.file_name.str.endswith('Audio Record'))].index + rec_index = df[ + (df.modality == 'Interviews') & (df.file_name.str.endswith('Audio Record')) + ].index rec_df = df.loc[rec_index] rec_df['file_check'] = True df.loc[rec_index] = rec_df + # ignore 'Zoomver.tag" files + zoomver_index = df[ + (df.modality == 'Interviews') & (df.file_name.str.endswith('Zoomver.tag')) + ].index + zoomver_df = df.loc[zoomver_index] + zoomver_df['file_check'] = True + df.loc[zoomver_index] = zoomver_df + + # Ignore files ending with '.log' + log_index = df[ + (df.modality == 'Interviews') & (df.file_name.str.endswith('.log')) + ].index + log_df = df.loc[log_index] + log_df['file_check'] = True + df.loc[log_index] = log_df def update_by_adding_notes(df: pd.DataFrame) -> None: @@ -230,13 +288,27 @@ def update_by_adding_notes(df: pd.DataFrame) -> None: def update_by_removing_unused_files(df: pd.DataFrame) -> None: - '''Remove unused files''' + ''' + Remove unused files. + + The following files will be removed: + - .DS_Store + - recording.conf + - chat.txt + - Files under TRANSCRIPTS/For review + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + None + ''' # .DS_Store ds_store_index = df[df.file_name == '.DS_Store'].index df.drop(ds_store_index, inplace=True) # recording.conf files - recording_conf_index = df[df.file_name == 'recording.conf'].index + recording_conf_index = df[df.file_name.str.endswith('recording.conf')].index df.drop(recording_conf_index, inplace=True) # chat.txt files From 80ea825bf3dc9ae373de26799977f8c1751dccfb Mon Sep 17 00:00:00 2001 From: Dheshan Mohandass Date: Mon, 12 Feb 2024 10:10:03 -0500 Subject: [PATCH 21/29] fix.: Make Zoomver.tag case insenstitve and relax recodring.conf matching --- lochness/utils/path_checker.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/lochness/utils/path_checker.py b/lochness/utils/path_checker.py index 1b9cf114..caf12624 100644 --- a/lochness/utils/path_checker.py +++ b/lochness/utils/path_checker.py @@ -156,21 +156,37 @@ def update_interviews_transcript_check(df: pd.DataFrame) -> pd.DataFrame: def update_interviews_video_check(df: pd.DataFrame) -> pd.DataFrame: - '''Check logics in rows for Interviews video''' + ''' + Check logics in rows for Interviews video + + Check the following: + - Zoom name in the parent directory + - Zoom name should be in the format of 'YYYY-MM-DD HH.MM.SS ' + - Video files should be under the zoom folder + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + pd.DataFrame: The updated DataFrame. + ''' # interviews transcript video_int_index = df[ - (df.modality=='Interviews') & - ((df.file_name.str.endswith('.mp4')) | - (df.file_name=='recording.conf')) - ].index + (df.modality == "Interviews") + & ( + (df.file_name.str.endswith(".mp4")) + | (df.file_name.str.endswith("recording.conf")) + ) + ].index video_int_df = df.loc[video_int_index] - video_int_df['zoom_name'] = video_int_df['parent_dir'].str.extract( - '\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2} (.+)') + video_int_df["zoom_name"] = video_int_df["parent_dir"].str.extract( + "\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2} (.+)" + ) # video files must be under the zoom folder - video_int_df['file_check'] = ~video_int_df['zoom_name'].isnull() + video_int_df["file_check"] = ~video_int_df["zoom_name"].isnull() df.loc[video_int_index] = video_int_df @@ -267,7 +283,7 @@ def update_interviews_audio_check(df: pd.DataFrame) -> pd.DataFrame: # ignore 'Zoomver.tag" files zoomver_index = df[ - (df.modality == 'Interviews') & (df.file_name.str.endswith('Zoomver.tag')) + (df.modality == 'Interviews') & (df.file_name.str.endswith('ver.tag')) ].index zoomver_df = df.loc[zoomver_index] zoomver_df['file_check'] = True From dfb9f2180679ac3a8bf471c2ecd5b9c9da9f114d Mon Sep 17 00:00:00 2001 From: kcho Date: Sat, 17 Feb 2024 03:26:12 +1100 Subject: [PATCH 22/29] feat: add 'shorten' option to send_out_daily_updates --- lochness/email/__init__.py | 45 +++++++++++++++++++++++-- tests/lochness_test/email/test_email.py | 8 +++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/lochness/email/__init__.py b/lochness/email/__init__.py index 35184866..6e4b56cc 100644 --- a/lochness/email/__init__.py +++ b/lochness/email/__init__.py @@ -1,3 +1,4 @@ +import re import os from pathlib import Path import string @@ -119,7 +120,8 @@ def send_detail(Lochness, def send_out_daily_updates(Lochness, days: int = 1, - test: bool = False, mailx: bool = True): + test: bool = False, mailx: bool = True, + shorten: bool = True): '''Send daily updates from Lochness''' s3_log = Path(Lochness['phoenix_root']) / 's3_log.csv' @@ -142,13 +144,16 @@ def send_out_daily_updates(Lochness, days: int = 1, s3_df_selected['date'] = s3_df_selected['timestamp'].apply( lambda x: x.date()) + + # remove logfile + s3_df_selected = s3_df_selected[s3_df_selected.filename != '.log'] count_df = s3_df_selected.groupby([ 'date', 'protected', 'study', - 'processed', 'subject', 'datatypes']).count()[['filename']] + 'processed', 'datatypes']).count()[['filename']] count_df.columns = ['file count'] count_df = count_df.reset_index() count_df.columns = ['Transfer date', 'Protected vs General', 'Site', - 'Raw vs Processed', 'Subject', 'Data type', + 'Raw vs Processed', 'Data type', 'Number of files transferred'] s3_df_selected.drop('date', axis=1, inplace=True) @@ -200,6 +205,40 @@ def send_out_daily_updates(Lochness, days: int = 1, in_mail_footer = 'Only the files transferred to NDA are shown.' + if shorten: + s3_df_tmp = s3_df_selected.copy() + for index, row in s3_df_tmp.iterrows(): + s3_df_tmp.loc[index, 'File name'] = re.sub( + row['Subject'], 'SUBJECT', row['File name']) + + mindlamp_act_patt = r'_activity_(\d{4}_\d{2}_\d{2}.+)$' + if re.search(mindlamp_act_patt, row['File name']): + rest_str = re.search(mindlamp_act_patt, + row['File name']).group(1) + s3_df_tmp.loc[index, 'File name'] = \ + f'SUBJECT_activity_{rest_str}' + + for patt in ['-diaryTranscriptQC-day', '_audioJournal_day', + '-diaryAudioQC-day']: + if re.search(patt, + row['File name']): + s3_df_tmp.loc[index, 'File name'] = \ + re.search(r'([a-zA-Z]+)[-_]day', patt).group(1) + + gb = s3_df_tmp.groupby('File name') + for unique_filename, table in gb: + s3_df_tmp.loc[table.index, 'Subject'] = \ + f'{len(table)} subjects' + s3_df_tmp.loc[table.index, 'Transfer time (UTC)'] = \ + table['Transfer time (UTC)'].iloc[0] + s3_df_tmp.loc[table.index, 'Download time (UTC)'] = \ + table['Download time (UTC)'].iloc[0] + s3_df_selected = s3_df_tmp.drop_duplicates() + rename_Subject = lambda x: 'Number of subjects' if x == 'Subject' \ + else x + s3_df_selected.columns = [rename_Subject(x) for x in + s3_df_selected.columns] + send_detail( Lochness, Lochness['sender'], diff --git a/tests/lochness_test/email/test_email.py b/tests/lochness_test/email/test_email.py index 28721e3b..9f3641cf 100644 --- a/tests/lochness_test/email/test_email.py +++ b/tests/lochness_test/email/test_email.py @@ -5,6 +5,7 @@ from pathlib import Path import sys +from lochness.config import load lochness_root = Path(lochness.__path__[0]).parent scripts_dir = lochness_root / 'scripts' test_dir = lochness_root / 'tests' @@ -54,3 +55,10 @@ def test_box_sync_module_mailx(args_and_Lochness_BIDS): send_out_daily_updates(Lochness) +def test_email_size(): + config_loc = '/mnt/prescient/Prescient_production/config.yml' + Lochness = load(config_loc) + Lochness['sender'] = 'kevincho@bwh.harvard.edu' + + Lochness['notify']['test'] = ['kevincho@bwh.harvard.edu'] + send_out_daily_updates(Lochness) From 11832e597ee6180237380ce18c677336eca1019d Mon Sep 17 00:00:00 2001 From: kcho Date: Sat, 17 Feb 2024 04:45:07 +1100 Subject: [PATCH 23/29] feat: keep the earliest consent date --- lochness/rpms/__init__.py | 20 ++++++++++++++++++-- tests/lochness_test/rpms/test_rpms.py | 21 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/lochness/rpms/__init__.py b/lochness/rpms/__init__.py index aeb2054d..eb3a451b 100644 --- a/lochness/rpms/__init__.py +++ b/lochness/rpms/__init__.py @@ -248,6 +248,7 @@ def initialize_metadata(Lochness: 'Lochness object', if not rpms_id_colname in df_measure_all_subj.columns: continue + consent_dict = {} for subject, df_table in df_measure_all_subj.groupby( rpms_id_colname): for index, df_measure in df_table.iterrows(): @@ -293,9 +294,24 @@ def initialize_metadata(Lochness: 'Lochness object', # Consent date if rpms_consent_colname in df_measure.index: - subject_dict['Consent'] = datetime.strptime( + consent_tmp = datetime.strptime( df_measure[rpms_consent_colname], - '%d/%m/%Y %I:%M:%S %p').strftime('%Y-%m-%d') + '%d/%m/%Y %I:%M:%S %p') + + if subject in consent_dict.keys(): + prev_consent = datetime.strptime( + consent_dict[subject], '%Y-%m-%d') + if prev_consent > consent_tmp: + subject_dict['Consent'] = consent_tmp.strftime( + '%Y-%m-%d') + consent_dict[subject] = subject_dict['Consent'] + else: + subject_dict['Consent'] = prev_consent.strftime( + '%Y-%m-%d') + else: + subject_dict['Consent'] = consent_tmp.strftime( + '%Y-%m-%d') + consent_dict[subject] = subject_dict['Consent'] # mediaflux source has its foldername as its subject ID subject_dict['RPMS'] = f'rpms.{study_name}:' + \ diff --git a/tests/lochness_test/rpms/test_rpms.py b/tests/lochness_test/rpms/test_rpms.py index 039bd622..bcb1a295 100644 --- a/tests/lochness_test/rpms/test_rpms.py +++ b/tests/lochness_test/rpms/test_rpms.py @@ -309,3 +309,24 @@ class SubjectTest(object): subject_df_dict = get_subject_data(all_df_dict, subject, id_colname) for key, table in subject_df_dict.items(): print(key, len(table)) + + +def test_subjects_with_multiple_consent_dates(): + Lochness = {} + Lochness['RPMS_PATH'] = '/mnt/prescient/RPMS_incoming' + Lochness['phoenix_root'] = 'test' + study_name = 'GW' + rpms_consent_colname = 'chric_consent_date' + rpms_id_colname = 'subjectkey' + Lochness['RPMS_id_colname'] = rpms_id_colname + out_metadata = Path(f'test/GENERAL/{study_name}/{study_name}_metadata.csv') + out_metadata.parent.mkdir(exist_ok=True) + + initialize_metadata(Lochness, + study_name, + rpms_id_colname, + rpms_consent_colname) + df = pd.read_csv(out_metadata) + with open('tmp_subject_id_to_check.txt', 'r') as fp: + subject_id = fp.read().strip() + print(df[df['Subject ID']==subject_id]['Consent']) From 555609600287c82fb6d0e8d10cfc65994d3e2211 Mon Sep 17 00:00:00 2001 From: kcho Date: Tue, 27 Feb 2024 03:40:36 +1100 Subject: [PATCH 24/29] fix: bracket in the wrong place --- lochness/utils/path_checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lochness/utils/path_checker.py b/lochness/utils/path_checker.py index caf12624..d84aba26 100644 --- a/lochness/utils/path_checker.py +++ b/lochness/utils/path_checker.py @@ -366,9 +366,9 @@ def update_by_checking_against_subject_list( subject_id_list: List[str]) -> None: '''pass''' df['exist_in_db'] = df['subject'].isin(subject_id_list).fillna(False) + missing_in_db_index = df[~df.exist_in_db].index.tolist() - df.loc[~df[df['exist_in_db']].index, - 'notes'] = 'Subject missing from database' + df.loc[missing_in_db_index, 'notes'] = 'Subject missing from database' def check_file_path_df(df: pd.DataFrame, From 7eb7aec9f4bf1c6ca298c4c8e54a1eecb7ef9143 Mon Sep 17 00:00:00 2001 From: kcho Date: Tue, 27 Feb 2024 03:40:57 +1100 Subject: [PATCH 25/29] fix: bracket in the wrong place --- .../lochness_test/utils/test_path_checker.py | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/lochness_test/utils/test_path_checker.py b/tests/lochness_test/utils/test_path_checker.py index cd8d4530..cec518a4 100644 --- a/tests/lochness_test/utils/test_path_checker.py +++ b/tests/lochness_test/utils/test_path_checker.py @@ -11,7 +11,8 @@ from lochness.utils.path_checker import nth_item_from_path, \ ampscz_id_validate, update_interviews_transcript_check, \ check_file_path_df, update_by_removing_unused_files, \ - update_eeg_check + update_eeg_check, update_by_checking_against_subject_list + from lochness.utils.source_check import collect_mediaflux_files_info, \ get_subject_list_from_metadata @@ -49,3 +50,38 @@ def test_update_interviews_transcript_check(): all_df = check_file_path_df(df, subject_id_list) df_tmp.to_csv('test.csv') + + +def test_update_by_checking_against_subject_list(): + from lochness.config import load + config_loc = '/mnt/prescient/Prescient_production/config.yml' + Lochness = load(config_loc) + + subject_id_list = get_subject_list_from_metadata(Lochness) + print(subject_id_list) + ignore_id_list = [] + + if Lochness.get('id_list_csv', False): + # Prescient network read full subject list from RPMS + check_only_subject_id_list = True + unique_subjects = [] + for csv in Path(Lochness['RPMS_PATH']).glob('*csv'): + try: + [unique_subjects.append(x) for x in + pd.read_csv(csv)[Lochness['RPMS_id_colname']] + if x not in unique_subjects] + except: + pass + ignore_id_list += [x for x in unique_subjects + if x not in subject_id_list] + ignore_id_list = [x for x in ignore_id_list if type(x) == str] + + if Path('mediaflux_df_test.csv').is_file(): + mediaflux_df = pd.read_csv('mediaflux_df_test.csv', index_col=0) + else: + mediaflux_df = collect_mediaflux_files_info(Lochness, ignore_id_list) + mediaflux_df.to_csv('mediaflux_df_test.csv') + + df = mediaflux_df.copy() + all_df = check_file_path_df(df, subject_id_list) + all_df.to_csv('test.csv') From f1c27c1444b558c7ec473e5b1d2e19491a93a3f4 Mon Sep 17 00:00:00 2001 From: Dheshan Mohandass Date: Thu, 7 Mar 2024 09:49:09 -0500 Subject: [PATCH 26/29] fix: Ignore all *.conf files for Interview modality --- lochness/utils/path_checker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lochness/utils/path_checker.py b/lochness/utils/path_checker.py index d84aba26..fac41585 100644 --- a/lochness/utils/path_checker.py +++ b/lochness/utils/path_checker.py @@ -175,7 +175,7 @@ def update_interviews_video_check(df: pd.DataFrame) -> pd.DataFrame: (df.modality == "Interviews") & ( (df.file_name.str.endswith(".mp4")) - | (df.file_name.str.endswith("recording.conf")) + | (df.file_name.str.endswith(".conf")) ) ].index @@ -309,7 +309,7 @@ def update_by_removing_unused_files(df: pd.DataFrame) -> None: The following files will be removed: - .DS_Store - - recording.conf + - *.conf - chat.txt - Files under TRANSCRIPTS/For review @@ -323,8 +323,8 @@ def update_by_removing_unused_files(df: pd.DataFrame) -> None: ds_store_index = df[df.file_name == '.DS_Store'].index df.drop(ds_store_index, inplace=True) - # recording.conf files - recording_conf_index = df[df.file_name.str.endswith('recording.conf')].index + # *.conf files + recording_conf_index = df[df.file_name.str.endswith('.conf')].index df.drop(recording_conf_index, inplace=True) # chat.txt files @@ -413,7 +413,7 @@ def check_file_path_df(df: pd.DataFrame, # ignore genetics and fluids update_by_removing_genetics_and_fluids(df) - # ignore .DS_Store, recording.conf, chat.txt + # ignore .DS_Store, *.conf, chat.txt update_by_removing_unused_files(df) # check if the subject exist in metadata From ac59b8539322f0dba934f02ef37553fa7aadad83 Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 19 Apr 2024 04:36:37 +1000 Subject: [PATCH 27/29] feat: metadata to be under PROTECTED --- README.md | 6 +++--- lochness/__init__.py | 13 +++++++------ lochness/redcap/__init__.py | 8 ++++---- scripts/lochness_create_template.py | 2 +- scripts/phoenix_generator.py | 2 +- tests/lochness_test/daris/test_daris.py | 2 +- tests/lochness_test/redcap/test_redcap.py | 12 ++++++++---- tests/lochness_test/rpms/test_rpms.py | 4 ++-- tests/lochness_test/transfer/test_transfer.py | 4 ++-- tests/test_lochness.py | 4 ++-- 10 files changed, 31 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 2defdb9c..8d005bbd 100644 --- a/README.md +++ b/README.md @@ -103,14 +103,14 @@ Running one of the commands above will create the structure below ├── 1_encrypt_command.sh ├── 2_sync_command.sh ├── PHOENIX -│   ├── GENERAL +│   ├── PROTECTED │   │   ├── PronetLA │   │   │   └── PronetLA_metadata.csv │   │   ├── PronetSL │   │   │   └── PronetSL_metadata.csv │   │   └── PronetWU │   │   └── PronetWU_metadata.csv -│   └── PROTECTED +│   └── GENERAL │   ├── PronetLA │   ├── PronetSL │   └── PronetWU @@ -131,7 +131,7 @@ has all information, it should be encrypted. (See step 3 below) non-sensitive information about the server, lochness instance and different data sources. -2. Either manually update the `PHOENIX/GENERAL/*/*_metadata.csv` or +2. Either manually update the `PHOENIX/PROTECTED/*/*_metadata.csv` or amend the field names in REDCap / RPMS sources correctly for lochness to automatically update the metadata files. diff --git a/lochness/__init__.py b/lochness/__init__.py index 2ed4b3b7..3d120bdb 100644 --- a/lochness/__init__.py +++ b/lochness/__init__.py @@ -121,17 +121,18 @@ def read_phoenix_metadata(Lochness, studies=None): studies = lochness.listdir(Lochness, general_folder) # iterate over studies for study_name in studies: - f = os.path.join(general_folder, - study_name, - f'{study_name}_metadata.csv') + metadata_loc = os.path.join(protected_folder, + study_name, + f'{study_name}_metadata.csv') if not os.path.exists(f): - logger.error('metadata file does not exist {0}'.format(f)) + logger.error('metadata file does not exist {0}'.format + (metadata_loc)) continue - logger.debug('reading metadata file {0}'.format(f)) + logger.debug('reading metadata file {0}'.format(metadata_loc)) try: # iterate over rows in metadata file for subject in _subjects(Lochness, study_name, general_folder, - protected_folder, f): + protected_folder, metadata_loc): yield subject except StudyMetadataError as e: logger.error(e) diff --git a/lochness/redcap/__init__.py b/lochness/redcap/__init__.py index 3dfa0ddc..4e162d32 100644 --- a/lochness/redcap/__init__.py +++ b/lochness/redcap/__init__.py @@ -81,8 +81,8 @@ def initialize_metadata(Lochness: 'Lochness object', project_name = study_name.split(site_code_study)[0] # 'Pronet' # metadata study location - general_path = Path(Lochness['phoenix_root']) / 'GENERAL' - metadata_study = general_path / study_name / f"{study_name}_metadata.csv" + protected_path = Path(Lochness['phoenix_root']) / 'PROTECTED' + metadata_study = protected_path / study_name / f"{study_name}_metadata.csv" # use redcap_project function to load the redcap keyrings for the project _, api_url, api_key = next(redcap_projects( @@ -237,8 +237,8 @@ def initialize_metadata_rm(Lochness: 'Lochness object', project_name = study_name.split(site_code_study)[0] # 'Pronet' # metadata study location - general_path = Path(Lochness['phoenix_root']) / 'GENERAL' - metadata_study = general_path / study_name / f"{study_name}_metadata.csv" + protected_path = Path(Lochness['phoenix_root']) / 'PROTECTED' + metadata_study = protected_path / study_name / f"{study_name}_metadata.csv" # use redcap_project function to load the redcap keyrings for the project _, api_url, api_key = next(redcap_projects( diff --git a/scripts/lochness_create_template.py b/scripts/lochness_create_template.py index 6a430947..f0ba0e51 100755 --- a/scripts/lochness_create_template.py +++ b/scripts/lochness_create_template.py @@ -46,7 +46,7 @@ def create_lochness_template(args): pg(argsForPheonix) except SystemExit: pass - metadata = phoenix_root / 'GENERAL' / study / f'{study}_metadata.csv' + metadata = phoenix_root / 'PROTECTED' / study / f'{study}_metadata.csv' # create example metadata create_example_meta_file_advanced(metadata, study, args.sources) diff --git a/scripts/phoenix_generator.py b/scripts/phoenix_generator.py index 66aa3f4e..6cca9d83 100755 --- a/scripts/phoenix_generator.py +++ b/scripts/phoenix_generator.py @@ -22,7 +22,7 @@ def main(args): general = os.path.join(args.dir, 'GENERAL', args.study) protected = os.path.join(args.dir, 'PROTECTED', args.study) - metadata = os.path.join(general, '{0}_metadata.csv'.format(args.study)) + metadata = os.path.join(protected, '{0}_metadata.csv'.format(args.study)) # check if folder or metadata file already exists for f in [general, protected, metadata]: diff --git a/tests/lochness_test/daris/test_daris.py b/tests/lochness_test/daris/test_daris.py index 1a61070d..754f62a8 100644 --- a/tests/lochness_test/daris/test_daris.py +++ b/tests/lochness_test/daris/test_daris.py @@ -99,7 +99,7 @@ def initialize_metadata(Lochness, study_name): 'Consent': '1988-09-16', 'Subject ID': 'subject01', 'Daris': 'daris.StudyA:5Yp0E'}) - df_loc = Path(Lochness['phoenix_root']) / 'GENERAL' / \ + df_loc = Path(Lochness['phoenix_root']) / 'PROTECTED' / \ study_name / f"{study_name}_metadata.csv" df.to_csv(df_loc, index=False) diff --git a/tests/lochness_test/redcap/test_redcap.py b/tests/lochness_test/redcap/test_redcap.py index 9fc89aa4..73aa2d50 100644 --- a/tests/lochness_test/redcap/test_redcap.py +++ b/tests/lochness_test/redcap/test_redcap.py @@ -108,7 +108,8 @@ def test_initialize_metadata_function_adding_new_data_to_csv( for study in args.studies: phoenix_path = Path(Lochness['phoenix_root']) general_path = phoenix_path / 'GENERAL' - metadata = general_path / study / f"{study}_metadata.csv" + protected_path = phoenix_path / 'PROTECTED' + metadata = protected_path / study / f"{study}_metadata.csv" assert len(pd.read_csv(metadata)) == 1 initialize_metadata(Lochness, study, 'record_id1', 'cons_date', False) @@ -125,7 +126,8 @@ def test_initialize_metadata_then_sync(args_and_Lochness): for study in args.studies: phoenix_path = Path(Lochness['phoenix_root']) general_path = phoenix_path / 'GENERAL' - metadata = general_path / study / f"{study}_metadata.csv" + protected_path = phoenix_path / 'PROTECTED' + metadata = protected_path / study / f"{study}_metadata.csv" initialize_metadata(Lochness, study, 'record_id1', 'cons_date', False) for subject in lochness.read_phoenix_metadata(Lochness, @@ -143,7 +145,8 @@ def LochnessMetadataInitialized(args_and_Lochness): for study in args.studies: phoenix_path = Path(Lochness['phoenix_root']) general_path = phoenix_path / 'GENERAL' - metadata = general_path / study / f"{study}_metadata.csv" + protected_path = phoenix_path / 'PROTECTED' + metadata = protected_path / study / f"{study}_metadata.csv" initialize_metadata(Lochness, study, 'record_id1', 'cons_date', False) @@ -156,7 +159,8 @@ def test_initialize_metadata_update_when_initialized_again( for study in args.studies: phoenix_path = Path(LochnessMetadataInitialized['phoenix_root']) general_path = phoenix_path / 'GENERAL' - metadata = general_path / study / f"{study}_metadata.csv" + protected_path = phoenix_path / 'PROTECTED' + metadata = protected_path / study / f"{study}_metadata.csv" prev_st_mtime = metadata.stat().st_mtime diff --git a/tests/lochness_test/rpms/test_rpms.py b/tests/lochness_test/rpms/test_rpms.py index bcb1a295..56b3c6a8 100644 --- a/tests/lochness_test/rpms/test_rpms.py +++ b/tests/lochness_test/rpms/test_rpms.py @@ -75,7 +75,7 @@ def test_initializing_based_on_rpms(Lochness): create_fake_rpms_repo() Lochness['RPMS_PATH'] = Path('RPMS_repo').absolute() initialize_metadata(Lochness, 'StudyA', 'subjectkey', 'Consent', False) - df = pd.read_csv('tmp_lochness/PHOENIX/GENERAL/StudyA/StudyA_metadata.csv') + df = pd.read_csv('tmp_lochness/PHOENIX/PROTECTED/StudyA/StudyA_metadata.csv') print(df) show_tree_then_delete('tmp_lochness') assert len(df) == 10 @@ -319,7 +319,7 @@ def test_subjects_with_multiple_consent_dates(): rpms_consent_colname = 'chric_consent_date' rpms_id_colname = 'subjectkey' Lochness['RPMS_id_colname'] = rpms_id_colname - out_metadata = Path(f'test/GENERAL/{study_name}/{study_name}_metadata.csv') + out_metadata = Path(f'test/PROTECTED/{study_name}/{study_name}_metadata.csv') out_metadata.parent.mkdir(exist_ok=True) initialize_metadata(Lochness, diff --git a/tests/lochness_test/transfer/test_transfer.py b/tests/lochness_test/transfer/test_transfer.py index ebeaffc2..97360213 100644 --- a/tests/lochness_test/transfer/test_transfer.py +++ b/tests/lochness_test/transfer/test_transfer.py @@ -67,8 +67,8 @@ def test_get_updated_files(Lochness): timestamp_a_day_ago, posttime) - assert Path('PHOENIX/GENERAL/StudyA/StudyA_metadata.csv') in file_lists - assert Path('PHOENIX/GENERAL/StudyB/StudyB_metadata.csv') in file_lists + assert Path('PHOENIX/PROTECTED/StudyA/StudyA_metadata.csv') in file_lists + assert Path('PHOENIX/PROTECTED/StudyB/StudyB_metadata.csv') in file_lists show_tree_then_delete('tmp_lochness') diff --git a/tests/test_lochness.py b/tests/test_lochness.py index fb1a0bbd..dcbdb995 100644 --- a/tests/test_lochness.py +++ b/tests/test_lochness.py @@ -396,8 +396,8 @@ def initialize_metadata_test(phoenix_root: 'phoenix root', df_final = df_final[main_cols + \ [x for x in df_final.columns if x not in main_cols]] - general_path = Path(phoenix_root) / 'GENERAL' - metadata_study = general_path / study_name / f"{study_name}_metadata.csv" + protected_path = Path(phoenix_root) / 'PROTECTED' + metadata_study = protected_path / study_name / f"{study_name}_metadata.csv" df_final.to_csv(metadata_study, index=False) From 7b7cd8c73ea5d73e183e9cf657332b863b1743b1 Mon Sep 17 00:00:00 2001 From: kcho Date: Fri, 19 Apr 2024 05:25:38 +1000 Subject: [PATCH 28/29] fix: metadata from RPMS to be under PROTECTED --- lochness/__init__.py | 2 +- lochness/rpms/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lochness/__init__.py b/lochness/__init__.py index 3d120bdb..a62ff79b 100644 --- a/lochness/__init__.py +++ b/lochness/__init__.py @@ -124,7 +124,7 @@ def read_phoenix_metadata(Lochness, studies=None): metadata_loc = os.path.join(protected_folder, study_name, f'{study_name}_metadata.csv') - if not os.path.exists(f): + if not os.path.exists(metadata_loc): logger.error('metadata file does not exist {0}'.format (metadata_loc)) continue diff --git a/lochness/rpms/__init__.py b/lochness/rpms/__init__.py index eb3a451b..196c3fee 100644 --- a/lochness/rpms/__init__.py +++ b/lochness/rpms/__init__.py @@ -348,7 +348,7 @@ def initialize_metadata(Lochness: 'Lochness object', df_final = df_final[main_cols + \ [x for x in df_final.columns if x not in main_cols]] - general_path = Path(Lochness['phoenix_root']) / 'GENERAL' + general_path = Path(Lochness['phoenix_root']) / 'PROTECTED' metadata_study = general_path / study_name / f"{study_name}_metadata.csv" df_final.to_csv(metadata_study, index=False) From e4b0af1cb2ec55db433b99d852fea1eb00e7f540 Mon Sep 17 00:00:00 2001 From: kcho Date: Sat, 20 Apr 2024 02:54:17 +1000 Subject: [PATCH 29/29] feat: metadata to be under PROTECTED --- lochness/transfer/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lochness/transfer/__init__.py b/lochness/transfer/__init__.py index 944409e1..f34eee77 100644 --- a/lochness/transfer/__init__.py +++ b/lochness/transfer/__init__.py @@ -93,12 +93,12 @@ def compress_list_of_files(phoenix_root: str, out_tar_ball: tar file to save eg) ``` - file_list = ['PHOENIX/GENERAL/StudyA/StudyA_metadata.csv', - 'PHOENIX/GENERAL/StudyB/StudyB_metadata.csv'] + file_list = ['PHOENIX/PROTECTED/StudyA/StudyA_metadata.csv', + 'PHOENIX/PROTECTED/StudyB/StudyB_metadata.csv'] ``` PHOENIX/ - └── GENERAL + └── PROTECTED ├── StudyA │   └── StudyA_metadata.csv └── StudyB @@ -339,7 +339,7 @@ def lochness_to_lochness_transfer_s3(Lochness, # metadata metadata_files = Path(Lochness['phoenix_root']).glob( - f'GENERAL/*/*_metadata.csv') + f'PROTECTED/*/*_metadata.csv') for metadata_file in metadata_files: metadata_dir = metadata_file.parent s3_phoenix_metadata = re.sub(Lochness['phoenix_root'],