diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..cbbc931d --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,7 @@ +version: 2 +formats: all +python: + version: 3 + install: + - requirements: docs/requirements.txt + system_packages: true diff --git a/README.md b/README.md index 2defdb9c..8d005bbd 100644 --- a/README.md +++ b/README.md @@ -103,14 +103,14 @@ Running one of the commands above will create the structure below ├── 1_encrypt_command.sh ├── 2_sync_command.sh ├── PHOENIX -│   ├── GENERAL +│   ├── PROTECTED │   │   ├── PronetLA │   │   │   └── PronetLA_metadata.csv │   │   ├── PronetSL │   │   │   └── PronetSL_metadata.csv │   │   └── PronetWU │   │   └── PronetWU_metadata.csv -│   └── PROTECTED +│   └── GENERAL │   ├── PronetLA │   ├── PronetSL │   └── PronetWU @@ -131,7 +131,7 @@ has all information, it should be encrypted. (See step 3 below) non-sensitive information about the server, lochness instance and different data sources. -2. Either manually update the `PHOENIX/GENERAL/*/*_metadata.csv` or +2. Either manually update the `PHOENIX/PROTECTED/*/*_metadata.csv` or amend the field names in REDCap / RPMS sources correctly for lochness to automatically update the metadata files. diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..815da534 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +pykernel +nbsphinx +sphinx-copyButton diff --git a/docs/source/box_source_structure.rst b/docs/source/box_source_structure.rst new file mode 100644 index 00000000..86de2986 --- /dev/null +++ b/docs/source/box_source_structure.rst @@ -0,0 +1,93 @@ +Box source +========== + +A Box app needs to be created with ``OAuth 2.0 with Client Credentials Grant`` +and needs to be authorized by Box admin. + + + +For Keyring +----------- +Make note of ``ENTERPRISE_ID``, ``CLIENT_ID`` and ``CLIENT_SECRET`` in the Boxp +developer webpage. + + + +Data structure on Box +--------------------- +Planned data structure on Box account (the source itself) looks like below :: + + ProNET + ├── PronetAB + │ ├── PronetAB_Interviews + │ │ ├── OPEN + │ │ │ └── AB00001 + │ │ │ └── 2021-12-10 16.01.56 Kevin Cho's Zoom Meeting + │ │ │ ├── video2515225130.mp4 + │ │ │ ├── video1515225130.mp4 + │ │ │ ├── audio2515225130.mp4 + │ │ │ ├── audio1515225130.mp4 + │ │ │ └── Audio Record + │ │ │ └── Audio Record + │ │ │ ├── audioKevinCho42515225130.m4a + │ │ │ ├── audioKevinCho21515225130.m4a + │ │ │ ├── audioAnotherPerson11515225130.m4a + │ │ │ └── audioAnotherPerson32515225130.m4a + │ │ ├── PSYCHS + │ │ │ ├── AB00001 + │ │ │ └── ... + │ │ └── transcripts + │ │ ├── Approved + │ │ │ ├── AB00001 + │ │ │ │ ├── PronetAB_AB00001_interviewAudioTranscript_open_day00001_session001.txt + │ │ │ │ └── PronetAB_AB00001_interviewAudioTranscript_open_day00001_session002.txt + │ │ │ └── ... + │ │ └── For_review + │ │ ├── AB00001 + │ │ │ ├── PronetAB_AB00001_interviewAudioTranscript_open_day00001_session001.txt + │ │ │ └── PronetAB_AB00001_interviewAudioTranscript_open_day00001_session002.txt + │ │ └── ... + │ ├── PronetAB_EEG + │ │ └── AB00001 + │ │ └── AB00001_eeg_20220101.zip + │ └── PronetAB_Actigraphy + │ │ └── AB00001 + │ │ └── ... + └── ... + + +Data structure on corresponding PHOENIX +--------------------------------------- +Planned data structure on Box account (the source itself) looks like below :: + + ProNET + ├── PronetAB + │ └── raw + │ ├── AB00009 + │ │   ├── actigraphy + │ │   ├── eeg + │ │   │   └── AB00009_eeg_20211102.eeg + │ │   ├── interviews + │ │   │   ├── open + │ │ │ │ └── 2021-12-10 16.01.56 Kevin Cho's Zoom Meeting + │ │ │ │ ├── Audio Record + │ │ │ │ │ └── Audio Record + │ │ │ │ │ ├── audioKevinCho42515225130.m4a + │ │ │ │ │ ├── audioKevinCho21515225130.m4a + │ │ │ │ │ ├── audioAnotherPerson11515225130.m4a + │ │ │ │ │ └── audioAnotherPerson32515225130.m4a + │ │ │ │ ├── video2515225130.mp4 + │ │ │ │ ├── audio2515225130.mp4 + │ │ │ │ ├── video1515225130.mp4 + │ │ │ │ └── audio1515225130.mp4 + │ │   │   ├── psychs + │    │   │   │ └── ... + │ │   │   └── transcripts + │ │ │ ├── PronetAB_AB00001_interviewAudioTranscript_open_day00001_session001.txt + │ │ │ └── PronetAB_AB00001_interviewAudioTranscript_open_day00001_session002.txt + │ │   └── surveys + │ │   └── AB00009.Pronet.json + │ └── ... + └── .. + + diff --git a/docs/source/conf.py b/docs/source/conf.py index c36eb5f1..ca583b67 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -42,7 +42,7 @@ ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['ntemplates'] +templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: @@ -74,7 +74,9 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +# html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' +html_logo = 'images/AMP SCZ Logo small.png' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/source/configuration_file.rst b/docs/source/configuration_file.rst index 963a78c1..87b60618 100644 --- a/docs/source/configuration_file.rst +++ b/docs/source/configuration_file.rst @@ -1,7 +1,78 @@ Configuration file ================== + The folling page will go over all of the configuration file sections, fields, -and what each one does +and what each one does. + +Example configuration file: + +.. code-block :: text + + keyring_file: /Users/kc244/data_sync/.lochness.enc + phoenix_root: /Users/kc244/data_sync/PHOENIX + pid: /Users/kc244/data_sync/lochness.pid + stderr: /Users/kc244/data_sync/lochness.stderr + stdout: /Users/kc244/data_sync/lochness.stdout + poll_interval: 86400 + ssh_user: kc244 + ssh_host: rgs09.research.partners.org + sender: kevincho@bwh.harvard.edu + mindlamp_days_to_pull: 100 + pii_table: /Users/kc244/data_sync/pii_table.csv + lochness_sync_history_csv: /Users/kc244/data_sync/sync_history.csv + + # REDCap + redcap_id_colname: chric_record_id + redcap_consent_colname: chric_consent_date + + # RPMS + RPMS_PATH: /Users/kc244/RPMS_incoming + RPMS_id_colname: subjectkey + RPMS_consent_colname: Consent + + AWS_BUCKET_NAME: prescient-test + AWS_BUCKET_ROOT: TEST_PHOENIX_ROOT_PRESCIENT + s3_selective_sync: mri,surveys,actigraphy + + redcap: + {study}: + deidentify: True + data_entry_trigger_csv: {args.det_csv} + update_metadata: True''' + box: + {study}: + base: ProNET/{study} + delete_on_success: False + file_patterns: + actigraphy: + - vendor: Activinsights + product: GENEActiv + data_dir: {study}_Actigraphy + pattern: '*.*' + eeg: + - product: eeg + data_dir: {study}_EEG + pattern: '*.*' + interviews: + - product: open + data_dir: {study}_Interviews/OPEN + out_dir: open + pattern: '*.*' + - product: psychs + data_dir: {study}_Interviews/PSYCHS + out_dir: psychs + pattern: '*.*' + - product: transcripts + data_dir: {study}_Interviews/transcripts/Approved + out_dir: transcripts + pattern: '*.*' + admins: + - kevincho@bwh.harvard.edu + notify: + __global__: + - kevincho@bwh.harvard.edu + + keyring_file ------------ @@ -18,33 +89,75 @@ should be a simple filesystem location :: phoenix_root: /data/PHOENIX + stdout ------ -This field determines the location of the Lochness process standard output +This field determines the location of the Lochness process standard output :: stdout: /logs/lochness.out + stderr ------ -This field determines the location of the Lochness process standard error +This field determines the location of the Lochness process standard error :: stderr: /logs/lochness.err + poll_interval ------------- This field determines the frequency at which Lochness will poll external data -sources for incoming data (in seconds) +sources for incoming data (in seconds) :: poll_interval: 43200 + +ssh_user +-------- +Occasionally, you may receive data on an external hard drive or flash drive. +If you want to use Lochness to transfer this data to your PHOENIX filesystem, +you can do this over ``rsync+ssh``. The ``ssh_user`` field determines the +username that will be used for this :: + + ssh_user: example + + +ssh_host +-------- +Occasionally, you may receive data on an external hard drive or flash drive. +If you want to use Lochness to transfer this data to your PHOENIXfilesystem, +you can do this over ``rsync+ssh``. The ``ssh_host`` field determines the +destination host you will connect to for this :: + + ssh_host: host.example.org + + +sender +------ +Whenever an email is sent by Lochness, use this field to determine the sender +address :: + + sender: lochness@host.example.org + + +mindlamp_days_to_pull +--------------------- +Mindlamp data can have a large size, which may require a long time to check the +database on the Mindlamp server. This field determines how many days, from the +day of running sync, to check the Mindlamp database for. :: + + mindlamp_days_to_pull: 100 + + pii_table -------------- +--------- This field determines the location of the csv file that has the mappings for each personally identifiable information (PII) to how to process them. It is -used to process the PII field values in both REDCap and RPMS sources. +used to process the PII field values in both REDCap and RPMS sources. :: poll_interval: ~/pii_convert_table.csv + lochness_sync_history_csv -------------------------- This field determines the location of the csv file that has the history of @@ -54,11 +167,63 @@ given location. lochness_sync_history_csv: /data/lochness_sync_history.csv + +redcap_id_colname and redcap_consent_colname +-------------------------------------------- +These fields determine the name of records on the REDCap database for the +unique subject ID and consent date. :: + + redcap_id_colname: chric_record_id + redcap_consent_colname: chric_consent_date + + +RPMS_PATH, RPMS_id_colname, and RPMS_consent_colname +---------------------------------------------------- +``RPMS_PATH`` determines the root path of RPMS database export directory, which +must be on the same server as Lochness. ``RPMS_id_colname`` and +``RPMS_consent_colname`` fields determine the name of records on the RPMS +database for the unique subject ID and consent date. :: + + RPMS_PATH: /Users/kc244/RPMS_incoming + RPMS_id_colname: subjectkey + RPMS_consent_colname: Consent + + +AWS_BUCKET_NAME and AWS_BUCKET_ROOT +----------------------------------- +These fields determine the name of AWS s3 bucket, and the path of the root to +transfer the data to. :: + + AWS_BUCKET_NAME: prescient-test + AWS_BUCKET_ROOT: TEST_PHOENIX_ROOT_PRESCIENT + + +s3_selective_sync +----------------- +This field determines the list of data types, that Lochness could transfer +the raw data without any processing, as they were downloaded from their data +sources. All the data, even the data under ``PROTECTED`` for these selected +datatypes will be transferred to s3 bucket. :: + + s3_selective_sync: mri,surveys,actigraphy + + +redcap +------ +This field determines the list of data types, that Lochness could transfer +the raw data without any processing, as they were downloaded from their data +sources. All the data, even the data under ``PROTECTED`` for these selected +datatypes will be transferred to s3 bucket. :: + + s3_selective_sync: mri,surveys,actigraphy + + beiwe ----- The ``beiwe`` section is used to configure how Lochness will behave while downloading data from the `Beiwe `_. + backfill_start ~~~~~~~~~~~~~~ The ``backfill_start`` field should be an ISO 8601 formatted timestamp. If you do not @@ -115,6 +280,7 @@ box The ``box`` section is used to configure how Lochness will behave when downloading data from `Box `_. + delete on success ~~~~~~~~~~~~~~~~~ You can add a ``delete_on_success: True`` field to indicate that any data successfully @@ -215,28 +381,3 @@ section :: __global__: - admin1@email.com -sender ------- -Whenever an email is sent by Lochness, use this field to determine the sender -address :: - - sender: lochness@host.example.org - -ssh_user --------- -Occasionally, you may receive data on an external hard drive or flash drive. -If you want to use Lochness to transfer this data to your PHOENIX filesystem, -you can do this over ``rsync+ssh``. The ``ssh_user`` field determines the -username that will be used for this :: - - ssh_user: example - -ssh_host --------- -Occasionally, you may receive data on an external hard drive or flash drive. -If you want to use Lochness to transfer this data to your PHOENIXfilesystem, -you can do this over ``rsync+ssh``. The ``ssh_host`` field determines the -destination host you will connect to for this :: - - ssh_host: host.example.org - diff --git a/docs/source/data_entry_trigger.rst b/docs/source/data_entry_trigger.rst new file mode 100644 index 00000000..8517ee09 --- /dev/null +++ b/docs/source/data_entry_trigger.rst @@ -0,0 +1,144 @@ + +REDCap data entry trigger +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unfortunately, REDCap does not support APIs for selectively downloading updated +data. So, if your REDCap server has a limit on how much you can download per +a given time, you will need to set up "data entry trigger" (DET). + + +First step: enabling DET on REDCap web settings +""""""""""""""""""""""""""""""""""""""""""""""" + + +Go to your REDCap project website, then + +1. Go to **Project Setup** +2. Go to **Additional customizations** under Enable optional modules and + customizations + + +.. image:: images/redcap_det_1.png + + + +Go to your REDCap project website, then + + +1. Scroll to the bottom, and enable **Data Entry Trigger** +2. Provide an address of your server, where you will be running the lochness. + Make sure your firewall is configured to receive signals from the REDCap + server. For an example, the REDCap data entry trigger could be configured to + send the signal to ``https://lochness.server.address/redcap/`` + +.. image:: images/redcap_det_2.png + + +.. note:: + + Please note the ``/`` at the end of the address being entered to the data + entry trigger target. + Please contact your REDCap admin if you do not see Data Entry Trigger on the + page. + + +Second step: set up nginx +"""""""""""""""""""""""""" + +The ``nginx`` on the server needs be configured to following +(create ssl certificate files and save it under ``/etc/nginx``) + +.. code-block:: shell + + server { + listen 443 ssl; + server_name lochness.server.address; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_certificate lochness.crt; + ssl_certificate_key lochness.key; + + location /redcap/ { + proxy_pass http://localhost:8080/; + } + } + + +.. note:: + + .. code-block:: shell + + proxy_pass http://localhost:8080/; + + This is directing the signal arriving at ``https://lochness.server.address/redcap/`` + to port 8080. + + Please note ``/`` at the end of ``/redcap/`` and ``:8080/`` + + + +Third step: run command to listen to the signal +""""""""""""""""""""""""""""""""""""""""""""""" + +Run the command below (and keep it running) to listen to any data entry +trigger. + + +.. code-block:: shell + + listen_to_redcap.py \ + --database_csv /data/pronet/data_sync_pronet/data_entry_trigger_database.csv \ + --port 8080 + + +.. note:: + + This command will listen to port 8080 and save database to + `data_entry_trigger_database.csv` file, which will be used by lochness to + determine if the REDCap data should be downloaded for the individual or not. + (Therefore, you will need to run this command on the server that is running + lochness) + + +.. image:: images/redcap_det_3.png + +1. Try pressing "Test" button on the data entry trigger page on the REDCap + website, and make sure the console of ``listen_to_redcap.py`` printing out + some lines whenever you press the "Test button" + +2. Save the settings. + + +.. note:: + + Try modifying a data field, and see if your ``data_entry_trigger_database.csv`` + saves this change correctly. + + + + +Last step: update your configuration file +""""""""""""""""""""""""""""""""""""""""" +Your lochness configuration file should include the path of the +``data_entry_trigger_database.csv``. See example below. + + +.. code-block:: shell + + AWS_BUCKET_NAME: pronet-test + AWS_BUCKET_ROOT: TEST_PHOENIX_ROOT_PRONET_PROD + s3_selective_sync: [surveys,mri,phone,eeg,actigraphy] + redcap: + PronetLA: + deidentify: True + data_entry_trigger_csv: /data/pronet/data_sync_pronet/data_entry_trigger_database.csv + update_metadata: True + PronetOR: + deidentify: True + data_entry_trigger_csv: /data/pronet/data_sync_pronet/data_entry_trigger_database.csv + update_metadata: True + ... + + + +If you have more than one REDCap server sending the data entry trigger signal, +``nginx`` and ``listen_to_redcap.py`` need to be configured accordingly. diff --git a/docs/source/data_sources.rst b/docs/source/data_sources.rst index 0b92d031..40fabb0f 100644 --- a/docs/source/data_sources.rst +++ b/docs/source/data_sources.rst @@ -1,149 +1,120 @@ Data sources ============ Lochness supports pulling data from a number of data sources. This page will -show you how to configure these data sources both in the `keyring `_ -and in the PHOENIX `metadata files `_. +show you how to configure these data sources both in the source, keyring, +and configuration file. -Beiwe ------ -For each Beiwe instance you will be pulling data from, add a new connection -details section to the root of your ``keyring``. This name of this section -can be whatever you like as long as it is valid JSON. Under your new section, -you must add the ``URL``, ``ACCESS_KEY``, and ``SECRET_KEY`` fields :: +As a reminder, to have AMPSCZ-Lochness download any data from given sources, +it requires either REDCap or RPMS linked to Lochness. Lochness loads list of +unique IDs from the chosen database, as well as the corresponding mindlamp IDs +for each subject. - { - "beiwe.example": { - "URL": "https://beiwe.example.org", - "ACCESS_KEY": "...", - "SECRET_KEY": "..." - } - } +AMP-SCZ subject ID follow the pattern of ``XX00000``, where -metadata file entry -~~~~~~~~~~~~~~~~~~~ -A valid metadata file entry for ``Beiwe`` should look as follows :: +* The first two capital letters represent unique site. +* The five unique digits for each subject. - Active,...,Beiwe,... - 1,...,beiwe.example:STUDY:SUBJECT,... -The ``beiwe.example`` should be a valid ``keyring`` section name, ``STUDY`` -should be the first few characters from the 24-character Study ID, and -``SUBJECT`` should be a valid Beiwe subject. +REDCap +------ -.. attention:: - The ``STUDY`` component of the Beiwe metadata file entry should be the - Study ID, **not** the Study Name. You do not need to enter all 24 - characters of the Study ID either. You only need enough characters - (e.g., the first 5 or so) to make it uniquely identifiable. - - -backfilling -~~~~~~~~~~~ -There is always the chance that you're deploying Lochness well after you've -started capturing data with Beiwe. Lochness will not attempt to download -*all* of your data in one enormous payload. That could be on the order of -many gigabytes and you will likely run into issues. Instead, Lochness will -request your data in day or week-sized chunks from some starting point up to -the current date. This process is called *backfilling*. - -While Lochness will politely download your data in these digestible chunks, -one detail it cannot easily predict is what date to start the backfill. The -user is expected to set this using the ``backfill_start`` field within the -Lochness configuration file. Please refer to the -configuration file `backfill_start documentation `_ +REDCap project settings +~~~~~~~~~~~~~~~~~~~~~~~ +On the REDCap project, the following fields are required. + +- ID +- Date of consent +- Mediaflux ID + +You need to have the exact the name of each field, which you can find out by +clinking 'H' button next to the field on the dictionary view page. + +.. image:: images/redcap_field_name.png + :width: 600 + + +Also, REDCap data entry trigger should be enabled to send a POST signal to the +same server hosting AMP-SCZ Lochness. This server should have a open port +available to listen to the POST signal from REDCap. Tick the box under +"Project Setup" >> "Additional customizations" >> "Data Entry Trigger" and give +it a address:port to the "URL of website" :: + + https://:9999 + + + +REDCap De-identification +~~~~~~~~~~~~~~~~~~~~~~~~ +For each PHOENIX study, you may add an entry to the Lochness configuration +file indicating that data from REDCap should be *de-identified* before being +saved to the filesystem. Please refer to the +`redcap configuration file documentation `_ for more details. -REDCap ------- -To have Lochness download data from REDCap, you need a few things. -redcap keyring section -~~~~~~~~~~~~~~~~~~~~~~ +REDCap keyring settings +~~~~~~~~~~~~~~~~~~~~~~~ First you need to create a section at the root of your ``keyring`` for your REDCap connection details. You can name this section whatever you like as long as it is valid JSON. Within this section you'll need to add a ``URL`` field and a subsection named ``API_TOKEN`` where you will store all of your -REDCap Project API tokens :: +REDCap Project API tokens. There are two parts required for the REDCap. - { - "redcap.demo": { - "URL": "https://redcap.demo.org", - "API_TOKEN": { - "Project 1": "...", - "Project 2": "..." - } - } - } -.. note:: - To generate a REDCap Project API Token, use the ``API`` section under your - REDCap Project Settings page. You must generate an API Token for each - project. +Part 1 at the top will link each site to ``redcap.Pronet`` credentials :: -lochness keyring subsection -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Within the ``keyring`` file, add a ``REDCAP`` subsection to the primary -``lochness`` section :: + {"lochness" : + {"REDCAP": + {"PronetXX": {"redcap.Pronet": [ "Pronet" ]}, + "PronetAA": {"redcap.Pronet": [ "Pronet" ]}} + } + } - { - "lochness": { - "REDCAP": { - } - } -Within this new ``REDCAP`` section, add a subsection for each PHOENIX -study name. In the following example, we'll assume the study name is -``StudyA`` :: +Part 2 will have the actual credentials of the ``redcap.Pronet`` REDCap +project :: - { - "lochness": { - "REDCAP": { - "StudyA": { - } - } - } + {"redcap.Pronet": + {"URL": "https://redcapynh-p11.ynhh.org", + "API_TOKEN": {"Pronet": "EAXBWERAAWETYZXCGOAWERY"}} } -Within the new ``StudyA`` section, create yet another section with the -name of your REDCap keyring section (e.g., ``redcap.example``) followed -by a list of REDCap projects you want Lochness to search and pull data -from :: - { - "lochness": { - "REDCAP": { - "StudyA": { - "redcap.example": [ - "Project 1", - "Project 2" - ] - } - } - } - } +.. note:: + To generate a REDCap Project API Token, use the ``API`` section under your + REDCap Project Settings page. -metadata file entry -~~~~~~~~~~~~~~~~~~~ -A valid metadata file entry should look as follows :: - Active,...,REDCap,... - 1,...,redcap.example:SUBJECT,... -Where ``redcap.example`` would be a valid ``keyring`` section and ``SUBJECT`` -would be a valid REDCap subject. +REDCap settings in the configuration file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -de-identification -~~~~~~~~~~~~~~~~~ -For each PHOENIX study, you may add an entry to the Lochness configuration -file indicating that data from REDCap should be *de-identified* before being -saved to the filesystem. Please refer to the -`redcap configuration file documentation `_ -for more details. +:: + + pii_table: {args.pii_csv} + lochness_sync_history_csv: {args.lochness_sync_history_csv} + redcap_id_colname: chric_record_id + redcap_consent_colname: chric_consent_date + + redcap: + PronetXX: + deidentify: True + data_entry_trigger_csv: {args.det_csv} + update_metadata: True + PronetAA: + deidentify: True + data_entry_trigger_csv: {args.det_csv} + update_metadata: True + + +Location of table to be used in deidentifying redcap fields +Lochness sync history database csv path +Redcap data entry trigger database csv path RPMS --------- +---- To have Lochness download data automatically from RPMS, you need a few things. @@ -167,17 +138,6 @@ Within your new section, you must add your RPMS RPMS_PATH :: } } -metadata file entry -~~~~~~~~~~~~~~~~~~~ -A valid metadata file entry should look as follows :: - - Active,...,RPMS,... - 1,...,rpms.xxxxx:SUBJECT,... - -Where ``rpms.xxxxx`` would be a valid ``keyring`` section and ``SUBJECT`` -would be a valid Subject raw in RPMS measures. This folder name does not -necessarily have to match the PHOENIX subject. - XNAT @@ -210,54 +170,6 @@ be a valid XNAT project, and ``SUBJECT`` would be a valid XNAT Subject. Subject, not just a MR Session. All MR Sessions for that XNAT Subject will be downloaded. -Dropbox -------- -To have Lochness download data automatically from Dropbox, you need a few -things. - -create access token -~~~~~~~~~~~~~~~~~~~ -First, you need to create an Access Token using the -`Dropbox App Console `_. The token should be a -64-character alphanumeric string. - -create keyring section -~~~~~~~~~~~~~~~~~~~~~~ -Next, you need to create a new ``keyring`` section for your Dropbox instance. -This section must be named ``dropbox.xxxxx`` where ``xxxxx`` can be any -string that is both valid JSON *and* valid as a Python module name. Behind the -scenes, Lochness will use this string to import a module. Within your new -section, you must add your Dropbox Acsess Token to an ``API_TOKEN`` field :: - - { - "dropbox.xxxxx": { - "API_TOKEN": "..." - } - } - -metadata file entry -~~~~~~~~~~~~~~~~~~~ -A valid metadata file entry should look as follows :: - - Active,...,Dropbox,... - 1,...,dropbox.xxxxx:SUBJECT,... - -Where ``dropbox.example`` would be a valid ``keyring`` section and ``SUBJECT`` -would be a valid Subject folder name in Dropbox. This folder name does not -necessarily have to match the PHOENIX subject. - -delete on success -~~~~~~~~~~~~~~~~~ -You can configure Lochness to delete files from Dropbox on successful download. -For details, please refer to the -`dropbox delete_on_success configuration file documentation `_ - -dropbox base -~~~~~~~~~~~~ -You can configure Lochness to begin searching your Dropbox account starting from -a specific subdirectory. For details, please refer to the -`dropbox base configuration file documentation `_. - Box --- @@ -312,6 +224,7 @@ a specific subdirectory. For details, please refer to the `box base configuration file documentation `_. + Mediaflux --------- A standalone documentation for the interaction between Mediaflux and lochness is available `here <./mediaflux.md>`_. @@ -321,6 +234,7 @@ from `https://wiki-rcs.unimelb.edu.au/display/RCS/Configuration+File `_ in 2020. -But since then, it has been greatly enhanced and actively maintained by DPACC for -the AMP-SCZ project. +to as building a "data lake" (hence the name). This repository is a fork from +`harvard-nrg lochness `_, updated and +maintained by DPACC for AMP-SCZ project. + + +AMP-SCZ Lochness has a number of extra functions, many of which are specific +to PRONET and PRESCIENT research networks within the AMP-SCZ. However, extra +functions to pull from the additional data sources such as +`Box `_, +`Mediaflux `_, +`RPMS `_, and +`Mindlamp `_ could be useful for other +studies as well. + + +The main difference between the AMP-SCZ Lochness verses the original +harvard-nrg Lochness, is that AMP-SCZ Lochness requies a database of unique +subject IDs in either ``REDCap`` or ``RPMS``, for lochness to automatically +create a list of subject IDs to be used with further down mechanisms of +Lochness (``metadata.csv``, which will be used in searching for any data to +download). So, if your project also has either ``REDCap`` or ``RPMS`` as the +main database for keeping the list of subjects involved in the study, AMP-SCZ +Lochness could also be useful and linked to your project! + + +List of supported data sources are + - REDCap or + - RPMS + - XNAT + - Box + - Mediaflux + - Mindlamp +.. note :: + + Currently, REDCap or RPMS data sources are the requirements for using + lochness, but future update will allow AMP-SCZ Lochness to pull data without + them, by allowing manual creation of the ``metadata.csv`` file. + + +Please report any bug or issue to our +`github repository `_. Thanks! + + +Contents +-------- + .. toctree:: :maxdepth: 2 - quick_start + installation + setting_up_lochness + running_lochness + phoenix_bids_example sync_in_detail - data_sources - phoenix + other_shell_functions configuration_file + data_sources + ..phoenix Indices and tables diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 00000000..78427abe --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,120 @@ +Installation +============ + + +Requirement +----------- + +Install ``python3`` and ``pip`` using ``Miniconda`` + +.. note :: + + Below is an example for Ubuntu 20.04 + + .. code-block:: shell + + apt-get update + apt-get install wget locales git -y + + echo "export LC_ALL=C" >> ~/.bashrc + echo 'export LANGUAGE="en_US.UTF-8"' >> ~/.bashrc + + cd ~/Downloads + wget https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh + bash Miniconda3-py38_4.10.3-Linux-x86_64.sh + + source ~/.bashrc + + +AMP-SCZ Lochness +---------------- + +Install ``AMP-SCZ Lochness`` and ``AMP-SCZ Yaxil`` using ``pip``. + +.. code-block:: shell + + pip install ampscz-lochness + + +.. note :: + For the most recent AMP-SCZ lochness verion for debugging, it might be + better off to pull most recent commits directly from github. + + .. code-block:: shell + + git clone https://github.com/AMP-SCZ/lochness + cd lochness + pip install -r requirements.txt + + echo "export PYTHONPATH=${PYTHONPATH}:lochness" >> ~/.bashrc # add path + echo "export PATH=${PATH}:lochness/scripts" >> ~/.bashrc # add path + source ~/.bashrc + + +After ``pip`` installation, the scripts below should be available from your +console. + +.. code-block:: shell + + sync.py -h + lochness_create_template.py -h + crypt.py -h + + + +Amazon Web Service (AWS) commandline tool +----------------------------------------- + +AMP-SCZ can also push the downloaded data to Amazon Web Service (AWS) s3 +bukcet. To use this functionality, AWS commandline tool needs be installed and +configured. Install ``awscli`` using ``apt-get`` :: + + sudo apt-get install awscli + +.. note :: + If you do not have sudo privileges, you can also download awscli excutables + `here `_. + + +Then configure ``AWS CLI`` with your AWS credentials for the s3 bucket. + +.. code-block:: shell + + aws configure + + AWS Access Key ID [None]: AKIAIOSFODNN7EXAMPLE + AWS Secret Access Key [None]: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + Default region name [None]: us-east + Default output format [None]: json + + +Your s3 bucket should be available from your shell environment. + +.. code-block:: shell + + $ aws s3 ls YourBucketName + + + +``mailx`` for sending out emails +------------------------------ + +Lochness can send out email updates. This requires ``mailx`` installed in the +data aggregation server. In the current version ``sync.py`` is configured to +use ``mailx`` as the default mechanism, but Google SMTP server can also be used +when ``sync.py`` is slightly tweaked. + + +.. note:: + + To use Google SMTP server, change the two lines + ``send_out_daily_updates(Lochness)`` to + ``send_out_daily_updates(Lochness, mailx=False)`` + + + In order to use Google SMTP in sending out the emails, you need to create + a google account and set the ``Less secure app access`` under the "Account + settings" to "ON". + + +Installation complete! diff --git a/docs/source/mediaflux_source_structure.rst b/docs/source/mediaflux_source_structure.rst new file mode 100644 index 00000000..a805a7ea --- /dev/null +++ b/docs/source/mediaflux_source_structure.rst @@ -0,0 +1,98 @@ +Mediaflux source +================ + +For Keyring +----------- +Please note following items required for mediaflux. :: + + "HOST": "mediaflux.researchsoftware.unimelb.edu.au", + "PORT": "443", + "TRANSPORT": "https", + "DOMAIN": "local", + "USER": "kevin.cho", + "PASSWORD": "****" + + +Data structure on Mediaflux +--------------------------- +Planned data structure on Mediaflux account (the source itself) looks like below :: + + Prescient + ├── PrescientAB + │ ├── PrescientAB_Interviews + │ │ ├── OPEN + │ │ │ └── AB00001 + │ │ │ └── 2021-12-10 16.01.56 Kevin Cho's Zoom Meeting + │ │ │ ├── video2515225130.mp4 + │ │ │ ├── video1515225130.mp4 + │ │ │ ├── audio2515225130.mp4 + │ │ │ ├── audio1515225130.mp4 + │ │ │ └── Audio Record + │ │ │ └── Audio Record + │ │ │ ├── audioKevinCho42515225130.m4a + │ │ │ ├── audioKevinCho21515225130.m4a + │ │ │ ├── audioAnotherPerson11515225130.m4a + │ │ │ └── audioAnotherPerson32515225130.m4a + │ │ ├── PSYCHS + │ │ │ ├── AB00001 + │ │ │ └── ... + │ │ └── transcripts + │ │ ├── Approved + │ │ │ ├── AB00001 + │ │ │ │ ├── PrescientAB_AB00001_interviewAudioTranscript_open_day00001_session001.txt + │ │ │ │ └── PrescientAB_AB00001_interviewAudioTranscript_open_day00001_session002.txt + │ │ │ └── ... + │ │ └── For_review + │ │ ├── AB00001 + │ │ │ ├── PrescientAB_AB00001_interviewAudioTranscript_open_day00001_session001.txt + │ │ │ └── PrescientAB_AB00001_interviewAudioTranscript_open_day00001_session002.txt + │ │ └── ... + │ ├── PrescientAB_EEG + │ │ └── AB00001 + │ │ └── AB00001_eeg_20220101.zip + │ ├── PrescientAB_MRI + │ │ └── AB00001 + │ │ └── AB00001_mri_20220101 + │ ├── PrescientAB_Actigraphy + │ │ └── AB00001 + │ │ └── ... + └── ... + + +Data structure on corresponding PHOENIX +--------------------------------------- +Planned data structure on Mediaflux account (the source itself) looks like below :: + + Prescient + ├── PrescientAB + │ └── raw + │ ├── AB00009 + │ │   ├── actigraphy + │ │   ├── eeg + │ │   │   └── AB00009_eeg_20211102.eeg + │ │   ├── mri + │ │   │   └── AB00001_mri_20220101 + │ │   ├── interviews + │ │   │   ├── open + │ │ │ │ └── 2021-12-10 16.01.56 Kevin Cho's Zoom Meeting + │ │ │ │ ├── Audio Record + │ │ │ │ │ └── Audio Record + │ │ │ │ │ ├── audioKevinCho42515225130.m4a + │ │ │ │ │ ├── audioKevinCho21515225130.m4a + │ │ │ │ │ ├── audioAnotherPerson11515225130.m4a + │ │ │ │ │ └── audioAnotherPerson32515225130.m4a + │ │ │ │ ├── video2515225130.mp4 + │ │ │ │ ├── audio2515225130.mp4 + │ │ │ │ ├── video1515225130.mp4 + │ │ │ │ └── audio1515225130.mp4 + │ │   │   ├── psychs + │    │   │   │ └── ... + │ │   │   └── transcripts + │ │ │ ├── PrescientAB_AB00001_interviewAudioTranscript_open_day00001_session001.txt + │ │ │ └── PrescientAB_AB00001_interviewAudioTranscript_open_day00001_session002.txt + │ │   └── surveys + │ │   └── AB00009.Prescient.json + │ └── ... + └── .. + + diff --git a/docs/source/other_shell_functions.rst b/docs/source/other_shell_functions.rst new file mode 100644 index 00000000..21780308 --- /dev/null +++ b/docs/source/other_shell_functions.rst @@ -0,0 +1,78 @@ +Other shell functions available in lochness +=========================================== + + +``lochness_check_config.py`` +---------------------------- + + +``phoenix_generator.py`` +------------------------ + + +``listen_to_redcap.py`` +----------------------- + + +``lochness_create_template.py`` +------------------------------- + + +``send_email_update.py`` +------------------------ + +Execute ``sync.py`` script to have lochness sync data + +**For Pronet network** :: + + sync.py \ + --config /data/pronet/data_sync_pronet/config.yml \ + --studies PronetLA PronetOR PronetBI PronetNL PronetNC PronetSD \ + PronetCA PronetYA PronetSF PronetPA PronetSI PronetPI \ + PronetNN PronetIR PronetTE PronetGA PronetWU PronetHA \ + PronetMT PronetKC PronetPV PronetMA PronetCM PronetMU \ + PronetSH PronetSL \ + --source redcap upenn box xnat mindlamp \ + --lochness_sync_send --s3 \ + --log-file /data/pronet/data_sync_pronet/log.txt \ + --daily_summary \ + --debug --continuous + + +**For Prescient network** :: + + sync.py \ + --config /data/prescient/data_sync_prescient/config.yml \ + --studies PronetLA PronetOR PronetBI PronetNL PronetNC PronetSD \ + PronetCA PronetYA PronetSF PronetPA PronetSI PronetPI \ + PronetNN PronetIR PronetTE PronetGA PronetWU PronetHA \ + PronetMT PronetKC PronetPV PronetMA PronetCM PronetMU \ + PronetSH PronetSL \ + --source rpms upenn mediaflux mindlamp \ + --lochness_sync_send --s3 \ + --log-file /data/prescient/data_sync_prescient/log.txt \ + --daily_summary \ + --debug --continuous + + +When you invoke this command, you will be prompted for the passphrase that +you used to encrypt your `keyring <#setup>`_. To sidestep the password prompt, +you can use an environment variable ``NRG_KEYRING_PASS``. + +This will run lochness sync function for each site (``studies``) for all +measures (``source``). It will upload newly downloaded data to the s3 bucket +after each data sweep for all sources. Then, this ``sync.py`` function will +execute these functions again after ``poll_interval`` stated in the +``config.yml``. + + +``lochness_create_template.py`` creates a template bash script that could be +used. :: + + bash 2_sync_command.sh + + +The downloaded data will be saved under the PHOENIX directory defined in the +``config.yml`` file. + +Good luck! diff --git a/docs/source/phoenix_bids_example.rst b/docs/source/phoenix_bids_example.rst new file mode 100644 index 00000000..b2f4eaff --- /dev/null +++ b/docs/source/phoenix_bids_example.rst @@ -0,0 +1,135 @@ +Example PHOENIX-BIDS structure +============================== + +U24 uses ``PHOENIX-BIDS`` structure, which is slightly different from the +``PHOENIX`` structure. ``PHOENIX-BIDS`` was used to have more similarity to the +``BIDS`` structure, while maintaining ``protected`` vs ``general`` and ``raw`` +vs ``processed`` concept of the ``PHOENIX``. + + +Summary of the structure +------------------------ + +``////`` :: + + PHOENIX/ + ├── PROTECTED + │   └── PronetAB + │      ├── raw + │    │ ├── AB00001 + │    │ │   ├── mri + │    │ │   ├── eeg + │    │ │   ├── interviews + │    │ │   └── ... + │    │ └── ... + │      └── processed + │    └── AB00001 + │       └── ... + └── GENERAL +       ├── raw +    │ ├── AB00001 +    │ │   ├── surveys +  │  │ │   └── AB00001.Pronet.json +    │ │   └── ... +    │ └── ... +       └── processed +    └── AB00001 + + +Different levels of the structure +--------------------------------- + +**Level 1**: ``GENERAL`` or ``PROTECTED``:: + + PHOENIX/ + ├── GENERAL + └── PROTECTED + + +**Level 2**: Sites (studies) :: + + PHOENIX/ + ├── GENERAL + │   ├── PronetAB + │   ├── ... + │   └── PronetCD + └── PROTECTED + ├── PronetAB + ├── ... + └── PronetCD + + +**Level 3**: ``raw`` or ``processed`` + +metadata files are also saved in this level. + +:: + + PHOENIX/ + ├── GENERAL + │   ├── PronetAB + │ │   ├── PronetAB_metadata.csv + │   │   ├── raw + │   │   └── processed + │   ├── ... + │   └── PronetCD + │    ├── PronetCD_metadata.csv + │      ├── raw + │      └── processed + └── PROTECTED +    ├── PronetAB +    │   ├── raw +    │   └── processed +    ├── ... +    └── PronetCD +       ├── raw +       └── processed + + +**Level 4**: Subject :: + + PHOENIX/ + ├── GENERAL + │   └── PronetAB + │      ├── raw + │    │ ├── AB00001 + │    │ ├── AB00002 + │    │ └── AB00003 + │      └── processed + │    ├── AB00001 + │    ├── AB00002 + │    └── AB00003 + └── PROTECTED +    └── ... + + +**Level 5**: Data types + +``surveys``, ``mri``, ``eeg``, ``interviews``, ``actigraphy`` etc. :: + + PHOENIX/ + ├── PROTECTED + │   └── PronetAB + │      ├── raw + │    │ ├── AB00001 + │    │ │   ├── surveys + │  │  │ │   └── AB00001.Pronet.json + │    │ │   ├── mri + │  │  │ │   ├── AB00001.Pronet.Run_sheet_mri.csv + │  │  │ │   └── AB00001_MR_2022_01_01_1 + │    │ │   ├── eeg + │  │  │ │   ├── AB00001.Pronet.Run_sheet_eeg.csv + │  │  │ │   └── AB00001_eeg_20220101.zip + │    │ │   ├── interviews + │  │  │ │   ├── open + │  │  │ │   ├── psychs + │  │  │ │   └── transcripts + │    │ │   └── actigraphy + │    │ └── ... + │      └── processed + │    └── ... + └── GENERAL +    └── ... + + + diff --git a/docs/source/running_lochness.rst b/docs/source/running_lochness.rst new file mode 100644 index 00000000..037a0a48 --- /dev/null +++ b/docs/source/running_lochness.rst @@ -0,0 +1,65 @@ +Running lochness +================ + +Run ``sync.py`` +--------------- + +Execute ``sync.py`` script to have lochness sync data + +**For Pronet network** + +.. code-block:: shell + + sync.py \ + --config /data/pronet/data_sync_pronet/config.yml \ + --studies PronetLA PronetOR PronetBI PronetNL PronetNC PronetSD \ + PronetCA PronetYA PronetSF PronetPA PronetSI PronetPI \ + PronetNN PronetIR PronetTE PronetGA PronetWU PronetHA \ + PronetMT PronetKC PronetPV PronetMA PronetCM PronetMU \ + PronetSH PronetSL \ + --source redcap upenn box xnat mindlamp \ + --lochness_sync_send --s3 \ + --log-file /data/pronet/data_sync_pronet/log.txt \ + --daily_summary \ + --debug --continuous + + +**For Prescient network** + +.. code-block:: shell + + sync.py \ + --config /data/prescient/data_sync_prescient/config.yml \ + --studies PrescientME PrescientSG PrescientAD PrescientAM PrescientBM \ + PrescientCL PrescientCP PrescientHC PrescientJE PrescientGW \ + PrescientLS \ + --source rpms upenn mediaflux mindlamp \ + --lochness_sync_send --s3 \ + --log-file /data/prescient/data_sync_prescient/log.txt \ + --daily_summary \ + --debug --continuous + + +When you execute this command, you will be prompted for the passphrase that +you used to encrypt your `keyring <#setup>`_. + +This will run lochness sync function for each site (``studies``) for all +measures (``source``) given to ``--source`` argument. The downloaded data will +be saved under the PHOENIX directory defined in the ``config.yml`` file. +``--lochness_sync_send`` with ``--s3`` argument, will make lochness upload +the newly downloaded data to the s3 bucket after each data sweep for all +sources and sites. Then, this ``sync.py`` function will execute these functions +again after the ``poll_interval`` stated in the ``config.yml``. + + +.. note :: + + ``lochness_create_template.py`` creates a template bash script that could be + used. + + .. code-block:: shell + + bash 2_sync_command.sh + + +Good luck! diff --git a/docs/source/setting_up_lochness.rst b/docs/source/setting_up_lochness.rst new file mode 100644 index 00000000..533d748d --- /dev/null +++ b/docs/source/setting_up_lochness.rst @@ -0,0 +1,483 @@ +Setting up Lochness +=================== + +The following items below are the step by step instructions to +semi-automatically setup the lochness environment. It is specific for +AMP-SCZ project, but it could also work for any other projects, if their main +subject database is maintained with ``REDCap`` or ``RPMS`` system. + +If you would like to set up your environment from scratch, please see the pages +for + +* :doc:`configuration file` +* :doc:`keyring file` +* :doc:`setting up data sources` + + +1. Create a template directory +------------------------------ +``lochness_create_template.py`` will help you create a starting point for your +lochness. ``config.yml``, ``PHOENIX``, and keyring files, as well as two bash +scripts for encrypting the keyring file and running the sync, will be created. + +**For Pronet network** + +.. code-block:: shell + + lochness_create_template.py \ + --outdir /data/pronet/data_sync_pronet \ + --studies PronetLA PronetOR PronetBI PronetNL PronetNC PronetSD \ + PronetCA PronetYA PronetSF PronetPA PronetSI PronetPI \ + PronetNN PronetIR PronetTE PronetGA PronetWU PronetHA \ + PronetMT PronetKC PronetPV PronetMA PronetCM PronetMU \ + PronetSH PronetSL \ + --sources redcap upenn box xnat mindlamp \ + --email kevincho@bwh.harvard.edu \ + --poll_interval 86400 \ + --ssh_host 123.456.789 \ + --ssh_user kc244 \ + --lochness_sync_send \ + --s3 \ + --s3_selective_sync surveys mri phone eeg actigraphy + + +**For Prescient network** + +.. code-block:: shell + + lochness_create_template.py \ + --outdir /data/prescient/data_sync_prescient \ + --studies PrescientME PrescientSG PrescientAD PrescientAM PrescientBM \ + PrescientCL PrescientCP PrescientHC PrescientJE PrescientGW \ + PrescientLS \ + --sources rpms upenn mediaflux mindlamp \ + --email kevincho@bwh.harvard.edu \ + --poll_interval 86400 \ + --ssh_host 123.456.789 \ + --ssh_user kc244 \ + --lochness_sync_send \ + --s3 \ + --s3_selective_sync surveys mri phone eeg actigraphy + + +.. note :: + + Add ``--enter_password`` option if you want ``lochness_create_template.py`` + to add your credentials to the template keyring file. + + + +Running the command above will create a directory which will look like this :: + + /data/pronet/data_sync_pronet + ├── 1_encrypt_command.sh + ├── 2_sync_command.sh + ├── PHOENIX + │   ├── GENERAL + │   │   ├── PronetAB + │   │   │   └── PronetAB_metadata.csv + │   │   ├── ... + │   │   └── PronetYA + │   │   └── PronetYA_metadata.csv + │   └── PROTECTED + │   ├── PronetAB + │   ├── ... + │   └── PronetYA + ├── config.yml + ├── lochness.json + └── pii_convert.csv + + +.. note :: + + To see detailed options of `lochness_create_template.py` + + .. code-block:: shell + + lochness_create_template.py -h + + +Step 1 completed. + + + +2. Edit credentials in the template keyring file +------------------------------------------------ + +Connecting to various external `data sources `_ +(REDCap, XNAT, Box, etc.) often requires a myriad of connection details +e.g., URLs, usernames, passwords, API tokens, etc. Lochness will only read +these pieces of information from an encrypted JSON file that we refer to as +the *keyring*. + + +These information needs be added to the ``lochness.json`` template + + +.. code-block:: shell + + cd /data/pronet/data_sync_pronet # the template directory created above + vim lochness.json + + +``lochness.json`` file looks like below. Add credentials to the fields marked +with ``*****`` + +.. code-block:: json + + { + "lochness": { + "REDCAP": { + "PronetLA": { + "redcap.Pronet": [ + "Pronet" + ], + "redcap.UPENN": [ + "UPENN" + ] + }, + ..., + }, + "SECRETS": { + "PronetLA": "LOCHNESS_SECRETS", + ..., + } + }, + "redcap.UPENN": { + "URL": "*****", + "API_TOKEN": { + "UPENN": "*****" + } + }, + "redcap.Pronet": { + "URL": "*****", + "API_TOKEN": { + "Pronet": "*****" + } + }, + "xnat.PronetLA": { + "URL": "*****", + "USERNAME": "*****", + "PASSWORD": "*****" + }, + ..., + "box.PronetLA": { + "CLIENT_ID": "*****", + "CLIENT_SECRET": "*****", + "ENTERPRISE_ID": "*****" + }, + ..., + "mindlamp.PronetLA": { + "URL": "*****", + "ACCESS_KEY": "*****", + "SECRET_KEY": "*****" + }, + ..., + } + + +.. note :: + + If you have used ``--enter_password`` option when creating the template + files, just check through your credentials if they are correctly entered to + the ``keyring.json`` file. + + +Example of completed ``lochness.json`` + +.. code-block:: json + + { + "lochness": { + "REDCAP": { + "PronetLA": { + "redcap.Pronet": [ + "Pronet" + ], + "redcap.UPENN": [ + "UPENN" + ] + }, + ..., + }, + "SECRETS": { + "PronetLA": "LOCHNESS_SECRETS", + ..., + } + }, + "redcap.UPENN": { + "URL": "https://redcap.med.upenn.edu", + "API_TOKEN": { + "UPENN": "BC6BEF2D2369BC8FE1233CAAAB20378D" + } + }, + "redcap.Pronet": { + "URL": "https://redcapynh-p11.ynhh.org" + "API_TOKEN": { + "Pronet": "AFBDCCD55934EE947A388541EED6A216" + } + }, + "xnat.PronetLA": { + "URL": "https://xnat.med.yale.edu", + "USERNAME": "kcho", + "PASSWORD": "whrkddlr8*90" + }, + ..., + "box.PronetLA": { + "CLIENT_ID": "e19fltqp9f9ftv4dydqjius4w20072cr", + "CLIENT_SECRET": "LrkDwYZvA49Q4dXVGv3g4aaSy4SQRobz", + "ENTERPRISE_ID": "756591" + }, + ..., + "mindlamp.PronetLA": { + "URL": "mindlamp.orygen.org.au", + "ACCESS_KEY": "kcho", + "SECRET_KEY": "0c5b0a5af972b2a1b2d6cd299dc37703c22e8ddd5dfd15f0d83ca7a1cb8bcce7" + }, + ..., + } + + + +.. note:: + + If you're using Google SMTP in sending out email, you need to add + ``"email_sender_pw": "PasswordForYourGoogleAccount"`` + + For an example, + + .. code:: json + + "SECRETS": { + "PronetLA": "LOCHNESS_SECRETS", + ..., + } + "email_sender_pw": "aaoiweytyEfhag189e7" + + + +3. Encrypt ``lochness.json`` to make a keyring file +--------------------------------------------------- + +Once required credentials are added to the template ``lochness.json`` keyring +file, it must be encrypted using a passphrase. At the moment, Lochness only +supports encrypting and decrypting files (including the keyring) using the +`cryptease `_ library. This library +should be installed automatically when you install Lochness, but you can +install it separately on another machine as well. + +Encrypt the temporary keyring file by running + +.. code-block:: shell + + crypt.py --encrypt lochness.json -o .lochness.enc + + +.. note :: + Or you could run `2_sync_command.sh`, which contains the same command + + .. code-block:: shell + + bash 1_encrypt_command.sh + + +.. attention:: + I'll leave it up to you to decide on which device you want to encrypt this + file. I will only recommend discarding the decrypted version as soon as + possible. + + +.. _edit_config :: + +4. Edit configuration file +-------------------------- +``config.yml`` file contains details of options to be used in Lochness. + +.. code-block:: console + + vim config.yml + + +Project name +~~~~~~~~~~~~~ +Name of the project. This string will be included in the daily email summary. + +.. code-block:: shell + + project_name: ProNET + or + project_name: Prescient + + + +REDCap or RPMS database column names +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Update names of the ``REDCap`` or ``RPMS`` columns that contain unique subject +ID and consent date of each stubject. + +For REDCap :: + + redcap_id_colname: chric_record_id + redcap_consent_colname: chric_consent_date + + +For RPMS :: + + RPMS_PATH: /mnt/prescient/RPMS_incoming + RPMS_id_colname: subjectkey + RPMS_consent_colname: Consent + +.. note :: + + ``RPMS_PATH`` is the directory where ``RPMS`` exports database as multiple + csv files. + +If there is a limit on how much data you can download in a given time on your +REDCap server, please see :doc:`data entry trigger`. + + + + +Amazon Web Services S3 bucket +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Update AWS s3 bucket name to your s3 bucket name and root directory :: + + AWS_BUCKET_NAME: pronet-test + AWS_BUCKET_ROOT: TEST_PHOENIX_ROOT_PRONET + + + +Remove old & already s3-transferred files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Lochness will read the following information from the configuration file to +remove already transferred files from the local PHOENIX directory. :: + + days_to_keep: 100 + removed_df_loc: /mnt/prescient/Prescient_data/PHOENIX/removed_files.csv + removed_phoenix_root: prescient/Prescient_data/track_removed_files_PHOENIX + + +Box +~~~ +See :doc:`here` for how to configure Box source. Then, +the configure file should have a ``box`` session that states which file +patterns to look for in each study. + +``base`` is the root of the data directory for the study under ``Box``. If your +data for ``PronetAB`` is saved under ``ProNET/PronetAB`` under the root of Box +source, the base for this study should be ``ProNET/PronetAB``. + +``delete_on_success`` is an option for removing the source files on the Box +once lochness successfully downloads them. ``True`` or ``False`` + +``file_patterns`` takes list of different datatypes to be captured in from the +Box. ``data_dir`` of each datatype is the name of the root directory that has +subject directories for this datatype. And each datatype can have more than one +product of files to look for. For an example, for ``interviews`` datatype, +``open``, ``psychs``, and ``transcripts`` products are searched for each +individual. ``out_dir`` can be specified if the files need to be saved under +a specific subdirectory for a product. + + +.. code-block:: shell + + box: + PronetAB: + base: ProNET/PronetAB + delete_on_success: False + file_patterns: + actigraphy: + - vendor: Activinsights + product: GENEActiv + data_dir: PronetAB_Actigraphy + pattern: '*.*' + eeg: + - product: eeg + data_dir: PronetAB_EEG + pattern: '*.*' + interviews: + - product: open + data_dir: PronetAB_Interviews/OPEN + out_dir: open + pattern: '*.*' + - product: psychs + data_dir: PronetAB_Interviews/PSYCHS + out_dir: psychs + pattern: '*.*' + - product: transcripts + data_dir: PronetAB_Interviews/transcripts/Approved + out_dir: transcripts + pattern: '*.*' + +See :doc:`here` for an example of output PHOENIX +structure from this configuration. + + +Mediaflux +~~~~~~~~~ + +See :doc:`here` for how to configure Box. Then, the +configure file should have a ``box`` session that states which file patterns +to look for in each study. + + +.. code-block:: shell + + box: + PrescientAB: + base: ProNET/PrescientAB + delete_on_success: False + file_patterns: + actigraphy: + - vendor: Activinsights + product: GENEActiv + data_dir: PrescientAB_Actigraphy + pattern: '*.*' + eeg: + - product: eeg + data_dir: PrescientAB_EEG + pattern: '*.*' + mri: + - product: mri + data_dir: PrescientAB_MRI + pattern: '*.*' + interviews: + - product: open + data_dir: PrescientAB_Interviews/OPEN + out_dir: open + pattern: '*.*' + - product: psychs + data_dir: PrescientAB_Interviews/PSYCHS + out_dir: psychs + pattern: '*.*' + - product: transcripts + data_dir: PrescientAB_Interviews/transcripts/Approved + out_dir: transcripts + pattern: '*.*' + +See :doc:`here` for an example of output PHOENIX +structure from this configuration. + + +Email function +~~~~~~~~~~~~~~ + +Update ``sender`` and ``notify`` fields. ``sender`` should be the google email +configured for sending emails with its relevant credentials in the keyring +file. List of emails, to which lochness should send the email should be added +under ``__global__`` field with ``-`` marking each email. :: + + + sender: kevincho.lochness@gmail.com + notify: + __global__: + - kevincho@bwh.harvard.edu + - another.person.to.receive.email.1@u24.com + - another.person.to.receive.email.2@u24.com + + +Now, your Lochness configuration is complete and ready to run! + diff --git a/docs/source/sync_in_detail.rst b/docs/source/sync_in_detail.rst index 4a1ff85c..2e709d97 100644 --- a/docs/source/sync_in_detail.rst +++ b/docs/source/sync_in_detail.rst @@ -1,24 +1,41 @@ Lochness ``sync.py`` function in detail ======================================= -Lochness provides a single command line tool (daemon) to periodically poll -and download data from various web services into a local directory. Out of -the box, there is support for pulling data from a multitude of -`data sources `_ including REDCap, XNAT, Dropbox, Box, -Mediaflux, RPMS, external hard drives, and more. Its steps are described below: +``sync.py`` is the main commandline shell script, which executes the data +sync pipelines of Lochness. This page goes through what the ``sync.py`` does +in more detail, so user can have a deeper understanding of the mechanims. -1. Load configuration file ---------------------------- -``Lochness`` loads and sets up configurations based on the information -contained in the ``config.yml``. +Loads configuration file, then keyring file +------------------------------------------- +Location of a configuration yaml file is one of the required input to the +``sync.py``. As briefly explained in the +:doc:`Setting up lochness `, unique information for +the server, various data sources, and etc. are included in the yaml file and +this file is loaded first, when ``sync.py`` is executed. +Please see the configuration file section for more information. -2. Automatically create and update metadata for each site ---------------------------------------------------------- -``Lochness`` connects to either ``REDCap`` or ``RPMS`` to download list of -subjects registered for each site (``study``) and creates :: +The encrypted keyring file, location of should have been included in +the configuration file, is also loaded by ``sync.py``. + +Please see the keyring file section for more information. + +Now Lochness is ready to pull the files. + + +Creates and updates metadata for each site +------------------------------------------ +``Lochness`` first needs to pull information from ``REDCap`` or ``RPMS`` to +get the list of subject IDs registered for each site (``study``). Using the +information loaded from the configuration and keyring files, ``Lochness`` looks +for unique subject IDs, consent date, and their unique mindlamp ID registered +in the ``REDCap`` or ``RPMS`` database. This step will create a +``{site}_metadata.csv`` file under each GENERAL site directory. + + +.. code-block:: shell PHOENIX └── GENERAL @@ -28,26 +45,76 @@ subjects registered for each site (``study``) and creates ::       └── PronetCD_metadata.csv -eg) ``PronetAB_metadata.csv`` :: +Here is an example of the ``metadata.csv``, created by Lochness. + +eg) ``PronetAB_metadata.csv`` + +.. csv-table:: + :header: "Active", "Consent", "Subject ID", "REDCap", "Box", "XNAT", "Mindlamp" + + 1,1900-01-01,AB00001,redcap.Pronet:AB00001;redcap.UPENN:AB00001,box.PronetAB:AB00001,xnat.PronetAB:`*`:AB00001,mindlamp.PronetAB:108230 + 1,1900-01-01,AB00002,redcap.Pronet:AB00002;redcap.UPENN:AB00002,box.PronetAB:AB00002,xnat.PronetAB:`*`:AB00002,mindlamp.PronetAB:801230 + 1,1900-01-01,AB00003,redcap.Pronet:AB00003;redcap.UPENN:AB00003,box.PronetAB:AB00003,xnat.PronetAB:`*`:AB00003,mindlamp.PronetAB:208103 + + +The columns for each data source get populated with the unique strings, which +are the combination of site (study) and subject ID, in the format that is +readable by Lochness. And this ``metadata.csv`` files are updated at every +sync circulation, therefore any new subjects added to the ``REDCap`` or +``RPMS`` will be populated into the ``metadata.csv``. + + +.. note :: + + These metadata files are automatically created and updated by ``lochness``, + so users should not manually update them. + + + +Pull data for each subject in ``metadata.csv`` +---------------------------------------------- +Then, ``Lochness`` goes over the list of data sources given to the ``sync.py`` +through ``--sources`` argument, checking for any available data that matches +unique subject ID patterns in the ``metadata.csv`` file. + + +List of data sources focused in AMP-SCZ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**For Pronet** + +* REDCap +* UPENN REDCap +* XNAT +* Box +* Mindlamp - Active,Consent,Subject ID,REDCap,Box,XNAT,Mindlamp - 1,1900-01-01,AB00001,redcap.Pronet:AB00001;redcap.UPENN:AB00001,box.PronetAB:AB00001,xnat.PronetAB:*:AB00001,mindlamp.PronetAB:108230 - 1,1900-01-01,AB00002,redcap.Pronet:AB00002;redcap.UPENN:AB00002,box.PronetAB:AB00002,xnat.PronetAB:*:AB00002,mindlamp.PronetAB:801230 - 1,1900-01-01,AB00003,redcap.Pronet:AB00003;redcap.UPENN:AB00003,box.PronetAB:AB00003,xnat.PronetAB:*:AB00003,mindlamp.PronetAB:208103 +**For Prescient** -These metadata files are automatically created and updated by ``lochness``, so -users should not manually update them. +* RPMS +* UPENN REDCap +* Mediaflux +* Mindlamp -3. Loop through sources for each subject in metadata ----------------------------------------------------- -``Lochness`` goes over each source, checking and pulling data for each subject available in ``*_metadata.csv`` +.. warning :: + Since this step depends on the ``metadata.csv`` file, any data from subjects + who are not included in the ``metadata.csv`` will not be downloaded by + Lochness. In another words, any data that belong to a subject who are + missing from ``REDCap`` or ``RPMS`` will not be downloaded by Lochness. +.. _transfer_selected_data_to_s3_bucket: -4. Transfer selected data to s3 bucket ---------------------------------------- -By the application of ``--s3`` option, ``Lochness`` can transfer files to AWS S3 bucket using AWS -CLI ``rsync``. +Transfer selected data to s3 bucket +------------------------------------ +With ``--s3`` option, ``Lochness`` can also transfer file to AWS s3 bucket +using AWS CLI ``rsync`` function. With this argument, at the end of every sync +citculation, the files data under ``PHOENIX/GENERAL`` directory will be +transferred to the s3 bucket. +If any raw data types is okay to be be transferred, as they were downloaded +from their data source, ``--selective_sync`` option can be used to select these +data types. Then all the data under both ``GENERAL`` and ``PROTECTED`` will be +transferred to s3 bucket. diff --git a/lochness/__init__.py b/lochness/__init__.py index 2ed4b3b7..a62ff79b 100644 --- a/lochness/__init__.py +++ b/lochness/__init__.py @@ -121,17 +121,18 @@ def read_phoenix_metadata(Lochness, studies=None): studies = lochness.listdir(Lochness, general_folder) # iterate over studies for study_name in studies: - f = os.path.join(general_folder, - study_name, - f'{study_name}_metadata.csv') - if not os.path.exists(f): - logger.error('metadata file does not exist {0}'.format(f)) + metadata_loc = os.path.join(protected_folder, + study_name, + f'{study_name}_metadata.csv') + if not os.path.exists(metadata_loc): + logger.error('metadata file does not exist {0}'.format + (metadata_loc)) continue - logger.debug('reading metadata file {0}'.format(f)) + logger.debug('reading metadata file {0}'.format(metadata_loc)) try: # iterate over rows in metadata file for subject in _subjects(Lochness, study_name, general_folder, - protected_folder, f): + protected_folder, metadata_loc): yield subject except StudyMetadataError as e: logger.error(e) diff --git a/lochness/__version__.py b/lochness/__version__.py index 19d0c304..672fca69 100644 --- a/lochness/__version__.py +++ b/lochness/__version__.py @@ -1,6 +1,6 @@ -__title__ = 'Lochness' +__title__ = 'ampscz-lochness' __description__ = 'Data aggregator' __url__ = 'https://github.com/AMP-SCZ/lochness' -__version__ = 'v0.1.2' +__version__ = 'v0.1.6' __author__ = 'DPACC' __author_email__ = 'kevincho@bwh.harvard.edu' diff --git a/lochness/box/__init__.py b/lochness/box/__init__.py index 49820ec6..e7b06b9e 100644 --- a/lochness/box/__init__.py +++ b/lochness/box/__init__.py @@ -575,7 +575,8 @@ def sync_module(Lochness: 'lochness.config', datatype, output_base, processed=processed, - BIDS=Lochness['BIDS']) + BIDS=Lochness['BIDS'], + makedirs=True) if 'out_dir' in product: output_dir = output_dir / product['out_dir'] diff --git a/lochness/cleaner/__init__.py b/lochness/cleaner/__init__.py index e60612d1..b4a4ff8f 100644 --- a/lochness/cleaner/__init__.py +++ b/lochness/cleaner/__init__.py @@ -105,7 +105,7 @@ def make_deleted_structure(phoenix_root: Path, def rm_transferred_files_under_phoenix( phoenix_root: Path, - days_to_keep: int = 15, + days_to_keep: int = 30, removed_df_loc: Path = None, removed_phoenix_root: Path = None) -> None: '''Remove s3 transferred files from PHOENIX @@ -118,10 +118,10 @@ def rm_transferred_files_under_phoenix( removed_phoenix_root: new phoenix directory to keep the track of removed files, Path. If None, the function does not create this new phoenix directory. - Returns: None ''' + phoenix_root = Path(phoenix_root) df_okay_to_remove = get_ok2remove_df_from_s3_log(Path(phoenix_root), days_to_keep) @@ -137,7 +137,9 @@ def rm_transferred_files_under_phoenix( for file in files: file_path = Path(root) / file if is_path_ok2remove(phoenix_root, file_path, df_okay_to_remove): + # os.remove(file_path) # safelock + df_removed_tmp = pd.DataFrame({ 'source': [file_path], 'removal_date': datetime.now()}) @@ -165,7 +167,7 @@ def is_transferred_and_removed(Lochness, bool: True if removed previously ''' if removed_df_loc is None: - removed_df_loc = Path(Lochness['phoenix_root']) / 'removed_files.csv' + removed_df_loc = Path(Lochness['removed_df_loc']) if Path(removed_df_loc).is_file(): df_removed = pd.read_csv(removed_df_loc, index_col=0) diff --git a/lochness/email/__init__.py b/lochness/email/__init__.py index 0814857e..6e4b56cc 100644 --- a/lochness/email/__init__.py +++ b/lochness/email/__init__.py @@ -1,3 +1,4 @@ +import re import os from pathlib import Path import string @@ -37,7 +38,8 @@ def send_detail(Lochness, second_message: str, code: List[str], in_mail_footer: str, test: bool = False, - mailx: bool = True) -> None: + mailx: bool = True, + recipients: list = []) -> None: '''Email Lochness updates This function uses Linux's mailx system by default. But when @@ -61,11 +63,12 @@ def send_detail(Lochness, ''' - recipients = [] - for study, study_recipients in recipients_for_each_study.items(): - for recipient in study_recipients: - if recipients not in recipients: - recipients.append(recipient) + if recipients == []: + for study, study_recipients in recipients_for_each_study.items(): + for recipient in study_recipients: + if recipients not in recipients: + recipients.append(recipient) + email_template_dir = os.path.join(__dir__) env = Environment(loader=FileSystemLoader(str(email_template_dir))) @@ -117,7 +120,8 @@ def send_detail(Lochness, def send_out_daily_updates(Lochness, days: int = 1, - test: bool = False, mailx: bool = True): + test: bool = False, mailx: bool = True, + shorten: bool = True): '''Send daily updates from Lochness''' s3_log = Path(Lochness['phoenix_root']) / 's3_log.csv' @@ -140,13 +144,16 @@ def send_out_daily_updates(Lochness, days: int = 1, s3_df_selected['date'] = s3_df_selected['timestamp'].apply( lambda x: x.date()) + + # remove logfile + s3_df_selected = s3_df_selected[s3_df_selected.filename != '.log'] count_df = s3_df_selected.groupby([ 'date', 'protected', 'study', - 'processed', 'subject', 'datatypes']).count()[['filename']] + 'processed', 'datatypes']).count()[['filename']] count_df.columns = ['file count'] count_df = count_df.reset_index() count_df.columns = ['Transfer date', 'Protected vs General', 'Site', - 'Raw vs Processed', 'Subject', 'Data type', + 'Raw vs Processed', 'Data type', 'Number of files transferred'] s3_df_selected.drop('date', axis=1, inplace=True) @@ -198,6 +205,40 @@ def send_out_daily_updates(Lochness, days: int = 1, in_mail_footer = 'Only the files transferred to NDA are shown.' + if shorten: + s3_df_tmp = s3_df_selected.copy() + for index, row in s3_df_tmp.iterrows(): + s3_df_tmp.loc[index, 'File name'] = re.sub( + row['Subject'], 'SUBJECT', row['File name']) + + mindlamp_act_patt = r'_activity_(\d{4}_\d{2}_\d{2}.+)$' + if re.search(mindlamp_act_patt, row['File name']): + rest_str = re.search(mindlamp_act_patt, + row['File name']).group(1) + s3_df_tmp.loc[index, 'File name'] = \ + f'SUBJECT_activity_{rest_str}' + + for patt in ['-diaryTranscriptQC-day', '_audioJournal_day', + '-diaryAudioQC-day']: + if re.search(patt, + row['File name']): + s3_df_tmp.loc[index, 'File name'] = \ + re.search(r'([a-zA-Z]+)[-_]day', patt).group(1) + + gb = s3_df_tmp.groupby('File name') + for unique_filename, table in gb: + s3_df_tmp.loc[table.index, 'Subject'] = \ + f'{len(table)} subjects' + s3_df_tmp.loc[table.index, 'Transfer time (UTC)'] = \ + table['Transfer time (UTC)'].iloc[0] + s3_df_tmp.loc[table.index, 'Download time (UTC)'] = \ + table['Download time (UTC)'].iloc[0] + s3_df_selected = s3_df_tmp.drop_duplicates() + rename_Subject = lambda x: 'Number of subjects' if x == 'Subject' \ + else x + s3_df_selected.columns = [rename_Subject(x) for x in + s3_df_selected.columns] + send_detail( Lochness, Lochness['sender'], diff --git a/lochness/email/bootdey_template.html b/lochness/email/bootdey_template.html index a48b8433..d0348827 100644 --- a/lochness/email/bootdey_template.html +++ b/lochness/email/bootdey_template.html @@ -1,7 +1,7 @@ - + @@ -16,7 +16,7 @@ @@ -75,4 +75,5 @@
-
+
{{ title }}
{{ subtitle }}
+ diff --git a/lochness/email/template.html b/lochness/email/template.html index 8fd0ac5c..19f91b97 100644 --- a/lochness/email/template.html +++ b/lochness/email/template.html @@ -45,7 +45,6 @@ - shield diff --git a/lochness/mediaflux/__init__.py b/lochness/mediaflux/__init__.py index a450e91b..0fbcba70 100644 --- a/lochness/mediaflux/__init__.py +++ b/lochness/mediaflux/__init__.py @@ -94,7 +94,7 @@ def sync_module(Lochness: 'lochness.config', # obtain mediaflux remote paths with tf.TemporaryDirectory() as tmpdir: - diff_path= pjoin(tmpdir,'diff.csv') + diff_path = pjoin(tmpdir,'diff.csv') cmd = (' ').join(['unimelb-mf-check', '--mf.config', mflux_cfg, '--nb-retries 5', @@ -112,12 +112,24 @@ def sync_module(Lochness: 'lochness.config', continue df = pd.read_csv(diff_path) - for remote in df['SRC_PATH'].values: - if remote is nan: + for _, row in df[~df['SRC_PATH'].isnull()].iterrows(): + remote = row.SRC_PATH + checksum = row.SRC_CHECKSUM + + if pd.isnull(checksum): continue - if not re.search(patt.replace('*', '(.+?)'), - remote): + # ignore mp3 or mp4 in non-interviews + if datatype != 'interviews': + if remote.endswith('mp3') or \ + remote.endswith('mp4'): + logger.warning('mp3 or mp4 file detected ' + f'in non-interviews datatype: {root}') + continue + + if not re.search( + patt.replace('*', '(.+?)').lower(), + remote.lower()): continue else: remote = remote.split(':')[1] @@ -147,12 +159,28 @@ def sync_module(Lochness: 'lochness.config', mf_local = tree.get(datatype, subj_dir, processed=processed, - BIDS=Lochness['BIDS']) + BIDS=Lochness['BIDS'], + makedirs=True) mf_local = str(mf_local / prod['out_dir'] \ if 'out_dir' in prod else mf_local) / \ subpath.parent + # do not re-download already downloaded data + prev_checksum_file = Path(mf_local) / \ + f'.checksum_{subpath.name}' + + + if prev_checksum_file.is_file(): + with open(prev_checksum_file, 'r') as fp: + prev_checksum = fp.read().strip() + + if prev_checksum == checksum: + continue + # else: + # pass + # os.remove(Path(mf_local) / subpath.name) + # do not re-download already transferred & # removed data if is_transferred_and_removed( @@ -168,24 +196,32 @@ def sync_module(Lochness: 'lochness.config', '--mf.config', mflux_cfg, '-o', f'"{mf_local}"', '--nb-retries 5', + '--overwrite', f'\"{remote}\"']) - p = Popen(cmd, shell=True, - stdout=DEVNULL, stderr=STDOUT) - - p.wait() - # verify checksum after download completes if - # checksum does not match, data will be downloaded - # again ENH should we verify checksum 5 times? - cmd += ' --csum-check' p = Popen(cmd, shell=True, stdout=DEVNULL, stderr=STDOUT) p.wait() + # write checksum to local + with open(prev_checksum_file, 'w') as fp: + fp.write(checksum) + # for A/V related files, force permission to 770 # so A/V pipeline can work if 'interviews' in str(mf_local): - for root, dirs, files in os.walk(mf_local): + interviews_root = tree.get( + datatype, + subj_dir, + processed=processed, + BIDS=Lochness['BIDS'], + makedirs=True) + for root, dirs, files in os.walk( + interviews_root): + dir_path = Path(root) + perm = oct(dir_path.stat().st_mode)[-3:] + if perm != '770': + os.chmod(dir_path, 0o0770) for file in files: file_p = Path(root) / file perm = oct(file_p.stat().st_mode)[-3:] diff --git a/lochness/mindlamp/__init__.py b/lochness/mindlamp/__init__.py index f8273fa5..7e48642a 100644 --- a/lochness/mindlamp/__init__.py +++ b/lochness/mindlamp/__init__.py @@ -121,7 +121,8 @@ def sync(Lochness: 'lochness.config', dst_folder = tree.get('mindlamp', subject.protected_folder, processed=False, - BIDS=Lochness['BIDS']) + BIDS=Lochness['BIDS'], + makedirs=True) # the loop below downloads all data from mindlamp from the current date # to (current date - 100 days), overwriting pre-downloaded files. @@ -143,7 +144,8 @@ def sync(Lochness: 'lochness.config', logger.debug(f'Mindlamp {subject_id} {date_str} data pull - start') # store both data types - for data_name in ['activity', 'sensor']: + # for data_name in ['activity', 'sensor']: + for data_name in ['activity']: dst = Path(dst_folder) / \ f'{subject_id}_{subject.study}_{data_name}_{date_str}.json' @@ -168,9 +170,14 @@ def sync(Lochness: 'lochness.config', # pull data from mindlamp begin = time.time() - data_dict = function_to_execute( - LAMP, subject_id, - from_ts=time_utc_00_ts, to_ts=time_utc_24_ts) + try: + data_dict = function_to_execute( + LAMP, subject_id, + from_ts=time_utc_00_ts, to_ts=time_utc_24_ts) + except Exception as e: + print(e) + continue + end = time.time() logger.debug( f'Mindlamp {subject_id} {date_str} {data_name} data pull' diff --git a/lochness/redcap/__init__.py b/lochness/redcap/__init__.py index 51d5a0e7..4e162d32 100644 --- a/lochness/redcap/__init__.py +++ b/lochness/redcap/__init__.py @@ -81,8 +81,8 @@ def initialize_metadata(Lochness: 'Lochness object', project_name = study_name.split(site_code_study)[0] # 'Pronet' # metadata study location - general_path = Path(Lochness['phoenix_root']) / 'GENERAL' - metadata_study = general_path / study_name / f"{study_name}_metadata.csv" + protected_path = Path(Lochness['phoenix_root']) / 'PROTECTED' + metadata_study = protected_path / study_name / f"{study_name}_metadata.csv" # use redcap_project function to load the redcap keyrings for the project _, api_url, api_key = next(redcap_projects( @@ -237,8 +237,8 @@ def initialize_metadata_rm(Lochness: 'Lochness object', project_name = study_name.split(site_code_study)[0] # 'Pronet' # metadata study location - general_path = Path(Lochness['phoenix_root']) / 'GENERAL' - metadata_study = general_path / study_name / f"{study_name}_metadata.csv" + protected_path = Path(Lochness['phoenix_root']) / 'PROTECTED' + metadata_study = protected_path / study_name / f"{study_name}_metadata.csv" # use redcap_project function to load the redcap keyrings for the project _, api_url, api_key = next(redcap_projects( @@ -643,7 +643,8 @@ def sync(Lochness, subject, dry=False): dst_folder = tree.get('surveys', subject.protected_folder, processed=False, - BIDS=Lochness['BIDS']) + BIDS=Lochness['BIDS'], + makedirs=True) fname = f'{redcap_subject}.{_redcap_project}.json' dst = Path(dst_folder) / fname @@ -651,7 +652,8 @@ def sync(Lochness, subject, dry=False): proc_folder = tree.get('surveys', subject.general_folder, processed=True, - BIDS=Lochness['BIDS']) + BIDS=Lochness['BIDS'], + makedirs=True) proc_dst = Path(proc_folder) / fname @@ -687,9 +689,9 @@ def sync(Lochness, subject, dry=False): contains_logic += [ f"contains([session_subid], '{subject_id}_{x}')" for x in digits_str] - # contains_logic += [ - # f"contains([session_subid], '{subject_id}={x}')" - # for x in digits_str] + contains_logic += [ + f"contains([session_subid], '{subject_id}={x}')" + for x in digits_str] record_query = { diff --git a/lochness/rpms/__init__.py b/lochness/rpms/__init__.py index 91312b01..196c3fee 100644 --- a/lochness/rpms/__init__.py +++ b/lochness/rpms/__init__.py @@ -12,6 +12,7 @@ from typing import List, Dict, Union import pandas as pd from datetime import datetime +from time import sleep import re from lochness.redcap.process_piis import process_and_copy_db pd.set_option('mode.chained_assignment', None) @@ -22,6 +23,28 @@ logger = logging.getLogger(__name__) +def _wait(): + + # RPMS dumps data at 20:00 (AEDT) or at 19:00 (AEST) + tz=os.getenv('TIME_ZONE','AEDT') + RPMS_DUMP_HOUR=20 if tz=='AEDT' else 19 + + # calculate _sleep in minutes + # it is between 15 (7:55 pm) and 0 (8:10 pm) + _sleep=0 + now=datetime.now() + if now.hour==RPMS_DUMP_HOUR-1: + if now.minute>=55: + _sleep=15-(now.minute-55) + + elif now.hour==RPMS_DUMP_HOUR: + if now.minute<=10: + _sleep=10-now.minute + + # argument of sleep() is in seconds + sleep(_sleep*60) + + def get_rpms_database(rpms_root_path: str) -> Dict[str, pd.DataFrame]: '''Return dictionary of RPMS database in pandas dataframes @@ -65,9 +88,14 @@ def get_rpms_database(rpms_root_path: str) -> Dict[str, pd.DataFrame]: 'measure_file_date', ascending=False).iterrows(): if n == 0: try: + # do not read CSV files within 7:55-8:10 pm + # they are exported by RPMS and modified by Tashrif's programs + # during this window + _wait() df_tmp = pd.read_csv(row.measure_file, dtype=str) except pd.errors.EmptyDataError: # ignore csv is empty - continue + shutil.move(row.measure_file, + rpms_old_files_root / row.measure_file.name) all_df_dict[measure_name] = df_tmp measure_date_dict[measure_name] = row.measure_file_date @@ -132,13 +160,13 @@ def get_run_sheets_for_datatypes(target_df_loc: Union[Path, str]) -> None: target_df = pd.read_csv(target_df_loc) target_df['timepoint'] = target_df['visit'].map(time_to_timepoint) for tp, table in target_df.groupby('timepoint'): - run_sheet_output = f'{run_sheet_output_prefix}_{tp}.csv' + run_sheet_output = f'{run_sheet_output_prefix}_{int(tp)}.csv' # compare existing table if Path(run_sheet_output).is_file(): run_sheet_prev = pd.read_csv( run_sheet_output, dtype=str).reset_index(drop=True) - same_df = table.astype(str).equals( + same_df = table.reset_index(drop=True).astype(str).equals( run_sheet_prev.astype(str)) if same_df: continue @@ -178,63 +206,125 @@ def initialize_metadata(Lochness: 'Lochness object', ids_with_consent = all_df_dict['informed_consent_run_sheet'][ ~all_df_dict['informed_consent_run_sheet'][ - rpms_consent_colname].isnull()].subjectkey.tolist() - - # all_df_dict - key: name of measure, value: pd.DataFrame of the whole file + rpms_consent_colname].isnull()][ + Lochness['RPMS_id_colname']].tolist() + + # test ids - if it's a production lochness ignore test IDs + if Lochness.get('ignore_id_csv', False): + ignore_id_list = pd.read_csv(Lochness.get('ignore_id_csv'))[ + 'id'].tolist() + else: + ignore_id_list = [] + + # inclusion list - if it's a development lochness only get IDs + if Lochness.get('id_list_csv', False): + id_list = pd.read_csv(Lochness.get('id_list_csv'))[ + 'id'].tolist() + else: + id_list = [] + + # case_with_consent + subject_with_consent = [] for measure, df_measure_all_subj in all_df_dict.items(): - # get the site information from the study name, eg. PrescientAD - site_code_study = study_name[-2:] # 'AD' - project_name = study_name.split(site_code_study)[0] # 'Prescient' + if rpms_consent_colname in df_measure_all_subj.columns: + subject_with_consent = df_measure_all_subj[ + ~df_measure_all_subj[rpms_consent_colname].isnull()][ + rpms_id_colname].unique() - # loop through each line of the RPMS database - for index, df_measure in df_measure_all_subj.iterrows(): - if not df_measure[rpms_id_colname] in ids_with_consent: - continue - # if multistudy: - # site of the subject for the line + # for measure, df_measure_all_subj in all_df_dict.items(): + for df_measure_all_subj in [ + all_df_dict['informed_consent_run_sheet'], + all_df_dict['digital_biomarkers_mindlamp_onboarding']]: + df_measure_all_subj = df_measure_all_subj[ + df_measure_all_subj[rpms_id_colname].isin(subject_with_consent) + ] - # if the rpms table is not ready (e.g.doesn't have the subject col) - if rpms_id_colname not in df_measure.index or \ - pd.isna(df_measure[rpms_id_colname]): - continue + # get the site information from the study name, eg. PrescientAD + site_code_study = study_name[-2:] # 'AD' + project_name = study_name.split(site_code_study)[0] # 'Prescient' - # print(df_measure) - site_code_rpms_id = df_measure[rpms_id_colname][:2] + # loop through each subject in the RPMS database + if not rpms_id_colname in df_measure_all_subj.columns: + continue - # if the subject does not belong to the site, pass it - if site_code_rpms_id != site_code_study: - continue + consent_dict = {} + for subject, df_table in df_measure_all_subj.groupby( + rpms_id_colname): + for index, df_measure in df_table.iterrows(): + if not df_measure[rpms_id_colname] in ids_with_consent: + continue - subject_dict = {'Subject ID': df_measure[rpms_id_colname], 'Study': site_code_study} + # ignore if testcase + if df_measure[rpms_id_colname] in ignore_id_list: + continue - # Consent date - if rpms_consent_colname in df_measure: - subject_dict['Consent'] = datetime.strptime( - df_measure[rpms_consent_colname], - '%d/%m/%Y %I:%M:%S %p').strftime('%Y-%m-%d') - else: - # subject_dict['Consent'] = '2021-10-01' # pseudo-random date - continue ## subject without consent date will be ignored + # if ids to pull exist + if len(id_list) > 0: + # don't pull if it's not in the list + if df_measure[rpms_id_colname] not in id_list: + continue - # mediaflux source has its foldername as its subject ID - subject_dict['RPMS'] = f'rpms.{study_name}:' + \ - df_measure[rpms_id_colname] - subject_dict['Mediaflux'] = f'mediaflux.{study_name}:' + \ - df_measure[rpms_id_colname] + # if the rpms table is not ready + # (e.g.doesn't have the subject col) + if rpms_id_colname not in df_measure.index or \ + pd.isna(df_measure[rpms_id_colname]): + continue - # if mindlamp_id exists in the rpms table - if 'chrdbb_lamp_id' in df_measure: - if not pd.isna(df_measure[f'chrdbb_lamp_id']): - subject_dict['Mindlamp'] = f'mindlamp.{study_name}:' \ - + df_measure[f'chrdbb_lamp_id'] + site_code_rpms_id = df_measure[rpms_id_colname][:2] - if upenn: - subject_dict['REDCap'] = \ - 'redcap.UPENN:' + df_measure[rpms_id_colname] + # if the subject does not belong to the site, pass it + if site_code_rpms_id != site_code_study: + continue - df_tmp = pd.DataFrame.from_dict(subject_dict, orient='index') - df = pd.concat([df, df_tmp.T]) + subject_dict = {'Subject ID': df_measure[rpms_id_colname], + 'Study': site_code_study} + + # if mindlamp_id exists in the rpms table + if 'chrdbb_lamp_id' in df_measure: + # most_recent_lamp_id = df_table + df_table['LastModifiedDate'] = pd.to_datetime( + df_table['LastModifiedDate']) + df_table = df_table.sort_values('LastModifiedDate') + most_recent_lamp_id = df_table.iloc[-1][ + 'chrdbb_lamp_id'] + if not pd.isna(most_recent_lamp_id): + subject_dict['Mindlamp'] = \ + f'mindlamp.{study_name}:{most_recent_lamp_id}' + + # Consent date + if rpms_consent_colname in df_measure.index: + consent_tmp = datetime.strptime( + df_measure[rpms_consent_colname], + '%d/%m/%Y %I:%M:%S %p') + + if subject in consent_dict.keys(): + prev_consent = datetime.strptime( + consent_dict[subject], '%Y-%m-%d') + if prev_consent > consent_tmp: + subject_dict['Consent'] = consent_tmp.strftime( + '%Y-%m-%d') + consent_dict[subject] = subject_dict['Consent'] + else: + subject_dict['Consent'] = prev_consent.strftime( + '%Y-%m-%d') + else: + subject_dict['Consent'] = consent_tmp.strftime( + '%Y-%m-%d') + consent_dict[subject] = subject_dict['Consent'] + + # mediaflux source has its foldername as its subject ID + subject_dict['RPMS'] = f'rpms.{study_name}:' + \ + df_measure[rpms_id_colname] + subject_dict['Mediaflux'] = f'mediaflux.{study_name}:' + \ + df_measure[rpms_id_colname] + + if upenn: + subject_dict['REDCap'] = \ + 'redcap.UPENN:' + df_measure[rpms_id_colname] + + df_tmp = pd.DataFrame.from_dict(subject_dict, orient='index') + df = pd.concat([df, df_tmp.T]) # if there is no data for the study, return without saving metadata if len(df) == 0: @@ -258,7 +348,7 @@ def initialize_metadata(Lochness: 'Lochness object', df_final = df_final[main_cols + \ [x for x in df_final.columns if x not in main_cols]] - general_path = Path(Lochness['phoenix_root']) / 'GENERAL' + general_path = Path(Lochness['phoenix_root']) / 'PROTECTED' metadata_study = general_path / study_name / f"{study_name}_metadata.csv" df_final.to_csv(metadata_study, index=False) @@ -280,20 +370,21 @@ def get_subject_data(all_df_dict: Dict[str, pd.DataFrame], # RPMS should overwrite the row whenever there is an update in any # field of the visit. The snippet below is a safety measure, to store # most recent visit row for each visit - if 'visit' in subject_df.columns: - for unique_visit, table in subject_df.groupby('visit'): - if len(table) == 1 or 'Row#' in subject_df: - pass - # entry_status form does not have LastModifiedDate - elif measure == 'entry_status': - pass - else: - most_recent_row_index = pd.to_datetime( - table['LastModifiedDate']).idxmax() - non_recent_row_index = [x for x in table.index - if x != most_recent_row_index] - print(f'RPMS export has duplicated rows for {measure}') - subject_df.drop(non_recent_row_index, inplace=True) + # if 'visit' in subject_df.columns: + # for unique_visit, table in subject_df.groupby('visit'): + # if len(table) == 1 or 'Row#' in subject_df: + # pass + # # entry_status form does not have LastModifiedDate + # elif measure == 'entry_status': + # pass + # else: + # pass + # most_recent_row_index = pd.to_datetime( + # table['LastModifiedDate']).idxmax() + # non_recent_row_index = [x for x in table.index + # if x != most_recent_row_index] + # print(f'RPMS export has duplicated rows for {measure}') + # subject_df.drop(non_recent_row_index, inplace=True) subject_df_dict[measure] = subject_df @@ -316,19 +407,25 @@ def sync(Lochness, subject, dry=False): Lochness['RPMS_id_colname']) for measure, source_df in subject_df_dict.items(): + if len(source_df) == 0: # do not save if the dataframe is empty + continue + # target data dirname = tree.get('surveys', subject.protected_folder, processed=False, - BIDS=Lochness['BIDS']) + BIDS=Lochness['BIDS'], + makedirs=True) target_df_loc = Path(dirname) / f"{subject_id}_{measure}.csv" proc_folder = tree.get('surveys', subject.general_folder, processed=True, - BIDS=Lochness['BIDS']) + BIDS=Lochness['BIDS'], + makedirs=True) proc_dst = Path(proc_folder) / f"{subject_id}_{measure}.csv" + get_run_sheets_for_datatypes(target_df_loc) # if the csv already exists, compare the dataframe if Path(target_df_loc).is_file(): # index might be different, so drop it before comparing it @@ -341,9 +438,6 @@ def sync(Lochness, subject, dry=False): if same_df: continue - if len(source_df) == 0: # do not save if the dataframe is empty - continue - if not dry: Path(dirname).mkdir(exist_ok=True) os.chmod(dirname, 0o0755) diff --git a/lochness/transfer/__init__.py b/lochness/transfer/__init__.py index f4da5c38..f34eee77 100644 --- a/lochness/transfer/__init__.py +++ b/lochness/transfer/__init__.py @@ -93,12 +93,12 @@ def compress_list_of_files(phoenix_root: str, out_tar_ball: tar file to save eg) ``` - file_list = ['PHOENIX/GENERAL/StudyA/StudyA_metadata.csv', - 'PHOENIX/GENERAL/StudyB/StudyB_metadata.csv'] + file_list = ['PHOENIX/PROTECTED/StudyA/StudyA_metadata.csv', + 'PHOENIX/PROTECTED/StudyB/StudyB_metadata.csv'] ``` PHOENIX/ - └── GENERAL + └── PROTECTED ├── StudyA │   └── StudyA_metadata.csv └── StudyB @@ -299,7 +299,7 @@ def send_file_to_s3_phoenix(Lochness, source_file: Path) -> None: command = f"aws s3 cp \ {source_file} s3://{s3_bucket_name}/{target_path} \ - --exclude '*.mp3' --exclude '.checksum*'" + --exclude '*.mp3' --exclude '.checksum*' --exclude '.check_sum*'" command_out = os.popen(command).read() # update s3 log @@ -339,7 +339,7 @@ def lochness_to_lochness_transfer_s3(Lochness, # metadata metadata_files = Path(Lochness['phoenix_root']).glob( - f'GENERAL/*/*_metadata.csv') + f'PROTECTED/*/*_metadata.csv') for metadata_file in metadata_files: metadata_dir = metadata_file.parent s3_phoenix_metadata = re.sub(Lochness['phoenix_root'], @@ -561,7 +561,8 @@ def lochness_to_lochness_transfer_s3_protected(Lochness, command = f"aws s3 sync \ {source_directory}/ \ s3://{s3_bucket_name}/{s3_phoenix_root_dtype} \ - --exclude '*.mp3' --exclude '.check_sum*'" + --exclude '*.mp3' --exclude '.check_sum*' \ + --exclude '.checksum*' " # logger.debug(re.sub(r'\s+', r' ', command)) @@ -628,6 +629,7 @@ def is_datatype_in_sources(datatype: str, sources: List[str]) -> bool: source_dtype_dict = { 'xnat': ['mri'], 'redcap': ['surveys'], + 'rpms': ['surveys'], 'mindlamp': ['phone'], 'box': ['actigraphy', 'eeg', 'interviews'], 'mediaflux': ['mri', 'eeg', 'actigraphy', 'interviews'], diff --git a/lochness/tree/__init__.py b/lochness/tree/__init__.py index 7600f55d..2521af59 100644 --- a/lochness/tree/__init__.py +++ b/lochness/tree/__init__.py @@ -4,6 +4,7 @@ from pathlib import Path from string import Template import re +from datetime import datetime Templates = { @@ -84,6 +85,9 @@ def get(data_type, base, **kwargs): if data_type not in Templates: raise TreeError('no tree templates defined for {0}'.format(data_type)) + general_folder = Path(base).parent.parent.parent / 'GENERAL' + protected_folder = Path(base).parent.parent.parent / 'PROTECTED' + raw_folder = None processed_folder = None @@ -113,18 +117,40 @@ def get(data_type, base, **kwargs): if 'processed' in Templates[data_type]: processed_folder = Templates[data_type]['processed'].substitute(base=base) + if kwargs.get('makedirs', True):# and \ + # processed_folder and not os.path.exists(processed_folder): + + protected_str = Path(re.sub('GENERAL', 'PROTECTED', + str(processed_folder))) + general_str = Path(re.sub('GENERAL', 'PROTECTED', + str(processed_folder))) + Path(protected_str).mkdir(exist_ok=True, parents=True) + Path(general_str).mkdir(exist_ok=True, parents=True) + os.chmod(protected_str, 0o01770) + os.chmod(general_str, 0o01770) + + for path in protected_str, general_str: + if not (path / '.log').is_file(): + with open(path / '.log', 'w') as fp: + fp.write(f'Created on: {datetime.today().isoformat()}') + + if kwargs.get('makedirs', True):# and \ + # raw_folder and not os.path.exists(raw_folder): + protected_str = Path(re.sub('GENERAL', 'PROTECTED', str(raw_folder))) + general_str = Path(re.sub('GENERAL', 'PROTECTED', str(raw_folder))) + protected_str.mkdir(exist_ok=True, parents=True) + general_str.mkdir(exist_ok=True, parents=True) + os.chmod(protected_str, 0o01770) + os.chmod(general_str, 0o01770) + + for path in protected_str, general_str: + if not (path / '.log').is_file(): + with open(path / '.log', 'w') as fp: + fp.write(f'Created on: {datetime.today().isoformat()}') + if kwargs.get('processed', True): - if kwargs.get('makedirs', True) and \ - processed_folder and not os.path.exists(processed_folder): - logger.debug(f'creating processed folder {processed_folder}') - os.makedirs(processed_folder) - os.chmod(processed_folder, 0o01777) return processed_folder else: - if kwargs.get('makedirs', True) and \ - raw_folder and not os.path.exists(raw_folder): - logger.debug(f'creating raw folder {raw_folder}') - os.makedirs(raw_folder) return raw_folder diff --git a/lochness/utils/path_checker.py b/lochness/utils/path_checker.py index dcb9d9f1..fac41585 100644 --- a/lochness/utils/path_checker.py +++ b/lochness/utils/path_checker.py @@ -1,3 +1,11 @@ +""" +Checks Mediaflux file paths for deviations from the standard operating +procedures (SOP). + +Note: This module checks files at source (Mediaflux) and not at the +destination (local storage). So this will check files that will be +pulled by Lochness and those that will not be pulled by Lochness. +""" import pandas as pd import re from pathlib import Path @@ -80,7 +88,7 @@ def update_mri_check(df: pd.DataFrame) -> pd.DataFrame: mri_index = df[df['modality'] == 'MRI'].index mri_df = df.loc[mri_index] mri_df['file_check'] = mri_df['file_name'].str.match( - '[A-Z]{2}\d{5}_MR_\d{4}_\d{2}_\d{2}_\d.zip') + '[A-Z]{2}\d{5}_MR_\d{4}_\d{2}_\d{2}_\d.[Zz][Ii][Pp]') df.loc[mri_index] = mri_df @@ -127,51 +135,126 @@ def update_interviews_transcript_check(df: pd.DataFrame) -> pd.DataFrame: # check site and AMPSCZ IDs in the transcript file name for index, row in transcript_int_df.iterrows(): if not row['subject_check']: - transcript_int_df.loc[index, 'subject'] = re.search( - r'[A-Z]{2}\d{5}', row['subject']).group(0) - row['subject'] = transcript_int_df.loc[index, 'subject'] - row['subject_check'] = ampscz_id_validate(row['subject']) - transcript_int_df.loc[index, - 'subject_check'] = row['subject_check'] + try: + #TODO: fix this + transcript_int_df.loc[index, 'subject'] = re.search( + r'[A-Z]{2}\d{5}', row['subject']).group(0) + row['subject'] = transcript_int_df.loc[index, 'subject'] + row['subject_check'] = ampscz_id_validate(row['subject']) + transcript_int_df.loc[index, + 'subject_check'] = row['subject_check'] + except: + pass subject = row['subject'] site = row['site'] transcript_int_df.loc[index, 'file_check'] = re.match( - f'{site}_{subject}_' - 'interviewAudioTranscript_open_day\d+_session\d+.txt', + f'(Prescient|){site}_{subject}_' + 'interviewAudioTranscript_(open|psychs)_day[-\d]+_session\d+.txt', row['file_name']) df.loc[transcript_int_index] = transcript_int_df def update_interviews_video_check(df: pd.DataFrame) -> pd.DataFrame: - '''Check logics in rows for Interviews video''' + ''' + Check logics in rows for Interviews video + + Check the following: + - Zoom name in the parent directory + - Zoom name should be in the format of 'YYYY-MM-DD HH.MM.SS ' + - Video files should be under the zoom folder + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + pd.DataFrame: The updated DataFrame. + ''' # interviews transcript video_int_index = df[ - (df.modality=='Interviews') & - ((df.file_name.str.endswith('.mp4')) | - (df.file_name=='recording.conf')) - ].index + (df.modality == "Interviews") + & ( + (df.file_name.str.endswith(".mp4")) + | (df.file_name.str.endswith(".conf")) + ) + ].index video_int_df = df.loc[video_int_index] - video_int_df['zoom_name'] = video_int_df['parent_dir'].str.extract( - '\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2} (.+)') + video_int_df["zoom_name"] = video_int_df["parent_dir"].str.extract( + "\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2} (.+)" + ) # video files must be under the zoom folder - video_int_df['file_check'] = ~video_int_df['zoom_name'].isnull() + video_int_df["file_check"] = ~video_int_df["zoom_name"].isnull() + + df.loc[video_int_index] = video_int_df + + +def update_interviews_teams_data_check(df: pd.DataFrame) -> pd.DataFrame: + ''' + Check logics in rows for Interviews with WAV audio files. + + Validate the following: + - The directory structure + - The file name pattern + - The file name should be in the format of 'YYYYMMDDHHMMSS.wav' + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + pd.DataFrame: The updated DataFrame. + ''' + # interviews transcript + video_int_index = df[ + (df.modality=='Interviews') & + (df.file_name.str.lower().str.endswith('.wav')) + ].index + + video_int_df = df.loc[video_int_index] + + # directory check + video_int_df['directory_check'] = video_int_df['parent_dir'] \ + == video_int_df['subject'] + # file name pattern check + video_int_df['file_pattern_check'] = video_int_df['file_name'].str.match( + r'\d{4}\d{2}\d{2}\d{6}(wav|WAV)' + ) + video_int_df['file_check'] = video_int_df['directory_check'] & \ + video_int_df['file_name'] df.loc[video_int_index] = video_int_df def update_interviews_audio_check(df: pd.DataFrame) -> pd.DataFrame: - '''Check logics in rows for Interviews audio''' + ''' + Check logics in rows for Interviews audio + + Skip the following files: + - 'playback.m3u' files + - 'Audio Record' directory + - 'Zoomver.tag' files + - files ending with '.log' + + Skip here means to set the 'file_check' column to True, essentially + ignoring these files. + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + pd.DataFrame: The updated DataFrame. + ''' # interviews audio - audio_int_index = df[(df.modality=='Interviews') & - (df.file_name.str.endswith('.m4a'))].index + audio_int_index = df[ + (df.modality == 'Interviews') & (df.file_name.str.endswith('.m4a')) + ].index audio_int_df = df.loc[audio_int_index] audio_int_df['zoom_name'] = audio_int_df['parent_dir'].str.extract( - '\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2} (.+)') + '\d{4}-\d{2}-\d{2} \d{2}\.\d{2}\.\d{2} (.+)' + ) # audio files need to be under either the zoom or Audio Record folder audio_int_df['file_check_1'] = ~audio_int_df['zoom_name'].isnull() @@ -182,14 +265,38 @@ def update_interviews_audio_check(df: pd.DataFrame) -> pd.DataFrame: df.loc[audio_int_index] = audio_int_df - # ignore playback.m3u audio files - audio_int_index = df[(df.modality=='Interviews') & - (df.file_name == 'playback.m3u')].index + audio_int_index = df[ + (df.modality == 'Interviews') & (df.file_name == 'playback.m3u') + ].index audio_int_df = df.loc[audio_int_index] audio_int_df['file_check'] = True df.loc[audio_int_index] = audio_int_df + # ignore 'Audio Record' folder + rec_index = df[ + (df.modality == 'Interviews') & (df.file_name.str.endswith('Audio Record')) + ].index + rec_df = df.loc[rec_index] + rec_df['file_check'] = True + df.loc[rec_index] = rec_df + + # ignore 'Zoomver.tag" files + zoomver_index = df[ + (df.modality == 'Interviews') & (df.file_name.str.endswith('ver.tag')) + ].index + zoomver_df = df.loc[zoomver_index] + zoomver_df['file_check'] = True + df.loc[zoomver_index] = zoomver_df + + # Ignore files ending with '.log' + log_index = df[ + (df.modality == 'Interviews') & (df.file_name.str.endswith('.log')) + ].index + log_df = df.loc[log_index] + log_df['file_check'] = True + df.loc[log_index] = log_df + def update_by_adding_notes(df: pd.DataFrame) -> None: '''Add notes to the table''' @@ -197,11 +304,55 @@ def update_by_adding_notes(df: pd.DataFrame) -> None: def update_by_removing_unused_files(df: pd.DataFrame) -> None: - '''Remove unused files''' + ''' + Remove unused files. + + The following files will be removed: + - .DS_Store + - *.conf + - chat.txt + - Files under TRANSCRIPTS/For review + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + None + ''' # .DS_Store ds_store_index = df[df.file_name == '.DS_Store'].index df.drop(ds_store_index, inplace=True) + # *.conf files + recording_conf_index = df[df.file_name.str.endswith('.conf')].index + df.drop(recording_conf_index, inplace=True) + + # chat.txt files + chat_txt_index = df[df.file_name == 'chat.txt'].index + df.drop(chat_txt_index, inplace=True) + + # Ignore For review + for_review_index = df[df.file_path.str.contains( + 'TRANSCRIPTS/For review')].index + df.drop(for_review_index, inplace=True) + +def update_skipped_av_files(df: pd.DataFrame) -> pd.DataFrame: + """ + Update the 'file_check' column in the DataFrame based on the 'parent_dir' column. + + Args: + df (pd.DataFrame): The DataFrame containing the data. + + Returns: + pd.DataFrame: The updated DataFrame. + """ + for idx, row in df.iterrows(): + if "Extraneous files" in row["file_path"]: + df.loc[idx, "file_check"] = True + elif "Additional interview files" in row["file_path"]: + df.loc[idx, "file_check"] = True + return df + def update_by_removing_genetics_and_fluids(df: pd.DataFrame) -> None: '''Remove files under GeneticsAndFluids directory''' @@ -215,9 +366,9 @@ def update_by_checking_against_subject_list( subject_id_list: List[str]) -> None: '''pass''' df['exist_in_db'] = df['subject'].isin(subject_id_list).fillna(False) + missing_in_db_index = df[~df.exist_in_db].index.tolist() - df.loc[df[df['exist_in_db']].index, - 'notes'] = 'Subject missing from database' + df.loc[missing_in_db_index, 'notes'] = 'Subject missing from database' def check_file_path_df(df: pd.DataFrame, @@ -254,13 +405,15 @@ def check_file_path_df(df: pd.DataFrame, update_interviews_check(df) update_interviews_transcript_check(df) + update_interviews_teams_data_check(df) update_interviews_video_check(df) update_interviews_audio_check(df) + update_skipped_av_files(df) # ignore genetics and fluids update_by_removing_genetics_and_fluids(df) - # ignore .DS_Store files + # ignore .DS_Store, *.conf, chat.txt update_by_removing_unused_files(df) # check if the subject exist in metadata diff --git a/lochness/utils/source_check.py b/lochness/utils/source_check.py index 57095da3..9e38b765 100644 --- a/lochness/utils/source_check.py +++ b/lochness/utils/source_check.py @@ -3,7 +3,6 @@ from typing import Union, List import xnat import json -import pandas as pd from pandas.api.types import CategoricalDtype from boxsdk import Client, OAuth2 from typing import List @@ -60,9 +59,11 @@ def get_box_client_from_box_keyrings(box_k: dict) -> Client: return client -def check_list_all_penn_cnb_subjects(project_name: str, - keyring: dict, - subject_list: List[str]) -> pd.DataFrame: +def check_list_all_penn_cnb_subjects( + project_name: str, + keyring: dict, + subject_list: List[str], + ignore_id_list: List[str] = []) -> pd.DataFrame: '''Return a dataframe of records in PENN CNB redcap''' api_url = keyring['redcap.UPENN']['URL'] + '/api/' @@ -83,6 +84,10 @@ def check_list_all_penn_cnb_subjects(project_name: str, if len(df) > 1: df.columns = ['site_orig', 'subject'] + # drop subjects included in ignore_id_list + df = df[~df.subject.str.lower().isin( + [x.lower() for x in ignore_id_list])] + # select records that start with the project name df = df[df.site_orig.str.lower().str.startswith(project_name.lower())] @@ -91,6 +96,16 @@ def check_list_all_penn_cnb_subjects(project_name: str, df['site_orig'].str.split('_').str[1].str.upper() df['modality'] = 'PENN_CNB' + + # PennCNB accepts subject IDs with '_\d' suffix that denotes different + # timepoint of the data submission + + # get only the first XX\d\d\d\d\d pattern + df['subject_orig'] = df['subject'].copy() + pattern = r'([A-Z]{2}\d{5}(?=_|=|$))' + df['subject'] = df['subject'].str.extract(pattern) + df['subject'] = df['subject'].fillna(df['subject_orig']) + df['subject_check'] = df['subject'].apply(ampscz_penn_validate) df['exist_in_db'] = df['subject'].str[:7].str.upper().isin( subject_list).fillna(False) @@ -315,6 +330,7 @@ def send_source_qc_summary(qc_fail_df: pd.DataFrame, lines, Lochness: 'lochness') -> None: '''Send summary of qc failed files in sources''' + site_email_csv = Lochness.get('site_email_csv', False) server_name = Lochness.get('project_name', 'Data aggregation server') if Lochness.get('production', False): @@ -322,11 +338,12 @@ def send_source_qc_summary(qc_fail_df: pd.DataFrame, else: title = f'{server_name}: List of files out of SOP' - table_str = '' cat_type = CategoricalDtype( categories=["REDCap", "MRI", "EEG", "Interviews", "Actigraphy", "PENN_CNB"], ordered=True) qc_fail_df['Data Type'] = qc_fail_df['Data Type'].astype(cat_type) + table_str = '' + for site, x in qc_fail_df.groupby('Site'): table_str += f'

{site}

' for dt, y in x.groupby('Data Type'): @@ -335,24 +352,66 @@ def send_source_qc_summary(qc_fail_df: pd.DataFrame, table_str += y.to_html(index=False) table_str += '
' + message = \ + 'Dear team,

Please find the list of files on the source, ' \ + 'which do not follow the SOP. Please move, rename or delete the ' \ + 'files according to the SOP. Let us know if you see any issues.' \ + '

Best wishes,
DPACC


' + send_detail( Lochness, Lochness['sender'], Lochness['file_check_notify'], 'Files on source out of SOP', f'Daily updates {datetime.now(tz).date()}', - 'Dear team,

Please find the list of files on the source, which ' - 'do not follow the SOP. Please move, rename or delete the files ' - 'according to the SOP. Please do not hesitate to get back to us. ' - '

Best wishes,
DPACC


', + message, table_str, lines, - 'Please let us know if any of the files above should have passed QC' - ) - - -def collect_mediaflux_files_info(Lochness: 'lochness') -> pd.DataFrame: - '''Collect list of files from mediaflux in a pandas dataframe''' + 'Please let us know if any of the files above should have passed QC') + + # copy the emails to sites + if site_email_csv: + site_email_df = pd.read_csv(site_email_csv) + for site, x in qc_fail_df.groupby('Site'): + site_code = site[-2:] + recipients = site_email_df[ + site_email_df['site_short'] == site_code].email.tolist() + + table_str = f'

{site}

' + for dt, y in x.groupby('Data Type'): + if len(y) >= 1: + table_str += f'

{site} - {dt}

' + table_str += y.to_html(index=False) + table_str += '
' + + send_detail( + Lochness, + Lochness['sender'], + Lochness['file_check_notify'], + 'Files on source out of SOP', + f'Daily updates {datetime.now(tz).date()}', + message, + table_str, + lines, + 'Please let us know if any of the files above should ' + 'have passed QC', + recipients=recipients) + + +def collect_mediaflux_files_info(Lochness: 'lochness', + ids_to_ignore: list = []) -> pd.DataFrame: + '''Collect list of files from mediaflux in a pandas dataframe + + Key arguments: + lochness: Lochness object, obj. + ids_to_ignore: list of AMP-SCZ ids to ignore. + + Returns: + pd.DataFrame + ids_to_ignore for production lochness would be the list of IDs used for + testing, and for test lochness the ids_to_ignore would be the list of real + participant IDs. + ''' mflux_cfg = Path(Lochness['phoenix_root']) / 'mflux.cfg' mf_remote_root = '/projects/proj-5070_prescient-1128.4.380' with tf.TemporaryDirectory() as tmpdir: @@ -369,7 +428,20 @@ def collect_mediaflux_files_info(Lochness: 'lochness') -> pd.DataFrame: p.wait() mediaflux_df = load_mediaflux_df(diff_path) - return mediaflux_df + + # remove rows for subjects included in the ids_to_ignore + mediaflux_df = mediaflux_df[ + ~mediaflux_df['file_path'].str.split('/').str[2].isin(ids_to_ignore)] + + # audio files have subdirectories + mediaflux_df = mediaflux_df[ + ~mediaflux_df['file_path'].str.split('/').str[3].isin(ids_to_ignore)] + + # transcript files have subdirectories + mediaflux_df = mediaflux_df[ + ~mediaflux_df['file_path'].str.split('/').str[4].isin(ids_to_ignore)] + + return mediaflux_df def get_subject_list_from_metadata(Lochness: 'lochness') -> List[str]: @@ -385,7 +457,7 @@ def get_subject_list_from_metadata(Lochness: 'lochness') -> List[str]: def get_all_rpms_subjects_with_consent(Lochness) -> pd.DataFrame: - '''Return df with subject column of all subjects IDs with consentID''' + '''Get df with the 'subject' column of subjects IDs from metadata file''' general_path = Path(Lochness['phoenix_root']) / 'GENERAL' metadata_file_paths = general_path.glob('*/*_metadata.csv') project_name = Lochness['project_name'] @@ -422,14 +494,36 @@ def check_source(Lochness: 'lochness', test: bool = False) -> None: subject_id_list = get_subject_list_from_metadata(Lochness) project_name = Lochness['project_name'] + # if the configuration provided ignore_id_csv field + if Lochness.get('ignore_id_csv', False): + ignore_id_list = pd.read_csv( + Lochness.get('ignore_id_csv'))['id'].tolist() + else: + ignore_id_list = [] + if project_name == 'Prescient': + # if the configuration provided id_list_csv + if Lochness.get('id_list_csv', False): + # Prescient network read full subject list from RPMS + check_only_subject_id_list = True + unique_subjects = [] + for csv in Path(Lochness['RPMS_PATH']).glob('*csv'): + try: + [unique_subjects.append(x) for x in + pd.read_csv(csv)[Lochness['RPMS_id_colname']] + if x not in unique_subjects] + except: + pass + ignore_id_list += [x for x in unique_subjects + if x not in subject_id_list] + ignore_id_list = [x for x in ignore_id_list if type(x) == str] + db_string = 'RPMS' - mediaflux_df = collect_mediaflux_files_info(Lochness) + mediaflux_df = collect_mediaflux_files_info(Lochness, ignore_id_list) # Penn CNB keyring = Lochness['keyring'] penn_cnb_df = check_list_all_penn_cnb_subjects( - project_name, keyring, subject_id_list) - + project_name, keyring, subject_id_list, ignore_id_list) all_df = check_file_path_df(mediaflux_df, subject_id_list) all_df = all_df[all_df['site'].str.startswith('Prescient')] all_df = pd.concat([all_df, penn_cnb_df]) @@ -550,7 +644,8 @@ def check_source(Lochness: 'lochness', test: bool = False) -> None: if __name__ == '__main__': # testing purposes - config_loc = '/mnt/ProNET/Lochness/config.yml' + config_loc = '/mnt/prescient/Prescient_production/config.yml' + # config_loc = '/mnt/prescient/Prescient_production/config.yml' # config_loc = '/opt/software/Pronet_data_sync/config.yml' Lochness = load(config_loc) Lochness['file_check_notify']['__global__'] = [ diff --git a/lochness/xnat/__init__.py b/lochness/xnat/__init__.py index d835b880..7783db94 100644 --- a/lochness/xnat/__init__.py +++ b/lochness/xnat/__init__.py @@ -37,7 +37,8 @@ def sync_old(Lochness, subject, dry=False): dirname = tree.get('mri', subject.protected_folder, processed=False, - BIDS=Lochness['BIDS']) + BIDS=Lochness['BIDS'], + makedirs=True) dst = os.path.join(dirname, experiment.label.upper()) # do not re-download already transferred & removed data diff --git a/scripts/lochness_create_template.py b/scripts/lochness_create_template.py index 0f61b4a3..f0ba0e51 100755 --- a/scripts/lochness_create_template.py +++ b/scripts/lochness_create_template.py @@ -46,7 +46,7 @@ def create_lochness_template(args): pg(argsForPheonix) except SystemExit: pass - metadata = phoenix_root / 'GENERAL' / study / f'{study}_metadata.csv' + metadata = phoenix_root / 'PROTECTED' / study / f'{study}_metadata.csv' # create example metadata create_example_meta_file_advanced(metadata, study, args.sources) @@ -233,22 +233,38 @@ def create_keyring_template(keyring_loc: Path, args: object) -> None: for study in args.studies: template_dict['lochness']['SECRETS'][study] = 'LOCHNESS_SECRETS' + if args.enter_passwords: + mediaflux_user = getpass.getpass('Mediaflux User: ') + mediaflux_password = getpass.getpass('Mediaflux Password: ') + else: + mediaflux_user = '*****' + mediaflux_password = '*****' + # lower part of the keyring template_dict[f'mediaflux.{study}'] = { 'HOST': 'mediaflux.researchsoftware.unimelb.edu.au', 'PORT': '443', 'TRANSPORT': 'https', 'DOMAIN': 'local', - 'USER': '**ID**', - 'PASSWORD': '**PASSWORD**'} + 'USER': mediaflux_user, + 'PASSWORD': mediaflux_password} if 'mindlamp' in args.sources: for study in args.studies: # lower part of the keyring + if args.enter_passwords: + mindlamp_url = getpass.getpass('Mindlamp URL: ') + mindlamp_ak = getpass.getpass('Mindlamp Access key: ') + mindlamp_sk = getpass.getpass('Mindlamp Secret key: ') + else: + mindlamp_url = '*****' + mindlamp_ak = '*****' + mindlamp_sk = '*****' + template_dict[f'mindlamp.{study}'] = { - "URL": "**api.lamp.digital**", - "ACCESS_KEY": args.email, - "SECRET_KEY": "**PASSWORD**"} + "URL": mindlamp_url, + "ACCESS_KEY": mindlamp_ak, + "SECRET_KEY": mindlamp_sk} if 'daris' in args.sources: for study in args.studies: @@ -321,6 +337,11 @@ def create_config_template(config_loc: Path, args: object) -> None: RPMS_id_colname: subjectkey RPMS_consent_colname: Consent ''' + if 'redcap' in args.sources: + config_example += '''redcap_id_colname: chric_record_id +redcap_consent_colname: chric_consent_date +''' + if args.s3: if 'rpms' in args.sources: diff --git a/scripts/phoenix_generator.py b/scripts/phoenix_generator.py index 66aa3f4e..6cca9d83 100755 --- a/scripts/phoenix_generator.py +++ b/scripts/phoenix_generator.py @@ -22,7 +22,7 @@ def main(args): general = os.path.join(args.dir, 'GENERAL', args.study) protected = os.path.join(args.dir, 'PROTECTED', args.study) - metadata = os.path.join(general, '{0}_metadata.csv'.format(args.study)) + metadata = os.path.join(protected, '{0}_metadata.csv'.format(args.study)) # check if folder or metadata file already exists for f in [general, protected, metadata]: diff --git a/scripts/sync.py b/scripts/sync.py index c6123b4d..64942c8b 100755 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -103,6 +103,8 @@ def main(): 'server side') parser.add_argument('-ds', '--daily_summary', action='store_true', help='Enable daily summary email function') + parser.add_argument('-cs', '--check_source', action='store_true', + help='Enable check source email function') parser.add_argument('--debug', action='store_true', help='Enable debug messages') parser.add_argument('-rof', '--remove_old_files', @@ -172,11 +174,13 @@ def main(): pass # no email elif datetime.today().isoweekday() == 1: # Monday days_to_summarize = 3 - check_source(Lochness) send_out_daily_updates(Lochness, days=days_to_summarize) + if args.check_source: + check_source(Lochness) else: - check_source(Lochness) send_out_daily_updates(Lochness) + if args.check_source: + check_source(Lochness) with open(email_dates_file, 'w') as fp: fp.write(str(date.today())) @@ -197,8 +201,9 @@ def main(): # email if args.daily_summary: - check_source(Lochness) - send_out_daily_updates(Lochness) + # check_source(Lochness) + if args.check_source: + check_source(Lochness) def do(args, Lochness): diff --git a/tests/lochness_test/cleaner/test_cleaner.py b/tests/lochness_test/cleaner/test_cleaner.py index 85e33c0f..d06a17cf 100644 --- a/tests/lochness_test/cleaner/test_cleaner.py +++ b/tests/lochness_test/cleaner/test_cleaner.py @@ -1,10 +1,12 @@ from pathlib import Path import os +import lochness.config as config from lochness.cleaner import is_transferred_and_removed from lochness.cleaner import get_ok2remove_df_from_s3_log from lochness.cleaner import rm_transferred_files_under_phoenix import pandas as pd -pd.set_option('max_columns', 50) + +# pd.set_option('max_columns', 50) def test_get_source_path_load_s3_log(): phoenix_root = Path('/opt/software/Pronet_data_sync/PHOENIX') @@ -26,3 +28,11 @@ def test_get_source_path_load_s3_log(): # 'removed_files.csv') # if is_removed: # print(file_path) + +def test_is_transferred_and_removed(): + file_path = Path('/mnt/prescient/Prescient_data_sync/PHOENIX/PROTECTED/PrescientME/raw/ME00005/mri/ME00005_MR_2021_12_08_1.zip') + config_file = '/mnt/prescient/Prescient_data_sync/config.yml' + Lochness = config.load(config_file, None) + var = is_transferred_and_removed(Lochness, file_path) + print(var) + diff --git a/tests/lochness_test/daris/test_daris.py b/tests/lochness_test/daris/test_daris.py index 1a61070d..754f62a8 100644 --- a/tests/lochness_test/daris/test_daris.py +++ b/tests/lochness_test/daris/test_daris.py @@ -99,7 +99,7 @@ def initialize_metadata(Lochness, study_name): 'Consent': '1988-09-16', 'Subject ID': 'subject01', 'Daris': 'daris.StudyA:5Yp0E'}) - df_loc = Path(Lochness['phoenix_root']) / 'GENERAL' / \ + df_loc = Path(Lochness['phoenix_root']) / 'PROTECTED' / \ study_name / f"{study_name}_metadata.csv" df.to_csv(df_loc, index=False) diff --git a/tests/lochness_test/email/test_email.py b/tests/lochness_test/email/test_email.py index 28721e3b..9f3641cf 100644 --- a/tests/lochness_test/email/test_email.py +++ b/tests/lochness_test/email/test_email.py @@ -5,6 +5,7 @@ from pathlib import Path import sys +from lochness.config import load lochness_root = Path(lochness.__path__[0]).parent scripts_dir = lochness_root / 'scripts' test_dir = lochness_root / 'tests' @@ -54,3 +55,10 @@ def test_box_sync_module_mailx(args_and_Lochness_BIDS): send_out_daily_updates(Lochness) +def test_email_size(): + config_loc = '/mnt/prescient/Prescient_production/config.yml' + Lochness = load(config_loc) + Lochness['sender'] = 'kevincho@bwh.harvard.edu' + + Lochness['notify']['test'] = ['kevincho@bwh.harvard.edu'] + send_out_daily_updates(Lochness) diff --git a/tests/lochness_test/mediaflux/test_mediaflux.py b/tests/lochness_test/mediaflux/test_mediaflux.py index fc33ab6a..fed17b4d 100644 --- a/tests/lochness_test/mediaflux/test_mediaflux.py +++ b/tests/lochness_test/mediaflux/test_mediaflux.py @@ -100,3 +100,24 @@ def test_sync_from_empty(args_and_Lochness): show_tree_then_delete('tmp_lochness') + +def test_config_patterns(): + from lochness.config import load + import re + config_loc = '/mnt/prescient/Prescient_data_sync/config.yml' + Lochness = load(config_loc) + + interview_products = Lochness['mediaflux'][ + 'PrescientCP']['file_patterns']['interviews'][0] + patts = interview_products['pattern'].split(',') + for patt in patts: + patt = patt.strip() + remote = '/projects/20220526000542_CP00091_OPEN.wav' + remote = "2022-05-20 14.35.34 S I's Personal Meeting Room" + if not re.search(patt.replace('*', '(.+?)'), + remote): + print(patt, ' not matched') + continue + print(remote) + + diff --git a/tests/lochness_test/mindlamp/test_mindlamp.py b/tests/lochness_test/mindlamp/test_mindlamp.py index 78d856ce..26d3ff49 100644 --- a/tests/lochness_test/mindlamp/test_mindlamp.py +++ b/tests/lochness_test/mindlamp/test_mindlamp.py @@ -472,18 +472,45 @@ def test_pronet_mindlamp(args): create_lochness_template(args) syncArgs.config = args.outdir / 'config.yml' # _ = KeyringAndEncryptMindlampAdminIP(args.outdir) + # _ = KeyringAndEncryptMindlampAdmin(args.outdir) _ = KeyringAndEncryptMindlamp(args.outdir) # _ = KeyringAndEncryptMindlampYoon(args.outdir) phoenix_root = args.outdir / 'PHOENIX' information_to_add_to_metadata = {'mindlamp': [ - {'subject_id': '1001', 'source_id': 'U2763080389'}, - {'subject_id': '1002', 'source_id': 'U4361826716'} + {'subject_id': '1001', 'source_id': 'U5891709819'}, ]} + # {'subject_id': '1001', 'source_id': 'U2763080389'}, + # {'subject_id': '1002', 'source_id': 'U4361826716'} initialize_metadata_test(phoenix_root, 'StudyA', information_to_add_to_metadata) Lochness = config_load_test(syncArgs.config) + + Lochness['mindlamp_days_to_pull'] = 3 + for subject in lochness.read_phoenix_metadata(Lochness, syncArgs.studies): + sync(Lochness, subject, False) + + +def test_sync_yoon_and_habib_feb_2022(args): + syncArgs = SyncArgs(args.outdir) + syncArgs.studies = ['StudyA'] + sources = ['mindlamp'] + syncArgs.update_source(sources) + + create_lochness_template(args) + syncArgs.config = args.outdir / 'config.yml' + # _ = KeyringAndEncryptMindlampAdminIP(args.outdir) + _ = KeyringAndEncryptMindlampAdmin(args.outdir) + information_to_add_to_metadata = {'mindlamp': [ + {'subject_id': '1003', 'source_id': 'U5891709819'}, + {'subject_id': '1004', 'source_id': 'U2862696942'} + ]} + phoenix_root = args.outdir / 'PHOENIX' + initialize_metadata_test(phoenix_root, 'StudyA', + information_to_add_to_metadata) + Lochness = config_load_test(syncArgs.config) + Lochness['mindlamp_days_to_pull'] = 8 for subject in lochness.read_phoenix_metadata(Lochness, syncArgs.studies): sync(Lochness, subject, False) @@ -512,4 +539,3 @@ def test_pronet_mindlamp_with_while(args): Lochness['mindlamp_days_to_pull'] = 14 for subject in lochness.read_phoenix_metadata(Lochness, syncArgs.studies): sync(Lochness, subject, False) - diff --git a/tests/lochness_test/redcap/test_redcap.py b/tests/lochness_test/redcap/test_redcap.py index 9fc89aa4..73aa2d50 100644 --- a/tests/lochness_test/redcap/test_redcap.py +++ b/tests/lochness_test/redcap/test_redcap.py @@ -108,7 +108,8 @@ def test_initialize_metadata_function_adding_new_data_to_csv( for study in args.studies: phoenix_path = Path(Lochness['phoenix_root']) general_path = phoenix_path / 'GENERAL' - metadata = general_path / study / f"{study}_metadata.csv" + protected_path = phoenix_path / 'PROTECTED' + metadata = protected_path / study / f"{study}_metadata.csv" assert len(pd.read_csv(metadata)) == 1 initialize_metadata(Lochness, study, 'record_id1', 'cons_date', False) @@ -125,7 +126,8 @@ def test_initialize_metadata_then_sync(args_and_Lochness): for study in args.studies: phoenix_path = Path(Lochness['phoenix_root']) general_path = phoenix_path / 'GENERAL' - metadata = general_path / study / f"{study}_metadata.csv" + protected_path = phoenix_path / 'PROTECTED' + metadata = protected_path / study / f"{study}_metadata.csv" initialize_metadata(Lochness, study, 'record_id1', 'cons_date', False) for subject in lochness.read_phoenix_metadata(Lochness, @@ -143,7 +145,8 @@ def LochnessMetadataInitialized(args_and_Lochness): for study in args.studies: phoenix_path = Path(Lochness['phoenix_root']) general_path = phoenix_path / 'GENERAL' - metadata = general_path / study / f"{study}_metadata.csv" + protected_path = phoenix_path / 'PROTECTED' + metadata = protected_path / study / f"{study}_metadata.csv" initialize_metadata(Lochness, study, 'record_id1', 'cons_date', False) @@ -156,7 +159,8 @@ def test_initialize_metadata_update_when_initialized_again( for study in args.studies: phoenix_path = Path(LochnessMetadataInitialized['phoenix_root']) general_path = phoenix_path / 'GENERAL' - metadata = general_path / study / f"{study}_metadata.csv" + protected_path = phoenix_path / 'PROTECTED' + metadata = protected_path / study / f"{study}_metadata.csv" prev_st_mtime = metadata.stat().st_mtime diff --git a/tests/lochness_test/rpms/test_rpms.py b/tests/lochness_test/rpms/test_rpms.py index 493a2e2b..56b3c6a8 100644 --- a/tests/lochness_test/rpms/test_rpms.py +++ b/tests/lochness_test/rpms/test_rpms.py @@ -15,7 +15,7 @@ from test_lochness import Tokens, KeyringAndEncrypt, args, Lochness from test_lochness import show_tree_then_delete, config_load_test from lochness.rpms import initialize_metadata, sync, get_rpms_database -from lochness.rpms import get_run_sheets_for_datatypes +from lochness.rpms import get_run_sheets_for_datatypes, get_subject_data import pytest @@ -75,7 +75,7 @@ def test_initializing_based_on_rpms(Lochness): create_fake_rpms_repo() Lochness['RPMS_PATH'] = Path('RPMS_repo').absolute() initialize_metadata(Lochness, 'StudyA', 'subjectkey', 'Consent', False) - df = pd.read_csv('tmp_lochness/PHOENIX/GENERAL/StudyA/StudyA_metadata.csv') + df = pd.read_csv('tmp_lochness/PHOENIX/PROTECTED/StudyA/StudyA_metadata.csv') print(df) show_tree_then_delete('tmp_lochness') assert len(df) == 10 @@ -243,3 +243,90 @@ def test_get_runsheets_for_datatypes(): sync(Lochness, subject, dry) show_tree_then_delete('tmp_lochness_2') + + +def test_loading_db(): + rpms_root_path = '/mnt/prescient/RPMS_incoming' + db_dict = get_rpms_database(rpms_root_path) + + +def test_repeated_measure_first(): + rpms_root_path = '/mnt/prescient/RPMS_incoming' + db_dict = get_rpms_database(rpms_root_path) + example_repeated_keyname = 'past_pharmaceutical_treatment' + example_repeated_keyname = 'chrdbb_lamp_id' + print(db_dict[example_repeated_keyname]) + + +def test_repeated_measure_second(): + rpms_root_path = '/mnt/prescient/RPMS_incoming' + all_df_dict = get_rpms_database(rpms_root_path) + example_repeated_keyname = 'past_pharmaceutical_treatment' + dup_id = 'ME08857' + + df_measure_all_subj = all_df_dict['informed_consent_run_sheet'] + for subject, table in df_measure_all_subj.groupby('subjectkey'): + table = table[['subjectkey', + 'chric_consent_date', 'LastModifiedDate']].drop_duplicates() + + if len(table) > 1: + table['LastModifiedDate'] = pd.to_datetime( + table['LastModifiedDate']) + table = table.sort_values('LastModifiedDate') + print(table) + print(table.iloc[-1]) + return + for measure, df_measure_all_subj in all_df_dict.items(): + # if not measure == example_repeated_keyname: + # continue + + # loop through each line of the RPMS database + # for index, df_measure in df_measure_all_subj.iterrows(): + for subject, df_measure in df_measure_all_subj.groupby('subjectkey'): + if len(df_measure) > 1: + print(df_measure) + break + # return + + # return + # if 'chrdbb_lamp_id' in df_measure: + # print(df_measure_all_subj) + # return + # print(df_measure['chrdbb_lamp_id']) + # if df_measure.subjectkey == dup_id: + # print(df_measure) + + + +def test_gets_subject_data(): + class SubjectTest(object): + pass + rpms_root_path = '/mnt/prescient/RPMS_incoming' + all_df_dict = get_rpms_database(rpms_root_path) + subject = SubjectTest() + subject.id = 'TE00001' + id_colname = 'subjectkey' + subject_df_dict = get_subject_data(all_df_dict, subject, id_colname) + for key, table in subject_df_dict.items(): + print(key, len(table)) + + +def test_subjects_with_multiple_consent_dates(): + Lochness = {} + Lochness['RPMS_PATH'] = '/mnt/prescient/RPMS_incoming' + Lochness['phoenix_root'] = 'test' + study_name = 'GW' + rpms_consent_colname = 'chric_consent_date' + rpms_id_colname = 'subjectkey' + Lochness['RPMS_id_colname'] = rpms_id_colname + out_metadata = Path(f'test/PROTECTED/{study_name}/{study_name}_metadata.csv') + out_metadata.parent.mkdir(exist_ok=True) + + initialize_metadata(Lochness, + study_name, + rpms_id_colname, + rpms_consent_colname) + df = pd.read_csv(out_metadata) + with open('tmp_subject_id_to_check.txt', 'r') as fp: + subject_id = fp.read().strip() + print(df[df['Subject ID']==subject_id]['Consent']) diff --git a/tests/lochness_test/transfer/test_transfer.py b/tests/lochness_test/transfer/test_transfer.py index ebeaffc2..97360213 100644 --- a/tests/lochness_test/transfer/test_transfer.py +++ b/tests/lochness_test/transfer/test_transfer.py @@ -67,8 +67,8 @@ def test_get_updated_files(Lochness): timestamp_a_day_ago, posttime) - assert Path('PHOENIX/GENERAL/StudyA/StudyA_metadata.csv') in file_lists - assert Path('PHOENIX/GENERAL/StudyB/StudyB_metadata.csv') in file_lists + assert Path('PHOENIX/PROTECTED/StudyA/StudyA_metadata.csv') in file_lists + assert Path('PHOENIX/PROTECTED/StudyB/StudyB_metadata.csv') in file_lists show_tree_then_delete('tmp_lochness') diff --git a/tests/lochness_test/utils/test_path_checker.py b/tests/lochness_test/utils/test_path_checker.py new file mode 100644 index 00000000..cec518a4 --- /dev/null +++ b/tests/lochness_test/utils/test_path_checker.py @@ -0,0 +1,87 @@ +import lochness +from pathlib import Path +import sys +import pandas as pd +lochness_root = Path(lochness.__path__[0]).parent +scripts_dir = lochness_root / 'scripts' +test_dir = lochness_root / 'tests' +sys.path.append(str(scripts_dir)) +sys.path.append(str(test_dir)) + +from lochness.utils.path_checker import nth_item_from_path, \ + ampscz_id_validate, update_interviews_transcript_check, \ + check_file_path_df, update_by_removing_unused_files, \ + update_eeg_check, update_by_checking_against_subject_list + +from lochness.utils.source_check import collect_mediaflux_files_info, \ + get_subject_list_from_metadata + + +def test_update_interviews_transcript_check(): + from lochness.config import load + config_loc = '/mnt/prescient/Prescient_production/config.yml' + Lochness = load(config_loc) + + subject_id_list = get_subject_list_from_metadata(Lochness) + ignore_id_list = [] + + if Lochness.get('id_list_csv', False): + # Prescient network read full subject list from RPMS + check_only_subject_id_list = True + unique_subjects = [] + for csv in Path(Lochness['RPMS_PATH']).glob('*csv'): + try: + [unique_subjects.append(x) for x in + pd.read_csv(csv)[Lochness['RPMS_id_colname']] + if x not in unique_subjects] + except: + pass + ignore_id_list += [x for x in unique_subjects + if x not in subject_id_list] + ignore_id_list = [x for x in ignore_id_list if type(x) == str] + + if Path('mediaflux_df_test.csv').is_file(): + mediaflux_df = pd.read_csv('mediaflux_df_test.csv', index_col=0) + else: + mediaflux_df = collect_mediaflux_files_info(Lochness, ignore_id_list) + mediaflux_df.to_csv('mediaflux_df_test.csv') + + df = mediaflux_df.copy() + all_df = check_file_path_df(df, subject_id_list) + + df_tmp.to_csv('test.csv') + + +def test_update_by_checking_against_subject_list(): + from lochness.config import load + config_loc = '/mnt/prescient/Prescient_production/config.yml' + Lochness = load(config_loc) + + subject_id_list = get_subject_list_from_metadata(Lochness) + print(subject_id_list) + ignore_id_list = [] + + if Lochness.get('id_list_csv', False): + # Prescient network read full subject list from RPMS + check_only_subject_id_list = True + unique_subjects = [] + for csv in Path(Lochness['RPMS_PATH']).glob('*csv'): + try: + [unique_subjects.append(x) for x in + pd.read_csv(csv)[Lochness['RPMS_id_colname']] + if x not in unique_subjects] + except: + pass + ignore_id_list += [x for x in unique_subjects + if x not in subject_id_list] + ignore_id_list = [x for x in ignore_id_list if type(x) == str] + + if Path('mediaflux_df_test.csv').is_file(): + mediaflux_df = pd.read_csv('mediaflux_df_test.csv', index_col=0) + else: + mediaflux_df = collect_mediaflux_files_info(Lochness, ignore_id_list) + mediaflux_df.to_csv('mediaflux_df_test.csv') + + df = mediaflux_df.copy() + all_df = check_file_path_df(df, subject_id_list) + all_df.to_csv('test.csv') diff --git a/tests/test_lochness.py b/tests/test_lochness.py index fb1a0bbd..dcbdb995 100644 --- a/tests/test_lochness.py +++ b/tests/test_lochness.py @@ -396,8 +396,8 @@ def initialize_metadata_test(phoenix_root: 'phoenix root', df_final = df_final[main_cols + \ [x for x in df_final.columns if x not in main_cols]] - general_path = Path(phoenix_root) / 'GENERAL' - metadata_study = general_path / study_name / f"{study_name}_metadata.csv" + protected_path = Path(phoenix_root) / 'PROTECTED' + metadata_study = protected_path / study_name / f"{study_name}_metadata.csv" df_final.to_csv(metadata_study, index=False)