Add pre_run/post_run hooks; make zipping an optional built-in hook

.codespellrc

@@ -1,2 +1,3 @@
 [codespell]
 skip = versioneer.py,_version.py,.git,*html
+ignore-words-list = copyin

babs/bootstrap.py

@@ -13,11 +13,13 @@
 
 from babs.base import BABS
 from babs.container import Container
+from babs.hooks import resolve_hooks
 from babs.input_datasets import InputDatasets
 from babs.status import create_initial_statuses, write_job_status_csv
 from babs.system import System, validate_queue
 from babs.utils import (
     get_datalad_version,
+    output_dir_from_config,
     validate_processing_level,
 )
 
@@ -270,8 +272,22 @@ def babs_bootstrap(
             )
             container = Container(container_ds, container_name, container_config)
 
-        # Copy in any other files needed:
-        self._init_import_files(container.config.get('imported_files', []))
+        # Materialize hooks (and copy in any other files needed): hook scripts
+        # (user `script:` and built-ins alike) are CopyIns riding the
+        # imported_files path. Destinations are relative to self.analysis_path,
+        # so they survive a configurable analysis_path. Pipeline configs have
+        # no `hooks:` block, so this is a no-op there (and output_dir -- which
+        # would hard-error on a pipeline config's legacy zip keys -- is only
+        # derived when hooks are configured).
+        hooks_config = container.config.get('hooks')
+        _, _, hook_materializations = resolve_hooks(
+            hooks_config,
+            output_dir=output_dir_from_config(container.config) if hooks_config else None,
+        )
+        self._init_import_files(
+            container.config.get('imported_files', [])
+            + [m.as_import() for m in hook_materializations]
+        )
         # _update_inclusion_dataframe() expects a DataFrame (or None).
         # If --list_sub_file was provided, use the parsed DataFrame
         # stored in initial_inclu_df by set_inclusion_dataframe() above.
@@ -412,12 +428,12 @@ def _bootstrap_single_app_scripts(
         """Bootstrap scripts for single BIDS app configuration."""
         container = Container(container_ds, container_name, container_config)
 
-        # Generate `<containerName>_zip.sh`: ----------------------------------
-        # which is a bash script of singularity run + zip
+        # Generate `<containerName>_run.sh`: ----------------------------------
+        # which is a bash script of singularity run
         # in folder: `analysis/code`
-        print('\nGenerating a bash script for running container and zipping the outputs...')
-        print('This bash script will be named as `' + container_name + '_zip.sh`')
-        bash_path = op.join(self.analysis_path, 'code', container_name + '_zip.sh')
+        print('\nGenerating a bash script for running the container...')
+        print('This bash script will be named as `' + container_name + '_run.sh`')
+        bash_path = op.join(self.analysis_path, 'code', container_name + '_run.sh')
         shared_group_mode = self.shared_group is not None
         container.generate_bash_run_bidsapp(
             bash_path,
@@ -426,7 +442,7 @@ def _bootstrap_single_app_scripts(
             shared_group_mode=shared_group_mode,
         )
         self.datalad_save(
-            path='code/' + container_name + '_zip.sh',
+            path='code/' + container_name + '_run.sh',
             message='Generate script of running container',
         )
 
@@ -564,6 +580,9 @@ def _init_import_files(self, file_list):
                     f'Requested imported file {imported_file["original_path"]} does not exist.'
                 )
             imported_location = op.join(self.analysis_path, imported_file['analysis_path'])
+            # Create the destination's parent dir if needed (e.g. hooks land in
+            # `code/hooks/`, which doesn't pre-exist like flat `code/` does).
+            os.makedirs(op.dirname(imported_location), exist_ok=True)
             # Copy the file using pure Python:
             with (
                 open(imported_file['original_path'], 'rb') as src,

babs/check_setup.py

@@ -156,7 +156,7 @@ def babs_check_setup(self, submit_a_test_job):
         else:
             list_files_code = [
                 'babs_proj_config.yaml',
-                container_name + '_zip.sh',
+                container_name + '_run.sh',
                 'participant_job.sh',
                 'submit_job_template.yaml',
             ]

babs/container.py

@@ -8,7 +8,8 @@
 
 from babs.generate_bidsapp_runscript import generate_bidsapp_runscript
 from babs.generate_submit_script import generate_submit_script, generate_test_submit_script
-from babs.utils import app_output_settings_from_config
+from babs.hooks import resolve_hooks
+from babs.utils import output_dir_from_config
 
 
 class Container:
@@ -127,16 +128,12 @@ def generate_bash_run_bidsapp(
         input_datasets = input_ds.as_records()
         templateflow_home = os.getenv('TEMPLATEFLOW_HOME')
 
-        # What should the outputs look like?
-        dict_zip_foldernames, bids_app_output_dir = app_output_settings_from_config(self.config)
-
         script_content = generate_bidsapp_runscript(
             input_datasets,
             processing_level,
             container_name=self.container_name,
             relative_container_path=self.container_path_relToAnalysis,
-            bids_app_output_dir=bids_app_output_dir,
-            dict_zip_foldernames=dict_zip_foldernames,
+            bids_app_output_dir=output_dir_from_config(self.config),
             bids_app_args=self.config.get('bids_app_args', None),
             singularity_args=self.config.get('singularity_args', []),
             templateflow_home=templateflow_home,
@@ -178,6 +175,10 @@ def generate_bash_participant_job(
             If True, align generated script permissions with shared-group mode.
         """
 
+        output_dir = output_dir_from_config(self.config)
+        hook_pre_run, hook_post_run, _ = resolve_hooks(
+            self.config.get('hooks'), output_dir=output_dir
+        )
         script_content = generate_submit_script(
             queue_system=system.type,
             cluster_resources_config=self.config['cluster_resources'],
@@ -186,8 +187,10 @@ def generate_bash_participant_job(
             input_datasets=input_ds.as_records(),
             processing_level=processing_level,
             container_name=self.container_name,
-            zip_foldernames=self.config['zip_foldernames'],
+            output_dir=output_dir,
             project_root=project_root,
+            hook_pre_run=hook_pre_run,
+            hook_post_run=hook_post_run,
         )
 
         with open(bash_path, 'w') as f:

babs/generate_bidsapp_runscript.py

@@ -14,7 +14,6 @@ def generate_bidsapp_runscript(
     container_name,
     relative_container_path,
     bids_app_output_dir,
-    dict_zip_foldernames,
     bids_app_args=None,
     singularity_args=None,
     templateflow_home=None,
@@ -43,7 +42,7 @@ def generate_bidsapp_runscript(
         The contents of the bash script that runs the BIDS App singularity image.
     """
 
-    from .constants import OUTPUT_MAIN_FOLDERNAME, PATH_FS_LICENSE_IN_CONTAINER
+    from .constants import PATH_FS_LICENSE_IN_CONTAINER
 
     # 1. check `bids_app_args` section:
     if bids_app_args is None:
@@ -73,9 +72,6 @@ def generate_bidsapp_runscript(
     # Get unzip commands for any zipped input datasets
     cmd_unzip_inputds = get_input_unzipping_cmds(input_datasets)
 
-    # Generate zip command
-    cmd_zip = get_output_zipping_cmds(dict_zip_foldernames, processing_level)
-
     # Render the template
     env = Environment(
         loader=PackageLoader('babs', 'templates'),
@@ -101,8 +97,6 @@ def generate_bidsapp_runscript(
         bids_app_input_dir=bids_app_input_dir,
         bids_app_output_dir=bids_app_output_dir,
         bids_app_args=bids_app_args,
-        cmd_zip=cmd_zip,
-        OUTPUT_MAIN_FOLDERNAME=OUTPUT_MAIN_FOLDERNAME,
         singularity_flags=singularity_args,
         subject_selection_flag=subject_selection_flag,
     )

babs/generate_submit_script.py

@@ -25,11 +25,14 @@ def generate_submit_script(
     input_datasets,
     processing_level,
     container_name,
-    zip_foldernames,
+    zip_foldernames=None,
+    output_dir=None,
     run_script_relpath=None,
     container_images=None,
     datalad_run_message=None,
     project_root=None,
+    hook_pre_run=None,
+    hook_post_run=None,
 ):
     """
     Generate a bash script that runs the BIDS App singularity image.
@@ -50,8 +53,13 @@ def generate_submit_script(
         Processing level ('subject' or 'session').
     container_name : str
         Name of the container.
-    zip_foldernames : dict
-        Dictionary mapping output names to versions.
+    zip_foldernames : dict, optional
+        Dictionary mapping output names to versions. Pipeline mode only: the
+        ``datalad run`` declares the corresponding per-subject zips as outputs.
+    output_dir : str, optional
+        Single-app mode: the app output folder, declared as the ``datalad run``
+        output (the run commits granular outputs; zipping is a ``post_run``
+        hook). Mutually exclusive with ``zip_foldernames``.
     run_script_relpath : str, optional
         Path to script executed by datalad run. None for single-app mode.
     container_images : list, optional
@@ -62,6 +70,12 @@ def generate_submit_script(
         Absolute path to the BABS project root (parent of `analysis/`).
         Passed to the template; used in the error message when PROJECT_ROOT
         is unset. If None, the placeholder ``{project_root}`` is shown.
+    hook_pre_run : list of str, optional
+        Shell snippets spliced into a subshell just before the ``datalad run``
+        wrapper. None (or empty) renders nothing. Snippets are emitted in order.
+    hook_post_run : list of str, optional
+        Shell snippets spliced into a subshell just after the ``datalad run``
+        wrapper, before the push. None (or empty) renders nothing.
 
     Returns
     -------
@@ -119,6 +133,7 @@ def generate_submit_script(
         zip_locator_text=zip_locator_text,
         container_name=container_name,
         zip_foldernames=zip_foldernames,
+        output_dir=output_dir,
         varname_jobid=varname_jobid,
         varname_taskid=varname_taskid,
         input_datasets=input_datasets,
@@ -129,6 +144,8 @@ def generate_submit_script(
         container_image_paths=container_image_paths,
         datalad_run_message=datalad_run_message,
         project_root=project_root,
+        hook_pre_run=hook_pre_run,
+        hook_post_run=hook_post_run,
     )