POC: zip as a built-in post_run hook (optional zipping)

babs/bootstrap.py

@@ -13,11 +13,13 @@
 
 from babs.base import BABS
 from babs.container import Container
+from babs.hooks import CopyIn, Render, resolve_hooks
 from babs.input_datasets import InputDatasets
 from babs.status import create_initial_statuses, write_job_status_csv
 from babs.system import System, validate_queue
 from babs.utils import (
     get_datalad_version,
+    output_dir_from_config,
     validate_processing_level,
 )
 
@@ -270,8 +272,24 @@
             )
             container = Container(container_ds, container_name, container_config)
 
-        # Copy in any other files needed:
-        self._init_import_files(container.config.get('imported_files', []))
+        # Materialize hooks (and copy in any other files needed). Built-in
+        # (Render) hooks are rendered from shipped templates; script (CopyIn)
+        # hooks reuse the imported_files path. Both destinations are relative
+        # to self.analysis_path, so they survive a configurable analysis_path.
+        # Pipeline configs have no `hooks:` block, so this is a no-op there
+        # (and output_dir -- which would hard-error on a pipeline config's
+        # legacy zip keys -- is only derived when hooks are configured).
+        hooks_config = container.config.get('hooks')
+        _, _, hook_materializations = resolve_hooks(
+            hooks_config,
+            output_dir=output_dir_from_config(container.config) if hooks_config else None,
+        )
+        self._init_render_hooks([m for m in hook_materializations if isinstance(m, Render)])
+        copy_ins = [m for m in hook_materializations if isinstance(m, CopyIn)]
+        self._init_import_files(
+            container.config.get('imported_files', [])
+            + [m.as_import() for m in copy_ins]
+        )
         # _update_inclusion_dataframe() expects a DataFrame (or None).
         # If --list_sub_file was provided, use the parsed DataFrame
         # stored in initial_inclu_df by set_inclusion_dataframe() above.
@@ -412,12 +430,12 @@
         """Bootstrap scripts for single BIDS app configuration."""
         container = Container(container_ds, container_name, container_config)
 
-        # Generate `<containerName>_zip.sh`: ----------------------------------
-        # which is a bash script of singularity run + zip
+        # Generate `<containerName>_run.sh`: ----------------------------------
+        # which is a bash script of singularity run
         # in folder: `analysis/code`
-        print('\nGenerating a bash script for running container and zipping the outputs...')
-        print('This bash script will be named as `' + container_name + '_zip.sh`')
-        bash_path = op.join(self.analysis_path, 'code', container_name + '_zip.sh')
+        print('\nGenerating a bash script for running the container...')
+        print('This bash script will be named as `' + container_name + '_run.sh`')
+        bash_path = op.join(self.analysis_path, 'code', container_name + '_run.sh')
         shared_group_mode = self.shared_group is not None
         container.generate_bash_run_bidsapp(
             bash_path,
@@ -426,7 +444,7 @@
             shared_group_mode=shared_group_mode,
         )
         self.datalad_save(
-            path='code/' + container_name + '_zip.sh',
+            path='code/' + container_name + '_run.sh',
             message='Generate script of running container',
         )
 
@@ -564,6 +582,9 @@
                     f'Requested imported file {imported_file["original_path"]} does not exist.'
                 )
             imported_location = op.join(self.analysis_path, imported_file['analysis_path'])
+            # Create the destination's parent dir if needed (e.g. hooks land in
+            # `code/hooks/`, which doesn't pre-exist like flat `code/` does).
+            os.makedirs(op.dirname(imported_location), exist_ok=True)
             # Copy the file using pure Python:
             with (
                 open(imported_file['original_path'], 'rb') as src,
@@ -582,6 +603,42 @@
                 message='Import files',
             )
 
+    def _init_render_hooks(self, renders):
+        """
+        Render built-in hook templates into ``code/hooks/`` and datalad save.
+
+        Parameters
+        ----------
+        renders: list of babs.hooks.Render
+            Built-in hooks to materialize. Each is rendered from its shipped
+            template (``babs/templates/hooks/``) with a context of the per-hook
+            config params plus the derived part supplied here
+            (``processing_level``).
+        """
+        if not renders:
+            return
+        env = Environment(
+            loader=PackageLoader('babs', 'templates'),
+            trim_blocks=True,
+            lstrip_blocks=True,
+            autoescape=False,
+            undefined=StrictUndefined,
+        )
+        rendered_files = []
+        for render in renders:
+            # Derived values win; a per-hook config param can't override them.
+            context = {**render.context, 'processing_level': self.processing_level}
+            content = env.get_template(render.template_path).render(**context)
+            destination = op.join(self.analysis_path, render.analysis_path)
+            os.makedirs(op.dirname(destination), exist_ok=True)
+            with open(destination, 'w') as f:
+                f.write(content)
+            rendered_files.append(render.analysis_path)
+        self.datalad_save(
+            path=rendered_files,
+            message='Materialize built-in hook scripts',
+        )
+
     def clean_up(self):
         """
         If `babs init` failed, this function cleans up the BABS project `babs init` creates.

babs/check_setup.py

@@ -156,7 +156,7 @@ def babs_check_setup(self, submit_a_test_job):
         else:
             list_files_code = [
                 'babs_proj_config.yaml',
-                container_name + '_zip.sh',
+                container_name + '_run.sh',
                 'participant_job.sh',
                 'submit_job_template.yaml',
             ]

babs/container.py

@@ -8,7 +8,8 @@
 
 from babs.generate_bidsapp_runscript import generate_bidsapp_runscript
 from babs.generate_submit_script import generate_submit_script, generate_test_submit_script
-from babs.utils import app_output_settings_from_config
+from babs.hooks import resolve_hooks
+from babs.utils import output_dir_from_config
 
 
 class Container:
@@ -127,16 +128,12 @@ def generate_bash_run_bidsapp(
         input_datasets = input_ds.as_records()
         templateflow_home = os.getenv('TEMPLATEFLOW_HOME')
 
-        # What should the outputs look like?
-        dict_zip_foldernames, bids_app_output_dir = app_output_settings_from_config(self.config)
-
         script_content = generate_bidsapp_runscript(
             input_datasets,
             processing_level,
             container_name=self.container_name,
             relative_container_path=self.container_path_relToAnalysis,
-            bids_app_output_dir=bids_app_output_dir,
-            dict_zip_foldernames=dict_zip_foldernames,
+            bids_app_output_dir=output_dir_from_config(self.config),
             bids_app_args=self.config.get('bids_app_args', None),
             singularity_args=self.config.get('singularity_args', []),
             templateflow_home=templateflow_home,
@@ -178,6 +175,10 @@ def generate_bash_participant_job(
             If True, align generated script permissions with shared-group mode.
         """
 
+        output_dir = output_dir_from_config(self.config)
+        hook_pre_run, hook_post_run, _ = resolve_hooks(
+            self.config.get('hooks'), output_dir=output_dir
+        )
         script_content = generate_submit_script(
             queue_system=system.type,
             cluster_resources_config=self.config['cluster_resources'],
@@ -186,8 +187,10 @@ def generate_bash_participant_job(
             input_datasets=input_ds.as_records(),
             processing_level=processing_level,
             container_name=self.container_name,
-            zip_foldernames=self.config['zip_foldernames'],
+            output_dir=output_dir,
             project_root=project_root,
+            hook_pre_run=hook_pre_run,
+            hook_post_run=hook_post_run,
         )
 
         with open(bash_path, 'w') as f:

babs/generate_bidsapp_runscript.py

@@ -14,7 +14,6 @@ def generate_bidsapp_runscript(
     container_name,
     relative_container_path,
     bids_app_output_dir,
-    dict_zip_foldernames,
     bids_app_args=None,
     singularity_args=None,
     templateflow_home=None,
@@ -43,7 +42,7 @@ def generate_bidsapp_runscript(
         The contents of the bash script that runs the BIDS App singularity image.
     """
 
-    from .constants import OUTPUT_MAIN_FOLDERNAME, PATH_FS_LICENSE_IN_CONTAINER
+    from .constants import PATH_FS_LICENSE_IN_CONTAINER
 
     # 1. check `bids_app_args` section:
     if bids_app_args is None:
@@ -73,9 +72,6 @@ def generate_bidsapp_runscript(
     # Get unzip commands for any zipped input datasets
     cmd_unzip_inputds = get_input_unzipping_cmds(input_datasets)
 
-    # Generate zip command
-    cmd_zip = get_output_zipping_cmds(dict_zip_foldernames, processing_level)
-
     # Render the template
     env = Environment(
         loader=PackageLoader('babs', 'templates'),
@@ -101,8 +97,6 @@ def generate_bidsapp_runscript(
         bids_app_input_dir=bids_app_input_dir,
         bids_app_output_dir=bids_app_output_dir,
         bids_app_args=bids_app_args,
-        cmd_zip=cmd_zip,
-        OUTPUT_MAIN_FOLDERNAME=OUTPUT_MAIN_FOLDERNAME,
         singularity_flags=singularity_args,
         subject_selection_flag=subject_selection_flag,
     )

babs/generate_submit_script.py

@@ -25,11 +25,14 @@ def generate_submit_script(
     input_datasets,
     processing_level,
     container_name,
-    zip_foldernames,
+    zip_foldernames=None,
+    output_dir=None,
     run_script_relpath=None,
     container_images=None,
     datalad_run_message=None,
     project_root=None,
+    hook_pre_run=None,
+    hook_post_run=None,
 ):
     """
     Generate a bash script that runs the BIDS App singularity image.
@@ -50,8 +53,13 @@ def generate_submit_script(
         Processing level ('subject' or 'session').
     container_name : str
         Name of the container.
-    zip_foldernames : dict
-        Dictionary mapping output names to versions.
+    zip_foldernames : dict, optional
+        Dictionary mapping output names to versions. Pipeline mode only: the
+        ``datalad run`` declares the corresponding per-subject zips as outputs.
+    output_dir : str, optional
+        Single-app mode: the app output folder, declared as the ``datalad run``
+        output (the run commits granular outputs; zipping is a ``post_run``
+        hook). Mutually exclusive with ``zip_foldernames``.
     run_script_relpath : str, optional
         Path to script executed by datalad run. None for single-app mode.
     container_images : list, optional
@@ -62,6 +70,12 @@ def generate_submit_script(
         Absolute path to the BABS project root (parent of `analysis/`).
         Passed to the template; used in the error message when PROJECT_ROOT
         is unset. If None, the placeholder ``{project_root}`` is shown.
+    hook_pre_run : list of str, optional
+        Shell snippets spliced into a subshell just before the ``datalad run``
+        wrapper. None (or empty) renders nothing. Snippets are emitted in order.
+    hook_post_run : list of str, optional
+        Shell snippets spliced into a subshell just after the ``datalad run``
+        wrapper, before the push. None (or empty) renders nothing.
 
     Returns
     -------
@@ -119,6 +133,7 @@ def generate_submit_script(
         zip_locator_text=zip_locator_text,
         container_name=container_name,
         zip_foldernames=zip_foldernames,
+        output_dir=output_dir,
         varname_jobid=varname_jobid,
         varname_taskid=varname_taskid,
         input_datasets=input_datasets,
@@ -129,6 +144,8 @@ def generate_submit_script(
         container_image_paths=container_image_paths,
         datalad_run_message=datalad_run_message,
         project_root=project_root,
+        hook_pre_run=hook_pre_run,
+        hook_post_run=hook_post_run,
     )