From bddfa7f776b2e2be0b9502279de0c5bc99ac921d Mon Sep 17 00:00:00 2001 From: Kelly McCusker Date: Fri, 1 May 2026 17:24:30 -0700 Subject: [PATCH 1/4] update R environment in container image --- docker/Dockerfile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index feec3317..db7a49a8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,14 +44,24 @@ RUN /factsVe/bin/pip install --no-cache-dir \ dask==2025.2.0 -# installs R for ubuntu 20.04 Focal -RUN apt-get update &&\ +# # installs R for ubuntu 20.04 Focal +# RUN apt-get update &&\ +# apt-get install -y --no-install-recommends r-base \ +# cmake \ +# libopenblas-dev \ +# gfortran + +# Add CRAN repository and install R 4.5 +RUN apt-get update && \ + apt-get install -y --no-install-recommends software-properties-common wget && \ + wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \ + echo "deb https://cloud.r-project.org/bin/linux/ubuntu noble-cran40/" >> /etc/apt/sources.list && \ + apt-get update && \ apt-get install -y --no-install-recommends r-base \ cmake \ libopenblas-dev \ gfortran - ARG NB_USER=jovyan ARG NB_UID=1001 ENV USER=${NB_USER} From b020f90eaecd042d41657eda6ed6460d44eb3601 Mon Sep 17 00:00:00 2001 From: Kelly McCusker Date: Fri, 1 May 2026 17:29:51 -0700 Subject: [PATCH 2/4] Update experiment files to work with symlinks properly --- modules/emulandice2/AIS/pipeline.global.2150.yml | 5 +++-- modules/emulandice2/AIS/pipeline.global.2300.yml | 5 +++-- modules/emulandice2/AIS/pipeline.global.yml | 5 +++-- modules/emulandice2/AIS/pipeline.yml | 8 +++++--- modules/emulandice2/GrIS/pipeline.global.2100.yml | 5 +++-- modules/emulandice2/GrIS/pipeline.global.2300.yml | 5 +++-- modules/emulandice2/GrIS/pipeline.global.yml | 5 +++-- modules/emulandice2/GrIS/pipeline.yml | 8 +++++--- modules/emulandice2/glaciers/pipeline.global.2100.yml | 5 +++-- modules/emulandice2/glaciers/pipeline.global.2300.yml | 5 +++-- modules/emulandice2/glaciers/pipeline.global.yml | 5 +++-- modules/emulandice2/glaciers/pipeline.yml | 8 +++++--- 12 files changed, 42 insertions(+), 27 deletions(-) diff --git a/modules/emulandice2/AIS/pipeline.global.2150.yml b/modules/emulandice2/AIS/pipeline.global.2150.yml index 2b5b4778..88b52e4d 100644 --- a/modules/emulandice2/AIS/pipeline.global.2150.yml +++ b/modules/emulandice2/AIS/pipeline.global.2150.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - AIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_AIS_project_2150_240214_data.tgz" diff --git a/modules/emulandice2/AIS/pipeline.global.2300.yml b/modules/emulandice2/AIS/pipeline.global.2300.yml index 9120c1f5..2752d2db 100644 --- a/modules/emulandice2/AIS/pipeline.global.2300.yml +++ b/modules/emulandice2/AIS/pipeline.global.2300.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - AIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_AIS_project_2300_240214_data.tgz" diff --git a/modules/emulandice2/AIS/pipeline.global.yml b/modules/emulandice2/AIS/pipeline.global.yml index 026ceae7..af3fa527 100644 --- a/modules/emulandice2/AIS/pipeline.global.yml +++ b/modules/emulandice2/AIS/pipeline.global.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - AIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_AIS_project_data.tgz" diff --git a/modules/emulandice2/AIS/pipeline.yml b/modules/emulandice2/AIS/pipeline.yml index e5a4f893..50654cb7 100644 --- a/modules/emulandice2/AIS/pipeline.yml +++ b/modules/emulandice2/AIS/pipeline.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - AIS @@ -23,7 +23,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_AIS_project_data.tgz" @@ -36,7 +37,7 @@ project: postprocess: task1: executable: "python3" - script: "emulandice_postprocess.py" + script_noupload: "emulandice_postprocess.py" arguments: - --ncfiles - "%PIPELINE_ID%_ALL_globalsl.nc" @@ -52,6 +53,7 @@ postprocess: - '%MODULE_PATH%/../shared/AssignFP.py' - '%MODULE_PATH%/../shared/ReadFingerprint.py' - '%MODULE_PATH%/../shared/grd_fingerprintmap.yml' + - '%MODULE_PATH%/../shared/emulandice_postprocess.py' upload_and_extract_input_data: - 'grd_fingerprints_data.tgz' copy_shared_data: diff --git a/modules/emulandice2/GrIS/pipeline.global.2100.yml b/modules/emulandice2/GrIS/pipeline.global.2100.yml index 66389cd2..1932fb9b 100644 --- a/modules/emulandice2/GrIS/pipeline.global.2100.yml +++ b/modules/emulandice2/GrIS/pipeline.global.2100.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_GrIS_project_2100_240214_data.tgz" diff --git a/modules/emulandice2/GrIS/pipeline.global.2300.yml b/modules/emulandice2/GrIS/pipeline.global.2300.yml index 67a01d88..9eba19f9 100644 --- a/modules/emulandice2/GrIS/pipeline.global.2300.yml +++ b/modules/emulandice2/GrIS/pipeline.global.2300.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_GrIS_project_2300_240214_data.tgz" diff --git a/modules/emulandice2/GrIS/pipeline.global.yml b/modules/emulandice2/GrIS/pipeline.global.yml index 6aedc4ae..c9150ab5 100644 --- a/modules/emulandice2/GrIS/pipeline.global.yml +++ b/modules/emulandice2/GrIS/pipeline.global.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_GrIS_project_data.tgz" diff --git a/modules/emulandice2/GrIS/pipeline.yml b/modules/emulandice2/GrIS/pipeline.yml index eaf12d4b..e5cf200a 100644 --- a/modules/emulandice2/GrIS/pipeline.yml +++ b/modules/emulandice2/GrIS/pipeline.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GIS @@ -23,7 +23,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_GrIS_project_data.tgz" @@ -36,7 +37,7 @@ project: postprocess: task1: executable: "python3" - script: "emulandice_postprocess.py" + script_noupload: "emulandice_postprocess.py" arguments: - --ncfiles - "%PIPELINE_ID%_ALL_globalsl.nc" @@ -52,6 +53,7 @@ postprocess: - '%MODULE_PATH%/../shared/AssignFP.py' - '%MODULE_PATH%/../shared/ReadFingerprint.py' - '%MODULE_PATH%/../shared/grd_fingerprintmap.yml' + - '%MODULE_PATH%/../shared/emulandice_postprocess.py' upload_and_extract_input_data: - 'grd_fingerprints_data.tgz' copy_shared_data: diff --git a/modules/emulandice2/glaciers/pipeline.global.2100.yml b/modules/emulandice2/glaciers/pipeline.global.2100.yml index 1b39153d..2d54f15b 100644 --- a/modules/emulandice2/glaciers/pipeline.global.2100.yml +++ b/modules/emulandice2/glaciers/pipeline.global.2100.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GLA @@ -61,7 +61,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_glaciers_project_2100_240223_data.tgz" diff --git a/modules/emulandice2/glaciers/pipeline.global.2300.yml b/modules/emulandice2/glaciers/pipeline.global.2300.yml index 00ad3792..47b0fad1 100644 --- a/modules/emulandice2/glaciers/pipeline.global.2300.yml +++ b/modules/emulandice2/glaciers/pipeline.global.2300.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GLA @@ -61,7 +61,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_glaciers_project_2300_240223_data.tgz" diff --git a/modules/emulandice2/glaciers/pipeline.global.yml b/modules/emulandice2/glaciers/pipeline.global.yml index 038e3859..dbaff656 100644 --- a/modules/emulandice2/glaciers/pipeline.global.yml +++ b/modules/emulandice2/glaciers/pipeline.global.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GLA @@ -61,7 +61,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_glaciers_project_data.tgz" diff --git a/modules/emulandice2/glaciers/pipeline.yml b/modules/emulandice2/glaciers/pipeline.yml index e876d6a2..4be08ffc 100644 --- a/modules/emulandice2/glaciers/pipeline.yml +++ b/modules/emulandice2/glaciers/pipeline.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GLA @@ -59,7 +59,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_glaciers_project_data.tgz" @@ -71,7 +72,7 @@ project: postprocess: task1: executable: "python3" - script: "emulandice_postprocess.py" + script_noupload: "emulandice_postprocess.py" arguments: - --ncfiles - "%PIPELINE_ID%_RGI01_globalsl.nc" @@ -105,6 +106,7 @@ postprocess: - '%MODULE_PATH%/../shared/AssignFP.py' - '%MODULE_PATH%/../shared/ReadFingerprint.py' - '%MODULE_PATH%/../shared/grd_fingerprintmap.yml' + - '%MODULE_PATH%/../shared/emulandice_postprocess.py' upload_and_extract_input_data: - 'grd_fingerprints_data.tgz' copy_shared_data: From 44d57b49c7f5a15fc3b7276682f1c632d7b72101 Mon Sep 17 00:00:00 2001 From: Kelly McCusker Date: Fri, 1 May 2026 17:30:38 -0700 Subject: [PATCH 3/4] update emulandice2 steer script to call main.R directly --- modules/emulandice2/shared/emulandice_steer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/emulandice2/shared/emulandice_steer.sh b/modules/emulandice2/shared/emulandice_steer.sh index 2f5d9be4..2fb29113 100644 --- a/modules/emulandice2/shared/emulandice_steer.sh +++ b/modules/emulandice2/shared/emulandice_steer.sh @@ -26,7 +26,7 @@ then exit 1 fi -Rscript -e "source('.Rprofile')" -e "library(emulandice2)" -e "source('emulandice_steer.R')" $ice_source $region $emu_file $climate_data_file $scenario $outdir $seed $pipeline_id +Rscript -e "source('.Rprofile')" -e "library(emulandice2)" -e "source('main.R')" $ice_source $region $emu_file $climate_data_file $scenario $outdir $seed $pipeline_id # Arguments to add ____________ # Read from emulator RData file: From 7712526bf8e93987283d466b319b58cabb21a2e3 Mon Sep 17 00:00:00 2001 From: pkjr Date: Mon, 11 May 2026 14:57:20 +0000 Subject: [PATCH 4/4] Fix emulandice dependency bundling failure handling --- modules/emulandice2/emulandice_config.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/emulandice2/emulandice_config.sh b/modules/emulandice2/emulandice_config.sh index 291309a5..0344473b 100755 --- a/modules/emulandice2/emulandice_config.sh +++ b/modules/emulandice2/emulandice_config.sh @@ -1,7 +1,8 @@ #!/bin/bash +set -e -ROOTDIR=`dirname $0` -MODE=$1 +ROOTDIR="$(dirname "$0")" +MODE=${1:-} case $MODE in "clean")