diff --git a/docker/Dockerfile b/docker/Dockerfile index feec3317..db7a49a8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,14 +44,24 @@ RUN /factsVe/bin/pip install --no-cache-dir \ dask==2025.2.0 -# installs R for ubuntu 20.04 Focal -RUN apt-get update &&\ +# # installs R for ubuntu 20.04 Focal +# RUN apt-get update &&\ +# apt-get install -y --no-install-recommends r-base \ +# cmake \ +# libopenblas-dev \ +# gfortran + +# Add CRAN repository and install R 4.5 +RUN apt-get update && \ + apt-get install -y --no-install-recommends software-properties-common wget && \ + wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \ + echo "deb https://cloud.r-project.org/bin/linux/ubuntu noble-cran40/" >> /etc/apt/sources.list && \ + apt-get update && \ apt-get install -y --no-install-recommends r-base \ cmake \ libopenblas-dev \ gfortran - ARG NB_USER=jovyan ARG NB_UID=1001 ENV USER=${NB_USER} diff --git a/modules/emulandice2/AIS/pipeline.global.2150.yml b/modules/emulandice2/AIS/pipeline.global.2150.yml index 2b5b4778..88b52e4d 100644 --- a/modules/emulandice2/AIS/pipeline.global.2150.yml +++ b/modules/emulandice2/AIS/pipeline.global.2150.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - AIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_AIS_project_2150_240214_data.tgz" diff --git a/modules/emulandice2/AIS/pipeline.global.2300.yml b/modules/emulandice2/AIS/pipeline.global.2300.yml index 9120c1f5..2752d2db 100644 --- a/modules/emulandice2/AIS/pipeline.global.2300.yml +++ b/modules/emulandice2/AIS/pipeline.global.2300.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - AIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_AIS_project_2300_240214_data.tgz" diff --git a/modules/emulandice2/AIS/pipeline.global.yml b/modules/emulandice2/AIS/pipeline.global.yml index 026ceae7..af3fa527 100644 --- a/modules/emulandice2/AIS/pipeline.global.yml +++ b/modules/emulandice2/AIS/pipeline.global.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - AIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_AIS_project_data.tgz" diff --git a/modules/emulandice2/AIS/pipeline.yml b/modules/emulandice2/AIS/pipeline.yml index e5a4f893..50654cb7 100644 --- a/modules/emulandice2/AIS/pipeline.yml +++ b/modules/emulandice2/AIS/pipeline.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - AIS @@ -23,7 +23,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_AIS_project_data.tgz" @@ -36,7 +37,7 @@ project: postprocess: task1: executable: "python3" - script: "emulandice_postprocess.py" + script_noupload: "emulandice_postprocess.py" arguments: - --ncfiles - "%PIPELINE_ID%_ALL_globalsl.nc" @@ -52,6 +53,7 @@ postprocess: - '%MODULE_PATH%/../shared/AssignFP.py' - '%MODULE_PATH%/../shared/ReadFingerprint.py' - '%MODULE_PATH%/../shared/grd_fingerprintmap.yml' + - '%MODULE_PATH%/../shared/emulandice_postprocess.py' upload_and_extract_input_data: - 'grd_fingerprints_data.tgz' copy_shared_data: diff --git a/modules/emulandice2/GrIS/pipeline.global.2100.yml b/modules/emulandice2/GrIS/pipeline.global.2100.yml index 66389cd2..1932fb9b 100644 --- a/modules/emulandice2/GrIS/pipeline.global.2100.yml +++ b/modules/emulandice2/GrIS/pipeline.global.2100.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_GrIS_project_2100_240214_data.tgz" diff --git a/modules/emulandice2/GrIS/pipeline.global.2300.yml b/modules/emulandice2/GrIS/pipeline.global.2300.yml index 67a01d88..9eba19f9 100644 --- a/modules/emulandice2/GrIS/pipeline.global.2300.yml +++ b/modules/emulandice2/GrIS/pipeline.global.2300.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_GrIS_project_2300_240214_data.tgz" diff --git a/modules/emulandice2/GrIS/pipeline.global.yml b/modules/emulandice2/GrIS/pipeline.global.yml index 6aedc4ae..c9150ab5 100644 --- a/modules/emulandice2/GrIS/pipeline.global.yml +++ b/modules/emulandice2/GrIS/pipeline.global.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GIS @@ -25,7 +25,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_GrIS_project_data.tgz" diff --git a/modules/emulandice2/GrIS/pipeline.yml b/modules/emulandice2/GrIS/pipeline.yml index eaf12d4b..e5cf200a 100644 --- a/modules/emulandice2/GrIS/pipeline.yml +++ b/modules/emulandice2/GrIS/pipeline.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GIS @@ -23,7 +23,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_GrIS_project_data.tgz" @@ -36,7 +37,7 @@ project: postprocess: task1: executable: "python3" - script: "emulandice_postprocess.py" + script_noupload: "emulandice_postprocess.py" arguments: - --ncfiles - "%PIPELINE_ID%_ALL_globalsl.nc" @@ -52,6 +53,7 @@ postprocess: - '%MODULE_PATH%/../shared/AssignFP.py' - '%MODULE_PATH%/../shared/ReadFingerprint.py' - '%MODULE_PATH%/../shared/grd_fingerprintmap.yml' + - '%MODULE_PATH%/../shared/emulandice_postprocess.py' upload_and_extract_input_data: - 'grd_fingerprints_data.tgz' copy_shared_data: diff --git a/modules/emulandice2/emulandice_config.sh b/modules/emulandice2/emulandice_config.sh index 291309a5..0344473b 100755 --- a/modules/emulandice2/emulandice_config.sh +++ b/modules/emulandice2/emulandice_config.sh @@ -1,7 +1,8 @@ #!/bin/bash +set -e -ROOTDIR=`dirname $0` -MODE=$1 +ROOTDIR="$(dirname "$0")" +MODE=${1:-} case $MODE in "clean") diff --git a/modules/emulandice2/glaciers/pipeline.global.2100.yml b/modules/emulandice2/glaciers/pipeline.global.2100.yml index 1b39153d..2d54f15b 100644 --- a/modules/emulandice2/glaciers/pipeline.global.2100.yml +++ b/modules/emulandice2/glaciers/pipeline.global.2100.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GLA @@ -61,7 +61,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_glaciers_project_2100_240223_data.tgz" diff --git a/modules/emulandice2/glaciers/pipeline.global.2300.yml b/modules/emulandice2/glaciers/pipeline.global.2300.yml index 00ad3792..47b0fad1 100644 --- a/modules/emulandice2/glaciers/pipeline.global.2300.yml +++ b/modules/emulandice2/glaciers/pipeline.global.2300.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GLA @@ -61,7 +61,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_glaciers_project_2300_240223_data.tgz" diff --git a/modules/emulandice2/glaciers/pipeline.global.yml b/modules/emulandice2/glaciers/pipeline.global.yml index 038e3859..dbaff656 100644 --- a/modules/emulandice2/glaciers/pipeline.global.yml +++ b/modules/emulandice2/glaciers/pipeline.global.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GLA @@ -61,7 +61,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_glaciers_project_data.tgz" diff --git a/modules/emulandice2/glaciers/pipeline.yml b/modules/emulandice2/glaciers/pipeline.yml index e876d6a2..4be08ffc 100644 --- a/modules/emulandice2/glaciers/pipeline.yml +++ b/modules/emulandice2/glaciers/pipeline.yml @@ -3,7 +3,7 @@ project: task1: python_dependencies: "xarray dask numpy scipy" executable: "python3" - script: "emulandice_project.py" + script_noupload: "emulandice_project.py" arguments: - --ice_source - GLA @@ -59,7 +59,8 @@ project: upload_input_data: - "%MODULE_PATH%/../shared/emulandice_environment.sh" - "%MODULE_PATH%/../shared/emulandice_steer.sh" - - "%MODULE_PATH%/../shared/emulandice_steer.R" + - "%MODULE_PATH%/../shared/emulandice2/main.R" + - "%MODULE_PATH%/../shared/emulandice_project.py" upload_and_extract_input_data: - "%MODULE_PATH%/../shared/emulandice_bundled_dependencies.tgz" - "emulandice2_glaciers_project_data.tgz" @@ -71,7 +72,7 @@ project: postprocess: task1: executable: "python3" - script: "emulandice_postprocess.py" + script_noupload: "emulandice_postprocess.py" arguments: - --ncfiles - "%PIPELINE_ID%_RGI01_globalsl.nc" @@ -105,6 +106,7 @@ postprocess: - '%MODULE_PATH%/../shared/AssignFP.py' - '%MODULE_PATH%/../shared/ReadFingerprint.py' - '%MODULE_PATH%/../shared/grd_fingerprintmap.yml' + - '%MODULE_PATH%/../shared/emulandice_postprocess.py' upload_and_extract_input_data: - 'grd_fingerprints_data.tgz' copy_shared_data: diff --git a/modules/emulandice2/shared/emulandice_steer.sh b/modules/emulandice2/shared/emulandice_steer.sh index 2f5d9be4..2fb29113 100644 --- a/modules/emulandice2/shared/emulandice_steer.sh +++ b/modules/emulandice2/shared/emulandice_steer.sh @@ -26,7 +26,7 @@ then exit 1 fi -Rscript -e "source('.Rprofile')" -e "library(emulandice2)" -e "source('emulandice_steer.R')" $ice_source $region $emu_file $climate_data_file $scenario $outdir $seed $pipeline_id +Rscript -e "source('.Rprofile')" -e "library(emulandice2)" -e "source('main.R')" $ice_source $region $emu_file $climate_data_file $scenario $outdir $seed $pipeline_id # Arguments to add ____________ # Read from emulator RData file: