diff --git a/lib/galaxy/config/sample/job_conf.sample.yml b/lib/galaxy/config/sample/job_conf.sample.yml index 3e11fa7582b6..55706e75be47 100644 --- a/lib/galaxy/config/sample/job_conf.sample.yml +++ b/lib/galaxy/config/sample/job_conf.sample.yml @@ -737,6 +737,30 @@ execution: # clusters. These don't need to be available on the Galaxy server # itself. + # Under most runners, Galaxy will set the `$GALAXY_SLOTS` environment variable in the + # tool execution environment to the number of cores allocated to the job by the + # scheduler. Tools can use this to control how many threads or processes they start. + # + # Under some runners, most notably Slurm and SGE, Galaxy will set environment + # variables related to the amount of memory allocated to the job by the scheduler, if + # possible. These are `$GALAXY_MEMORY_MB`, `$GALAXY_MEMORY_GB`, + # `$GALAXY_MEMORY_MB_PER_SLOT`, and `$GALAXY_MEMORY_GB_PER_SLOT`. You can override + # these on a per-environment basis as shown below. + # + # Additionally, you can specify two variables that will indicate a tool should use + # less memory than allocated. Typically some overhead is required or else the Linux + # OOM Killer will simply kill the job's processes if they exceed they amount of memory + # allocated by the scheduler. + job_vars_example: + runner: drmaa + env: + # Sets `$GALAXY_MEMORY_MB` to 2GB less than what has been allocated + - name: GALAXY_MEMORY_MB_OVERHEAD + value: "2048" + # Floor value for `$GALAXY_MEMORY_MB` if (allocated - overhead < floor) + - name: GALAXY_MEMORY_MB_FLOOR + value: "1024" + # Following three environments demonstrate setting up per-job temp directory handling. # In these cases TEMP, TMP, and TMPDIR will be set for each job dispatched to these # environments. diff --git a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT_TEMPLATE.sh index a04087672bd1..20cbe2094364 100644 --- a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT_TEMPLATE.sh @@ -5,10 +5,27 @@ if [ -n "$SGE_HGR_h_vmem" ]; then GALAXY_MEMORY_MB_PER_SLOT=`echo "$SGE_HGR_h_vmem" | sed 's/G$/ * 1024/' | bc | cut -d"." -f1` 2>$metadata_directory/memory_statement.log fi +if [ -n "$GALAXY_MEMORY_MB" -a -n "${GALAXY_MEMORY_MB_OVERHEAD:-}" ]; then + GALAXY_MEMORY_MB=$(($GALAXY_MEMORY_MB - GALAXY_MEMORY_MB_OVERHEAD)) + [ "$GALAXY_MEMORY_MB" -gt "${GALAXY_MEMORY_MB_FLOOR:=256}" ] || GALAXY_MEMORY_MB=$GALAXY_MEMORY_MB_FLOOR +fi + if [ -z "$GALAXY_MEMORY_MB_PER_SLOT" -a -n "$GALAXY_MEMORY_MB" ]; then GALAXY_MEMORY_MB_PER_SLOT=$(($GALAXY_MEMORY_MB / $GALAXY_SLOTS)) elif [ -z "$GALAXY_MEMORY_MB" -a -n "$GALAXY_MEMORY_MB_PER_SLOT" ]; then GALAXY_MEMORY_MB=$(($GALAXY_MEMORY_MB_PER_SLOT * $GALAXY_SLOTS)) fi + +if [ -n "$GALAXY_MEMORY_MB" -a -z "$GALAXY_MEMORY_GB" ]; then + GALAXY_MEMORY_GB=$(($GALAXY_MEMORY_MB / 1024)) + [ "$GALAXY_MEMORY_GB" -gt 0 ] || GALAXY_MEMORY_GB=1 +fi +if [ -n "$GALAXY_MEMORY_GB" -a -z "$GALAXY_MEMORY_GB_PER_SLOT" ]; then + GALAXY_MEMORY_GB_PER_SLOT=$(($GALAXY_MEMORY_GB / $GALAXY_SLOTS)) + [ "$GALAXY_MEMORY_GB_PER_SLOT" -gt 0 ] || GALAXY_MEMORY_GB_PER_SLOT=1 +fi + [ "${GALAXY_MEMORY_MB--1}" -gt 0 ] 2>>$metadata_directory/memory_statement.log && export GALAXY_MEMORY_MB || unset GALAXY_MEMORY_MB [ "${GALAXY_MEMORY_MB_PER_SLOT--1}" -gt 0 ] 2>>$metadata_directory/memory_statement.log && export GALAXY_MEMORY_MB_PER_SLOT || unset GALAXY_MEMORY_MB_PER_SLOT +[ "${GALAXY_MEMORY_GB--1}" -gt 0 ] 2>>$metadata_directory/memory_statement.log && export GALAXY_MEMORY_GB || unset GALAXY_MEMORY_GB +[ "${GALAXY_MEMORY_GB_PER_SLOT--1}" -gt 0 ] 2>>$metadata_directory/memory_statement.log && export GALAXY_MEMORY_GB_PER_SLOT || unset GALAXY_MEMORY_GB_PER_SLOT