diff --git a/eval/evalchemy/delphi_eval.sbatch b/eval/evalchemy/delphi_eval.sbatch index 49b583f93..1058dbb40 100755 --- a/eval/evalchemy/delphi_eval.sbatch +++ b/eval/evalchemy/delphi_eval.sbatch @@ -5,7 +5,14 @@ #SBATCH --gres=gpu:4 #SBATCH --time=08:00:00 #SBATCH --job-name=delphi-eval -#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/%x-%j.log +#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log +# +# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT. +# SLURM opens the log file before the job script runs and does NOT create intermediate +# directories -- a missing dir fails the job instantly with "Unable to open file", which +# looks like a launch bug rather than a missing mkdir. It already exists for the original +# operator; a NEW operator runs this once: +# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval # # ============================================================================== # delphi_eval.sbatch — THIN delphi wrapper around evalchemy_eval.sbatch (#6279). @@ -33,15 +40,58 @@ set -euo pipefail # so BASH_SOURCE[0]/dirname resolves to the spool dir (where evalchemy_eval.sbatch does NOT exist) — # that produced a 4-sec ExitCode-127 "No such file or directory". Pin to the repo clone instead, # falling back to the BASH_SOURCE dir only for interactive/non-SLURM invocation. -REPO_EVAL_DIR="/leonardo_work/AIFAC_5C0_290/bfeuer00/code/OpenThoughts-Agent/eval/evalchemy" -if [[ -f "${REPO_EVAL_DIR}/evalchemy_eval.sbatch" ]]; then - SELF_DIR="${REPO_EVAL_DIR}" -else - SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# $USER-scoped so a second operator runs THEIR checkout, not another user's. Resolves to the +# original bfeuer00 paths byte-identically when USER=bfeuer00. +# +# Resolution is by PROBE, not by assumption: try each candidate and take the first that actually +# contains the marker file. A bare "$WORK/$USER/code/OpenThoughts-Agent" default only works for +# operators who cloned at exactly that path; anyone else fell through to the BASH_SOURCE branch, +# which under SLURM is the /var/spool/slurmd/jobNNN/ copy — i.e. it re-created the very +# ExitCode-127 "No such file or directory" this block was written to prevent. It also silently +# mis-pointed CHAT_TEMPLATE_OVERRIDE below (and THINK_TEMPLATE in delphi_rl_think_eval.sbatch). +_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" +_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" +_OTA_MARKER="eval/evalchemy/evalchemy_eval.sbatch" +# Walk up from a directory looking for the marker: SLURM_SUBMIT_DIR is wherever the operator ran +# sbatch, which is often a subdirectory of the repo rather than its root. +_ota_up() { + local d="${1:-}" + [ -n "$d" ] || return 1 + while [ -n "$d" ] && [ "$d" != "/" ]; do + [ -f "$d/${_OTA_MARKER}" ] && { printf '%s' "$d"; return 0; } + d="$(dirname "$d")" + done + return 1 +} +_ota_first_repo() { + local c + for c in "$@"; do + [ -n "$c" ] && [ -f "$c/${_OTA_MARKER}" ] && { printf '%s' "$c"; return 0; } + done + return 1 +} +_OTA_REPO="$(_ota_first_repo \ + "${OTAGENT_REPO_DIR:-}" \ + "${_OTA_WORK}/code/OpenThoughts-Agent" \ + "$(_ota_up "${SLURM_SUBMIT_DIR:-}" || true)" \ + "${_OTA_SHARED}/code/OpenThoughts-Agent" \ + "$(_ota_up "$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || true)" || true)" \ + || true)" +if [ -z "${_OTA_REPO}" ]; then + echo "ERROR: cannot locate an OpenThoughts-Agent checkout containing ${_OTA_MARKER}." >&2 + echo " tried, in order:" >&2 + echo " OTAGENT_REPO_DIR = ${OTAGENT_REPO_DIR:-}" >&2 + echo " \$USER checkout = ${_OTA_WORK}/code/OpenThoughts-Agent" >&2 + echo " SLURM_SUBMIT_DIR = ${SLURM_SUBMIT_DIR:-} (and parents)" >&2 + echo " shared checkout = ${_OTA_SHARED}/code/OpenThoughts-Agent" >&2 + echo " script dir = $(dirname "${BASH_SOURCE[0]}") (and parents)" >&2 + echo " Set OTAGENT_REPO_DIR=/path/to/OpenThoughts-Agent and resubmit." >&2 + exit 1 fi +SELF_DIR="${_OTA_REPO}/eval/evalchemy" # Delphi-specific defaults (overridable from the environment). -export CHAT_TEMPLATE_OVERRIDE="${CHAT_TEMPLATE_OVERRIDE:-/leonardo_work/AIFAC_5C0_290/bfeuer00/code/OpenThoughts-Agent/chat_templates/delphi_v0.jinja2}" +export CHAT_TEMPLATE_OVERRIDE="${CHAT_TEMPLATE_OVERRIDE:-${_OTA_REPO}/chat_templates/delphi_v0.jinja2}" export MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" export MAX_GEN_TOKS="${MAX_GEN_TOKS:-3584}" export JOB_PREFIX="${JOB_PREFIX:-delphi-eval}" diff --git a/eval/evalchemy/delphi_rl_think_eval.sbatch b/eval/evalchemy/delphi_rl_think_eval.sbatch index cc7d8ac5a..6597e8e6a 100644 --- a/eval/evalchemy/delphi_rl_think_eval.sbatch +++ b/eval/evalchemy/delphi_rl_think_eval.sbatch @@ -5,7 +5,14 @@ #SBATCH --gres=gpu:4 #SBATCH --time=08:00:00 #SBATCH --job-name=delphi-think-eval -#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/%x-%j.log +#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log +# +# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT. +# SLURM opens the log file before the job script runs and does NOT create intermediate +# directories -- a missing dir fails the job instantly with "Unable to open file", which +# looks like a launch bug rather than a missing mkdir. It already exists for the original +# operator; a NEW operator runs this once: +# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval # # Held-out math eval for the 4 FORCED-THINKING RL grid cells (consolidated local HF dirs). # IDENTICAL harness to eval/evalchemy/evalchemy_eval.sbatch (same tasks, seeds, gen_kwargs, @@ -21,10 +28,58 @@ # Usage: sbatch --job-name=delphi-think-eval- delphi_rl_think_eval.sbatch set -euo pipefail +# ----- $USER-scoped paths (multi-operator) ----------------------------------------------------- +# Resolves to the original bfeuer00 strings byte-identically when USER=bfeuer00. Read-only shared +# runtime (conda, evalchemy-marin clone) falls back to the bfeuer00 copy; WRITE paths are per-user. +_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" +_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" +_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } +# Conda: pick the root that actually CONTAINS the env. Testing only for a miniforge3 dir is not +# enough -- an operator can have a partial install (e.g. otagent only) and would then fail with +# "EnvironmentNameNotFound: evalchemy-marin" instead of falling back to the shared root. +_pick_conda() { [ -d "${_OTA_WORK}/miniforge3/envs/$1" ] && echo "${_OTA_WORK}/miniforge3" || echo "${_OTA_SHARED}/miniforge3"; } +_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick_conda evalchemy-marin)}" +# Repo root by PROBE, not assumption — see the long note in delphi_eval.sbatch. A bare +# "$WORK/$USER/code/OpenThoughts-Agent" default silently mis-points THINK_TEMPLATE below for any +# operator who cloned elsewhere, and lm-eval then fails on a missing chat template rather than +# saying the repo was not found. +_OTA_MARKER="eval/evalchemy/evalchemy_eval.sbatch" +_ota_up() { + local d="${1:-}" + [ -n "$d" ] || return 1 + while [ -n "$d" ] && [ "$d" != "/" ]; do + [ -f "$d/${_OTA_MARKER}" ] && { printf '%s' "$d"; return 0; } + d="$(dirname "$d")" + done + return 1 +} +_ota_first_repo() { + local c + for c in "$@"; do + [ -n "$c" ] && [ -f "$c/${_OTA_MARKER}" ] && { printf '%s' "$c"; return 0; } + done + return 1 +} +_OTA_REPO="$(_ota_first_repo \ + "${OTAGENT_REPO_DIR:-}" \ + "${_OTA_WORK}/code/OpenThoughts-Agent" \ + "$(_ota_up "${SLURM_SUBMIT_DIR:-}" || true)" \ + "${_OTA_SHARED}/code/OpenThoughts-Agent" \ + "$(_ota_up "$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || true)" || true)" \ + || true)" +if [ -z "${_OTA_REPO}" ]; then + echo "ERROR: cannot locate an OpenThoughts-Agent checkout containing ${_OTA_MARKER}." >&2 + echo " Set OTAGENT_REPO_DIR=/path/to/OpenThoughts-Agent and resubmit." >&2 + echo " tried: OTAGENT_REPO_DIR=${OTAGENT_REPO_DIR:-}, ${_OTA_WORK}/code/OpenThoughts-Agent," >&2 + echo " SLURM_SUBMIT_DIR=${SLURM_SUBMIT_DIR:-} (and parents), ${_OTA_SHARED}/code/OpenThoughts-Agent" >&2 + exit 1 +fi + + # Clean any propagated python env that can shadow conda's base-python stdlib # (intermittent "ImportError: cannot import name 'JSONEncoder' from 'json'" on some compute nodes). unset PYTHONHOME PYTHONPATH PYTHONSTARTUP 2>/dev/null || true -source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh +source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh" # Activate with a small retry — the JSONEncoder import flake is transient (GPFS/conda). for try in 1 2 3 4 5; do conda activate evalchemy-marin 2>/dev/null || true @@ -32,20 +87,20 @@ for try in 1 2 3 4 5; do echo ">>> conda evalchemy-marin active (try $try): $(which python)"; break fi echo ">>> conda activate flake on try $try; retrying in 10s"; conda deactivate 2>/dev/null || true; sleep 10 - source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh + source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh" done python -c "from json import JSONEncoder; import sys; assert 'evalchemy-marin' in sys.executable" \ || { echo "FATAL: could not activate evalchemy-marin cleanly after retries"; exit 17; } -cd /leonardo_work/AIFAC_5C0_290/bfeuer00/code/evalchemy-marin +cd "${EVALCHEMY_DIR:-$(_pick "${_OTA_WORK}/code/evalchemy-marin" "${_OTA_SHARED}/code/evalchemy-marin")}" source ~/secrets.env -export HF_HOME=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub -export HF_HUB_CACHE=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub +export HF_HOME="${HF_HOME:-${_OTA_WORK}/data/hub}" +export HF_HUB_CACHE="${HF_HUB_CACHE:-${_OTA_WORK}/data/hub}" export HF_HUB_OFFLINE=1 export HF_ALLOW_CODE_EVAL=1 export TOKENIZERS_PARALLELISM=false -CACHE_ROOT=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/.cache +CACHE_ROOT="${CACHE_ROOT:-${_OTA_WORK}/experiments/delphi-eval/.cache}" mkdir -p "$CACHE_ROOT"/{flashinfer,triton,inductor,vllm,xdg,hf_local} export HOME="$CACHE_ROOT/hf_local" export XDG_CACHE_HOME="$CACHE_ROOT/xdg" @@ -57,8 +112,8 @@ mkdir -p "$HOME/.cache/flashinfer" MODEL_DIR="${1:?usage: sbatch ... }" RUN_NAME="${2:?usage: sbatch ... }" -THINK_TEMPLATE="${THINK_TEMPLATE:-/leonardo_work/AIFAC_5C0_290/bfeuer00/code/OpenThoughts-Agent/chat_templates/delphi_v0_think.jinja2}" -OUT_ROOT="/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval" +THINK_TEMPLATE="${THINK_TEMPLATE:-${_OTA_REPO}/chat_templates/delphi_v0_think.jinja2}" +OUT_ROOT="${OUT_ROOT:-${_OTA_WORK}/experiments/delphi-eval}" OUT="${OUT_ROOT}/${RUN_NAME}" TP_SIZE=2 MAX_MODEL_LEN=4096 diff --git a/eval/evalchemy/evalchemy_eval.sbatch b/eval/evalchemy/evalchemy_eval.sbatch index e239a8e36..6c87a7b61 100755 --- a/eval/evalchemy/evalchemy_eval.sbatch +++ b/eval/evalchemy/evalchemy_eval.sbatch @@ -5,7 +5,14 @@ #SBATCH --gres=gpu:4 #SBATCH --time=08:00:00 #SBATCH --job-name=evalchemy-eval -#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/%x-%j.log +#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log +# +# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT. +# SLURM opens the log file before the job script runs and does NOT create intermediate +# directories -- a missing dir fails the job instantly with "Unable to open file", which +# looks like a launch bug rather than a missing mkdir. It already exists for the original +# operator; a NEW operator runs this once: +# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval # # ============================================================================== # evalchemy_eval.sbatch — GENERIC evalchemy downstream-eval runner (MATH500 + gsm8k + AIME24) @@ -42,19 +49,36 @@ set -euo pipefail +# ----- $USER-scoped paths (multi-operator) ----------------------------------------------------- +# Every path below resolves to the ORIGINAL bfeuer00 string byte-identically when USER=bfeuer00, +# so this is a no-op for the original operator. Read-only shared runtime (conda env, the canonical +# evalchemy-marin clone, nltk data) falls back to the bfeuer00 copy when the submitter has none — +# a second operator does NOT need to duplicate a 20 GB env to run an eval. WRITE paths +# (OUT_ROOT, CACHE_ROOT, HF_HOME) are always per-user: bfeuer00's dirs are mode 755. +_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" +_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" +_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } +# Conda: pick the root that actually CONTAINS the env. Testing only for a miniforge3 dir is not +# enough -- an operator can have a partial install (e.g. otagent only) and would then fail with +# "EnvironmentNameNotFound: evalchemy-marin" instead of falling back to the shared root. +_pick_conda() { [ -d "${_OTA_WORK}/miniforge3/envs/$1" ] && echo "${_OTA_WORK}/miniforge3" || echo "${_OTA_SHARED}/miniforge3"; } # own copy if present, else shared + # ----- env: conda env `evalchemy-marin` (lm-eval; the env every recent SCORES.md row used) ----- -source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh +_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick_conda evalchemy-marin)}" +source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh" conda activate evalchemy-marin # Must cd into the evalchemy-MARIN clone (matches the env's editable install). evalchemy-marin's # eval/eval.py imports parse_eval_args/setup_parser from its eval/lm_eval_compat.py shim — unlike # the now-removed `code/evalchemy` clone, whose eval/eval.py did `from lm_eval.__main__ import # parse_eval_args, setup_parser` (both dropped in lm-eval 0.4.12 -> ImportError at startup). # (`code/evalchemy` + its `evalchemy-resume-test` worktree were deleted 2026-06-18; this is the sole clone.) -cd /leonardo_work/AIFAC_5C0_290/bfeuer00/code/evalchemy-marin +cd "${EVALCHEMY_DIR:-$(_pick "${_OTA_WORK}/code/evalchemy-marin" "${_OTA_SHARED}/code/evalchemy-marin")}" source ~/secrets.env # HF_TOKEN etc. (BEFORE HOME override) -export HF_HOME=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub -export HF_HUB_CACHE=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub +# HF cache is per-user and WRITABLE (model weights may be symlinked in from another operator's cache; +# HF's python resolver follows symlinks — datasets must be real-copied, see ops/leonardo). +export HF_HOME="${HF_HOME:-${_OTA_WORK}/data/hub}" +export HF_HUB_CACHE="${HF_HUB_CACHE:-${_OTA_WORK}/data/hub}" export HF_HUB_OFFLINE=1 # repos pre-cached on login node; no compute-node internet export HF_ALLOW_CODE_EVAL=1 # required for gsm8k execution grading export TOKENIZERS_PARALLELISM=false @@ -64,12 +88,12 @@ export HF_HUB_ENABLE_HF_TRANSFER=1 # only matters for any pre-download; harmles # point NLTK_DATA at the pre-downloaded copy under $WORK (populated once on a login node: # python -c 'import nltk; d="$WORK/data/nltk_data"; nltk.download("punkt_tab",download_dir=d); nltk.download("punkt",download_dir=d)' # ). Harmless for the math tasks (they don't import nltk). -export NLTK_DATA="${NLTK_DATA:-/leonardo_work/AIFAC_5C0_290/bfeuer00/data/nltk_data}" +export NLTK_DATA="${NLTK_DATA:-$(_pick "${_OTA_WORK}/data/nltk_data" "${_OTA_SHARED}/data/nltk_data")}" # ----- writable caches (Leonardo: $HOME is READ-ONLY from compute AND login nodes) ----- # vLLM's flashinfer/triton/inductor JIT caches default under $HOME/.cache and hard-crash every worker # on a read-only HOME. Redirect HOME + every cache to the writable work filesystem. -CACHE_ROOT=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/.cache +CACHE_ROOT="${CACHE_ROOT:-${_OTA_WORK}/experiments/delphi-eval/.cache}" # must be WRITABLE mkdir -p "$CACHE_ROOT"/{flashinfer,triton,inductor,vllm,xdg,hf_local} export HOME="$CACHE_ROOT/hf_local" # flashinfer derives FLASHINFER_WORKSPACE_DIR from $HOME/.cache export XDG_CACHE_HOME="$CACHE_ROOT/xdg" @@ -83,7 +107,7 @@ mkdir -p "$HOME/.cache/flashinfer" MODEL_REPO="${1:?usage: sbatch --job-name=- evalchemy_eval.sbatch [STAGE]}" RUN_NAME="${2:?usage: sbatch --job-name=- evalchemy_eval.sbatch [STAGE]}" STAGE="${3:-sft}" -OUT_ROOT="${OUT_ROOT:-/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval}" +OUT_ROOT="${OUT_ROOT:-${_OTA_WORK}/experiments/delphi-eval}" JOB_PREFIX="${JOB_PREFIX:-evalchemy-eval}" MAX_GEN_TOKS="${MAX_GEN_TOKS:-30720}" GPUS_PER_NODE="${GPUS_PER_NODE:-4}" diff --git a/eval/evalchemy/qwen3_eval.sbatch b/eval/evalchemy/qwen3_eval.sbatch index 2fd35eea7..3c8f2efc1 100755 --- a/eval/evalchemy/qwen3_eval.sbatch +++ b/eval/evalchemy/qwen3_eval.sbatch @@ -5,7 +5,14 @@ #SBATCH --gres=gpu:4 #SBATCH --time=23:59:00 #SBATCH --job-name=qwen3-baseline -#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/qwen3-baseline/%x-%j.log +#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/qwen3-baseline/%x-%j.log +# +# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT. +# SLURM opens the log file before the job script runs and does NOT create intermediate +# directories -- a missing dir fails the job instantly with "Unable to open file", which +# looks like a launch bug rather than a missing mkdir. It already exists for the original +# operator; a NEW operator runs this once: +# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval/qwen3-baseline # # Qwen3 dense POST-TRAINED baseline — ONE downstream-eval job (MATH500 + gsm8k + AIME24). # SAME fixed protocol as delphi_eval.sbatch (EVAL_CONVENTION.md §2.2-§2.5, §3.3), with the Qwen3 @@ -33,22 +40,36 @@ set -euo pipefail +# ----- $USER-scoped paths (multi-operator) ----------------------------------------------------- +# Resolves to the original bfeuer00 strings byte-identically when USER=bfeuer00. Read-only shared +# runtime (conda, evalchemy-marin clone) falls back to the bfeuer00 copy; WRITE paths are per-user +# (bfeuer00's dirs are mode 755, so a second operator cannot write them). +_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" +_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" +_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } +# Conda: pick the root that actually CONTAINS the env. Testing only for a miniforge3 dir is not +# enough -- an operator can have a partial install (e.g. otagent only) and would then fail with +# "EnvironmentNameNotFound: evalchemy-marin" instead of falling back to the shared root. +_pick_conda() { [ -d "${_OTA_WORK}/miniforge3/envs/$1" ] && echo "${_OTA_WORK}/miniforge3" || echo "${_OTA_SHARED}/miniforge3"; } +_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick_conda evalchemy-marin)}" + + # ----- env: conda env `evalchemy-marin` (lm-eval v0.4.12 standard; cutover 2026-06-15 from # `evalchemy`=0.4.9.1 after MATH500 validation 9.6% vs 10.2%, grader byte-identical) ----- -source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh +source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh" conda activate evalchemy-marin -cd /leonardo_work/AIFAC_5C0_290/bfeuer00/code/evalchemy-marin +cd "${EVALCHEMY_DIR:-$(_pick "${_OTA_WORK}/code/evalchemy-marin" "${_OTA_SHARED}/code/evalchemy-marin")}" source ~/secrets.env # HF_TOKEN etc. (BEFORE HOME override) -export HF_HOME=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub -export HF_HUB_CACHE=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub +export HF_HOME="${HF_HOME:-${_OTA_WORK}/data/hub}" +export HF_HUB_CACHE="${HF_HUB_CACHE:-${_OTA_WORK}/data/hub}" export HF_HUB_OFFLINE=1 # repos pre-cached on login node; no compute-node internet export HF_ALLOW_CODE_EVAL=1 # required for gsm8k execution grading (evals.py:64) export TOKENIZERS_PARALLELISM=false export HF_HUB_ENABLE_HF_TRANSFER=1 # only matters for any pre-download; harmless here # ----- writable caches (Leonardo: $HOME is READ-ONLY from compute AND login nodes) (§3.0 amendment) ----- -CACHE_ROOT=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/.cache +CACHE_ROOT="${CACHE_ROOT:-${_OTA_WORK}/experiments/delphi-eval/.cache}" mkdir -p "$CACHE_ROOT"/{flashinfer,triton,inductor,vllm,xdg,hf_local} export HOME="$CACHE_ROOT/hf_local" # flashinfer derives FLASHINFER_WORKSPACE_DIR from $HOME/.cache export XDG_CACHE_HOME="$CACHE_ROOT/xdg" @@ -66,7 +87,7 @@ TP_SIZE="${3:-4}" # max_model_len=32768 OOMs warming up 256 dummy sampler requests (job 46868341) — pass a smaller # value (e.g. 32) for big models. 4th positional arg keeps the default for the small cells unchanged. MAX_NUM_SEQS="${4:-256}" -OUT="/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/qwen3-baseline/${RUN_NAME}" +OUT="${OUT_ROOT:-${_OTA_WORK}/experiments/delphi-eval/qwen3-baseline}/${RUN_NAME}" # ----- model-specific naming + jobid->model map ----- mkdir -p "${OUT}"