From da55d6a90e1c4fbf8b34c0b584190c199a69b28a Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Wed, 29 Jul 2026 11:46:50 -0700 Subject: [PATCH 1/5] Non-agentic evals: parameterize evalchemy sbatch paths by $USER (shared runtime falls back) Co-Authored-By: Claude Opus 5 --- eval/evalchemy/delphi_eval.sbatch | 10 +++++--- eval/evalchemy/delphi_rl_think_eval.sbatch | 28 ++++++++++++++------- eval/evalchemy/evalchemy_eval.sbatch | 29 ++++++++++++++++------ 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/eval/evalchemy/delphi_eval.sbatch b/eval/evalchemy/delphi_eval.sbatch index 49b583f93..3bb839cef 100755 --- a/eval/evalchemy/delphi_eval.sbatch +++ b/eval/evalchemy/delphi_eval.sbatch @@ -5,7 +5,7 @@ #SBATCH --gres=gpu:4 #SBATCH --time=08:00:00 #SBATCH --job-name=delphi-eval -#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/%x-%j.log +#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log # # ============================================================================== # delphi_eval.sbatch — THIN delphi wrapper around evalchemy_eval.sbatch (#6279). @@ -33,7 +33,11 @@ set -euo pipefail # so BASH_SOURCE[0]/dirname resolves to the spool dir (where evalchemy_eval.sbatch does NOT exist) — # that produced a 4-sec ExitCode-127 "No such file or directory". Pin to the repo clone instead, # falling back to the BASH_SOURCE dir only for interactive/non-SLURM invocation. -REPO_EVAL_DIR="/leonardo_work/AIFAC_5C0_290/bfeuer00/code/OpenThoughts-Agent/eval/evalchemy" +# $USER-scoped so a second operator runs THEIR checkout, not another user's. Resolves to the +# original bfeuer00 paths byte-identically when USER=bfeuer00. Override with OTAGENT_REPO_DIR. +_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" +_OTA_REPO="${OTAGENT_REPO_DIR:-${_OTA_WORK}/code/OpenThoughts-Agent}" +REPO_EVAL_DIR="${_OTA_REPO}/eval/evalchemy" if [[ -f "${REPO_EVAL_DIR}/evalchemy_eval.sbatch" ]]; then SELF_DIR="${REPO_EVAL_DIR}" else @@ -41,7 +45,7 @@ else fi # Delphi-specific defaults (overridable from the environment). -export CHAT_TEMPLATE_OVERRIDE="${CHAT_TEMPLATE_OVERRIDE:-/leonardo_work/AIFAC_5C0_290/bfeuer00/code/OpenThoughts-Agent/chat_templates/delphi_v0.jinja2}" +export CHAT_TEMPLATE_OVERRIDE="${CHAT_TEMPLATE_OVERRIDE:-${_OTA_REPO}/chat_templates/delphi_v0.jinja2}" export MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" export MAX_GEN_TOKS="${MAX_GEN_TOKS:-3584}" export JOB_PREFIX="${JOB_PREFIX:-delphi-eval}" diff --git a/eval/evalchemy/delphi_rl_think_eval.sbatch b/eval/evalchemy/delphi_rl_think_eval.sbatch index cc7d8ac5a..83fb9fa3b 100644 --- a/eval/evalchemy/delphi_rl_think_eval.sbatch +++ b/eval/evalchemy/delphi_rl_think_eval.sbatch @@ -5,7 +5,7 @@ #SBATCH --gres=gpu:4 #SBATCH --time=08:00:00 #SBATCH --job-name=delphi-think-eval -#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/%x-%j.log +#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log # # Held-out math eval for the 4 FORCED-THINKING RL grid cells (consolidated local HF dirs). # IDENTICAL harness to eval/evalchemy/evalchemy_eval.sbatch (same tasks, seeds, gen_kwargs, @@ -21,10 +21,20 @@ # Usage: sbatch --job-name=delphi-think-eval- delphi_rl_think_eval.sbatch set -euo pipefail +# ----- $USER-scoped paths (multi-operator) ----------------------------------------------------- +# Resolves to the original bfeuer00 strings byte-identically when USER=bfeuer00. Read-only shared +# runtime (conda, evalchemy-marin clone) falls back to the bfeuer00 copy; WRITE paths are per-user. +_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" +_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" +_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } +_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick "${_OTA_WORK}/miniforge3" "${_OTA_SHARED}/miniforge3")}" +_OTA_REPO="${OTAGENT_REPO_DIR:-${_OTA_WORK}/code/OpenThoughts-Agent}" + + # Clean any propagated python env that can shadow conda's base-python stdlib # (intermittent "ImportError: cannot import name 'JSONEncoder' from 'json'" on some compute nodes). unset PYTHONHOME PYTHONPATH PYTHONSTARTUP 2>/dev/null || true -source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh +source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh" # Activate with a small retry — the JSONEncoder import flake is transient (GPFS/conda). for try in 1 2 3 4 5; do conda activate evalchemy-marin 2>/dev/null || true @@ -32,20 +42,20 @@ for try in 1 2 3 4 5; do echo ">>> conda evalchemy-marin active (try $try): $(which python)"; break fi echo ">>> conda activate flake on try $try; retrying in 10s"; conda deactivate 2>/dev/null || true; sleep 10 - source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh + source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh" done python -c "from json import JSONEncoder; import sys; assert 'evalchemy-marin' in sys.executable" \ || { echo "FATAL: could not activate evalchemy-marin cleanly after retries"; exit 17; } -cd /leonardo_work/AIFAC_5C0_290/bfeuer00/code/evalchemy-marin +cd "${EVALCHEMY_DIR:-$(_pick "${_OTA_WORK}/code/evalchemy-marin" "${_OTA_SHARED}/code/evalchemy-marin")}" source ~/secrets.env -export HF_HOME=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub -export HF_HUB_CACHE=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub +export HF_HOME="${HF_HOME:-${_OTA_WORK}/data/hub}" +export HF_HUB_CACHE="${HF_HUB_CACHE:-${_OTA_WORK}/data/hub}" export HF_HUB_OFFLINE=1 export HF_ALLOW_CODE_EVAL=1 export TOKENIZERS_PARALLELISM=false -CACHE_ROOT=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/.cache +CACHE_ROOT="${CACHE_ROOT:-${_OTA_WORK}/experiments/delphi-eval/.cache}" mkdir -p "$CACHE_ROOT"/{flashinfer,triton,inductor,vllm,xdg,hf_local} export HOME="$CACHE_ROOT/hf_local" export XDG_CACHE_HOME="$CACHE_ROOT/xdg" @@ -57,8 +67,8 @@ mkdir -p "$HOME/.cache/flashinfer" MODEL_DIR="${1:?usage: sbatch ... }" RUN_NAME="${2:?usage: sbatch ... }" -THINK_TEMPLATE="${THINK_TEMPLATE:-/leonardo_work/AIFAC_5C0_290/bfeuer00/code/OpenThoughts-Agent/chat_templates/delphi_v0_think.jinja2}" -OUT_ROOT="/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval" +THINK_TEMPLATE="${THINK_TEMPLATE:-${_OTA_REPO}/chat_templates/delphi_v0_think.jinja2}" +OUT_ROOT="${OUT_ROOT:-${_OTA_WORK}/experiments/delphi-eval}" OUT="${OUT_ROOT}/${RUN_NAME}" TP_SIZE=2 MAX_MODEL_LEN=4096 diff --git a/eval/evalchemy/evalchemy_eval.sbatch b/eval/evalchemy/evalchemy_eval.sbatch index e239a8e36..bc05cd1fb 100755 --- a/eval/evalchemy/evalchemy_eval.sbatch +++ b/eval/evalchemy/evalchemy_eval.sbatch @@ -5,7 +5,7 @@ #SBATCH --gres=gpu:4 #SBATCH --time=08:00:00 #SBATCH --job-name=evalchemy-eval -#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/%x-%j.log +#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log # # ============================================================================== # evalchemy_eval.sbatch — GENERIC evalchemy downstream-eval runner (MATH500 + gsm8k + AIME24) @@ -42,19 +42,32 @@ set -euo pipefail +# ----- $USER-scoped paths (multi-operator) ----------------------------------------------------- +# Every path below resolves to the ORIGINAL bfeuer00 string byte-identically when USER=bfeuer00, +# so this is a no-op for the original operator. Read-only shared runtime (conda env, the canonical +# evalchemy-marin clone, nltk data) falls back to the bfeuer00 copy when the submitter has none — +# a second operator does NOT need to duplicate a 20 GB env to run an eval. WRITE paths +# (OUT_ROOT, CACHE_ROOT, HF_HOME) are always per-user: bfeuer00's dirs are mode 755. +_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" +_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" +_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } # own copy if present, else shared + # ----- env: conda env `evalchemy-marin` (lm-eval; the env every recent SCORES.md row used) ----- -source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh +_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick "${_OTA_WORK}/miniforge3" "${_OTA_SHARED}/miniforge3")}" +source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh" conda activate evalchemy-marin # Must cd into the evalchemy-MARIN clone (matches the env's editable install). evalchemy-marin's # eval/eval.py imports parse_eval_args/setup_parser from its eval/lm_eval_compat.py shim — unlike # the now-removed `code/evalchemy` clone, whose eval/eval.py did `from lm_eval.__main__ import # parse_eval_args, setup_parser` (both dropped in lm-eval 0.4.12 -> ImportError at startup). # (`code/evalchemy` + its `evalchemy-resume-test` worktree were deleted 2026-06-18; this is the sole clone.) -cd /leonardo_work/AIFAC_5C0_290/bfeuer00/code/evalchemy-marin +cd "${EVALCHEMY_DIR:-$(_pick "${_OTA_WORK}/code/evalchemy-marin" "${_OTA_SHARED}/code/evalchemy-marin")}" source ~/secrets.env # HF_TOKEN etc. (BEFORE HOME override) -export HF_HOME=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub -export HF_HUB_CACHE=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub +# HF cache is per-user and WRITABLE (model weights may be symlinked in from another operator's cache; +# HF's python resolver follows symlinks — datasets must be real-copied, see ops/leonardo). +export HF_HOME="${HF_HOME:-${_OTA_WORK}/data/hub}" +export HF_HUB_CACHE="${HF_HUB_CACHE:-${_OTA_WORK}/data/hub}" export HF_HUB_OFFLINE=1 # repos pre-cached on login node; no compute-node internet export HF_ALLOW_CODE_EVAL=1 # required for gsm8k execution grading export TOKENIZERS_PARALLELISM=false @@ -64,12 +77,12 @@ export HF_HUB_ENABLE_HF_TRANSFER=1 # only matters for any pre-download; harmles # point NLTK_DATA at the pre-downloaded copy under $WORK (populated once on a login node: # python -c 'import nltk; d="$WORK/data/nltk_data"; nltk.download("punkt_tab",download_dir=d); nltk.download("punkt",download_dir=d)' # ). Harmless for the math tasks (they don't import nltk). -export NLTK_DATA="${NLTK_DATA:-/leonardo_work/AIFAC_5C0_290/bfeuer00/data/nltk_data}" +export NLTK_DATA="${NLTK_DATA:-$(_pick "${_OTA_WORK}/data/nltk_data" "${_OTA_SHARED}/data/nltk_data")}" # ----- writable caches (Leonardo: $HOME is READ-ONLY from compute AND login nodes) ----- # vLLM's flashinfer/triton/inductor JIT caches default under $HOME/.cache and hard-crash every worker # on a read-only HOME. Redirect HOME + every cache to the writable work filesystem. -CACHE_ROOT=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/.cache +CACHE_ROOT="${CACHE_ROOT:-${_OTA_WORK}/experiments/delphi-eval/.cache}" # must be WRITABLE mkdir -p "$CACHE_ROOT"/{flashinfer,triton,inductor,vllm,xdg,hf_local} export HOME="$CACHE_ROOT/hf_local" # flashinfer derives FLASHINFER_WORKSPACE_DIR from $HOME/.cache export XDG_CACHE_HOME="$CACHE_ROOT/xdg" @@ -83,7 +96,7 @@ mkdir -p "$HOME/.cache/flashinfer" MODEL_REPO="${1:?usage: sbatch --job-name=- evalchemy_eval.sbatch [STAGE]}" RUN_NAME="${2:?usage: sbatch --job-name=- evalchemy_eval.sbatch [STAGE]}" STAGE="${3:-sft}" -OUT_ROOT="${OUT_ROOT:-/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval}" +OUT_ROOT="${OUT_ROOT:-${_OTA_WORK}/experiments/delphi-eval}" JOB_PREFIX="${JOB_PREFIX:-evalchemy-eval}" MAX_GEN_TOKS="${MAX_GEN_TOKS:-30720}" GPUS_PER_NODE="${GPUS_PER_NODE:-4}" From 450e1450f2e66f3b233bcdaa39343a26dccfbd62 Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Wed, 29 Jul 2026 14:40:36 -0700 Subject: [PATCH 2/5] Non-agentic evals: parameterize qwen3_eval.sbatch by $USER (missed in prior commit) Co-Authored-By: Claude Opus 5 --- eval/evalchemy/qwen3_eval.sbatch | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/eval/evalchemy/qwen3_eval.sbatch b/eval/evalchemy/qwen3_eval.sbatch index 2fd35eea7..655143fbb 100755 --- a/eval/evalchemy/qwen3_eval.sbatch +++ b/eval/evalchemy/qwen3_eval.sbatch @@ -5,7 +5,7 @@ #SBATCH --gres=gpu:4 #SBATCH --time=23:59:00 #SBATCH --job-name=qwen3-baseline -#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/qwen3-baseline/%x-%j.log +#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/qwen3-baseline/%x-%j.log # # Qwen3 dense POST-TRAINED baseline — ONE downstream-eval job (MATH500 + gsm8k + AIME24). # SAME fixed protocol as delphi_eval.sbatch (EVAL_CONVENTION.md §2.2-§2.5, §3.3), with the Qwen3 @@ -33,22 +33,32 @@ set -euo pipefail +# ----- $USER-scoped paths (multi-operator) ----------------------------------------------------- +# Resolves to the original bfeuer00 strings byte-identically when USER=bfeuer00. Read-only shared +# runtime (conda, evalchemy-marin clone) falls back to the bfeuer00 copy; WRITE paths are per-user +# (bfeuer00's dirs are mode 755, so a second operator cannot write them). +_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" +_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" +_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } +_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick "${_OTA_WORK}/miniforge3" "${_OTA_SHARED}/miniforge3")}" + + # ----- env: conda env `evalchemy-marin` (lm-eval v0.4.12 standard; cutover 2026-06-15 from # `evalchemy`=0.4.9.1 after MATH500 validation 9.6% vs 10.2%, grader byte-identical) ----- -source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh +source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh" conda activate evalchemy-marin -cd /leonardo_work/AIFAC_5C0_290/bfeuer00/code/evalchemy-marin +cd "${EVALCHEMY_DIR:-$(_pick "${_OTA_WORK}/code/evalchemy-marin" "${_OTA_SHARED}/code/evalchemy-marin")}" source ~/secrets.env # HF_TOKEN etc. (BEFORE HOME override) -export HF_HOME=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub -export HF_HUB_CACHE=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub +export HF_HOME="${HF_HOME:-${_OTA_WORK}/data/hub}" +export HF_HUB_CACHE="${HF_HUB_CACHE:-${_OTA_WORK}/data/hub}" export HF_HUB_OFFLINE=1 # repos pre-cached on login node; no compute-node internet export HF_ALLOW_CODE_EVAL=1 # required for gsm8k execution grading (evals.py:64) export TOKENIZERS_PARALLELISM=false export HF_HUB_ENABLE_HF_TRANSFER=1 # only matters for any pre-download; harmless here # ----- writable caches (Leonardo: $HOME is READ-ONLY from compute AND login nodes) (§3.0 amendment) ----- -CACHE_ROOT=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/.cache +CACHE_ROOT="${CACHE_ROOT:-${_OTA_WORK}/experiments/delphi-eval/.cache}" mkdir -p "$CACHE_ROOT"/{flashinfer,triton,inductor,vllm,xdg,hf_local} export HOME="$CACHE_ROOT/hf_local" # flashinfer derives FLASHINFER_WORKSPACE_DIR from $HOME/.cache export XDG_CACHE_HOME="$CACHE_ROOT/xdg" @@ -66,7 +76,7 @@ TP_SIZE="${3:-4}" # max_model_len=32768 OOMs warming up 256 dummy sampler requests (job 46868341) — pass a smaller # value (e.g. 32) for big models. 4th positional arg keeps the default for the small cells unchanged. MAX_NUM_SEQS="${4:-256}" -OUT="/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/qwen3-baseline/${RUN_NAME}" +OUT="${OUT_ROOT:-${_OTA_WORK}/experiments/delphi-eval/qwen3-baseline}/${RUN_NAME}" # ----- model-specific naming + jobid->model map ----- mkdir -p "${OUT}" From 38f286a4f454f2feb8ed1828dba25f56a5e97aa0 Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Wed, 29 Jul 2026 15:12:19 -0700 Subject: [PATCH 3/5] evalchemy sbatch: conda fallback must validate the ENV exists, not just the miniforge3 root Co-Authored-By: Claude Opus 5 --- eval/evalchemy/delphi_rl_think_eval.sbatch | 6 +++++- eval/evalchemy/evalchemy_eval.sbatch | 8 ++++++-- eval/evalchemy/qwen3_eval.sbatch | 6 +++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/eval/evalchemy/delphi_rl_think_eval.sbatch b/eval/evalchemy/delphi_rl_think_eval.sbatch index 83fb9fa3b..80d81265a 100644 --- a/eval/evalchemy/delphi_rl_think_eval.sbatch +++ b/eval/evalchemy/delphi_rl_think_eval.sbatch @@ -27,7 +27,11 @@ set -euo pipefail _OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" _OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" _pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } -_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick "${_OTA_WORK}/miniforge3" "${_OTA_SHARED}/miniforge3")}" +# Conda: pick the root that actually CONTAINS the env. Testing only for a miniforge3 dir is not +# enough -- an operator can have a partial install (e.g. otagent only) and would then fail with +# "EnvironmentNameNotFound: evalchemy-marin" instead of falling back to the shared root. +_pick_conda() { [ -d "${_OTA_WORK}/miniforge3/envs/$1" ] && echo "${_OTA_WORK}/miniforge3" || echo "${_OTA_SHARED}/miniforge3"; } +_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick_conda evalchemy-marin)}" _OTA_REPO="${OTAGENT_REPO_DIR:-${_OTA_WORK}/code/OpenThoughts-Agent}" diff --git a/eval/evalchemy/evalchemy_eval.sbatch b/eval/evalchemy/evalchemy_eval.sbatch index bc05cd1fb..a92403d7f 100755 --- a/eval/evalchemy/evalchemy_eval.sbatch +++ b/eval/evalchemy/evalchemy_eval.sbatch @@ -50,10 +50,14 @@ set -euo pipefail # (OUT_ROOT, CACHE_ROOT, HF_HOME) are always per-user: bfeuer00's dirs are mode 755. _OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" _OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" -_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } # own copy if present, else shared +_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } +# Conda: pick the root that actually CONTAINS the env. Testing only for a miniforge3 dir is not +# enough -- an operator can have a partial install (e.g. otagent only) and would then fail with +# "EnvironmentNameNotFound: evalchemy-marin" instead of falling back to the shared root. +_pick_conda() { [ -d "${_OTA_WORK}/miniforge3/envs/$1" ] && echo "${_OTA_WORK}/miniforge3" || echo "${_OTA_SHARED}/miniforge3"; } # own copy if present, else shared # ----- env: conda env `evalchemy-marin` (lm-eval; the env every recent SCORES.md row used) ----- -_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick "${_OTA_WORK}/miniforge3" "${_OTA_SHARED}/miniforge3")}" +_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick_conda evalchemy-marin)}" source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh" conda activate evalchemy-marin # Must cd into the evalchemy-MARIN clone (matches the env's editable install). evalchemy-marin's diff --git a/eval/evalchemy/qwen3_eval.sbatch b/eval/evalchemy/qwen3_eval.sbatch index 655143fbb..c342f903e 100755 --- a/eval/evalchemy/qwen3_eval.sbatch +++ b/eval/evalchemy/qwen3_eval.sbatch @@ -40,7 +40,11 @@ set -euo pipefail _OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" _OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" _pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } -_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick "${_OTA_WORK}/miniforge3" "${_OTA_SHARED}/miniforge3")}" +# Conda: pick the root that actually CONTAINS the env. Testing only for a miniforge3 dir is not +# enough -- an operator can have a partial install (e.g. otagent only) and would then fail with +# "EnvironmentNameNotFound: evalchemy-marin" instead of falling back to the shared root. +_pick_conda() { [ -d "${_OTA_WORK}/miniforge3/envs/$1" ] && echo "${_OTA_WORK}/miniforge3" || echo "${_OTA_SHARED}/miniforge3"; } +_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick_conda evalchemy-marin)}" # ----- env: conda env `evalchemy-marin` (lm-eval v0.4.12 standard; cutover 2026-06-15 from From 3e74bc22ca6270ebfae46b662e06cb4bfd867497 Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Tue, 4 Aug 2026 12:03:29 -0700 Subject: [PATCH 4/5] evalchemy sbatch: document that the $USER --output dir must pre-exist The --output paths are now $USER-scoped via SLURM's %u. That is correct, but SLURM opens the log file BEFORE the job script runs and does not create intermediate directories, so a new operator whose /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval/ does not exist yet gets an instant job failure ('Unable to open file') that reads like a launch bug rather than a missing mkdir. The job script cannot self-heal this -- its own mkdir -p runs far too late. So document the one-time setup inline, next to the path it applies to, including the qwen3-baseline subdirectory that only that sbatch needs. No behaviour change: comments only. Byte-identical for the original operator, whose directories already exist. Co-Authored-By: Claude Opus 5 --- eval/evalchemy/delphi_eval.sbatch | 7 +++++++ eval/evalchemy/delphi_rl_think_eval.sbatch | 7 +++++++ eval/evalchemy/evalchemy_eval.sbatch | 7 +++++++ eval/evalchemy/qwen3_eval.sbatch | 7 +++++++ 4 files changed, 28 insertions(+) diff --git a/eval/evalchemy/delphi_eval.sbatch b/eval/evalchemy/delphi_eval.sbatch index 3bb839cef..416c6a18b 100755 --- a/eval/evalchemy/delphi_eval.sbatch +++ b/eval/evalchemy/delphi_eval.sbatch @@ -7,6 +7,13 @@ #SBATCH --job-name=delphi-eval #SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log # +# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT. +# SLURM opens the log file before the job script runs and does NOT create intermediate +# directories -- a missing dir fails the job instantly with "Unable to open file", which +# looks like a launch bug rather than a missing mkdir. It already exists for the original +# operator; a NEW operator runs this once: +# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval +# # ============================================================================== # delphi_eval.sbatch — THIN delphi wrapper around evalchemy_eval.sbatch (#6279). # diff --git a/eval/evalchemy/delphi_rl_think_eval.sbatch b/eval/evalchemy/delphi_rl_think_eval.sbatch index 80d81265a..16d939fe6 100644 --- a/eval/evalchemy/delphi_rl_think_eval.sbatch +++ b/eval/evalchemy/delphi_rl_think_eval.sbatch @@ -7,6 +7,13 @@ #SBATCH --job-name=delphi-think-eval #SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log # +# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT. +# SLURM opens the log file before the job script runs and does NOT create intermediate +# directories -- a missing dir fails the job instantly with "Unable to open file", which +# looks like a launch bug rather than a missing mkdir. It already exists for the original +# operator; a NEW operator runs this once: +# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval +# # Held-out math eval for the 4 FORCED-THINKING RL grid cells (consolidated local HF dirs). # IDENTICAL harness to eval/evalchemy/evalchemy_eval.sbatch (same tasks, seeds, gen_kwargs, # gsm8k-via-lm_eval, AIME24 10-seed, MATH500 via eval.eval) EXCEPT: diff --git a/eval/evalchemy/evalchemy_eval.sbatch b/eval/evalchemy/evalchemy_eval.sbatch index a92403d7f..6c87a7b61 100755 --- a/eval/evalchemy/evalchemy_eval.sbatch +++ b/eval/evalchemy/evalchemy_eval.sbatch @@ -7,6 +7,13 @@ #SBATCH --job-name=evalchemy-eval #SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log # +# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT. +# SLURM opens the log file before the job script runs and does NOT create intermediate +# directories -- a missing dir fails the job instantly with "Unable to open file", which +# looks like a launch bug rather than a missing mkdir. It already exists for the original +# operator; a NEW operator runs this once: +# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval +# # ============================================================================== # evalchemy_eval.sbatch — GENERIC evalchemy downstream-eval runner (MATH500 + gsm8k + AIME24) # diff --git a/eval/evalchemy/qwen3_eval.sbatch b/eval/evalchemy/qwen3_eval.sbatch index c342f903e..3c8f2efc1 100755 --- a/eval/evalchemy/qwen3_eval.sbatch +++ b/eval/evalchemy/qwen3_eval.sbatch @@ -7,6 +7,13 @@ #SBATCH --job-name=qwen3-baseline #SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/qwen3-baseline/%x-%j.log # +# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT. +# SLURM opens the log file before the job script runs and does NOT create intermediate +# directories -- a missing dir fails the job instantly with "Unable to open file", which +# looks like a launch bug rather than a missing mkdir. It already exists for the original +# operator; a NEW operator runs this once: +# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval/qwen3-baseline +# # Qwen3 dense POST-TRAINED baseline — ONE downstream-eval job (MATH500 + gsm8k + AIME24). # SAME fixed protocol as delphi_eval.sbatch (EVAL_CONVENTION.md §2.2-§2.5, §3.3), with the Qwen3 # adaptations documented in baseline_evals/grid.md. This is a FORK of delphi_eval.sbatch; the deltas From dc3108dbefdeef69c19dbe1723f050cbf3b235d5 Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Tue, 4 Aug 2026 14:17:19 -0700 Subject: [PATCH 5/5] evalchemy: resolve the repo root by probe, not by assuming a canonical checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review catch (P2). _OTA_REPO defaulted to $WORK/$USER/code/OpenThoughts-Agent, which only holds for operators who cloned at exactly that path. Anyone else fell through to the BASH_SOURCE branch — which under SLURM is the /var/spool/slurmd/jobNNN/ copy — so it re-created the very ExitCode-127 'No such file or directory' that the comment three lines above says this block exists to prevent. The blast radius was wider than the exec target. _OTA_REPO also feeds CHAT_TEMPLATE_OVERRIDE (delphi_eval) and THINK_TEMPLATE (delphi_rl_think_eval), so a noncanonical checkout silently pointed lm-eval at a missing chat template instead of reporting that the repo was not found. Resolve by PROBING for a marker file (eval/evalchemy/evalchemy_eval.sbatch), taking the first candidate that actually contains it: 1. OTAGENT_REPO_DIR explicit override 2. $WORK/$USER/code/... canonical per-user checkout 3. SLURM_SUBMIT_DIR + parents where the operator actually ran sbatch 4. shared bfeuer00 checkout same borrow pattern as conda/evalchemy-marin 5. script dir + parents interactive / non-SLURM only Walking up from SLURM_SUBMIT_DIR matters because operators typically submit from a subdirectory of the repo, not its root. If nothing matches, fail LOUDLY listing every candidate tried, instead of exec'ing a nonexistent path and leaving a bare 127. Unchanged for the original operator: candidate 2 hits first when USER=bfeuer00. Verified: canonical operator, noncanonical checkout via SLURM_SUBMIT_DIR, noncanonical with only the spool dir (falls back to shared), explicit override, and the nothing-found path. Co-Authored-By: Claude Opus 5 --- eval/evalchemy/delphi_eval.sbatch | 53 +++++++++++++++++++--- eval/evalchemy/delphi_rl_think_eval.sbatch | 36 ++++++++++++++- 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/eval/evalchemy/delphi_eval.sbatch b/eval/evalchemy/delphi_eval.sbatch index 416c6a18b..1058dbb40 100755 --- a/eval/evalchemy/delphi_eval.sbatch +++ b/eval/evalchemy/delphi_eval.sbatch @@ -41,15 +41,54 @@ set -euo pipefail # that produced a 4-sec ExitCode-127 "No such file or directory". Pin to the repo clone instead, # falling back to the BASH_SOURCE dir only for interactive/non-SLURM invocation. # $USER-scoped so a second operator runs THEIR checkout, not another user's. Resolves to the -# original bfeuer00 paths byte-identically when USER=bfeuer00. Override with OTAGENT_REPO_DIR. +# original bfeuer00 paths byte-identically when USER=bfeuer00. +# +# Resolution is by PROBE, not by assumption: try each candidate and take the first that actually +# contains the marker file. A bare "$WORK/$USER/code/OpenThoughts-Agent" default only works for +# operators who cloned at exactly that path; anyone else fell through to the BASH_SOURCE branch, +# which under SLURM is the /var/spool/slurmd/jobNNN/ copy — i.e. it re-created the very +# ExitCode-127 "No such file or directory" this block was written to prevent. It also silently +# mis-pointed CHAT_TEMPLATE_OVERRIDE below (and THINK_TEMPLATE in delphi_rl_think_eval.sbatch). _OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}" -_OTA_REPO="${OTAGENT_REPO_DIR:-${_OTA_WORK}/code/OpenThoughts-Agent}" -REPO_EVAL_DIR="${_OTA_REPO}/eval/evalchemy" -if [[ -f "${REPO_EVAL_DIR}/evalchemy_eval.sbatch" ]]; then - SELF_DIR="${REPO_EVAL_DIR}" -else - SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00" +_OTA_MARKER="eval/evalchemy/evalchemy_eval.sbatch" +# Walk up from a directory looking for the marker: SLURM_SUBMIT_DIR is wherever the operator ran +# sbatch, which is often a subdirectory of the repo rather than its root. +_ota_up() { + local d="${1:-}" + [ -n "$d" ] || return 1 + while [ -n "$d" ] && [ "$d" != "/" ]; do + [ -f "$d/${_OTA_MARKER}" ] && { printf '%s' "$d"; return 0; } + d="$(dirname "$d")" + done + return 1 +} +_ota_first_repo() { + local c + for c in "$@"; do + [ -n "$c" ] && [ -f "$c/${_OTA_MARKER}" ] && { printf '%s' "$c"; return 0; } + done + return 1 +} +_OTA_REPO="$(_ota_first_repo \ + "${OTAGENT_REPO_DIR:-}" \ + "${_OTA_WORK}/code/OpenThoughts-Agent" \ + "$(_ota_up "${SLURM_SUBMIT_DIR:-}" || true)" \ + "${_OTA_SHARED}/code/OpenThoughts-Agent" \ + "$(_ota_up "$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || true)" || true)" \ + || true)" +if [ -z "${_OTA_REPO}" ]; then + echo "ERROR: cannot locate an OpenThoughts-Agent checkout containing ${_OTA_MARKER}." >&2 + echo " tried, in order:" >&2 + echo " OTAGENT_REPO_DIR = ${OTAGENT_REPO_DIR:-}" >&2 + echo " \$USER checkout = ${_OTA_WORK}/code/OpenThoughts-Agent" >&2 + echo " SLURM_SUBMIT_DIR = ${SLURM_SUBMIT_DIR:-} (and parents)" >&2 + echo " shared checkout = ${_OTA_SHARED}/code/OpenThoughts-Agent" >&2 + echo " script dir = $(dirname "${BASH_SOURCE[0]}") (and parents)" >&2 + echo " Set OTAGENT_REPO_DIR=/path/to/OpenThoughts-Agent and resubmit." >&2 + exit 1 fi +SELF_DIR="${_OTA_REPO}/eval/evalchemy" # Delphi-specific defaults (overridable from the environment). export CHAT_TEMPLATE_OVERRIDE="${CHAT_TEMPLATE_OVERRIDE:-${_OTA_REPO}/chat_templates/delphi_v0.jinja2}" diff --git a/eval/evalchemy/delphi_rl_think_eval.sbatch b/eval/evalchemy/delphi_rl_think_eval.sbatch index 16d939fe6..6597e8e6a 100644 --- a/eval/evalchemy/delphi_rl_think_eval.sbatch +++ b/eval/evalchemy/delphi_rl_think_eval.sbatch @@ -39,7 +39,41 @@ _pick() { [ -e "$1" ] && echo "$1" || echo "$2"; } # "EnvironmentNameNotFound: evalchemy-marin" instead of falling back to the shared root. _pick_conda() { [ -d "${_OTA_WORK}/miniforge3/envs/$1" ] && echo "${_OTA_WORK}/miniforge3" || echo "${_OTA_SHARED}/miniforge3"; } _OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick_conda evalchemy-marin)}" -_OTA_REPO="${OTAGENT_REPO_DIR:-${_OTA_WORK}/code/OpenThoughts-Agent}" +# Repo root by PROBE, not assumption — see the long note in delphi_eval.sbatch. A bare +# "$WORK/$USER/code/OpenThoughts-Agent" default silently mis-points THINK_TEMPLATE below for any +# operator who cloned elsewhere, and lm-eval then fails on a missing chat template rather than +# saying the repo was not found. +_OTA_MARKER="eval/evalchemy/evalchemy_eval.sbatch" +_ota_up() { + local d="${1:-}" + [ -n "$d" ] || return 1 + while [ -n "$d" ] && [ "$d" != "/" ]; do + [ -f "$d/${_OTA_MARKER}" ] && { printf '%s' "$d"; return 0; } + d="$(dirname "$d")" + done + return 1 +} +_ota_first_repo() { + local c + for c in "$@"; do + [ -n "$c" ] && [ -f "$c/${_OTA_MARKER}" ] && { printf '%s' "$c"; return 0; } + done + return 1 +} +_OTA_REPO="$(_ota_first_repo \ + "${OTAGENT_REPO_DIR:-}" \ + "${_OTA_WORK}/code/OpenThoughts-Agent" \ + "$(_ota_up "${SLURM_SUBMIT_DIR:-}" || true)" \ + "${_OTA_SHARED}/code/OpenThoughts-Agent" \ + "$(_ota_up "$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || true)" || true)" \ + || true)" +if [ -z "${_OTA_REPO}" ]; then + echo "ERROR: cannot locate an OpenThoughts-Agent checkout containing ${_OTA_MARKER}." >&2 + echo " Set OTAGENT_REPO_DIR=/path/to/OpenThoughts-Agent and resubmit." >&2 + echo " tried: OTAGENT_REPO_DIR=${OTAGENT_REPO_DIR:-}, ${_OTA_WORK}/code/OpenThoughts-Agent," >&2 + echo " SLURM_SUBMIT_DIR=${SLURM_SUBMIT_DIR:-} (and parents), ${_OTA_SHARED}/code/OpenThoughts-Agent" >&2 + exit 1 +fi # Clean any propagated python env that can shadow conda's base-python stdlib