From e2d5008b71ca9e29a1e42f0300937628dd4cc7c6 Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Thu, 23 Jul 2026 11:53:36 -0700 Subject: [PATCH 1/5] TACC/vista: parameterize user-scoped eval paths by $SCRATCH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same fix as the Leonardo $USER commit, for the vista (TACC) eval cluster view. The write/code paths (project_root, hf_cache, eval_jobs_dir, harbor_src, datasets_dirs, secrets_file) hardcoded penfever's /scratch/10635/penfever, so no other operator could run the listener there (another user's dirs aren't writable; secrets_file pointed at penfever's keys.env). Parameterize them with $SCRATCH, which to_eval_cluster_view()/load_cluster_config() expandvars-expand at load time. $SCRATCH (not $USER) because TACC scratch is /scratch// and the alloc-number differs per user — only $SCRATCH encodes both. For penfever $SCRATCH=/scratch/10635/penfever, so this renders byte-identical to the prior hardcoded values (verified: view rendered as penfever == old literals). conda_envs.otagent and hardware.cuda_home stay pinned to penfever's prefix on purpose — it's world-readable and shared read-only, so a new operator borrows it without rebuilding the aarch64 env. Point at your own once built. hpc.py 'tacc' view and the eval/clusters/tacc.yaml shim kept consistent (paths parity verified rendered as mkumar73). Co-Authored-By: Claude Opus 4.8 --- eval/clusters/tacc.yaml | 15 +++++++++------ hpc/hpc.py | 26 ++++++++++++++++---------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/eval/clusters/tacc.yaml b/eval/clusters/tacc.yaml index 4f50d7852..0bab19072 100644 --- a/eval/clusters/tacc.yaml +++ b/eval/clusters/tacc.yaml @@ -15,17 +15,20 @@ conda_envs: otagent: /scratch/10635/penfever/miniconda3/envs/otagent paths: - project_root: /scratch/10635/penfever/OpenThoughts-Agent - hf_cache: /scratch/10635/penfever/hub - eval_jobs_dir: /scratch/10635/penfever/eval_jobs + # $SCRATCH-parameterized (load_cluster_config expands $SCRATCH at load time); each operator's + # $SCRATCH encodes their /scratch// root. Byte-identical for penfever + # ($SCRATCH=/scratch/10635/penfever). conda_envs/cuda_home stay penfever's = shared read-only borrow. + project_root: $SCRATCH/OpenThoughts-Agent + hf_cache: $SCRATCH/hub + eval_jobs_dir: $SCRATCH/eval_jobs eval_logs_dir: eval/tacc/logs listener_logs_dir: experiments/listener_logs sbatch_script: eval/tacc/eval_harbor.sbatch dp_sbatch_script: eval/tacc/eval_harbor.sbatch - harbor_src: /scratch/10635/penfever/harbor/src + harbor_src: $SCRATCH/harbor/src datasets_dirs: - - /scratch/10635/penfever/hub - secrets_file: /scratch/10635/penfever/keys.env + - $SCRATCH/hub + secrets_file: $SCRATCH/keys.env proxy: enabled: false diff --git a/hpc/hpc.py b/hpc/hpc.py index e9ff5c160..6c19fec20 100644 --- a/hpc/hpc.py +++ b/hpc/hpc.py @@ -1470,10 +1470,16 @@ def get_pre_run_commands(self) -> str: # name "tacc"; the view's cluster_name is "tacc" (mirroring tacc.yaml) and the # listener's _resolve_cluster_view_by_name() matches EITHER c.name ("vista") OR the # view's cluster_name ("tacc"), so `--cluster-config tacc` AND `--cluster-config vista` - # both resolve. User-scoped paths mirror that yaml (penfever's). Vista-specific vs - # leonardo: hardware_profile gh200, 1 GPU/node, NO mem_per_node_mb (RealMemory is - # misreported as 1 MB → request whole nodes), and hardware.gpu_gres false (GPUs are - # not a SLURM gres here). Compute nodes have full egress so proxy is disabled. + # both resolve. User-scoped write/code paths are $SCRATCH-parameterized (to_eval_cluster_view + # expandvars-expands them at call time on the login node, where $SCRATCH is always set); each + # operator's $SCRATCH already encodes their /scratch// root, so $USER alone is + # insufficient here (the alloc-number differs per user). For penfever $SCRATCH=/scratch/10635/ + # penfever, so this renders byte-identical to the prior hardcoded values. The conda env / + # cuda_home stay pinned to penfever's prefix ON PURPOSE — it is world-readable and shared + # read-only, so a new operator borrows it without rebuilding (point at your own once built). + # Vista-specific vs leonardo: hardware_profile gh200, 1 GPU/node, NO mem_per_node_mb (RealMemory + # is misreported as 1 MB → request whole nodes), and hardware.gpu_gres false (GPUs are not a + # SLURM gres here). Compute nodes have full egress so proxy is disabled. eval_cluster_view={ "cluster_name": "tacc", "baseline_model_configs": "eval/clusters/tacc_baseline_model_configs.yaml", @@ -1487,16 +1493,16 @@ def get_pre_run_commands(self) -> str: "otagent": "/scratch/10635/penfever/miniconda3/envs/otagent", }, "paths": { - "project_root": "/scratch/10635/penfever/OpenThoughts-Agent", - "hf_cache": "/scratch/10635/penfever/hub", - "eval_jobs_dir": "/scratch/10635/penfever/eval_jobs", + "project_root": "$SCRATCH/OpenThoughts-Agent", + "hf_cache": "$SCRATCH/hub", + "eval_jobs_dir": "$SCRATCH/eval_jobs", "eval_logs_dir": "eval/tacc/logs", "listener_logs_dir": "experiments/listener_logs", "sbatch_script": "eval/tacc/eval_harbor.sbatch", "dp_sbatch_script": "eval/tacc/eval_harbor.sbatch", - "harbor_src": "/scratch/10635/penfever/harbor/src", - "datasets_dirs": ["/scratch/10635/penfever/hub"], - "secrets_file": "/scratch/10635/penfever/keys.env", + "harbor_src": "$SCRATCH/harbor/src", + "datasets_dirs": ["$SCRATCH/hub"], + "secrets_file": "$SCRATCH/keys.env", }, "proxy": {"enabled": False}, "hardware": { From 6484a947803bc7ab75c904e1c7c4a50c0c8e928b Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Thu, 23 Jul 2026 14:40:39 -0700 Subject: [PATCH 2/5] TACC eval sbatch: EVAL_SKIP_UPLOAD guard (local-only smoke/canary safety) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the leonardo sbatch's EVAL_SKIP_UPLOAD guard to the TACC path — main's TACC sbatch had neither EVAL_SKIP_UPLOAD nor EVAL_SKIP_DB. Without it a subset/canary run would call upload_eval_results() (HF upload + Supabase register in one call) and either register a below-gate canary as a leaderboard result or crash on the bare os.environ["HF_TOKEN"] when no token is set. The case-guard exit 0s before the uploader, preserving results on disk. Needed for any TACC smoke/canary. Co-Authored-By: Claude Opus 4.8 --- eval/tacc/eval_harbor.sbatch | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eval/tacc/eval_harbor.sbatch b/eval/tacc/eval_harbor.sbatch index 307c3ff64..7ff0abebb 100644 --- a/eval/tacc/eval_harbor.sbatch +++ b/eval/tacc/eval_harbor.sbatch @@ -790,6 +790,18 @@ export UPLOAD_USERNAME="${UPLOAD_USERNAME:-$USER}" export UPLOAD_MODE="${UPLOAD_MODE:-skip_on_error}" export RUN_TAG +# Local-only smoke/canary guard (mirrors the leonardo sbatch): EVAL_SKIP_UPLOAD=1 preserves the +# results on disk and skips the HF/DB uploader entirely, so a subset/canary is NEVER registered as a +# full-benchmark leaderboard result. upload_eval_results() does HF upload AND Supabase registration in +# one call, so this single exit covers both. Also side-steps the bare os.environ["HF_TOKEN"] below when +# no token is configured (canary path). +case "${EVAL_SKIP_UPLOAD:-0}" in + 1|true|TRUE|yes|YES) + echo "EVAL_SKIP_UPLOAD=${EVAL_SKIP_UPLOAD}: preserving local results at $RUN_DIR, skipping HF/DB upload" + exit 0 + ;; +esac + UPLOAD_LOG="$RUN_DIR/upload.log" echo "Uploading results from: $RUN_DIR" | tee -a "$UPLOAD_LOG" From 5b8087403ae5c0337784df157929cdf9feb8fde1 Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Thu, 23 Jul 2026 15:56:47 -0700 Subject: [PATCH 3/5] TACC sbatch: conda activate via OTAGENT_CONDA_PREFIX (borrow-safe, default unchanged) --- eval/tacc/eval_harbor.sbatch | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/eval/tacc/eval_harbor.sbatch b/eval/tacc/eval_harbor.sbatch index 7ff0abebb..b51c9434f 100644 --- a/eval/tacc/eval_harbor.sbatch +++ b/eval/tacc/eval_harbor.sbatch @@ -146,9 +146,16 @@ module purge 2>/dev/null || true module load gcc/13.2.0 cuda/12.8 2>/dev/null || true # --- Conda activation: otagent (harbor + fork vLLM + DB client + uploader) --- -source "${SCRATCH}/miniconda3/etc/profile.d/conda.sh" -conda activate otagent -echo "Conda env: otagent (prefix $CONDA_PREFIX)" +# OTAGENT_CONDA_PREFIX (full path to the env) lets a borrowing operator point at a shared read-only +# env instead of building their own $SCRATCH/miniconda3. Defaults to the operator's own +# $SCRATCH/miniconda3/envs/otagent — byte-identical for penfever, and picks up an operator's own env +# automatically once built. Activate by full prefix path (not by bare name, which needs the env +# registered in the active conda root). The conda root is derived from the prefix. +OTAGENT_CONDA_PREFIX="${OTAGENT_CONDA_PREFIX:-${SCRATCH}/miniconda3/envs/otagent}" +_OTAGENT_CONDA_ROOT="${OTAGENT_CONDA_PREFIX%/envs/*}" +source "${_OTAGENT_CONDA_ROOT}/etc/profile.d/conda.sh" +conda activate "${OTAGENT_CONDA_PREFIX}" +echo "Conda env: ${OTAGENT_CONDA_PREFIX} (prefix $CONDA_PREFIX)" # Prepend conda's lib so its libstdc++ wins over the system/module versions. export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" From 38ee890f4e890e7227c422ba48242487221093e0 Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Mon, 3 Aug 2026 10:23:38 -0700 Subject: [PATCH 4/5] TACC eval sbatch: fall back to the listener's OTAGENT_DIR when $SCRATCH has no otagent env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Job 886379 died 9s in with EnvironmentLocationNotFound: Not a conda environment: /scratch/11694/mkumar73/miniconda3/envs/otagent The operator had a bare $SCRATCH/miniconda3 with no otagent env inside it. The listener had ALREADY passed a perfectly good shared env in OTAGENT_DIR (the vista eval_cluster_view pins conda_envs.otagent to the world-readable /scratch/10635/penfever/miniconda3/envs/otagent, precisely so a borrowing operator does not have to build their own), but the sbatch never read that var — it defaulted OTAGENT_CONDA_PREFIX on the variable being UNSET, so a present -but-empty miniconda root beat the env that actually existed. Resolve in order OTAGENT_CONDA_PREFIX > OTAGENT_DIR > $SCRATCH default, and select on the env EXISTING (/bin/python) rather than on the var being set. Unchanged for an operator who has built their own env. Same defect class as 2442d5e (evalchemy: validate the ENV, not just the miniforge3 root). Co-Authored-By: Claude Opus 5 --- eval/tacc/eval_harbor.sbatch | 43 +++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/eval/tacc/eval_harbor.sbatch b/eval/tacc/eval_harbor.sbatch index b51c9434f..03955262e 100644 --- a/eval/tacc/eval_harbor.sbatch +++ b/eval/tacc/eval_harbor.sbatch @@ -147,11 +147,44 @@ module load gcc/13.2.0 cuda/12.8 2>/dev/null || true # --- Conda activation: otagent (harbor + fork vLLM + DB client + uploader) --- # OTAGENT_CONDA_PREFIX (full path to the env) lets a borrowing operator point at a shared read-only -# env instead of building their own $SCRATCH/miniconda3. Defaults to the operator's own -# $SCRATCH/miniconda3/envs/otagent — byte-identical for penfever, and picks up an operator's own env -# automatically once built. Activate by full prefix path (not by bare name, which needs the env -# registered in the active conda root). The conda root is derived from the prefix. -OTAGENT_CONDA_PREFIX="${OTAGENT_CONDA_PREFIX:-${SCRATCH}/miniconda3/envs/otagent}" +# env instead of building their own $SCRATCH/miniconda3. Activate by full prefix path (not by bare +# name, which needs the env registered in the active conda root). The conda root is derived from +# the prefix. +# +# Resolution order, first EXISTING env wins: +# 1. OTAGENT_CONDA_PREFIX — explicit operator override +# 2. OTAGENT_DIR — sent by the listener from the cluster view's conda_envs +# (hpc.py vista eval_cluster_view pins the shared read-only env) +# 3. $SCRATCH/miniconda3/envs/otagent — the operator's own env, once built +# +# Falling back on EXISTENCE (not just on the var being unset) is the load-bearing part: a borrowing +# operator can have a bare $SCRATCH/miniconda3 with no otagent env in it, and the old +# unset-only default picked that non-existent path over the perfectly good shared env the listener +# had already passed in OTAGENT_DIR. That failed job 886379 nine seconds in with +# "EnvironmentLocationNotFound: Not a conda environment: $SCRATCH/miniconda3/envs/otagent". +# Same defect class as the evalchemy sbatch fix: validate the ENV, not just its root. +_pick_conda_prefix() { + for _c in "$@"; do + [ -n "$_c" ] && [ -x "${_c}/bin/python" ] && { printf '%s' "$_c"; return 0; } + done + # Nothing exists — echo the first non-empty candidate so the error names what we wanted. + for _c in "$@"; do [ -n "$_c" ] && { printf '%s' "$_c"; return 1; }; done + return 1 +} +# Snapshot the configured values first: the assignment below overwrites +# OTAGENT_CONDA_PREFIX even on failure, so the error message must not read it back. +_want_prefix="${OTAGENT_CONDA_PREFIX:-}" +_want_dir="${OTAGENT_DIR:-}" +OTAGENT_CONDA_PREFIX="$(_pick_conda_prefix \ + "${OTAGENT_CONDA_PREFIX:-}" \ + "${OTAGENT_DIR:-}" \ + "${SCRATCH}/miniconda3/envs/otagent")" || { + echo "ERROR: no usable otagent conda env found (no /bin/python). Tried, in order:" >&2 + echo " 1. OTAGENT_CONDA_PREFIX = ${_want_prefix}" >&2 + echo " 2. OTAGENT_DIR = ${_want_dir} (from the cluster view's conda_envs)" >&2 + echo " 3. \$SCRATCH default = ${SCRATCH}/miniconda3/envs/otagent" >&2 + exit 1 +} _OTAGENT_CONDA_ROOT="${OTAGENT_CONDA_PREFIX%/envs/*}" source "${_OTAGENT_CONDA_ROOT}/etc/profile.d/conda.sh" conda activate "${OTAGENT_CONDA_PREFIX}" From 45d1ed1767fc981b3587462704b47463095b64e7 Mon Sep 17 00:00:00 2001 From: Mrinal Kumar Date: Thu, 6 Aug 2026 09:23:09 -0700 Subject: [PATCH 5/5] TACC eval sbatch: make the trace-upload HF org configurable (EVAL_UPLOAD_HF_ORG) The TACC uploader hardcoded DCAgent2/. An operator without DCAgent2 write gets a 403 ('Make sure your token has the correct permissions'), the leg logs 'HF upload failed but DB records uploaded successfully', and hf_traces_link is never set -- so the leaderboard row lands with no traces. Leonardo has had EVAL_UPLOAD_HF_ORG since 06ef8f9; TACC was missed. Same default (DCAgent2), so this is byte-identical for anyone who already had write there. Co-Authored-By: Claude Opus 5 --- eval/tacc/eval_harbor.sbatch | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/eval/tacc/eval_harbor.sbatch b/eval/tacc/eval_harbor.sbatch index 03955262e..5964c64c8 100644 --- a/eval/tacc/eval_harbor.sbatch +++ b/eval/tacc/eval_harbor.sbatch @@ -871,7 +871,14 @@ def sanitize_hf_repo_id(repo_id: str, max_length: int = 96) -> str: run_dir = os.environ["RUN_DIR"]; run_tag = os.environ["RUN_TAG"] username = os.environ.get("UPLOAD_USERNAME", os.environ.get("USER", "tacc")) error_mode = os.environ.get("UPLOAD_MODE", "skip_on_error") -hf_repo_id = sanitize_hf_repo_id(f"DCAgent2/{run_tag}") +# Trace-upload HF org. Default DCAgent2 (byte-identical to before), overridable via +# EVAL_UPLOAD_HF_ORG so an operator without DCAgent2 write can land traces in an org they +# CAN write (e.g. laion). Without this the upload dies with a 403 "Make sure your token has +# the correct permissions", the leg logs "HF upload failed but DB records uploaded", and the +# hf_traces_link is never set — so the leaderboard row has no traces. Ported from +# eval/leonardo/eval_harbor.sbatch, which has had this since 06ef8f9; TACC was missed. +hf_upload_org = os.environ.get("EVAL_UPLOAD_HF_ORG", "DCAgent2") +hf_repo_id = sanitize_hf_repo_id(f"{hf_upload_org}/{run_tag}") hf_token = os.environ["HF_TOKEN"] dataset_hf = os.environ.get("REPO_ID", "") benchmark_name = os.environ.get("BENCHMARK_NAME", "") or (dataset_hf.split("/")[-1] if "/" in dataset_hf else dataset_hf)