TACC/Vista eval: multi-operator support

eval/clusters/tacc.yaml

@@ -15,17 +15,20 @@ conda_envs:
   otagent: /scratch/10635/penfever/miniconda3/envs/otagent
 
 paths:
-  project_root: /scratch/10635/penfever/OpenThoughts-Agent
-  hf_cache: /scratch/10635/penfever/hub
-  eval_jobs_dir: /scratch/10635/penfever/eval_jobs
+  # $SCRATCH-parameterized (load_cluster_config expands $SCRATCH at load time); each operator's
+  # $SCRATCH encodes their /scratch/<allocnum>/<user> root. Byte-identical for penfever
+  # ($SCRATCH=/scratch/10635/penfever). conda_envs/cuda_home stay penfever's = shared read-only borrow.
+  project_root: $SCRATCH/OpenThoughts-Agent
+  hf_cache: $SCRATCH/hub
+  eval_jobs_dir: $SCRATCH/eval_jobs
   eval_logs_dir: eval/tacc/logs
   listener_logs_dir: experiments/listener_logs
   sbatch_script: eval/tacc/eval_harbor.sbatch
   dp_sbatch_script: eval/tacc/eval_harbor.sbatch
-  harbor_src: /scratch/10635/penfever/harbor/src
+  harbor_src: $SCRATCH/harbor/src
   datasets_dirs:
-    - /scratch/10635/penfever/hub
-  secrets_file: /scratch/10635/penfever/keys.env
+    - $SCRATCH/hub
+  secrets_file: $SCRATCH/keys.env
 
 proxy:
   enabled: false

eval/tacc/eval_harbor.sbatch

@@ -146,9 +146,49 @@ module purge 2>/dev/null || true
 module load gcc/13.2.0 cuda/12.8 2>/dev/null || true
 
 # --- Conda activation: otagent (harbor + fork vLLM + DB client + uploader) ---
-source "${SCRATCH}/miniconda3/etc/profile.d/conda.sh"
-conda activate otagent
-echo "Conda env: otagent (prefix $CONDA_PREFIX)"
+# OTAGENT_CONDA_PREFIX (full path to the env) lets a borrowing operator point at a shared read-only
+# env instead of building their own $SCRATCH/miniconda3. Activate by full prefix path (not by bare
+# name, which needs the env registered in the active conda root). The conda root is derived from
+# the prefix.
+#
+# Resolution order, first EXISTING env wins:
+#   1. OTAGENT_CONDA_PREFIX  — explicit operator override
+#   2. OTAGENT_DIR           — sent by the listener from the cluster view's conda_envs
+#                              (hpc.py vista eval_cluster_view pins the shared read-only env)
+#   3. $SCRATCH/miniconda3/envs/otagent — the operator's own env, once built
+#
+# Falling back on EXISTENCE (not just on the var being unset) is the load-bearing part: a borrowing
+# operator can have a bare $SCRATCH/miniconda3 with no otagent env in it, and the old
+# unset-only default picked that non-existent path over the perfectly good shared env the listener
+# had already passed in OTAGENT_DIR. That failed job 886379 nine seconds in with
+# "EnvironmentLocationNotFound: Not a conda environment: $SCRATCH/miniconda3/envs/otagent".
+# Same defect class as the evalchemy sbatch fix: validate the ENV, not just its root.
+_pick_conda_prefix() {
+    for _c in "$@"; do
+        [ -n "$_c" ] && [ -x "${_c}/bin/python" ] && { printf '%s' "$_c"; return 0; }
+    done
+    # Nothing exists — echo the first non-empty candidate so the error names what we wanted.
+    for _c in "$@"; do [ -n "$_c" ] && { printf '%s' "$_c"; return 1; }; done
+    return 1
+}
+# Snapshot the configured values first: the assignment below overwrites
+# OTAGENT_CONDA_PREFIX even on failure, so the error message must not read it back.
+_want_prefix="${OTAGENT_CONDA_PREFIX:-<unset>}"
+_want_dir="${OTAGENT_DIR:-<unset>}"
+OTAGENT_CONDA_PREFIX="$(_pick_conda_prefix \
+    "${OTAGENT_CONDA_PREFIX:-}" \
+    "${OTAGENT_DIR:-}" \
+    "${SCRATCH}/miniconda3/envs/otagent")" || {
+    echo "ERROR: no usable otagent conda env found (no <prefix>/bin/python). Tried, in order:" >&2
+    echo "  1. OTAGENT_CONDA_PREFIX = ${_want_prefix}" >&2
+    echo "  2. OTAGENT_DIR          = ${_want_dir}  (from the cluster view's conda_envs)" >&2
+    echo "  3. \$SCRATCH default     = ${SCRATCH}/miniconda3/envs/otagent" >&2
+    exit 1
+}
+_OTAGENT_CONDA_ROOT="${OTAGENT_CONDA_PREFIX%/envs/*}"
+source "${_OTAGENT_CONDA_ROOT}/etc/profile.d/conda.sh"
+conda activate "${OTAGENT_CONDA_PREFIX}"
+echo "Conda env: ${OTAGENT_CONDA_PREFIX} (prefix $CONDA_PREFIX)"
 
 # Prepend conda's lib so its libstdc++ wins over the system/module versions.
 export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
@@ -790,6 +830,18 @@ export UPLOAD_USERNAME="${UPLOAD_USERNAME:-$USER}"
 export UPLOAD_MODE="${UPLOAD_MODE:-skip_on_error}"
 export RUN_TAG
 
+# Local-only smoke/canary guard (mirrors the leonardo sbatch): EVAL_SKIP_UPLOAD=1 preserves the
+# results on disk and skips the HF/DB uploader entirely, so a subset/canary is NEVER registered as a
+# full-benchmark leaderboard result. upload_eval_results() does HF upload AND Supabase registration in
+# one call, so this single exit covers both. Also side-steps the bare os.environ["HF_TOKEN"] below when
+# no token is configured (canary path).
+case "${EVAL_SKIP_UPLOAD:-0}" in
+    1|true|TRUE|yes|YES)
+        echo "EVAL_SKIP_UPLOAD=${EVAL_SKIP_UPLOAD}: preserving local results at $RUN_DIR, skipping HF/DB upload"
+        exit 0
+        ;;
+esac
+
 UPLOAD_LOG="$RUN_DIR/upload.log"
 echo "Uploading results from: $RUN_DIR" | tee -a "$UPLOAD_LOG"
 
@@ -819,7 +871,14 @@ def sanitize_hf_repo_id(repo_id: str, max_length: int = 96) -> str:
 run_dir = os.environ["RUN_DIR"]; run_tag = os.environ["RUN_TAG"]
 username = os.environ.get("UPLOAD_USERNAME", os.environ.get("USER", "tacc"))
 error_mode = os.environ.get("UPLOAD_MODE", "skip_on_error")
-hf_repo_id = sanitize_hf_repo_id(f"DCAgent2/{run_tag}")
+# Trace-upload HF org. Default DCAgent2 (byte-identical to before), overridable via
+# EVAL_UPLOAD_HF_ORG so an operator without DCAgent2 write can land traces in an org they
+# CAN write (e.g. laion). Without this the upload dies with a 403 "Make sure your token has
+# the correct permissions", the leg logs "HF upload failed but DB records uploaded", and the
+# hf_traces_link is never set — so the leaderboard row has no traces. Ported from
+# eval/leonardo/eval_harbor.sbatch, which has had this since 06ef8f9; TACC was missed.
+hf_upload_org = os.environ.get("EVAL_UPLOAD_HF_ORG", "DCAgent2")
+hf_repo_id = sanitize_hf_repo_id(f"{hf_upload_org}/{run_tag}")
 hf_token = os.environ["HF_TOKEN"]
 dataset_hf = os.environ.get("REPO_ID", "")
 benchmark_name = os.environ.get("BENCHMARK_NAME", "") or (dataset_hf.split("/")[-1] if "/" in dataset_hf else dataset_hf)

hpc/hpc.py

@@ -1470,10 +1470,16 @@ def get_pre_run_commands(self) -> str:
     # name "tacc"; the view's cluster_name is "tacc" (mirroring tacc.yaml) and the
     # listener's _resolve_cluster_view_by_name() matches EITHER c.name ("vista") OR the
     # view's cluster_name ("tacc"), so `--cluster-config tacc` AND `--cluster-config vista`
-    # both resolve. User-scoped paths mirror that yaml (penfever's). Vista-specific vs
-    # leonardo: hardware_profile gh200, 1 GPU/node, NO mem_per_node_mb (RealMemory is
-    # misreported as 1 MB → request whole nodes), and hardware.gpu_gres false (GPUs are
-    # not a SLURM gres here). Compute nodes have full egress so proxy is disabled.
+    # both resolve. User-scoped write/code paths are $SCRATCH-parameterized (to_eval_cluster_view
+    # expandvars-expands them at call time on the login node, where $SCRATCH is always set); each
+    # operator's $SCRATCH already encodes their /scratch/<allocnum>/<user> root, so $USER alone is
+    # insufficient here (the alloc-number differs per user). For penfever $SCRATCH=/scratch/10635/
+    # penfever, so this renders byte-identical to the prior hardcoded values. The conda env /
+    # cuda_home stay pinned to penfever's prefix ON PURPOSE — it is world-readable and shared
+    # read-only, so a new operator borrows it without rebuilding (point at your own once built).
+    # Vista-specific vs leonardo: hardware_profile gh200, 1 GPU/node, NO mem_per_node_mb (RealMemory
+    # is misreported as 1 MB → request whole nodes), and hardware.gpu_gres false (GPUs are not a
+    # SLURM gres here). Compute nodes have full egress so proxy is disabled.
     eval_cluster_view={
         "cluster_name": "tacc",
         "baseline_model_configs": "eval/clusters/tacc_baseline_model_configs.yaml",
@@ -1487,16 +1493,16 @@ def get_pre_run_commands(self) -> str:
             "otagent": "/scratch/10635/penfever/miniconda3/envs/otagent",
         },
         "paths": {
-            "project_root": "/scratch/10635/penfever/OpenThoughts-Agent",
-            "hf_cache": "/scratch/10635/penfever/hub",
-            "eval_jobs_dir": "/scratch/10635/penfever/eval_jobs",
+            "project_root": "$SCRATCH/OpenThoughts-Agent",
+            "hf_cache": "$SCRATCH/hub",
+            "eval_jobs_dir": "$SCRATCH/eval_jobs",
             "eval_logs_dir": "eval/tacc/logs",
             "listener_logs_dir": "experiments/listener_logs",
             "sbatch_script": "eval/tacc/eval_harbor.sbatch",
             "dp_sbatch_script": "eval/tacc/eval_harbor.sbatch",
-            "harbor_src": "/scratch/10635/penfever/harbor/src",
-            "datasets_dirs": ["/scratch/10635/penfever/hub"],
-            "secrets_file": "/scratch/10635/penfever/keys.env",
+            "harbor_src": "$SCRATCH/harbor/src",
+            "datasets_dirs": ["$SCRATCH/hub"],
+            "secrets_file": "$SCRATCH/keys.env",
         },
         "proxy": {"enabled": False},
         "hardware": {