From 7decf3e284dfaa636019bddad71b89722dc1ad87 Mon Sep 17 00:00:00 2001 From: "ilkosare@amd.com" Date: Sat, 4 Jul 2026 19:07:51 +0000 Subject: [PATCH 1/2] Fix SGLang disagg on ionic/RoCE fabrics: preserve AINIC transport env MOONCAKE_COOKBOOK's set_env_vars.sh assumes a Mellanox/mlx5 fabric and, on ionic-based RoCE clusters (mia1), disables RDMA (NCCL_IB_DISABLE=1), hardcodes mlx5 device names, and mangles NCCL_SOCKET_IFNAME. Save the launcher-provided transport iface before sourcing, then re-assert NCCL_IB_DISABLE=0, IBDEVICES from IB_DEVICES, and NCCL/GLOO socket ifnames afterward. Also derive host_ip from the transport interface instead of the first hostname -I address. Validated on mia1 (DeepSeek-R1-0528, 1P+1D disagg, TP=8, RCCL 78e8ba0 overlay + AINIC ionic RDMA), SLURM job 11211: both servers fired up, router bound on 2322, full benchmark sweep 100% request success. Co-Authored-By: Claude Opus 4 --- scripts/sglang_disagg/sglang_disagg_server.sh | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/sglang_disagg/sglang_disagg_server.sh b/scripts/sglang_disagg/sglang_disagg_server.sh index 3db53d7a..2507d0d9 100755 --- a/scripts/sglang_disagg/sglang_disagg_server.sh +++ b/scripts/sglang_disagg/sglang_disagg_server.sh @@ -33,11 +33,33 @@ esac pip install py-spy pip install --ignore-installed --force-reinstall flask +# AINIC/ionic RoCE fix: MOONCAKE_COOKBOOK's set_env_vars.sh assumes a Mellanox/mlx5 +# fabric. On ionic-based RoCE clusters (e.g. mia1) it (1) hardcodes mlx5 IB device +# names, (2) forces NCCL_IB_DISABLE=1, and (3) mangles NCCL_SOCKET_IFNAME via an +# `ip route` awk parse. Save the transport iface set by the launcher, then re-assert +# the correct ionic values after sourcing so RDMA over ionic actually engages. +_SAVED_NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-}" + source $MOONCAKE_COOKBOOK_PATH/set_env_vars.sh #trap 'echo "Error occurred. Cleaning up..."; exit 0' ERR -host_ip=$(hostname -I | awk '{print $1}') +# Re-assert RDMA/socket env clobbered by set_env_vars.sh (see note above). +export NCCL_IB_DISABLE=0 +[[ -n "${IB_DEVICES:-}" ]] && export IBDEVICES="${IB_DEVICES}" +export NCCL_SOCKET_IFNAME="${_SAVED_NCCL_SOCKET_IFNAME:-eno0}" +export GLOO_SOCKET_IFNAME="${_SAVED_NCCL_SOCKET_IFNAME%%,*}" + +# Derive host_ip from the actual transport interface rather than the first +# `hostname -I` address (which on multi-homed nodes may be the wrong NIC). +_XPORT_IFACE="${_SAVED_NCCL_SOCKET_IFNAME%%,*}" +host_ip="" +if [[ -n "$_XPORT_IFACE" ]]; then + host_ip=$(ip -4 -o addr show "$_XPORT_IFACE" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -n1) +fi +[[ -z "$host_ip" ]] && host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src"){print $(i+1); exit}}') +[[ -z "$host_ip" ]] && host_ip=$(hostname -I | awk '{print $1}') host_name=$(hostname) +unset _SAVED_NCCL_SOCKET_IFNAME _XPORT_IFACE # ============================================================================= # Model-Specific Configuration Maps From 581a61a0a1599285912f774abe4484089c7d4069 Mon Sep 17 00:00:00 2001 From: "ilkosare@amd.com" Date: Tue, 7 Jul 2026 13:47:20 +0000 Subject: [PATCH 2/2] sglang_disagg_server: source host_ip from IPADDRS[NODE_RANK] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-deriving the node IP via ip addr/hostname -I risks picking a different NIC on multi-homed nodes, causing a mismatch between what peers registered in the router/barrier (built from IPADDRS) and what --host / socket_barrier --local-ip binds to. Source host_ip from IPADDRS[NODE_RANK] instead — the rank-ordered, post-rendezvous list that run.sh already resolved and passed in. Keep a hostname -I fallback for environments where IPADDRS is unset. Remove the now-redundant IFS/read of IP_ARRAY in the Cluster Topology section; the array is already populated at host_ip derivation time. Suggested-by: mkuznet1 Co-Authored-By: Claude Sonnet 4 --- scripts/sglang_disagg/sglang_disagg_server.sh | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/scripts/sglang_disagg/sglang_disagg_server.sh b/scripts/sglang_disagg/sglang_disagg_server.sh index 2507d0d9..51d859c9 100755 --- a/scripts/sglang_disagg/sglang_disagg_server.sh +++ b/scripts/sglang_disagg/sglang_disagg_server.sh @@ -49,17 +49,15 @@ export NCCL_IB_DISABLE=0 export NCCL_SOCKET_IFNAME="${_SAVED_NCCL_SOCKET_IFNAME:-eno0}" export GLOO_SOCKET_IFNAME="${_SAVED_NCCL_SOCKET_IFNAME%%,*}" -# Derive host_ip from the actual transport interface rather than the first -# `hostname -I` address (which on multi-homed nodes may be the wrong NIC). -_XPORT_IFACE="${_SAVED_NCCL_SOCKET_IFNAME%%,*}" -host_ip="" -if [[ -n "$_XPORT_IFACE" ]]; then - host_ip=$(ip -4 -o addr show "$_XPORT_IFACE" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -n1) -fi -[[ -z "$host_ip" ]] && host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src"){print $(i+1); exit}}') -[[ -z "$host_ip" ]] && host_ip=$(hostname -I | awk '{print $1}') +# This node's IP is already resolved by run.sh and handed to us (rank-ordered, +# post-rendezvous) via IPADDRS. Reuse IPADDRS[NODE_RANK] so the address we bind +# and advertise (--host, socket_barrier --local-ip) matches exactly what peers +# registered for us in the router/barrier; re-deriving it here can pick a +# different NIC on multi-homed nodes and desync the two. +IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" +host_ip="${IP_ARRAY[NODE_RANK]:-$(hostname -I | awk '{print $1}')}" host_name=$(hostname) -unset _SAVED_NCCL_SOCKET_IFNAME _XPORT_IFACE +unset _SAVED_NCCL_SOCKET_IFNAME # ============================================================================= # Model-Specific Configuration Maps @@ -134,7 +132,7 @@ python $MOONCAKE_COOKBOOK_PATH/socket_barrier.py \ # Cluster Topology Configuration # ============================================================================= -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" +# IP_ARRAY was already parsed from IPADDRS above (where host_ip is derived). PREFILL_ARGS="" DECODE_ARGS=""