From aa3b32902b544d229abb1053e2755499fb1cbbb8 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 17:00:32 -0700 Subject: [PATCH] start.sh: detect the fabric port on each Spark and say which one it uses The default was hardcoded to rocep1s0f0, so a pair cabled on the second port (enp1s0f1np1) failed with no hint. start.sh now lists each Spark's RoCE ports whose link is up and that have an IPv4 address, picks the pair on the same subnet (the cable between them; with several cables this skips the others), falls back to the first active port with a warning, and prints its choice and how to override it. FABRIC_HCA in .env still overrides both ranks. Fixes #44. Co-Authored-By: Claude Opus 5.5 --- .env.example | 5 +++- docs/INSTALL-MANUAL.md | 4 ++- start.sh | 57 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 3b56c52..f7c34fa 100644 --- a/.env.example +++ b/.env.example @@ -55,7 +55,10 @@ WORKER=you@192.0.2.2 # Only for private Hugging Face mirrors; the pinned checkpoints are public. #HF_TOKEN= -# Detected from the RoCE device; set only if detection picks the wrong port. +# The fabric port is detected on each Spark: start.sh lists the RoCE ports whose +# link is up and that have an IPv4 address, picks the pair on the same subnet (the +# cable between the two Sparks), and prints its choice. Set these only if it picks +# the wrong port, e.g. rocep1s0f1 / enp1s0f1np1 for a cable on the second port. #FABRIC_HCA=rocep1s0f0 #FABRIC_INTERFACE=enp1s0f0np0 # The fabric also uses the RDMA device behind the ConnectX-7's other PCIe domain diff --git a/docs/INSTALL-MANUAL.md b/docs/INSTALL-MANUAL.md index 8169507..369cbb0 100644 --- a/docs/INSTALL-MANUAL.md +++ b/docs/INSTALL-MANUAL.md @@ -74,7 +74,9 @@ matches that converter; otherwise it moves any older overlay aside. Start rank **1** (the worker) first, then rank **0** (the leader). The `--leader-address` is rank 0's IPv4 address on the direct cable. Each rank finds its own fabric interface from the RoCE device (`--fabric-hca`, default -`rocep1s0f0`). Pass `--fabric-interface` to override it. +`rocep1s0f0`, the first port). If the cable is on the second port, pass +`--fabric-hca rocep1s0f1` on that Spark (`./start.sh` detects this itself). +Pass `--fabric-interface` to override the interface. ```sh # on rank 1: diff --git a/start.sh b/start.sh index 743fe71..3843a77 100755 --- a/start.sh +++ b/start.sh @@ -14,7 +14,6 @@ if [[ -f .env ]]; then set -a; . ./.env; set +a; fi : "${WORKER:?set WORKER in .env to the ssh destination of the other Spark (see .env.example)}" MODEL_ROOT=${MODEL_ROOT:-$HOME/models/sparkglm} PROFILE=${PROFILE:-4x1m} -FABRIC_HCA=${FABRIC_HCA:-rocep1s0f0} IMAGE=${IMAGE:-$(install/build.sh --tag)} CHECKPOINT=nvidia/GLM-5.3-Flash-NVFP4@423acf37583782c51c142d145aef733d72943d93 DRAFTER=incoai/GLM-5.3-Flash-DFlash2@7d74cdd881ed7e32c31175984a67823127b66cfe @@ -27,6 +26,47 @@ on_worker() { ssh -o BatchMode=yes "$WORKER" "$@"; } script_on_worker() { local script=$1; shift; on_worker "bash -s -- $(printf '%q ' "$@")" < "$script"; } has_image() { docker image inspect "$IMAGE" >/dev/null 2>&1; } +# Each Spark's RoCE ports whose link is up and that have an IPv4 address, one +# "
" line each. Only PCIe domain 0 (rocep*): +# the engine adds the device behind the other domain of the same port itself. +fabric_ports() { + local dev net cidr + for dev in /sys/class/infiniband/rocep*; do + grep -qs ACTIVE "$dev/ports/1/state" || continue + for net in "$dev"/device/net/*; do + cidr=$(ip -4 -o addr show dev "${net##*/}" 2>/dev/null | awk '{print $4; exit}') + [[ -n $cidr ]] && echo "${dev##*/} ${net##*/} $cidr" + done + done +} + +# Pick the fabric port on each Spark: the pair of active ports on the same IPv4 +# subnet is the cable between them. Sets hca0/iface0/ip0 (here) and hca1/ip1 +# (the worker). FABRIC_HCA in .env overrides it for both. +pick_fabric() { + local here there choice + here=$(fabric_ports) + there=$(on_worker "bash -s" <<< "$(declare -f fabric_ports); fabric_ports") + [[ -n $here ]] || { echo "no RoCE port with its link up and an IPv4 address on this Spark; check the cable and the port's address, or set FABRIC_HCA in .env" >&2; exit 2; } + [[ -n $there ]] || { echo "no RoCE port with its link up and an IPv4 address on $WORKER; check the cable and the port's address, or set FABRIC_HCA in .env" >&2; exit 2; } + choice=$(python3 - "$here" "$there" <<'PY' +import ipaddress, sys +parse = lambda text: [line.split() for line in text.splitlines() if len(line.split()) == 3] +here, there = parse(sys.argv[1]), parse(sys.argv[2]) +net = lambda cidr: ipaddress.ip_interface(cidr).network +match = [(a, b) for a in here for b in there if net(a[2]) == net(b[2])] +a, b = match[0] if match else (here[0], there[0]) +print(a[0], a[1], a[2].split('/')[0], b[0], b[1], b[2].split('/')[0], 'subnet' if match else 'first') +PY +) + read -r hca0 iface0 ip0 hca1 iface1 ip1 how <<< "$choice" + if [[ $how == subnet ]]; then + say "fabric: $hca0 ($iface0, $ip0) here, $hca1 ($iface1, $ip1) on $WORKER; set FABRIC_HCA in .env to choose another port" + else + say "fabric: no active port pair shares a subnet; using $hca0 ($iface0, $ip0) here and $hca1 ($iface1, $ip1) on $WORKER. If they are not cabled to each other, set FABRIC_HCA in .env" + fi +} + build() { say "image $IMAGE" if ! has_image; then @@ -89,13 +129,18 @@ state() { docker inspect -f '{{.State.Status}}' "${NAME}0" 2>/dev/null || echo a worker_state() { on_worker docker inspect -f '{{.State.Status}}' "${NAME}1" 2>/dev/null || echo absent; } serve() { - local iface address common - iface=${FABRIC_INTERFACE:-$(ls "/sys/class/infiniband/$FABRIC_HCA/device/net" 2>/dev/null | head -1)} + local iface address common hca0 iface0 ip0 hca1 iface1 ip1 how + if [[ -n ${FABRIC_HCA:-} ]]; then + hca0=$FABRIC_HCA hca1=$FABRIC_HCA + else + pick_fabric + fi + iface=${FABRIC_INTERFACE:-$(ls "/sys/class/infiniband/$hca0/device/net" 2>/dev/null | head -1)} address=${LEADER_ADDRESS:-} [[ -n $address ]] || address=$(ip -4 -o addr show dev "$iface" 2>/dev/null | awk '{sub("/.*", "", $4); print $4; exit}') [[ -n $address ]] || { echo "no IPv4 address on the fabric interface '$iface'; set LEADER_ADDRESS" >&2; exit 2; } common=(--leader-address "$address" --model-root "$MODEL_ROOT" --image "$IMAGE" - --profile "$PROFILE" --fabric-hca "$FABRIC_HCA" ${FABRIC_INTERFACE:+--fabric-interface "$FABRIC_INTERFACE"} + --profile "$PROFILE" ${FABRIC_INTERFACE:+--fabric-interface "$FABRIC_INTERFACE"} ${GPU_MEMORY_UTILIZATION:+--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"} ${PREFIX_CACHE_DIR:+--prefix-cache-dir "$PREFIX_CACHE_DIR"} ${PREFIX_CACHE_GB:+--prefix-cache-gb "$PREFIX_CACHE_GB"}) case ${DISPLAY_CARVEOUT:-0} in @@ -115,8 +160,8 @@ serve() { esac say "start rank 1 on $WORKER, then rank 0 here (leader $address, profile $PROFILE)" stop - script_on_worker install/start-node.sh --rank 1 "${common[@]}" - install/start-node.sh --rank 0 "${common[@]}" + script_on_worker install/start-node.sh --rank 1 --fabric-hca "$hca1" "${common[@]}" + install/start-node.sh --rank 0 --fabric-hca "$hca0" "${common[@]}" say "loading (about 2.5 minutes)" for ((i = 0; i < 90; i++)); do if curl -sf "$API/health" >/dev/null; then