diff --git a/.env.example b/.env.example index 3b56c52..f7c34fa 100644 --- a/.env.example +++ b/.env.example @@ -55,7 +55,10 @@ WORKER=you@192.0.2.2 # Only for private Hugging Face mirrors; the pinned checkpoints are public. #HF_TOKEN= -# Detected from the RoCE device; set only if detection picks the wrong port. +# The fabric port is detected on each Spark: start.sh lists the RoCE ports whose +# link is up and that have an IPv4 address, picks the pair on the same subnet (the +# cable between the two Sparks), and prints its choice. Set these only if it picks +# the wrong port, e.g. rocep1s0f1 / enp1s0f1np1 for a cable on the second port. #FABRIC_HCA=rocep1s0f0 #FABRIC_INTERFACE=enp1s0f0np0 # The fabric also uses the RDMA device behind the ConnectX-7's other PCIe domain diff --git a/docs/INSTALL-MANUAL.md b/docs/INSTALL-MANUAL.md index 8169507..369cbb0 100644 --- a/docs/INSTALL-MANUAL.md +++ b/docs/INSTALL-MANUAL.md @@ -74,7 +74,9 @@ matches that converter; otherwise it moves any older overlay aside. Start rank **1** (the worker) first, then rank **0** (the leader). The `--leader-address` is rank 0's IPv4 address on the direct cable. Each rank finds its own fabric interface from the RoCE device (`--fabric-hca`, default -`rocep1s0f0`). Pass `--fabric-interface` to override it. +`rocep1s0f0`, the first port). If the cable is on the second port, pass +`--fabric-hca rocep1s0f1` on that Spark (`./start.sh` detects this itself). +Pass `--fabric-interface` to override the interface. ```sh # on rank 1: diff --git a/start.sh b/start.sh index 743fe71..3843a77 100755 --- a/start.sh +++ b/start.sh @@ -14,7 +14,6 @@ if [[ -f .env ]]; then set -a; . ./.env; set +a; fi : "${WORKER:?set WORKER in .env to the ssh destination of the other Spark (see .env.example)}" MODEL_ROOT=${MODEL_ROOT:-$HOME/models/sparkglm} PROFILE=${PROFILE:-4x1m} -FABRIC_HCA=${FABRIC_HCA:-rocep1s0f0} IMAGE=${IMAGE:-$(install/build.sh --tag)} CHECKPOINT=nvidia/GLM-5.3-Flash-NVFP4@423acf37583782c51c142d145aef733d72943d93 DRAFTER=incoai/GLM-5.3-Flash-DFlash2@7d74cdd881ed7e32c31175984a67823127b66cfe @@ -27,6 +26,47 @@ on_worker() { ssh -o BatchMode=yes "$WORKER" "$@"; } script_on_worker() { local script=$1; shift; on_worker "bash -s -- $(printf '%q ' "$@")" < "$script"; } has_image() { docker image inspect "$IMAGE" >/dev/null 2>&1; } +# Each Spark's RoCE ports whose link is up and that have an IPv4 address, one +# "
" line each. Only PCIe domain 0 (rocep*): +# the engine adds the device behind the other domain of the same port itself. +fabric_ports() { + local dev net cidr + for dev in /sys/class/infiniband/rocep*; do + grep -qs ACTIVE "$dev/ports/1/state" || continue + for net in "$dev"/device/net/*; do + cidr=$(ip -4 -o addr show dev "${net##*/}" 2>/dev/null | awk '{print $4; exit}') + [[ -n $cidr ]] && echo "${dev##*/} ${net##*/} $cidr" + done + done +} + +# Pick the fabric port on each Spark: the pair of active ports on the same IPv4 +# subnet is the cable between them. Sets hca0/iface0/ip0 (here) and hca1/ip1 +# (the worker). FABRIC_HCA in .env overrides it for both. +pick_fabric() { + local here there choice + here=$(fabric_ports) + there=$(on_worker "bash -s" <<< "$(declare -f fabric_ports); fabric_ports") + [[ -n $here ]] || { echo "no RoCE port with its link up and an IPv4 address on this Spark; check the cable and the port's address, or set FABRIC_HCA in .env" >&2; exit 2; } + [[ -n $there ]] || { echo "no RoCE port with its link up and an IPv4 address on $WORKER; check the cable and the port's address, or set FABRIC_HCA in .env" >&2; exit 2; } + choice=$(python3 - "$here" "$there" <<'PY' +import ipaddress, sys +parse = lambda text: [line.split() for line in text.splitlines() if len(line.split()) == 3] +here, there = parse(sys.argv[1]), parse(sys.argv[2]) +net = lambda cidr: ipaddress.ip_interface(cidr).network +match = [(a, b) for a in here for b in there if net(a[2]) == net(b[2])] +a, b = match[0] if match else (here[0], there[0]) +print(a[0], a[1], a[2].split('/')[0], b[0], b[1], b[2].split('/')[0], 'subnet' if match else 'first') +PY +) + read -r hca0 iface0 ip0 hca1 iface1 ip1 how <<< "$choice" + if [[ $how == subnet ]]; then + say "fabric: $hca0 ($iface0, $ip0) here, $hca1 ($iface1, $ip1) on $WORKER; set FABRIC_HCA in .env to choose another port" + else + say "fabric: no active port pair shares a subnet; using $hca0 ($iface0, $ip0) here and $hca1 ($iface1, $ip1) on $WORKER. If they are not cabled to each other, set FABRIC_HCA in .env" + fi +} + build() { say "image $IMAGE" if ! has_image; then @@ -89,13 +129,18 @@ state() { docker inspect -f '{{.State.Status}}' "${NAME}0" 2>/dev/null || echo a worker_state() { on_worker docker inspect -f '{{.State.Status}}' "${NAME}1" 2>/dev/null || echo absent; } serve() { - local iface address common - iface=${FABRIC_INTERFACE:-$(ls "/sys/class/infiniband/$FABRIC_HCA/device/net" 2>/dev/null | head -1)} + local iface address common hca0 iface0 ip0 hca1 iface1 ip1 how + if [[ -n ${FABRIC_HCA:-} ]]; then + hca0=$FABRIC_HCA hca1=$FABRIC_HCA + else + pick_fabric + fi + iface=${FABRIC_INTERFACE:-$(ls "/sys/class/infiniband/$hca0/device/net" 2>/dev/null | head -1)} address=${LEADER_ADDRESS:-} [[ -n $address ]] || address=$(ip -4 -o addr show dev "$iface" 2>/dev/null | awk '{sub("/.*", "", $4); print $4; exit}') [[ -n $address ]] || { echo "no IPv4 address on the fabric interface '$iface'; set LEADER_ADDRESS" >&2; exit 2; } common=(--leader-address "$address" --model-root "$MODEL_ROOT" --image "$IMAGE" - --profile "$PROFILE" --fabric-hca "$FABRIC_HCA" ${FABRIC_INTERFACE:+--fabric-interface "$FABRIC_INTERFACE"} + --profile "$PROFILE" ${FABRIC_INTERFACE:+--fabric-interface "$FABRIC_INTERFACE"} ${GPU_MEMORY_UTILIZATION:+--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"} ${PREFIX_CACHE_DIR:+--prefix-cache-dir "$PREFIX_CACHE_DIR"} ${PREFIX_CACHE_GB:+--prefix-cache-gb "$PREFIX_CACHE_GB"}) case ${DISPLAY_CARVEOUT:-0} in @@ -115,8 +160,8 @@ serve() { esac say "start rank 1 on $WORKER, then rank 0 here (leader $address, profile $PROFILE)" stop - script_on_worker install/start-node.sh --rank 1 "${common[@]}" - install/start-node.sh --rank 0 "${common[@]}" + script_on_worker install/start-node.sh --rank 1 --fabric-hca "$hca1" "${common[@]}" + install/start-node.sh --rank 0 --fabric-hca "$hca0" "${common[@]}" say "loading (about 2.5 minutes)" for ((i = 0; i < 90; i++)); do if curl -sf "$API/health" >/dev/null; then