Skip to content

start.sh: detect the fabric port on each Spark (fixes #44) #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
data-angel merged 1 commit into from
Oct 12, 2026
Merged

start.sh: detect the fabric port on each Spark (fixes #44) #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .env.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,10 @@ WORKER=you@192.0.2.2
# Only for private Hugging Face mirrors; the pinned checkpoints are public.
#HF_TOKEN=

# Detected from the RoCE device; set only if detection picks the wrong port.
# The fabric port is detected on each Spark: start.sh lists the RoCE ports whose
# link is up and that have an IPv4 address, picks the pair on the same subnet (the
# cable between the two Sparks), and prints its choice. Set these only if it picks
# the wrong port, e.g. rocep1s0f1 / enp1s0f1np1 for a cable on the second port.
#FABRIC_HCA=rocep1s0f0
#FABRIC_INTERFACE=enp1s0f0np0
# The fabric also uses the RDMA device behind the ConnectX-7's other PCIe domain
Expand Down
4 changes: 3 additions & 1 deletion docs/INSTALL-MANUAL.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,9 @@ matches that converter; otherwise it moves any older overlay aside.
Start rank **1** (the worker) first, then rank **0** (the leader). The
`--leader-address` is rank 0's IPv4 address on the direct cable. Each rank
finds its own fabric interface from the RoCE device (`--fabric-hca`, default
`rocep1s0f0`). Pass `--fabric-interface` to override it.
`rocep1s0f0`, the first port). If the cable is on the second port, pass
`--fabric-hca rocep1s0f1` on that Spark (`./start.sh` detects this itself).
Pass `--fabric-interface` to override the interface.

```sh
# on rank 1:
Expand Down
57 changes: 51 additions & 6 deletions start.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ if [[ -f .env ]]; then set -a; . ./.env; set +a; fi
: "${WORKER:?set WORKER in .env to the ssh destination of the other Spark (see .env.example)}"
MODEL_ROOT=${MODEL_ROOT:-$HOME/models/sparkglm}
PROFILE=${PROFILE:-4x1m}
FABRIC_HCA=${FABRIC_HCA:-rocep1s0f0}
IMAGE=${IMAGE:-$(install/build.sh --tag)}
CHECKPOINT=nvidia/GLM-5.3-Flash-NVFP4@423acf37583782c51c142d145aef733d72943d93
DRAFTER=incoai/GLM-5.3-Flash-DFlash2@7d74cdd881ed7e32c31175984a67823127b66cfe
Expand All @@ -27,6 +26,47 @@ on_worker() { ssh -o BatchMode=yes "$WORKER" "$@"; }
script_on_worker() { local script=$1; shift; on_worker "bash -s -- $(printf '%q ' "$@")" < "$script"; }
has_image() { docker image inspect "$IMAGE" >/dev/null 2>&1; }

# Each Spark's RoCE ports whose link is up and that have an IPv4 address, one
# "<hca> <interface> <address/prefix>" line each. Only PCIe domain 0 (rocep*):
# the engine adds the device behind the other domain of the same port itself.
fabric_ports() {
local dev net cidr
for dev in /sys/class/infiniband/rocep*; do
grep -qs ACTIVE "$dev/ports/1/state" || continue
for net in "$dev"/device/net/*; do
cidr=$(ip -4 -o addr show dev "${net##*/}" 2>/dev/null | awk '{print $4; exit}')
[[ -n $cidr ]] && echo "${dev##*/} ${net##*/} $cidr"
done
done
}

# Pick the fabric port on each Spark: the pair of active ports on the same IPv4
# subnet is the cable between them. Sets hca0/iface0/ip0 (here) and hca1/ip1
# (the worker). FABRIC_HCA in .env overrides it for both.
pick_fabric() {
local here there choice
here=$(fabric_ports)
there=$(on_worker "bash -s" <<< "$(declare -f fabric_ports); fabric_ports")
[[ -n $here ]] || { echo "no RoCE port with its link up and an IPv4 address on this Spark; check the cable and the port's address, or set FABRIC_HCA in .env" >&2; exit 2; }
[[ -n $there ]] || { echo "no RoCE port with its link up and an IPv4 address on $WORKER; check the cable and the port's address, or set FABRIC_HCA in .env" >&2; exit 2; }
choice=$(python3 - "$here" "$there" <<'PY'
import ipaddress, sys
parse = lambda text: [line.split() for line in text.splitlines() if len(line.split()) == 3]
here, there = parse(sys.argv[1]), parse(sys.argv[2])
net = lambda cidr: ipaddress.ip_interface(cidr).network
match = [(a, b) for a in here for b in there if net(a[2]) == net(b[2])]
a, b = match[0] if match else (here[0], there[0])
print(a[0], a[1], a[2].split('/')[0], b[0], b[1], b[2].split('/')[0], 'subnet' if match else 'first')
PY
)
read -r hca0 iface0 ip0 hca1 iface1 ip1 how <<< "$choice"
if [[ $how == subnet ]]; then
say "fabric: $hca0 ($iface0, $ip0) here, $hca1 ($iface1, $ip1) on $WORKER; set FABRIC_HCA in .env to choose another port"
else
say "fabric: no active port pair shares a subnet; using $hca0 ($iface0, $ip0) here and $hca1 ($iface1, $ip1) on $WORKER. If they are not cabled to each other, set FABRIC_HCA in .env"
fi
}

build() {
say "image $IMAGE"
if ! has_image; then
Expand Down Expand Up @@ -89,13 +129,18 @@ state() { docker inspect -f '{{.State.Status}}' "${NAME}0" 2>/dev/null || echo a
worker_state() { on_worker docker inspect -f '{{.State.Status}}' "${NAME}1" 2>/dev/null || echo absent; }

serve() {
local iface address common
iface=${FABRIC_INTERFACE:-$(ls "/sys/class/infiniband/$FABRIC_HCA/device/net" 2>/dev/null | head -1)}
local iface address common hca0 iface0 ip0 hca1 iface1 ip1 how
if [[ -n ${FABRIC_HCA:-} ]]; then
hca0=$FABRIC_HCA hca1=$FABRIC_HCA
else
pick_fabric
fi
iface=${FABRIC_INTERFACE:-$(ls "/sys/class/infiniband/$hca0/device/net" 2>/dev/null | head -1)}
address=${LEADER_ADDRESS:-}
[[ -n $address ]] || address=$(ip -4 -o addr show dev "$iface" 2>/dev/null | awk '{sub("/.*", "", $4); print $4; exit}')
[[ -n $address ]] || { echo "no IPv4 address on the fabric interface '$iface'; set LEADER_ADDRESS" >&2; exit 2; }
common=(--leader-address "$address" --model-root "$MODEL_ROOT" --image "$IMAGE"
--profile "$PROFILE" --fabric-hca "$FABRIC_HCA" ${FABRIC_INTERFACE:+--fabric-interface "$FABRIC_INTERFACE"}
--profile "$PROFILE" ${FABRIC_INTERFACE:+--fabric-interface "$FABRIC_INTERFACE"}
${GPU_MEMORY_UTILIZATION:+--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"}
${PREFIX_CACHE_DIR:+--prefix-cache-dir "$PREFIX_CACHE_DIR"} ${PREFIX_CACHE_GB:+--prefix-cache-gb "$PREFIX_CACHE_GB"})
case ${DISPLAY_CARVEOUT:-0} in
Expand All @@ -115,8 +160,8 @@ serve() {
esac
say "start rank 1 on $WORKER, then rank 0 here (leader $address, profile $PROFILE)"
stop
script_on_worker install/start-node.sh --rank 1 "${common[@]}"
install/start-node.sh --rank 0 "${common[@]}"
script_on_worker install/start-node.sh --rank 1 --fabric-hca "$hca1" "${common[@]}"
install/start-node.sh --rank 0 --fabric-hca "$hca0" "${common[@]}"
say "loading (about 2.5 minutes)"
for ((i = 0; i < 90; i++)); do
if curl -sf "$API/health" >/dev/null; then
Expand Down