From 5485824d32b1a418c9c243d5dcdcdeb77c1147a9 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Tue, 13 Oct 2026 00:49:14 -0700 Subject: [PATCH] Pass explicit per-node Atlas RDMA rails and reject ambiguous discovery --- docs/recipes.md | 38 +++++++++- ...linux-nvidia-dgx-spark-2x-glm53-atlas.json | 14 ++-- src/cluster.mjs | 23 +++++- src/init.mjs | 23 ++++++ test/atlas-sparkglm-recipe.test.mjs | 73 +++++++++++++++++-- test/cluster.test.mjs | 36 +++++++++ 6 files changed, 190 insertions(+), 17 deletions(-) diff --git a/docs/recipes.md b/docs/recipes.md index 5477980..e5d86d4 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -448,7 +448,7 @@ vector from `/opt/atlas/profiles/4x512k.json` (its default `SPARKGLM_PROFILE`). | `MODEL_PATH` | mounted converted overlay root, `${installRoot}/atlas-overlay` | | `SERVED_MODEL_NAME` | `glm-5.3-flash-atlas` | | `ATLAS_WORLD_SIZE` / `ATLAS_TP_SIZE` / `ATLAS_EP_SIZE` | `2` | -| `NCCL_*` | IB transport with `NCCL_IB_HCA=rocep1s0f0`, `AF_INET`, `NCCL_CROSS_NIC=0` | +| `NCCL_*` | IB transport with per-node `NCCL_IB_HCA=${fabricHca}`, `AF_INET`, `NCCL_CROSS_NIC=0` | Listeners are private and loopback-bound per node: leader `127.0.0.1:8893`, worker `127.0.0.1:8894`. The worker starts first (`order` 10, rank 1) with @@ -523,3 +523,39 @@ curl http://127.0.0.1:8100/v1/chat/completions \ Responses clients can send the same media URLs as `input_image` and `input_video` content parts; LLooM converts those parts to the matching Chat Completions form before forwarding the request. + +### Per-node RDMA ports for Atlas SparkGLM + +The Atlas recipe requires `fabricInterface` and `fabricHca` on each participating +`cluster.nodes` entry. `fabricHca` is a comma-separated list of exact local RDMA +device names, without spaces or `:port` suffixes. Names and physical port numbers +can differ between nodes. For example: + +```json +{ + "cluster": { + "leaderNode": "left", + "nodes": { + "left": { "backendHost": "192.0.2.1", "fabricInterface": "enp1s0f1np1", "fabricHca": "rocep1s0f1,roceP2p1s0f1" }, + "right": { "backendHost": "192.0.2.2", "fabricInterface": "enp1s0f0np0", "fabricHca": "rocep1s0f0,roceP2p1s0f0", "labels": { "role": "worker" } } + } + } +} +``` + +Check each host's `/sys/class/infiniband//device/net` and active RoCE +IPv4 GIDs against the pair's network interfaces. Include only rails to the other +participating node; a backchannel to a third node is not a serving rail. NVIDIA +Sync discovery supplies network inventory, but does not prove which additional +RDMA rails should participate. Missing or malformed settings stop recipe +materialization with the node and field named in the error. + +The recipe passes each node's exact list as `FABRIC_HCA` and `NCCL_IB_HCA`, and +sets `FABRIC_SIBLINGS=0` so the image cannot add another device. The pinned image +sets Atlas's direct RDMA rails from the same list. Existing running containers +are unchanged; set these fields before reapplying the updated recipe. No image +rebuild is required for this LLooM configuration change. + +Applying NVIDIA Sync discovery refuses ambiguous peer/interface observations. +An explicit per-node `fabricHca` selection also preserves that node's configured +interface and backend address when refreshing discovery. diff --git a/recipes/linux-nvidia-dgx-spark-2x-glm53-atlas.json b/recipes/linux-nvidia-dgx-spark-2x-glm53-atlas.json index 7220002..0c85476 100644 --- a/recipes/linux-nvidia-dgx-spark-2x-glm53-atlas.json +++ b/recipes/linux-nvidia-dgx-spark-2x-glm53-atlas.json @@ -70,7 +70,7 @@ "testedMachine": "Two directly connected DGX Spark systems", "notes": [ "Candidate lane: no live hardware qualification has been recorded through LLooM yet. The profile and capability claims below come from the source build and focused checks; gateway and two-node serving canaries remain required before treating this recipe as qualified.", - "TP2 with expert parallelism 2 across the private ConnectX RoCE fabric. The direct-fabric address and NIC are discovered per node and passed as MASTER_ADDR and FABRIC_INTERFACE; no host-specific addresses are pinned.", + "TP2 with expert parallelism 2 across the private ConnectX RoCE fabric. The direct-fabric address and NIC are discovered per node and passed as MASTER_ADDR and FABRIC_INTERFACE; the RoCE HCA is declared explicitly per node as cluster.nodes[ID].fabricHca (a comma-separated list of exact device names, no :port syntax) and passed as FABRIC_HCA and NCCL_IB_HCA with FABRIC_SIBLINGS=0, so the engine never silently adds a backchannel. No host-specific addresses or port names are pinned.", "The worker is admitted first (rank 1) and readiness-checked through its container state, because the Atlas engine has no separate worker HTTP surface. The leader starts second (rank 0) and owns routing.", "The NVFP4 checkpoint requires a per-node overlay conversion before it can be served. LLooM treats the conversion as incomplete unless conversion.complete.json plus its recorded verification are present.", "Licenses: the Atlas engine in the image is AGPL-3.0-only (Atlas-Inf/atlas with SparkGLM's work on Enntity/atlas; not included in LLooM). The default DFlash2 drafter incoai/GLM-5.3-Flash-DFlash2 is CC BY-NC-ND 4.0 (non-commercial); nvidia/GLM-5.3-Flash-NVFP4 is MIT per SparkGLM docs/LICENSING.md. Read each model card before use.", @@ -333,7 +333,9 @@ "-e", "FABRIC_INTERFACE=${fabricInterface}", "-e", - "FABRIC_HCA=rocep1s0f0", + "FABRIC_HCA=${fabricHca}", + "-e", + "FABRIC_SIBLINGS=0", "-e", "NODE_ADDRESS=${nodeAddress}", "-e", @@ -367,7 +369,7 @@ "-e", "GLOO_SOCKET_IFNAME=${fabricInterface}", "-e", - "NCCL_IB_HCA=rocep1s0f0", + "NCCL_IB_HCA=${fabricHca}", "-e", "NCCL_IB_ADDR_FAMILY=AF_INET", "-e", @@ -490,7 +492,9 @@ "-e", "FABRIC_INTERFACE=${fabricInterface}", "-e", - "FABRIC_HCA=rocep1s0f0", + "FABRIC_HCA=${fabricHca}", + "-e", + "FABRIC_SIBLINGS=0", "-e", "NODE_ADDRESS=${nodeAddress}", "-e", @@ -524,7 +528,7 @@ "-e", "GLOO_SOCKET_IFNAME=${fabricInterface}", "-e", - "NCCL_IB_HCA=rocep1s0f0", + "NCCL_IB_HCA=${fabricHca}", "-e", "NCCL_IB_ADDR_FAMILY=AF_INET", "-e", diff --git a/src/cluster.mjs b/src/cluster.mjs index 8038946..5771a98 100644 --- a/src/cluster.mjs +++ b/src/cluster.mjs @@ -214,6 +214,7 @@ export function clusterNodes(config, env = process.env) { endpoint: node?.endpoint ? trimSlash(node.endpoint) : null, backendHost: node?.backendHost ?? null, fabricInterface: node?.fabricInterface ?? null, + fabricHca: node?.fabricHca ?? null, sshAlias: node?.sshAlias ?? null, apiKeyEnv: node?.apiKeyEnv ?? config.cluster?.apiKeyEnv ?? null, labels: asObject(node?.labels), @@ -344,7 +345,7 @@ function syncPeerMatches(peers, localAddresses, localAddressSet = new Set()) { (address) => address.address?.startsWith('10.100.') && sameSubnet(address.address, privateAddress, address.prefixlen) ) - .map((address) => [address.interface ?? address.address, address]) + .map((address) => [`${address.interface ?? ''}|${address.address}`, address]) ).values() ]; const candidateAddresses = candidates.map((address) => address.address); @@ -405,7 +406,11 @@ function nodeRailsForPeers(matches) { export function buildNvidiaSyncDiscovery({ peers = [], localAddresses = [], localNodeId, hostname } = {}) { const localAddressSet = new Set(localAddresses.map((address) => address.address).filter(Boolean)); const { matches, ambiguous } = syncPeerMatches(peers, localAddresses, localAddressSet); - if (!matches.length) return null; + if (!matches.length) { + const conflict = ambiguous.find((entry) => entry.reason === 'ambiguous-local-interface'); + if (conflict) throw new Error(`Ambiguous NVIDIA Sync discovery: ${conflict.message}`); + return null; + } const id = localNodeId || hostname || os.hostname(); if (matches.some((match) => friendlyNodeId(match.peer.clusterAlias) === id)) { throw new Error(`Sync peer alias conflicts with local node id ${id}`); @@ -557,6 +562,13 @@ export function mergeNvidiaSyncClusterDiscovery({ discovery, cluster = {}, id, a if (!discovery?.detected || discovery.provider !== 'nvidia-sync') { throw new Error('NVIDIA Sync cluster was not detected'); } + if (discovery.ambiguousPeers?.length) { + throw new Error( + `Ambiguous NVIDIA Sync discovery; resolve conflicting interfaces before applying: ${discovery.ambiguousPeers + .map((entry) => entry.message) + .join('; ')}` + ); + } const current = asObject(cluster); const nodeId = localNodeId ?? current.nodeId ?? discovery.nodeId; if (current.nodeId && !String(current.nodeId).includes('${') && current.nodeId !== nodeId) { @@ -584,6 +596,7 @@ export function mergeNvidiaSyncClusterDiscovery({ discovery, cluster = {}, id, a let endpoint = existing.endpoint; let backendHost = existing.backendHost ?? observed.backendHost; if ( + !existing.fabricHca && existing.endpoint && /^10\.100\./.test(existing.backendHost ?? '') && endpointHostMatches(existing.endpoint, existing.backendHost) @@ -604,8 +617,10 @@ export function mergeNvidiaSyncClusterDiscovery({ discovery, cluster = {}, id, a name: existing.name ?? targetId, ...(endpoint ? { endpoint } : {}), backendHost, - fabricAddress: observed.backendHost, - ...(observed.fabricInterface ? { fabricInterface: observed.fabricInterface } : {}), + fabricAddress: existing.fabricHca ? backendHost : observed.backendHost, + ...(observed.fabricInterface && !(existing.fabricHca && existing.fabricInterface) + ? { fabricInterface: observed.fabricInterface } + : {}), ...(observed.sshAlias ? { sshAlias: observed.sshAlias } : {}), ...(observed.rails ? { rails: observed.rails } : {}), labels: { diff --git a/src/init.mjs b/src/init.mjs index 32defc8..9f40cd5 100644 --- a/src/init.mjs +++ b/src/init.mjs @@ -230,6 +230,27 @@ function templateValue(value, variables) { return value; } +// Recipes that consume fabricHca require exact, node-local device names. +// Do not guess an HCA from a NIC name: PCI naming and cabling differ by host. +function validateNodeFabric(node, nodeId, member) { + if (!JSON.stringify(member.runtimeSettings ?? {}).includes('${fabricHca}')) return; + const field = `cluster.nodes[${nodeId}]`; + const hcas = typeof node.fabricHca === 'string' ? node.fabricHca.split(',') : []; + if ( + !hcas.length || + hcas.some((name) => !/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(name)) || + new Set(hcas).size !== hcas.length + ) { + throw new Error( + `${field}.fabricHca must list unique, exact RDMA device names separated by commas ` + + '(no spaces or :port suffixes); select only rails connecting this pair' + ); + } + if (typeof node.fabricInterface !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/.test(node.fabricInterface)) { + throw new Error(`${field}.fabricInterface must name the network interface for this pair`); + } +} + function primitiveSettings(settings) { return Object.fromEntries( Object.entries(settings).filter( @@ -667,6 +688,7 @@ function ensureRecipeConfigEntries(config, recipe, { modelRoot, sessionCacheRoot const node = configuredNodes[nodeId]; if (!node) throw new Error(`recipe model ${modelId} references unknown cluster node ${nodeId}`); if (!node.backendHost) throw new Error(`cluster node ${nodeId} requires backendHost for distributed recipes`); + validateNodeFabric(node, nodeId, member); const memberRuntimeId = selectedNodes.length > 1 ? `${member.runtime}-${slug(nodeId)}` : member.runtime; if (!memberRuntimeId) throw new Error(`recipe model ${modelId} has a distributed member without runtime`); const nodeRank = nodeId === leaderNode ? 0 : selectedWorkerNodes.indexOf(nodeId) + 1; @@ -682,6 +704,7 @@ function ensureRecipeConfigEntries(config, recipe, { modelRoot, sessionCacheRoot nodeRank, nodeAddress: node.backendHost, fabricInterface: node.fabricInterface ?? '', + fabricHca: node.fabricHca ?? '', leaderNode, leaderAddress: configuredNodes[leaderNode].backendHost, clusterNodeCount: selectedNodeIds.length, diff --git a/test/atlas-sparkglm-recipe.test.mjs b/test/atlas-sparkglm-recipe.test.mjs index 5a63f95..d222007 100644 --- a/test/atlas-sparkglm-recipe.test.mjs +++ b/test/atlas-sparkglm-recipe.test.mjs @@ -164,12 +164,13 @@ for (const member of members) { 'type=bind,src=${modelRoot}/nvidia--GLM-5.3-Flash-NVFP4,dst=${modelRoot}/nvidia--GLM-5.3-Flash-NVFP4,readonly', 'type=bind,src=${installRoot}/atlas-overlay,dst=${installRoot}/atlas-overlay,readonly', 'MODEL_PATH=${installRoot}/atlas-overlay', - 'FABRIC_HCA=rocep1s0f0', + 'FABRIC_HCA=${fabricHca}', + '-e FABRIC_SIBLINGS=0', 'NODE_RANK=${nodeRank}', 'MASTER_ADDR=${leaderAddress}', 'MASTER_PORT=29510', 'FABRIC_INTERFACE=${fabricInterface}', - '-e FABRIC_HCA=rocep1s0f0', + '-e FABRIC_HCA=${fabricHca}', 'ATLAS_WORLD_SIZE=2', 'ATLAS_TP_SIZE=2', 'ATLAS_EP_SIZE=2', @@ -177,7 +178,7 @@ for (const member of members) { 'type=bind,src=${modelRoot}/incoai--GLM-5.3-Flash-DFlash2,dst=${modelRoot}/incoai--GLM-5.3-Flash-DFlash2,readonly', 'DRAFTER_PATH=${modelRoot}/incoai--GLM-5.3-Flash-DFlash2', 'SERVED_MODEL_NAME=glm-5.3-flash-atlas', - 'NCCL_IB_HCA=rocep1s0f0', + 'NCCL_IB_HCA=${fabricHca}', 'NCCL_IB_ADDR_FAMILY=AF_INET', 'NCCL_CROSS_NIC=0', 'NCCL_SOCKET_IFNAME=${fabricInterface}', @@ -384,8 +385,8 @@ const materialize = (recipeDocument) => cluster: { leaderNode: 'spark01', nodes: { - spark01: { backendHost: '10.0.0.1', fabricInterface: 'eth0' }, - spark02: { backendHost: '10.0.0.2', fabricInterface: 'eth0', labels: { role: 'worker' } } + spark01: { backendHost: '10.0.0.1', fabricInterface: 'eth0', fabricHca: 'mlx5_0' }, + spark02: { backendHost: '10.0.0.2', fabricInterface: 'eth0', fabricHca: 'mlx5_0', labels: { role: 'worker' } } } } }, @@ -412,8 +413,8 @@ const additiveExisting = deriveUserConfig( cluster: { leaderNode: 'spark01', nodes: { - spark01: { backendHost: '10.0.0.1', fabricInterface: 'eth0' }, - spark02: { backendHost: '10.0.0.2', fabricInterface: 'eth0', labels: { role: 'worker' } } + spark01: { backendHost: '10.0.0.1', fabricInterface: 'eth0', fabricHca: 'mlx5_0' }, + spark02: { backendHost: '10.0.0.2', fabricInterface: 'eth0', fabricHca: 'mlx5_0', labels: { role: 'worker' } } } } }, @@ -454,6 +455,64 @@ for (const runtimeId of ['glm53-flash-atlas-worker', 'glm53-flash-atlas-head']) assert(args.includes('-e SPARKGLM_KV_SHARD=1 ')); assert(args.includes('-e SPARKGLM_PROFILE=4x1m ')); } +// Each host can use unrelated HCA names/ports. Explicit lists stay exact. +const fabricConfig = { + cluster: { + leaderNode: 'left', + nodes: { + left: { backendHost: '192.0.2.1', fabricInterface: 'enp1s0f1np1', fabricHca: 'rocep1s0f1,roceP2p1s0f1' }, + right: { + backendHost: '192.0.2.2', + fabricInterface: 'ens7', + fabricHca: 'mlx5_7,mlx5_9', + labels: { role: 'worker' } + } + } + } +}; +const deriveFabric = (config, document = recipe) => + deriveUserConfig(config, document, { + modelRoot: '/models', + additive: true, + backendVariables: { repoRoot: root, backendRoot: '/backend', installRoot: '/install' } + }); +const beforeFabric = structuredClone(fabricConfig); +const perNode = deriveFabric(fabricConfig); +assert.deepEqual(fabricConfig, beforeFabric, 'planning must not mutate input'); +for (const [id, expected, iface] of [ + ['glm53-flash-atlas-head', 'rocep1s0f1,roceP2p1s0f1', 'enp1s0f1np1'], + ['glm53-flash-atlas-worker', 'mlx5_7,mlx5_9', 'ens7'] +]) { + const args = perNode.runtimes[id].bootstrap.createArgs; + for (const env of [ + `FABRIC_HCA=${expected}`, + `NCCL_IB_HCA=${expected}`, + `FABRIC_INTERFACE=${iface}`, + 'FABRIC_SIBLINGS=0' + ]) { + const index = args.indexOf(env); + assert(index > 0 && args[index - 1] === '-e', `${id}: ${env} must be a Docker environment argument`); + } + assert(!args.some((arg) => arg.includes('rocep1s0f0'))); +} +for (const invalid of [undefined, '', 'mlx5_0,', 'mlx5_0,mlx5_0', 'mlx5_0:1', 'mlx5_0;true', ' mlx5_0', ['mlx5_0']]) { + const config = structuredClone(fabricConfig); + config.cluster.nodes.right.fabricHca = invalid; + const snapshot = structuredClone(config); + assert.throws(() => deriveFabric(config), /cluster.nodes\[right\].fabricHca/); + assert.deepEqual(config, snapshot); +} +for (const invalid of [undefined, '', 'eth0;true', ['eth0']]) { + const config = structuredClone(fabricConfig); + config.cluster.nodes.right.fabricInterface = invalid; + assert.throws(() => deriveFabric(config), /cluster.nodes\[right\].fabricInterface/); +} +// Recipes that do not consume fabricHca keep their existing configuration contract. +const legacyRecipe = JSON.parse(JSON.stringify(recipe).replaceAll('${fabricHca}', 'mlx5_0')); +const legacyConfig = structuredClone(fabricConfig); +delete legacyConfig.cluster.nodes.right.fabricHca; +assert.doesNotThrow(() => deriveFabric(legacyConfig, legacyRecipe)); + // A size reaches both ranks alike (the mount stays either way). const diskCacheRecipe = structuredClone(recipe); diskCacheRecipe.models[0].settings.prefixCacheGb = 0; diff --git a/test/cluster.test.mjs b/test/cluster.test.mjs index 263ec62..d00d704 100644 --- a/test/cluster.test.mjs +++ b/test/cluster.test.mjs @@ -1172,6 +1172,42 @@ assert.deepEqual( [['spark-node-02-lan', 'ambiguous-local-interface']] ); +assert.throws( + () => + buildNvidiaSyncDiscovery({ + peers: [{ alias: 'peer', createdBySync: true, clusterAlias: 'peer', hostname: '10.100.192.1' }], + localAddresses: [ + { interface: 'same-nic', address: '10.100.192.2', prefixlen: 24 }, + { interface: 'same-nic', address: '10.100.192.3', prefixlen: 24 } + ], + localNodeId: 'local' + }), + /Ambiguous NVIDIA Sync discovery/ +); +const ambiguousBefore = structuredClone(ambiguousDiscovery); +assert.throws( + () => mergeNvidiaSyncClusterDiscovery({ discovery: ambiguousDiscovery }), + /Ambiguous NVIDIA Sync discovery/ +); +assert.deepEqual(ambiguousDiscovery, ambiguousBefore); +const explicitFabric = { + nodeId: 'spark-node-01', + leaderNode: 'spark-node-01', + nodes: { + 'spark-node-01': { + backendHost: '10.100.196.1', + endpoint: 'http://10.100.196.1:8100', + fabricInterface: 'enp1s0f1np1', + fabricHca: 'rocep1s0f1,roceP2p1s0f1' + } + } +}; +const explicitAfter = mergeNvidiaSyncClusterDiscovery({ discovery: fixtureDiscovery, cluster: explicitFabric }); +assert.equal(explicitAfter.nodes['spark-node-01'].fabricInterface, 'enp1s0f1np1'); +assert.equal(explicitAfter.nodes['spark-node-01'].fabricHca, 'rocep1s0f1,roceP2p1s0f1'); +assert.equal(explicitAfter.nodes['spark-node-01'].backendHost, '10.100.196.1'); +assert.equal(explicitAfter.nodes['spark-node-01'].endpoint, 'http://10.100.196.1:8100'); + // --------------------------------------------------------------------------- // Merge: preserve every existing node/endpoint/catalog/leader; migrate only the // unambiguous old-fabric endpoint; never duplicate the local hostname.