CLI: federation proxy timeout, init backend port range, explicit recipes root

bin/lloom.mjs

@@ -70,7 +70,7 @@ import {
   stopGatewayService,
   uninstallGatewayService
 } from '../src/service-install.mjs';
-import { addAuthenticatedNode, readBoundedStdinCredential } from '../src/node-onboarding.mjs';
+import { addAuthenticatedNode, parsePositiveTimeoutMs, readBoundedStdinCredential } from '../src/node-onboarding.mjs';
 import { retargetGatewayRelay } from '../src/gateway-relay.mjs';
 import { retargetClientFile } from '../src/client-retarget.mjs';
 import { applyModelImport, applyModelImportGo } from '../src/model-intake.mjs';
@@ -246,7 +246,7 @@ Backends and runtimes:
   lloom deployment rollback --plan FILE --nodes FILE --operation-id ID --apply --yes [--generation N] [--journal FILE] [--json]
   deployment plans require the reviewed archive and manifest at exact absolute paths on every target
   lloom node-agent <phase> --json   # authenticated remote gateway release phase; request JSON on stdin
-  lloom cluster add-node <id> <url> [--namespace NAME|--merge] [--include-external] [--api-key-env NAME|--api-key-stdin] [--telemetry-only] [--apply --yes]
+  lloom cluster add-node <id> <url> [--namespace NAME|--merge] [--include-external] [--api-key-env NAME|--api-key-stdin] [--timeout-ms N] [--telemetry-only] [--apply --yes]
   lloom service install [--enable-linger] [--apply --yes] [--json]
   lloom service doctor [--json]
   lloom service stop [--apply --yes] [--json]
@@ -1720,6 +1720,7 @@ async function main() {
     },
     init: async ({ args, config, command: _command }) => {
       const recipeId = argValue(args, '--recipe');
+      const recipesRoot = argValue(args, '--recipes-root');
       const home = argValue(args, '--home') ?? process.env.HOME;
       const configOut = argValue(args, '--config-out') ?? defaultUserConfigPath(home);
       const modelRoot = argValue(args, '--model-root');
@@ -1748,6 +1749,7 @@ async function main() {
             integrate,
             additive,
             backendVariables: backendVariablesForCli(args),
+            ...(recipesRoot ? { recipesRoot } : {}),
             backendCatalogPath: argValue(args, '--backend-catalog')
           }),
           null,
@@ -1758,6 +1760,7 @@ async function main() {
     bootstrap: async ({ args, config, command: _command }) => {
       const recipeId = argValue(args, '--recipe');
       const modelRoot = argValue(args, '--model-root');
+      const recipesRoot = argValue(args, '--recipes-root');
       const clientId = argValue(args, '--client') ?? 'all';
       const statePath = argValue(args, '--state');
       const apply = hasFlag(args, '--apply');
@@ -1770,6 +1773,7 @@ async function main() {
               modelRoot,
               clientId,
               backendVariables: backendVariablesForCli(args),
+              ...(recipesRoot ? { recipesRoot } : {}),
               benchmarksRoot: argValue(args, '--benchmarks-root'),
               backendCatalogPath: argValue(args, '--backend-catalog')
             }),
@@ -1785,6 +1789,7 @@ async function main() {
             recipeId,
             modelRoot,
             clientId,
+            ...(recipesRoot ? { recipesRoot } : {}),
             dryRun: false,
             yes,
             ...(statePath ? { statePath } : {}),
@@ -2343,7 +2348,7 @@ async function main() {
         process.exitCode = 2;
         return;
       }
-      const recipe = await loadRecipeById(recipeId);
+      const recipe = await loadRecipeById(recipeId, argValue(args, '--recipes-root'));
       const catalog = await loadBackendCatalog(argValue(args, '--backend-catalog'));
       const modelRoot = argValue(args, '--model-root') ?? '${LLOOM_MODEL_ROOT}';
       const {
@@ -2374,7 +2379,7 @@ async function main() {
         process.exitCode = 2;
         return;
       }
-      const recipe = await loadRecipeById(recipeId);
+      const recipe = await loadRecipeById(recipeId, argValue(args, '--recipes-root'));
       const modelRoot = argValue(args, '--model-root') ?? process.env.LLOOM_MODEL_ROOT;
       const statePath = argValue(args, '--state');
       const onlyStep = argValue(args, '--step');
@@ -2715,11 +2720,15 @@ async function main() {
         const endpoint = positional(args)[3];
         if (!nodeId || !endpoint) {
           throw new Error(
-            'Usage: lloom cluster add-node <id> <url> [--namespace NAME|--merge] [--include-external] [--api-key-env NAME|--api-key-stdin] [--telemetry-only] [--apply --yes]'
+            'Usage: lloom cluster add-node <id> <url> [--namespace NAME|--merge] [--include-external] [--api-key-env NAME|--api-key-stdin] [--timeout-ms N] [--telemetry-only] [--apply --yes]'
           );
         }
         if (hasFlag(args, '--api-key-stdin') && argValue(args, '--api-key-env'))
           throw new Error('Choose --api-key-stdin or --api-key-env');
+        const timeoutArg = argValue(args, '--timeout-ms');
+        if (hasFlag(args, '--timeout-ms') && timeoutArg === undefined)
+          throw new Error('--timeout-ms requires a positive integer');
+        const timeoutMs = parsePositiveTimeoutMs(timeoutArg);
         const apiKeyEnv = hasFlag(args, '--api-key-stdin')
           ? null
           : (argValue(args, '--api-key-env') ?? config.cluster?.apiKeyEnv ?? 'LLOOM_CLUSTER_KEY');
@@ -2737,6 +2746,7 @@ async function main() {
               namespace: argValue(args, '--namespace') ?? nodeId,
               merge: hasFlag(args, '--merge'),
               includeExternal: hasFlag(args, '--include-external'),
+              timeoutMs,
               apply: hasFlag(args, '--apply'),
               yes: hasFlag(args, '--yes')
             }),

docs/clusters.md

@@ -17,11 +17,13 @@ Expose the joining LLooM only on a trusted private network, require authenticati
 
 ```bash
 export LLOOM_CLUSTER_KEY='the key accepted by the joining LLooM'
-lloom cluster add-node macbook-local http://macbook-private:8100 --apply --yes
+lloom cluster add-node macbook-local http://macbook-private:8100 --timeout-ms 3600000 --apply --yes
 ```
 
 The command reads the joining node's `/gateway/node` snapshot, records its architecture, accelerators, memory, and directly hosted model catalog, and creates deterministic gateway names such as `macbook-local/local/qwen`. Imported targets retain the joining gateway's runtime ID as observational metadata, so the central topology and replica selection can reflect remote lifecycle and load without taking ownership of that process. It never copies credentials into config: the node and generated backend retain only `apiKeyEnv`. Config reload activates the node without restarting the central gateway.
 
+`--timeout-ms` sets the positive-integer request deadline for the joining node's federation proxy and updates its existing dedicated backend timeout while preserving the backend URL and authentication fields. If another node shares that backend ID, the apply is refused rather than changing a shared deadline. Omitting the option preserves an existing timeout.
+
 The declarative form is useful for review or hand editing:
 
 ```json

package.json

@@ -74,7 +74,7 @@
     "community:check": "node scripts/check-community-deploy.mjs",
     "package:check": "node scripts/check-package.mjs",
     "test:adoption": "node test/resource-fit.test.mjs && node test/model-acquisition.test.mjs && node test/dashboard-status.test.mjs",
-    "test:unit": "node --test test/deployment-fence.test.mjs test/cluster-deployment.test.mjs test/cluster-deployment-cli.test.mjs test/cluster-deployment-e2e.test.mjs test/node-release-agent.test.mjs test/node-gateway-adapter.test.mjs test/node-gateway-adapter-e2e.test.mjs test/deployment-bundle.test.mjs test/node-agent-cli.test.mjs test/ssh-deployment-transport.test.mjs && node test/runtime-health-ownership.test.mjs && node test/long-prefill-transport.test.mjs && node test/sparkglm-managed.test.mjs && node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/stream-metrics.test.mjs && node test/responses-codex.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/config-profiles.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-policy-config.test.mjs && node --test test/request-scheduler.test.mjs test/adaptive-priority.test.mjs test/rate-limit.test.mjs test/runtime-queue.test.mjs test/foreground-priority.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node --test test/standby.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && node test/hear-recipe.test.mjs && npm run test:adoption && node --test test/performance-sampler.test.mjs && node --test test/audio-generations.test.mjs && node test/video-providers.test.mjs && node --test test/web-functions.test.mjs && node --test test/config-mutation.test.mjs && node --test test/model-maintenance.test.mjs && node --test test/model-maintenance-control.test.mjs && node --test test/route-maintenance-concurrency.test.mjs && node --test test/model-maintenance-integration.test.mjs && node --test test/schedule.test.mjs test/schedule-integration.test.mjs && node test/comfyui-media.test.mjs && node test/qwen-image-diffusers.test.mjs && node test/ace-step-diffusers.test.mjs && node test/runtime-residency.test.mjs && node test/spark-audio-recipes.test.mjs && node --test test/video-contracts.test.mjs && node test/ltx-pipelines-recipe.test.mjs && npm run test:ux && npm run test:memory-safety && node --test test/dashboard-memory.test.mjs test/runtime-memory-usage.test.mjs && node --test test/speech-stream.test.mjs && node --test test/audio-providers.test.mjs && node test/openrouter-provider.test.mjs && node test/atlas-sparkglm-recipe.test.mjs && node test/atlas-sparkqwen-recipe.test.mjs && node --test test/head-preparation.test.mjs test/head-transfer.test.mjs test/service-control.test.mjs && node --test test/gateway-relay.test.mjs test/client-retarget.test.mjs && node --test test/head-profile-acceptance.test.mjs test/head-promotion.test.mjs test/head-promotion-transfer.test.mjs test/head-transfer-acceptance.test.mjs && node --test test/cli-diagnostics.test.mjs && node --test test/service-install.test.mjs",
+    "test:unit": "node --test test/deployment-fence.test.mjs test/cluster-deployment.test.mjs test/cluster-deployment-cli.test.mjs test/cluster-deployment-e2e.test.mjs test/node-release-agent.test.mjs test/node-gateway-adapter.test.mjs test/node-gateway-adapter-e2e.test.mjs test/deployment-bundle.test.mjs test/node-agent-cli.test.mjs test/ssh-deployment-transport.test.mjs && node test/runtime-health-ownership.test.mjs && node test/long-prefill-transport.test.mjs && node test/sparkglm-managed.test.mjs && node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/stream-metrics.test.mjs && node test/responses-codex.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/config-profiles.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-policy-config.test.mjs && node --test test/request-scheduler.test.mjs test/adaptive-priority.test.mjs test/rate-limit.test.mjs test/runtime-queue.test.mjs test/foreground-priority.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node --test test/standby.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && node test/hear-recipe.test.mjs && npm run test:adoption && node --test test/performance-sampler.test.mjs && node --test test/audio-generations.test.mjs && node test/video-providers.test.mjs && node --test test/web-functions.test.mjs && node --test test/config-mutation.test.mjs && node --test test/model-maintenance.test.mjs && node --test test/model-maintenance-control.test.mjs && node --test test/route-maintenance-concurrency.test.mjs && node --test test/model-maintenance-integration.test.mjs && node --test test/schedule.test.mjs test/schedule-integration.test.mjs && node test/comfyui-media.test.mjs && node test/qwen-image-diffusers.test.mjs && node test/ace-step-diffusers.test.mjs && node test/runtime-residency.test.mjs && node test/spark-audio-recipes.test.mjs && node --test test/video-contracts.test.mjs && node test/ltx-pipelines-recipe.test.mjs && npm run test:ux && npm run test:memory-safety && node --test test/dashboard-memory.test.mjs test/runtime-memory-usage.test.mjs && node --test test/speech-stream.test.mjs && node --test test/audio-providers.test.mjs && node test/openrouter-provider.test.mjs && node test/atlas-sparkglm-recipe.test.mjs && node test/atlas-sparkqwen-recipe.test.mjs && node --test test/head-preparation.test.mjs test/head-transfer.test.mjs test/service-control.test.mjs && node --test test/gateway-relay.test.mjs test/client-retarget.test.mjs && node --test test/head-profile-acceptance.test.mjs test/head-promotion.test.mjs test/head-promotion-transfer.test.mjs test/head-transfer-acceptance.test.mjs && node --test test/cli-diagnostics.test.mjs test/init-backend-timeout.test.mjs test/recipes-root-cli.test.mjs && node --test test/service-install.test.mjs",
     "test:cluster": "node test/cluster.test.mjs && node --test test/cluster-availability.test.mjs test/cluster-readiness.test.mjs",
     "test": "npm run test:cluster && npm run test:unit && npm run test:entity-stagger && npm run smoke && npm run community:check",
     "test:entity-stagger": "node test/entity-stagger-benchmark.test.mjs",

src/cluster.mjs

@@ -45,6 +45,30 @@ function slug(value) {
   );
 }
 
+export function federatedBackendId(nodeId, node = {}) {
+  return asObject(node?.proxy).backend ?? `lloom-node-${slug(nodeId)}`;
+}
+
+function sharedFederatedBackendNodeIds(config, nodeId, backendId) {
+  return Object.entries(asObject(config.cluster?.nodes))
+    .filter(
+      ([candidateId, candidate]) => candidateId !== nodeId && federatedBackendId(candidateId, candidate) === backendId
+    )
+    .map(([candidateId]) => candidateId);
+}
+
+export function updateFederatedBackendTimeout(config, { nodeId, node, timeoutMs } = {}) {
+  const backendId = federatedBackendId(nodeId, node);
+  const sharedNodeIds = sharedFederatedBackendNodeIds(config, nodeId, backendId);
+  if (sharedNodeIds.length) return { backendId, updated: false, sharedNodeIds };
+  const backend = config.backends?.[backendId];
+  if (!backend || typeof backend !== 'object' || Array.isArray(backend)) {
+    return { backendId, updated: false, sharedNodeIds: [] };
+  }
+  backend.timeoutMs = timeoutMs;
+  return { backendId, updated: true, sharedNodeIds: [] };
+}
+
 function proxyModels(node = {}) {
   const proxy = asObject(node.proxy);
   return (Array.isArray(proxy.models) ? proxy.models : []).map((entry) =>
@@ -61,7 +85,7 @@ export function isFederatedGatewayBackend(config, backendId) {
       proxy.enabled !== false &&
       proxyModels(node).length > 0 &&
       Boolean(node.endpoint || proxy.baseUrl) &&
-      backendId === (proxy.backend ?? `lloom-node-${slug(nodeId)}`)
+      backendId === federatedBackendId(nodeId, node)
     );
   });
 }
@@ -74,7 +98,7 @@ export function materializeFederatedNodes(config) {
     const advertised = proxyModels(node);
     if (proxy.enabled === false || !advertised.length) continue;
     if (!node.endpoint && !proxy.baseUrl) continue;
-    const backendId = proxy.backend ?? `lloom-node-${slug(nodeId)}`;
+    const backendId = federatedBackendId(nodeId, node);
     config.backends[backendId] ??= {
       type: 'openai',
       baseUrl: proxy.baseUrl ?? `${trimSlash(node.endpoint)}/v1`,