diff --git a/CHANGELOG.md b/CHANGELOG.md index 91d88ac..58459c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to LLooM will be documented in this file. The format follows ### Added +- `lloom pool plan`: a dry-run placement plan across every cluster node. It proposes which lanes run where, which residency they get and how each alias routes, and diffs that against what runs now. It covers every required alias first, then minimises demand-weighted latency and the load cost of changes, within per-domain memory headroom, direct-link tensor parallelism, compute architecture and pins. Nodes report a pool inventory through `/gateway/node?inventory=1`. Recipes may declare `requirements.computeArchs`, `requirements.tensorParallel` and `settings.maxSequences`. `/gateway/metrics` adds `requested` buckets, which give per-alias demand. See `docs/pool-planning.md`. + - LLooM Hear CPU audio analysis, with structured estimates, optional dashboard images, and opt-in upstream interpretation. File and URL inputs require operator configuration. - Four standalone NVIDIA Spark audio recipes (Qwen3-TTS 1.7B CustomVoice, Base voice cloning and VoiceDesign, and Whisper large-v3-turbo). Each runs one pinned, integrity-checked checkpoint in its own container from a locally built `spark-audio` image that replaces a host-local prebuilt image. diff --git a/README.md b/README.md index e32c43b..cd07a57 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ lloom route q38fn # inspe lloom route q38fn --suspend-member qwen3.8-flash-next --apply --yes # keep stable traffic on cloud lloom runtime-stop qwen38-flash-next-cluster # stop local TP-2 after suspension lloom route q38fn --resume-member qwen3.8-flash-next --apply --yes # restore preferred-local recovery +lloom pool plan --cordon spark-node-02 # dry-run placement across the cluster lloom integrate omp --apply --yes lloom add-model mlx-community/Qwen3.6-27B-OptiQ-4bit --keep-warm --default --go # Mac: install, download, start lloom remove-model mlx-community/Qwen3.6-27B-OptiQ-4bit # preview safe cleanup diff --git a/bin/lloom.mjs b/bin/lloom.mjs index b599222..00cc0be 100755 --- a/bin/lloom.mjs +++ b/bin/lloom.mjs @@ -36,6 +36,7 @@ import { selectedRecipeIdFromCommunityPlan } from '../src/community-client.mjs'; import { loadConfig } from '../src/config.mjs'; +import { POOL_USAGE, createPoolPlanReport, formatPoolPlanReport, poolPlanOptions } from '../src/pool-cli.mjs'; import { mutateConfigSource } from '../src/config-mutation.mjs'; import { runtimeControlTimeoutMs } from '../src/control-timeout.mjs'; import { createDoctorReport } from '../src/doctor.mjs'; @@ -136,6 +137,7 @@ const COMMAND_REGISTRY = [ { name: 'runtimes', aliases: ['runtime-status'], tier: 'advanced', needsInstalledConfig: true }, { name: 'service', aliases: [], tier: 'advanced', needsInstalledConfig: true }, { name: 'cluster', aliases: ['cluster-status'], tier: 'advanced', needsInstalledConfig: true }, + { name: 'pool', aliases: [], tier: 'advanced', needsInstalledConfig: true }, { name: 'deployment', aliases: ['rollout'], tier: 'advanced', needsInstalledConfig: false }, { name: 'node-agent', aliases: [], tier: 'advanced', needsInstalledConfig: false }, { name: 'runtime-plan', aliases: [], tier: 'advanced', needsInstalledConfig: true }, @@ -228,6 +230,7 @@ Serving and discovery: Backends and runtimes: lloom down lloom cluster [status|doctor|discover] [--apply] [--id NAME] [--api-key-env NAME] [--json] + lloom pool plan [--cordon NODE[,NODE]] [--pin ALIAS=MODEL] [--profile NAME] [--consider-installs] [--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json] lloom deployment plan --plan FILE [--json] lloom deployment apply --plan FILE --nodes FILE [--operation-id ID] --apply --yes [--journal FILE] [--json] lloom deployment status --operation-id ID [--journal FILE] [--json] @@ -370,6 +373,7 @@ function commandLine(command, parts = []) { const INSTALLED_CONFIG_COMMANDS = new Set([ 'add-model', + 'pool', 'bootstrap', 'service', 'cluster', @@ -404,6 +408,7 @@ const INSTALLED_CONFIG_COMMANDS = new Set([ const OPERATIONAL_CONFIG_COMMANDS = new Set([ 'add-model', + 'pool', 'service', 'cluster', 'cluster-status', @@ -2495,6 +2500,25 @@ async function main() { if (!result) throw new Error(`route switch failed through ${gatewayUrlFor(config)}`); console.log(JSON.stringify({ ...result, applied: true }, null, 2)); }, + pool: async ({ args, config }) => { + const action = positional(args)[1] ?? 'plan'; + if (action !== 'plan') throw new Error(POOL_USAGE); + const options = poolPlanOptions({ + cordon: argValues(args, '--cordon'), + pins: argValues(args, '--pin'), + profile: argValue(args, '--profile'), + considerInstalls: hasFlag(args, '--consider-installs'), + offline: hasFlag(args, '--offline'), + headroomGb: argValue(args, '--headroom-gb'), + churnWeight: argValue(args, '--churn-weight'), + external: argValue(args, '--external') + }); + const report = await createPoolPlanReport(config, options, { + gatewayRequest: (target, pathname) => gatewayRequest(target, pathname, { args, timeoutMs: 30000 }), + gatewayUrl: gatewayUrlFor(config) + }); + console.log(hasFlag(args, '--json') ? JSON.stringify(report, null, 2) : formatPoolPlanReport(report)); + }, fleet: async ({ args, config }) => { const action = positional(args)[1] ?? 'list'; const name = positional(args)[2]; diff --git a/docs/architecture.md b/docs/architecture.md index 19e9744..4829b02 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -220,7 +220,7 @@ The gateway expands `voice: "sample-speaker"` into the profile model, on-disk `r `/gateway/doctor` wraps setup status into a product-readiness report. It separates hard `blockers` from non-fatal `warnings`, groups install state into registry, recipe, backend, models, client, runtime, and benchmark phases, and returns ordered next actions that a UI can show directly. -`/gateway/metrics` is the operational feed for dashboards, local tuning, and recipe evidence gathering. Aggregate totals plus 400 days of daily rollups are persisted by default in `~/.lloom/metrics-history.json` with atomic replacement and restored when the gateway starts, while active connections, rolling windows, and the bounded recent-request list remain process-local. `period=today|7d|30d|all` selects the durable window, and the topology HUD exposes the same choices. The persistence file contains aggregate counts and timing only—never prompts, responses, raw headers, callers, or client IPs. Before the first persistence-aware deployment, `node scripts/capture-live-metrics.mjs` can capture the old gateway's in-memory snapshot so its counters become the initial durable baseline. +`/gateway/metrics` is the operational feed for dashboards, local tuning, and recipe evidence gathering. Aggregate totals plus 400 days of daily rollups are persisted by default in `~/.lloom/metrics-history.json` with atomic replacement and restored when the gateway starts, while active connections, rolling windows, and the bounded recent-request list remain process-local. Buckets are kept per resolved model, per HTTP route, and per requested id (`requested`, usually an alias), which is the demand signal for [pool planning](pool-planning.md). `period=today|7d|30d|all` selects the durable window, and the topology HUD exposes the same choices. The persistence file contains aggregate counts and timing only—never prompts, responses, raw headers, callers, or client IPs. Before the first persistence-aware deployment, `node scripts/capture-live-metrics.mjs` can capture the old gateway's in-memory snapshot so its counters become the initial durable baseline. Model-facing routes record requested model ID, resolved model ID, upstream model ID, kind, backend, runtime, status, stream flag, duration, first-content latency, last-content latency, response bytes, and normalized usage when the upstream reports it. Each completed streaming decode measurement follows the Spark Arena / `llama-benchy` timing convention: `(generated tokens - 1) / (last content token time - first content token time)`. The dashboard header shows `totals.activeDecodeTokensPerSecond`: completed decode tokens divided by the union of their first-to-last-content generation intervals within the selected period. Overlapping streams contribute tokens together but count elapsed generation time once; idle gaps and prefill are excluded. It updates as requests complete and remains visible during idle or buffered work. Daily windows select requests by completion date, as with token totals. Merged generation intervals are persisted but omitted from the HTTP metrics response. Legacy history without interval timing contributes its summed generation duration as an estimate; `activeDecodeRateEstimated` flags this or estimated token counts, and the HUD prefixes the rate with `~`. Model topology cards show instantaneous output rates; the model inspector shows `decodeTokensPerSecond`, the rolling mean of the latest ten measured streaming request rates in the selected period. LLooM uses reported output tokens when available and otherwise estimates them from observed output characters. Embedding responses have no generative token count, so their output contribution is estimated from observed response bytes for cumulative volume only. `estimatedDecodeSamples` makes estimated decode samples explicit. Non-streaming generative responses are excluded from decode speed because LLooM cannot observe their token-by-token decode interval; their tokens and end-to-end duration remain available in the other aggregate fields. Connection telemetry derives a short caller label from explicit client-name headers, recognized User-Agent families, or the SDK language header, but never stores raw headers or client IPs; unknown callers retain `conn_N`. Client disconnects before completion are recorded as status `499` so local timeouts do not look like backend crashes. The endpoint aggregates durable totals by model and route while retaining live-only rolling and recent windows. diff --git a/docs/clusters.md b/docs/clusters.md index a941081..38f29e6 100644 --- a/docs/clusters.md +++ b/docs/clusters.md @@ -256,6 +256,8 @@ before contacting any node. The live dashboard renders node cards between the main LLooM chassis and model cards. Each node card reports architecture, accelerator type, CPU, RAM, GPU compute, GPU memory, temperature, and power when the host can report them. Unsupported fields render as unavailable rather than inventing values; Apple Silicon still reports its profile and shared-memory pressure even when per-GPU utilization is unavailable. Select a node for its complete profile. Replicated models connect to every target node and distributed models connect to every member node. +`GET /gateway/node?inventory=1` and `GET /gateway/cluster?inventory=1` add each node's pool inventory (compute architectures, model-root disk, cached recipe weights, installed images and fabric links). `lloom pool plan` uses it to propose placements across the whole cluster; see [`pool-planning.md`](pool-planning.md). + Before sending model traffic, use `lloom cluster doctor --runtime ID` to verify the required nodes and serving endpoint. Whole-topology doctor reports optional offline nodes and missing memory telemetry as warnings; they do not make unrelated runtimes unavailable. Then start a small replicated lane, verify requests alternate under equal load, and only then canary the distributed engine. ## A note on example identifiers diff --git a/docs/pool-planning.md b/docs/pool-planning.md new file mode 100644 index 0000000..78ab5c7 --- /dev/null +++ b/docs/pool-planning.md @@ -0,0 +1,113 @@ +# Pool planning + +`lloom pool plan` treats every node in the cluster as one pool and proposes which model lanes run where, how each alias routes, and what would change from today. It is a dry run: it never starts, stops or reroutes anything, and there is no apply step yet. Act on a plan with `lloom fleet`, `lloom route`, `lloom keep-warm` and `lloom suspend`/`resume`. + +```bash +lloom pool plan # live status + 7-day demand from the gateway +lloom pool plan --cordon spark-node-01,spark-node-02 # what if the pair is taken? +lloom pool plan --profile standalone # keep a fleet profile's routes fixed +lloom pool plan --pin omp=qwen38-27b-nvfp4 # keep one alias fixed +lloom pool plan --consider-installs # also place bundled recipes not installed yet +lloom pool plan --offline --json # config + this machine only +``` + +The planner sits above the per-node admission planner (`runtime-policy.mjs`), and does not replace it. Admission still decides at request time whether a start fits in live memory. The pool plan decides which lanes should be resident and where. + +## Inputs + +**Nodes.** Each node comes from `GET /gateway/cluster?inventory=1`. `GET /gateway/node?inventory=1` adds the node's inventory, which is collected on request and cached for five minutes. The inventory contains: + +- structured compute architectures (`sm_121` for GB10, `apple-metal` for Apple Silicon); +- memory domains; +- free disk per model root; +- recipes whose weights are already complete on disk; +- installed commands and Docker images; +- direct fabric links to other nodes, from NVIDIA Sync discovery. + +A gateway that predates inventory still reports its machine profile, and the planner works from that, without weights or images. A node that does not answer is planned around as unavailable. Without a running gateway, the plan uses the config's node resources plus this machine's own inventory. + +**Variants (the catalog).** A variant is one way to serve a gateway model. Variants are grouped by underlying model, and come from four sources: + +- **Managed runtimes:** each runtime is fixed to its configured `node` or `placement`. Its memory per node is the configured estimate, raised to the measured resident size when that is larger. +- **Federated targets:** a node's proxy model with a `remoteRuntime`. +- **Unmanaged lanes:** declared with a `pool` block on the model, for example a hand-run TP2 server on the pair: + + ```json + { "id": "qwen3.8-flash-next-atlas", "backend": "atlas", "pool": { "nodes": ["spark-node-01", "spark-node-02"], "memoryGbPerNode": 110 } } + ``` + + LLooM cannot start or stop these, so the plan lists them as manual actions. +- **External models:** cloud and other models with no node. + +With `--consider-installs`, bundled recipes add variants that may go on any eligible node set. + +Each variant has structured requirements: node count, tensor-parallel degree, compute architectures, platform, backend, context window and scheduler sequences. Recipes can state these with `requirements.computeArchs`, `requirements.tensorParallel` and `models[].settings.maxSequences`. When they are absent, they are inferred from launch settings (`CUTE_DSL_ARCH`, `TORCH_CUDA_ARCH_LIST`, `--tensor-parallel-size`, `MAX_NUM_SEQS`) or from the accelerators and cluster node count. Performance comes from live gateway metrics when a model has at least five requests in the window, otherwise from the best matching benchmark result. + +**Workloads.** Every alias is a workload. Its candidates are its current members plus the members of all its route profiles, expanded through nested aliases, with suspended members left out. Demand is the alias's request rate over the last seven days, from the `requested` buckets in `/gateway/metrics`. A gateway that predates those buckets reports demand per resolved model only, so each model's demand is split evenly across the aliases routed to it. + +Residency is the larger of two values: + +- **Demand-based:** at least 1 request/hour means `always`; at least 0.125 means `preferred`; otherwise `auto`. +- **Current-target:** the residency of the runtime the alias routes to now. + +Override any of this per alias in config: + +```json +"pool": { + "cordon": ["spark-node-02"], + "pins": { "routes": { "omp": "qwen38-27b-nvfp4" } }, + "workloads": { + "embeddings": { "residency": "always" }, + "video": { "required": false, "slo": { "ttftMs": 2000, "contextWindow": 131072 } } + }, + "policy": { "headroomGb": 4, "churnWeight": 10, "external": "fallback" } +} +``` + +**Pins are hard constraints.** A pin is any of the following: + +- a runtime's `node` or `placement`; +- `keepWarm` (residency `always`); +- `pool.pins.routes` and `--pin`; +- the routes and `always` residency of a `--profile`. + +The currently active routes are not pins; they are what the diff is measured against. To keep a hand-chosen route, pin it. + +## Constraints and objective + +- **Memory per domain.** Unified memory on Spark and Apple Silicon. Device memory, plus system memory for host-side state, on discrete GPUs. + - The budget follows admission: `memoryBudgetGb`, else capacity × `maxMemoryUtilization`, else capacity minus `reserveMemoryGb`. Pool planning then subtracts `headroomGb` (default 4 GB), because exhausting unified memory hangs a Spark. + - `always` and `preferred` lanes must fit together. Every on-demand lane must fit next to the `always` set. + - A `preferred` lane that does not fit becomes on demand, with a warning. +- **Tensor parallel only across directly linked nodes.** Every pair in a multi-node set must share a reported fabric link. +- **Compute architecture, platform and accelerators** must match the node. +- **Cordoned and unavailable nodes** host nothing. + +Plans are compared in this order: + +1. Every required workload and pin covered. +2. Fewest workloads on external models, which are a fallback by default. `--external allow` lets them compete on latency; `never` excludes them. +3. Fewest latency SLO misses. +4. Lowest cost. Cost is demand × latency over a 24-hour horizon, plus each change's load time × `churnWeight`. + +Latency is time to first token plus decode time for 256 output tokens, or the measured request duration for media. Load time is estimated from the variant's memory, or taken from its `loadSeconds`. Missing weights add download time, a missing image adds pull time, and an unmanaged lane adds a fixed manual-action time. + +The search places workloads greedily, pinned and hot ones first, sharing a placement wherever several aliases can use the same lane. Local search then retries each workload with each alternative candidate and keeps any change that makes the whole plan better. For example, local search can move one alias to a smaller model so that another required alias still fits, or move all cognition aliases to a faster lane together once one of them pays its load cost. + +## Output + +The text output lists: + +- each node's memory budget and resident set; +- every placement by tier, with the aliases it serves; +- the changes from now: + - route changes; + - lanes to start; + - manual starts and stops for unmanaged lanes; + - lanes no workload needs any more (`release`); + - residency changes; +- uncovered workloads, with reasons. + +`--json` returns the full report. It includes the plan's profile-shaped `routes` and `residency`, `placements`, per-node memory, the `diff`, and the `score`. + +On a three-Spark fleet, the planner's fixture tests (`test/fixtures/pool/spark-fleet.json`) reproduce the hand-written fleet profiles. With every node present, it gives the `sparkqwen-pair-rc16` routes and residency. With the 01/02 pair cordoned or offline, it gives `standalone`: cognition moves to the 27B on spark-node-03, which becomes `always`, and the SparkQwen lane gets a manual stop. diff --git a/docs/recipes.md b/docs/recipes.md index f614043..2cdcde8 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -73,7 +73,7 @@ limits, permissions, tests, and measurement limitations. Recipes are JSON documents with these top-level sections: -- `requirements`: platform, memory, disk, and command prerequisites. +- `requirements`: platform, memory, disk, and command prerequisites. Optional `computeArchs` (for example `["sm_121"]`) and `tensorParallel` state what [pool planning](pool-planning.md) otherwise infers from launch settings. - `backend`: the runtime family being configured. - `setup.steps`: ordered checks, downloads, commands, and tuning operations. - `models`: role-to-artifact mappings plus gateway model/runtime IDs. @@ -89,7 +89,7 @@ Model entries should be specific enough for LLooM to create the local runtime wi - `backendConfig`: optional stable backend config ID when the recipe should not use an auto-generated ID. - `input` and `output`: modalities such as `text`, `image`, `video`, `audio`, `embedding`, or `scores`. - `capabilities`: the API contract the model supports, for example `responses`, `anthropic-messages`, `tools`, `reasoning`, `vision`, `mtp`, or `long-context`. Media capabilities select the model kind and therefore the serving route: `image-generation`/`image-editing` give `image`, `video-generation` gives `video`, `audio-generation`/`music-generation`/`audio-music-generation` give `audio_generation`, and `audio-speech`/`tts` give `audio_speech`. A music capability wins over a speech capability when a recipe declares both. -- `settings`: runtime knobs consumed during config materialization, including `contextWindow`, `maxOutputTokens`, `maxActiveRequests`, `profile`, `draftDepth`, `reasoning`, `preserveThinking`, `batchingPreset`, `memoryGb`, `startupTimeoutMs`, `priority`, `keepWarm`, and session-cache fields. `keepWarm` is the single hard residency pin and is valid only for a managed internal runtime. `priority` orders only unpinned eviction candidates; lower values are evicted first. +- `settings`: runtime knobs consumed during config materialization, including `contextWindow`, `maxOutputTokens`, `maxActiveRequests`, `maxSequences` (backend scheduler sequences, reported by pool planning), `profile`, `draftDepth`, `reasoning`, `preserveThinking`, `batchingPreset`, `memoryGb`, `startupTimeoutMs`, `priority`, `keepWarm`, and session-cache fields. `keepWarm` is the single hard residency pin and is valid only for a managed internal runtime. `priority` orders only unpinned eviction candidates; lower values are evicted first. - `settings.runtime`: optional explicit launcher data for recipes that need backend-specific command lines or bootstrap-only managed containers. LLooM templates `command`, `args`, `env`, `bootstrap`, `healthPath` or `healthUrl`, `warmup`, `watchdog`, and session-cache hints with variables such as `${modelRoot}`, `${modelPath}`, `${modelId}`, `${port}`, `${contextWindow}`, `${maxOutputTokens}`, `${maxActiveRequests}`, `${runtimeId}`, and `${sessionCacheDir}`. If it is absent, LLooM uses the built-in defaults for known backends. - `settings.placement`: optional cluster placement. `{"mode":"replicated"}` materializes one local target on every configured node (or the `nodes` subset) plus leader-side LLooM proxy targets for remote replicas; distributed logical runtimes use ordered delegated `members` with per-node resource estimates. See [`clusters.md`](clusters.md). - `setDefault`: optional explicit instruction to select that model as the installed default for its output modality. This also applies during additive recipe updates, so use it only when the recipe intentionally owns the default choice. diff --git a/examples/interchange/recipe.v1.json b/examples/interchange/recipe.v1.json index ead31f1..6f95167 100644 --- a/examples/interchange/recipe.v1.json +++ b/examples/interchange/recipe.v1.json @@ -32,7 +32,9 @@ "memoryGb": 48, "diskGb": 45, "commands": ["mtplx"], - "accelerators": ["apple-gpu"] + "accelerators": ["apple-gpu"], + "computeArchs": ["apple-metal"], + "tensorParallel": 1 }, "backend": { "id": "mtplx", @@ -66,6 +68,7 @@ "contextWindow": 262144, "maxOutputTokens": 32768, "draftDepth": 2, + "maxSequences": 8, "maxActiveRequests": 8, "interactiveReservedSlots": 1, "interactiveReservedQueueSlots": 1, diff --git a/package.json b/package.json index 1fe3596..329499b 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "check": "npm run check:cluster && npm run check:js && npm run check:entity-stagger && npm run check:python && npm run check:shell && npm run check:spark-deploy && npm run check:workers", "check:cluster": "node --check src/cluster.mjs && node --check test/cluster.test.mjs && node --check test/cluster-availability.test.mjs && node --check test/cluster-readiness.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs", "check:entity-stagger": "node --check scripts/entity-stagger-benchmark.mjs && node --check test/entity-stagger-benchmark.test.mjs", - "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/request-scheduler.mjs && node --check src/rate-limit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/deployment-fence.mjs && node --check test/deployment-fence.test.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/standby.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check test/spark-audio-recipes.test.mjs && node --check src/runtime-residency.mjs && node --check test/ltx-pipelines.test.mjs && node --check test/ltx-pipelines-recipe.test.mjs && node --check src/video-contracts.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs && node --check src/runtime-policy-config.mjs && node --check test/runtime-policy-config.test.mjs && node --check src/audio-providers.mjs && node --check src/protocol/openrouter-provider.mjs && node --check test/audio-providers.test.mjs && node --check test/openrouter-provider.test.mjs && node --check backends/atlas-sparkglm/verify-pins.mjs && node --check test/atlas-sparkglm-recipe.test.mjs && node --check src/head-preparation.mjs && node --check src/head-transfer.mjs && node --check src/service-control.mjs && node --check src/gateway-relay.mjs && node --check src/client-retarget.mjs && node --check src/head-promotion.mjs && node --check src/head-promotion-transfer.mjs && node --check src/gateway-client.mjs && node --check test/cli-diagnostics.test.mjs && node --check src/service-install.mjs && node --check test/service-install.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check src/node-release-agent.mjs && node --check src/node-gateway-adapter.mjs && node --check src/node-agent-cli.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs && node --check test/node-release-agent.test.mjs && node --check test/node-gateway-adapter.test.mjs && node --check test/node-gateway-adapter-e2e.test.mjs && node --check test/deployment-bundle.test.mjs && node --check test/node-agent-cli.test.mjs && node --check test/ssh-deployment-transport.test.mjs && node --check scripts/build-deployment-bundle.mjs", + "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/request-scheduler.mjs && node --check src/rate-limit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/deployment-fence.mjs && node --check test/deployment-fence.test.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/standby.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check test/spark-audio-recipes.test.mjs && node --check src/runtime-residency.mjs && node --check test/ltx-pipelines.test.mjs && node --check test/ltx-pipelines-recipe.test.mjs && node --check src/video-contracts.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs && node --check src/pool-planner.mjs && node --check src/pool-catalog.mjs && node --check src/pool-inventory.mjs && node --check src/pool-cli.mjs && node --check test/pool-planner.test.mjs && node --check test/pool-catalog.test.mjs && node --check test/pool-inventory.test.mjs && node --check test/pool-cli.test.mjs && node --check src/runtime-policy-config.mjs && node --check test/runtime-policy-config.test.mjs && node --check src/audio-providers.mjs && node --check src/protocol/openrouter-provider.mjs && node --check test/audio-providers.test.mjs && node --check test/openrouter-provider.test.mjs && node --check backends/atlas-sparkglm/verify-pins.mjs && node --check test/atlas-sparkglm-recipe.test.mjs && node --check src/head-preparation.mjs && node --check src/head-transfer.mjs && node --check src/service-control.mjs && node --check src/gateway-relay.mjs && node --check src/client-retarget.mjs && node --check src/head-promotion.mjs && node --check src/head-promotion-transfer.mjs && node --check src/gateway-client.mjs && node --check test/cli-diagnostics.test.mjs && node --check src/service-install.mjs && node --check test/service-install.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check src/node-release-agent.mjs && node --check src/node-gateway-adapter.mjs && node --check src/node-agent-cli.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs && node --check test/node-release-agent.test.mjs && node --check test/node-gateway-adapter.test.mjs && node --check test/node-gateway-adapter-e2e.test.mjs && node --check test/deployment-bundle.test.mjs && node --check test/node-agent-cli.test.mjs && node --check test/ssh-deployment-transport.test.mjs && node --check scripts/build-deployment-bundle.mjs", "check:python": "python3 -m py_compile backends/dspark-vllm/apply-patch-pack.py backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/vision_exp/*.py backends/qwen38-vllm/apply-ple-fp8-patch.py backends/qwen38-sglang/*.py backends/glm53-exl3/*.py backends/mlx-audio/lloom_audio_server.py backends/hear/lloom_hear_server.py backends/chatterbox/lloom_chatterbox_server.py patches/apply_mtplx_longctx_fix.py backends/comfyui-media/bridge/*.py backends/comfyui-media/graphs/*.py backends/comfyui-media/build/*.py backends/comfyui-media/install.py backends/qwen-image-diffusers/*.py backends/ace-step-diffusers/*.py backends/ace-step-diffusers/tests/*.py backends/hear/test/test_hear_security.py backends/spark-audio/install.py backends/spark-audio/lloom_audio_cuda_server.py backends/spark-audio/test_lloom_audio_cuda_server.py backends/ltx-pipelines/*.py src/darwin-memory-usage.py", "check:shell": "bash -n backends/dspark-vllm/entrypoint.sh backends/dspark-vllm/runtime/build-hardened-image.sh backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/apply-runtime.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.sh backends/qwen38-vllm/entrypoint.sh backends/qwen38-sglang/entrypoint.sh backends/glm53-exl3/entrypoint.sh backends/mlx-audio/install.sh backends/hear/install.sh backends/chatterbox/install.sh clients/examples/lloom-claude clients/examples/lloom-codex clients/examples/lloom-hermes clients/examples/lloom-opencode clients/examples/lloom-zero scripts/remote-install-spark.sh scripts/remote-stage-spark-worker.sh backends/atlas-sparkglm/install.sh backends/atlas-sparkglm/convert-overlay.sh", "check:spark-deploy": "node --check test/spark-deploy.test.mjs && node test/spark-deploy.test.mjs", @@ -73,7 +73,7 @@ "community:check": "node scripts/check-community-deploy.mjs", "package:check": "node scripts/check-package.mjs", "test:adoption": "node test/resource-fit.test.mjs && node test/model-acquisition.test.mjs && node test/dashboard-status.test.mjs", - "test:unit": "node --test test/deployment-fence.test.mjs test/cluster-deployment.test.mjs test/cluster-deployment-cli.test.mjs test/cluster-deployment-e2e.test.mjs test/node-release-agent.test.mjs test/node-gateway-adapter.test.mjs test/node-gateway-adapter-e2e.test.mjs test/deployment-bundle.test.mjs test/node-agent-cli.test.mjs test/ssh-deployment-transport.test.mjs && node test/runtime-health-ownership.test.mjs && node test/long-prefill-transport.test.mjs && node test/sparkglm-managed.test.mjs && node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/stream-metrics.test.mjs && node test/responses-codex.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/config-profiles.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-policy-config.test.mjs && node --test test/request-scheduler.test.mjs test/adaptive-priority.test.mjs test/rate-limit.test.mjs test/runtime-queue.test.mjs test/foreground-priority.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node --test test/standby.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && node test/hear-recipe.test.mjs && npm run test:adoption && node --test test/performance-sampler.test.mjs && node --test test/audio-generations.test.mjs && node test/video-providers.test.mjs && node --test test/web-functions.test.mjs && node --test test/config-mutation.test.mjs && node --test test/model-maintenance.test.mjs && node --test test/model-maintenance-control.test.mjs && node --test test/route-maintenance-concurrency.test.mjs && node --test test/model-maintenance-integration.test.mjs && npm run test:workers && node test/comfyui-media.test.mjs && node test/qwen-image-diffusers.test.mjs && node test/ace-step-diffusers.test.mjs && node test/runtime-residency.test.mjs && node test/spark-audio-recipes.test.mjs && node --test test/video-contracts.test.mjs && node test/ltx-pipelines-recipe.test.mjs && npm run test:ux && npm run test:memory-safety && node --test test/dashboard-memory.test.mjs test/runtime-memory-usage.test.mjs && node --test test/speech-stream.test.mjs && node --test test/audio-providers.test.mjs && node test/openrouter-provider.test.mjs && node test/atlas-sparkglm-recipe.test.mjs && node --test test/head-preparation.test.mjs test/head-transfer.test.mjs test/service-control.test.mjs && node --test test/gateway-relay.test.mjs test/client-retarget.test.mjs && node --test test/head-profile-acceptance.test.mjs test/head-promotion.test.mjs test/head-promotion-transfer.test.mjs test/head-transfer-acceptance.test.mjs && node --test test/cli-diagnostics.test.mjs && node --test test/service-install.test.mjs", + "test:unit": "node --test test/deployment-fence.test.mjs test/cluster-deployment.test.mjs test/cluster-deployment-cli.test.mjs test/cluster-deployment-e2e.test.mjs test/node-release-agent.test.mjs test/node-gateway-adapter.test.mjs test/node-gateway-adapter-e2e.test.mjs test/deployment-bundle.test.mjs test/node-agent-cli.test.mjs test/ssh-deployment-transport.test.mjs && node test/runtime-health-ownership.test.mjs && node test/long-prefill-transport.test.mjs && node test/sparkglm-managed.test.mjs && node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/stream-metrics.test.mjs && node test/responses-codex.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/config-profiles.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-policy-config.test.mjs && node --test test/pool-planner.test.mjs test/pool-catalog.test.mjs test/pool-inventory.test.mjs test/pool-cli.test.mjs && node --test test/request-scheduler.test.mjs test/adaptive-priority.test.mjs test/rate-limit.test.mjs test/runtime-queue.test.mjs test/foreground-priority.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node --test test/standby.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && node test/hear-recipe.test.mjs && npm run test:adoption && node --test test/performance-sampler.test.mjs && node --test test/audio-generations.test.mjs && node test/video-providers.test.mjs && node --test test/web-functions.test.mjs && node --test test/config-mutation.test.mjs && node --test test/model-maintenance.test.mjs && node --test test/model-maintenance-control.test.mjs && node --test test/route-maintenance-concurrency.test.mjs && node --test test/model-maintenance-integration.test.mjs && npm run test:workers && node test/comfyui-media.test.mjs && node test/qwen-image-diffusers.test.mjs && node test/ace-step-diffusers.test.mjs && node test/runtime-residency.test.mjs && node test/spark-audio-recipes.test.mjs && node --test test/video-contracts.test.mjs && node test/ltx-pipelines-recipe.test.mjs && npm run test:ux && npm run test:memory-safety && node --test test/dashboard-memory.test.mjs test/runtime-memory-usage.test.mjs && node --test test/speech-stream.test.mjs && node --test test/audio-providers.test.mjs && node test/openrouter-provider.test.mjs && node test/atlas-sparkglm-recipe.test.mjs && node --test test/head-preparation.test.mjs test/head-transfer.test.mjs test/service-control.test.mjs && node --test test/gateway-relay.test.mjs test/client-retarget.test.mjs && node --test test/head-profile-acceptance.test.mjs test/head-promotion.test.mjs test/head-promotion-transfer.test.mjs test/head-transfer-acceptance.test.mjs && node --test test/cli-diagnostics.test.mjs && node --test test/service-install.test.mjs", "test:cluster": "node test/cluster.test.mjs && node --test test/cluster-availability.test.mjs test/cluster-readiness.test.mjs", "test": "npm run test:cluster && npm run test:unit && npm run test:entity-stagger && npm run smoke && npm run community:check", "test:entity-stagger": "node test/entity-stagger-benchmark.test.mjs", diff --git a/schemas/recipe.v1.schema.json b/schemas/recipe.v1.schema.json index cbdf183..06b97ad 100644 --- a/schemas/recipe.v1.schema.json +++ b/schemas/recipe.v1.schema.json @@ -125,6 +125,20 @@ "type": "string" }, "uniqueItems": true + }, + "computeArchs": { + "type": "array", + "description": "Compute architectures the lane's kernels are built for, such as sm_121 or apple-metal. Pool planning only places the lane on nodes that report one of them; when absent, it is inferred from launch settings or accelerators.", + "items": { + "type": "string", + "pattern": "^(sm_[0-9]+[af]?|apple-metal)$" + }, + "uniqueItems": true + }, + "tensorParallel": { + "type": "integer", + "minimum": 1, + "description": "Tensor-parallel degree across the lane's GPUs. Multi-node lanes are only placed on directly linked nodes; when absent, it is inferred from launch arguments or the cluster node count." } } }, @@ -344,6 +358,11 @@ "type": "integer", "minimum": 0 }, + "maxSequences": { + "type": "integer", + "minimum": 1, + "description": "Concurrent sequences the backend scheduler is configured for. Pool planning reports it with the lane; when absent, it is inferred from launch arguments or maxActiveRequests." + }, "contextWindow": { "type": "integer", "minimum": 1 diff --git a/src/cluster.mjs b/src/cluster.mjs index 8038946..dd5bb45 100644 --- a/src/cluster.mjs +++ b/src/cluster.mjs @@ -978,7 +978,8 @@ export class ClusterCoordinator { logger = console, telemetry = null, profile = null, - models = null + models = null, + inventory = null } = {} ) { this.config = config; @@ -988,6 +989,7 @@ export class ClusterCoordinator { this.telemetry = telemetry; this.profile = profile; this.models = models; + this.inventory = inventory; this.runtimeManager = null; this.targetCursor = new Map(); this.targetLoads = new Map(); @@ -1062,9 +1064,15 @@ export class ClusterCoordinator { } } - async localNodeStatus({ runtimeStatus = null, includeMemoryUsage = false } = {}) { + async localNodeStatus({ runtimeStatus = null, includeMemoryUsage = false, includeInventory = false } = {}) { const profile = typeof this.profile === 'function' ? await this.profile() : this.profile; const models = typeof this.models === 'function' ? await this.models() : this.models; + const inventory = + includeInventory && this.inventory + ? typeof this.inventory === 'function' + ? await this.inventory() + : this.inventory + : undefined; return { id: this.nodeId, name: this.nodes[this.nodeId]?.name ?? this.nodeId, @@ -1086,20 +1094,22 @@ export class ClusterCoordinator { runtimeStatus ?? (this.runtimeManager ? await this.runtimeManager.status({ localOnly: true, includeMemoryUsage }) - : { runtimes: {} }) + : { runtimes: {} }), + ...(inventory ? { inventory } : {}) }; } - async nodeStatus(nodeId, { refresh = false } = {}) { - if (this.isLocalNode(nodeId)) return this.localNodeStatus(); + async nodeStatus(nodeId, { refresh = false, inventory = false } = {}) { + if (this.isLocalNode(nodeId)) return this.localNodeStatus({ includeInventory: inventory }); const node = this.nodes[nodeId]; const ttlMs = Math.max(0, Number(this.config.cluster?.statusCacheMs ?? 1000)); - const cached = this.nodeCache.get(nodeId); + const cacheKey = inventory ? `${nodeId}\u0000inventory` : nodeId; + const cached = this.nodeCache.get(cacheKey); if (!refresh && cached && Date.now() - cached.at < ttlMs) return cached.value; if (!refresh && cached?.pending) return cached.pending; const pending = (async () => { try { - const result = await this.requestNode(nodeId, '/gateway/node?memoryUsage=1'); + const result = await this.requestNode(nodeId, `/gateway/node?memoryUsage=1${inventory ? '&inventory=1' : ''}`); const claimedNodeId = result?.node?.id ?? null; if (claimedNodeId !== nodeId) { return { @@ -1155,19 +1165,19 @@ export class ClusterCoordinator { }; } })(); - this.nodeCache.set(nodeId, { at: cached?.at ?? 0, value: cached?.value, pending }); + this.nodeCache.set(cacheKey, { at: cached?.at ?? 0, value: cached?.value, pending }); const value = await pending; - this.nodeCache.set(nodeId, { at: Date.now(), value, pending: null }); + this.nodeCache.set(cacheKey, { at: Date.now(), value, pending: null }); return value; } - async status({ localRuntimeStatus = null } = {}) { + async status({ localRuntimeStatus = null, inventory = false } = {}) { const entries = await Promise.all( Object.keys(this.nodes).map(async (id) => [ id, this.isLocalNode(id) && localRuntimeStatus - ? await this.localNodeStatus({ runtimeStatus: localRuntimeStatus }) - : await this.nodeStatus(id) + ? await this.localNodeStatus({ runtimeStatus: localRuntimeStatus, includeInventory: inventory }) + : await this.nodeStatus(id, { inventory }) ]) ); const scopes = Object.keys(this.config.runtimes ?? {}) @@ -1336,6 +1346,7 @@ export class ClusterCoordinator { signal }); this.nodeCache.delete(nodeId); + this.nodeCache.delete(`${nodeId}\u0000inventory`); return result; } diff --git a/src/pool-catalog.mjs b/src/pool-catalog.mjs new file mode 100644 index 0000000..291df8d --- /dev/null +++ b/src/pool-catalog.mjs @@ -0,0 +1,538 @@ +// Pool catalog and planner inputs. Turns recipes, the installed config, node +// status and gateway metrics into the plain data pool-planner.mjs consumes: +// a model-centric variant catalog, node inventories, workloads, pins and the +// current state. Read-only: nothing here contacts nodes or changes config. +import { aliasRoutableMemberIds, expandedAliasMemberIds } from './alias-resolution.mjs'; +import { benchmarkRankingEligible, benchmarkScore } from './benchmarks.mjs'; +import { clusterNodes, modelTargets, runtimeResourcesByNode } from './cluster.mjs'; +import { resolveRouteTarget } from './config-profiles.mjs'; +import { inventoryFromProfile } from './pool-inventory.mjs'; +import { residencyPolicy } from './runtime-preferences.mjs'; + +export const POOL_CATALOG_VERSION = 1; +const LIVE_METRIC_MIN_REQUESTS = 5; +const PERIOD_HOURS = { today: 24, '7d': 168, '30d': 720 }; +const RESIDENCY_RANK = { auto: 0, preferred: 1, always: 2 }; +const ARCH_BY_ACCELERATOR = { gb10: 'sm_121', 'dgx-spark': 'sm_121' }; + +function object(value) { + return value && typeof value === 'object' && !Array.isArray(value) ? value : {}; +} + +function num(value) { + if (value == null || value === '') return null; + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +function round(value, places = 1) { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +function unique(values) { + return [...new Set(values.filter((value) => value != null && value !== ''))]; +} + +function compact(value) { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry != null)); +} + +function archFromCapability(value) { + const match = String(value).match(/(\d+)\.(\d+)/); + return match ? `sm_${match[1]}${match[2]}` : null; +} + +function normalizeArch(value) { + return String(value) + .toLowerCase() + .replace(/^(sm_\d+)[af]$/, '$1'); +} + +// Launch settings carry facts the recipe does not yet state structurally: +// container images, tensor-parallel size, scheduler sequences and target arch. +function launchSignals(settings) { + const signals = { images: [], env: {}, flags: {} }; + const visit = (value, key) => { + if (typeof value === 'string') { + if (key === 'image') signals.images.push(value); + const env = value.match(/^([A-Z][A-Z0-9_]*)=(.*)$/); + if (env) signals.env[env[1]] ??= env[2]; + return; + } + if (Array.isArray(value)) { + value.forEach((item, index) => { + if (typeof item === 'string' && item.startsWith('--') && typeof value[index + 1] === 'string') { + signals.flags[item] ??= value[index + 1]; + } + visit(item); + }); + return; + } + if (value && typeof value === 'object') { + for (const [childKey, child] of Object.entries(value)) visit(child, childKey); + } + }; + visit(settings); + return signals; +} + +export function recipeServingRequirements(recipe, model = {}) { + const requirements = object(recipe?.requirements); + const settings = object(model.settings); + const signals = launchSignals(settings); + const inferred = []; + const infer = (field, value) => { + if (value != null && !(Array.isArray(value) && !value.length)) inferred.push(field); + return value; + }; + const nodeCount = Math.max(1, Math.floor(num(requirements.cluster?.nodes ?? requirements.cluster?.minNodes) ?? 1)); + const explicitArchs = Array.isArray(requirements.computeArchs) ? requirements.computeArchs : null; + const signalArchs = unique([ + signals.env.CUTE_DSL_ARCH ? normalizeArch(signals.env.CUTE_DSL_ARCH) : null, + ...String(signals.env.TORCH_CUDA_ARCH_LIST ?? '') + .split(/[;,\s]+/) + .map(archFromCapability) + ]); + const acceleratorArchs = unique((requirements.accelerators ?? []).map((name) => ARCH_BY_ACCELERATOR[name])); + const explicitTp = num(requirements.tensorParallel ?? settings.tensorParallel); + const signalTp = num( + signals.flags['--tensor-parallel-size'] ?? signals.env.TENSOR_PARALLEL_SIZE ?? signals.env.TP_SIZE + ); + const explicitSequences = num(settings.maxSequences); + const signalSequences = num(signals.flags['--max-num-seqs'] ?? signals.env.MAX_NUM_SEQS); + const placementMemory = Math.max( + 0, + ...(settings.placement?.members ?? []).map((member) => num(member?.resources?.memoryGb) ?? 0) + ); + return { + recipeId: recipe?.id ?? null, + nodeCount, + tensorParallel: explicitTp ?? infer('tensorParallel', signalTp ?? nodeCount), + computeArchs: explicitArchs + ? explicitArchs.map(normalizeArch) + : (infer('computeArchs', signalArchs.length ? signalArchs : acceleratorArchs) ?? []), + platforms: requirements.platforms ?? [], + accelerators: requirements.accelerators ?? [], + backend: recipe?.backend?.id ?? null, + contextWindow: num(settings.contextWindow ?? model.contextWindow), + maxSequences: explicitSequences ?? infer('maxSequences', signalSequences ?? num(settings.maxActiveRequests)), + memoryGbPerNode: num(settings.memoryGb) ?? (placementMemory || num(requirements.memoryGb)), + diskGb: num(requirements.diskGb), + images: unique(signals.images).sort(), + inferred + }; +} + +// /gateway/metrics returns id-keyed bucket arrays; accept a plain map too. +function metricBucket(buckets, id) { + if (Array.isArray(buckets)) return buckets.find((bucket) => bucket?.id === id) ?? null; + return object(buckets)[id] ?? null; +} + +function livePerformance(metrics, gatewayModel) { + const bucket = metricBucket(metrics?.models, gatewayModel); + if (!bucket || (num(bucket.requests) ?? 0) < LIVE_METRIC_MIN_REQUESTS) return null; + return compact({ + ttftMs: num(bucket.avgFirstContentMs), + decodeTps: num(bucket.decodeTokensPerSecond) ?? num(bucket.activeDecodeTokensPerSecond), + durationMs: num(bucket.avgDurationMs) || null, + source: 'live-metrics' + }); +} + +function benchmarkPerformance(benchmarks, { gatewayModel, recipeId, model }) { + const best = benchmarks + .filter((result) => benchmarkRankingEligible(result)) + .filter( + (result) => + (result.gatewayModel && result.gatewayModel === gatewayModel) || + (recipeId && result.recipeId === recipeId && (!model || result.model === model)) + ) + .sort((left, right) => benchmarkScore(right) - benchmarkScore(left))[0]; + if (!best) return null; + const metrics = object(best.metrics); + return compact({ + ttftMs: num(metrics.timeToFirstTokenMs), + decodeTps: num(metrics.generationTokPerSec), + durationMs: num(metrics.generationSeconds) != null ? num(metrics.generationSeconds) * 1000 : null, + source: `benchmark:${best.id}` + }); +} + +function measuredMemoryGb(nodeStatuses, nodes, runtimeIds) { + let largest = 0; + for (const nodeId of nodes) { + const runtimes = object(nodeStatuses?.[nodeId]?.runtimeManager?.runtimes); + for (const runtimeId of runtimeIds) { + const bytes = num(runtimes[runtimeId]?.memoryUsage?.residentBytes); + if (bytes) largest = Math.max(largest, bytes / 1024 ** 3); + } + } + return largest ? round(largest) : null; +} + +function memberRuntimeIds(runtime, runtimeId) { + const members = runtime?.placement?.mode === 'distributed' ? (runtime.placement.members ?? []) : []; + return members.length ? members.map((member) => member.runtime).filter(Boolean) : [runtimeId]; +} + +function configVariants(config, { nodeStatuses, env }) { + const variants = []; + for (const model of config.models ?? []) { + if (!model?.id) continue; + const base = { + gatewayModel: model.id, + model: model.pool?.model ?? model.upstreamModel ?? model.id, + kind: model.kind ?? 'chat', + capabilities: model.capabilities ?? [], + source: 'config' + }; + if (model.pool?.nodes?.length) { + const nodes = [...model.pool.nodes].sort(); + variants.push({ + ...base, + id: `${model.id}@${nodes.join('+')}`, + managed: model.pool.managed === true, + nodes, + requirements: compact({ + nodeCount: nodes.length, + memoryGbPerNode: num(model.pool.memoryGbPerNode), + contextWindow: num(model.contextWindow) + }) + }); + continue; + } + const runtime = model.runtime ? config.runtimes?.[model.runtime] : null; + if (model.runtime) { + if (!runtime || runtime.enabled === false) continue; + const resources = runtimeResourcesByNode(runtime, config, env); + const nodes = Object.keys(resources).sort(); + if (!nodes.length) continue; + const configuredGb = Math.max(0, ...nodes.map((nodeId) => resources[nodeId].memoryGb)); + const measuredGb = measuredMemoryGb(nodeStatuses, nodes, memberRuntimeIds(runtime, model.runtime)); + variants.push({ + ...base, + id: `${model.id}@${nodes.join('+')}`, + runtime: model.runtime, + managed: runtime.managed !== false && runtime.management !== 'external', + nodes, + requirements: compact({ + nodeCount: nodes.length, + memoryGbPerNode: Math.max(configuredGb, measuredGb ?? 0) || null, + contextWindow: num(model.contextWindow ?? runtime.contextWindow), + maxSequences: num(runtime.maxActiveRequests), + recipeId: runtime.recipe?.id ?? null, + images: runtime.bootstrap?.image ? [runtime.bootstrap.image] : null + }), + ...(measuredGb ? { measured: { memoryGbPerNode: measuredGb, source: 'live-rss' } } : {}) + }); + continue; + } + const federated = modelTargets(model).filter((target) => target.node && target.remoteRuntime); + if (federated.length) { + for (const target of federated) { + const remote = object(nodeStatuses?.[target.node]?.runtimeManager?.runtimes?.[target.remoteRuntime]); + const members = remote.placement?.mode === 'distributed' ? (remote.placement.members ?? []) : []; + const nodes = unique(members.length ? members.map((member) => member.node) : [target.node]).sort(); + const memberMemory = members.map((member) => num(member.resources?.memoryGb) ?? 0); + variants.push({ + ...base, + id: `${model.id}@${nodes.join('+')}`, + remoteRuntime: target.remoteRuntime, + controlNode: target.node, + managed: true, + nodes, + requirements: compact({ + nodeCount: nodes.length, + memoryGbPerNode: Math.max(num(remote.memoryGb) ?? 0, ...memberMemory) || null, + contextWindow: num(model.contextWindow) + }) + }); + } + continue; + } + variants.push({ ...base, id: model.id, external: true, managed: false, requirements: {} }); + } + return variants; +} + +function recipeVariants(recipes) { + return recipes.flatMap((recipe) => + (recipe.models ?? []).map((model) => { + const gatewayModel = model.gatewayModel ?? model.model; + return { + id: `recipe:${recipe.id}:${gatewayModel}`, + gatewayModel, + model: model.model, + kind: model.kind ?? (model.capabilities?.includes('chat') ? 'chat' : (recipe.kind ?? 'chat')), + capabilities: model.capabilities ?? recipe.capabilities ?? [], + source: 'recipe', + managed: true, + requirements: recipeServingRequirements(recipe, model) + }; + }) + ); +} + +export function buildPoolCatalog({ + config = {}, + recipes = [], + benchmarks = [], + metrics = null, + nodeStatuses = {}, + env = process.env +} = {}) { + const variants = [...configVariants(config, { nodeStatuses, env }), ...recipeVariants(recipes)] + .map((variant) => { + const performance = + livePerformance(metrics, variant.gatewayModel) ?? + benchmarkPerformance(benchmarks, { + gatewayModel: variant.gatewayModel, + recipeId: variant.requirements?.recipeId, + model: variant.source === 'recipe' ? variant.model : null + }); + return performance ? { ...variant, performance } : variant; + }) + .sort((left, right) => left.id.localeCompare(right.id)); + const groups = new Map(); + for (const variant of variants) { + if (!groups.has(variant.model)) { + groups.set(variant.model, { + model: variant.model, + kind: variant.kind, + capabilities: variant.capabilities, + variants: [] + }); + } + groups.get(variant.model).variants.push(variant); + } + return { + version: POOL_CATALOG_VERSION, + variants, + models: [...groups.values()].sort((left, right) => left.model.localeCompare(right.model)) + }; +} + +function aliasLeaves(config, ids) { + const aliases = object(config.aliases); + const modelIds = new Set((config.models ?? []).map((model) => model.id)); + return unique( + ids.flatMap((id) => { + if (!Object.hasOwn(aliases, id)) return modelIds.has(id) ? [id] : []; + try { + return expandedAliasMemberIds(id, aliases, modelIds).filter((leaf) => modelIds.has(leaf)); + } catch { + return []; + } + }) + ); +} + +function currentRouteModel(config, alias) { + return aliasLeaves(config, aliasRoutableMemberIds(alias))[0] ?? null; +} + +function maxResidency(left, right) { + return RESIDENCY_RANK[right] > RESIDENCY_RANK[left] ? right : left; +} + +export function poolWorkloadsFromConfig(config = {}, { metrics = null, periodHours } = {}) { + const pool = object(config.pool); + const policy = object(pool.policy); + const hours = num(periodHours) ?? PERIOD_HOURS[metrics?.period] ?? null; + const alwaysAt = num(policy.alwaysAtRequestsPerHour) ?? 1; + const preferredAt = num(policy.preferredAtRequestsPerHour) ?? 0.125; + // Older gateways only aggregate by resolved model; split each model's demand + // evenly across the aliases currently routed to it. + const perAlias = Array.isArray(metrics?.requested); + const routedTo = new Map(); + for (const alias of Object.values(object(config.aliases))) { + const model = currentRouteModel(config, alias); + if (model) routedTo.set(model, (routedTo.get(model) ?? 0) + 1); + } + const requestsFor = (aliasId, alias) => { + if (perAlias) return num(metricBucket(metrics.requested, aliasId)?.requests) ?? 0; + const model = currentRouteModel(config, alias); + // A model with no bucket in the window had no traffic. + return model ? (num(metricBucket(metrics.models, model)?.requests) ?? 0) / routedTo.get(model) : 0; + }; + const workloads = []; + for (const [aliasId, alias] of Object.entries(object(config.aliases))) { + const profileMembers = Object.values(object(alias?.routeProfiles)).flatMap((profile) => + Array.isArray(profile?.members) ? profile.members : [] + ); + const candidates = aliasLeaves(config, [...aliasRoutableMemberIds(alias), ...profileMembers]); + if (!candidates.length) continue; + const override = object(object(pool.workloads)[aliasId]); + const requests = metrics ? requestsFor(aliasId, alias) : null; + const demandPerHour = hours && requests != null ? round(requests / hours, 3) : null; + const demandResidency = + demandPerHour == null + ? 'auto' + : demandPerHour >= alwaysAt + ? 'always' + : demandPerHour >= preferredAt + ? 'preferred' + : 'auto'; + const target = (config.models ?? []).find((model) => model.id === currentRouteModel(config, alias)); + const targetRuntime = target?.runtime ? config.runtimes?.[target.runtime] : null; + const currentResidency = targetRuntime ? residencyPolicy(targetRuntime) : 'auto'; + workloads.push( + compact({ + id: aliasId, + candidates, + residency: + RESIDENCY_RANK[override.residency] != null + ? override.residency + : maxResidency(demandResidency, currentResidency), + demandPerHour, + required: override.required === false ? false : true, + outputTokens: num(override.outputTokens), + slo: override.slo && typeof override.slo === 'object' ? override.slo : null + }) + ); + } + return workloads.sort((left, right) => left.id.localeCompare(right.id)); +} + +function pinnedModel(config, aliasId, target) { + const alias = object(config.aliases)[aliasId]; + if (!alias) return target; + try { + return aliasLeaves(config, resolveRouteTarget(alias, target).members)[0] ?? target; + } catch { + return target; + } +} + +export function poolPinsFromConfig(config = {}, { profile = null, cordon = [], routes = {} } = {}) { + const pool = object(config.pool); + const pinned = { ...object(object(pool.pins).routes), ...object(profile?.routes), ...object(routes) }; + const residency = {}; + for (const [runtimeId, runtime] of Object.entries(object(config.runtimes))) { + if (runtime?.enabled === false) continue; + if (residencyPolicy(runtime) === 'always') residency[runtimeId] = 'always'; + } + for (const [runtimeId, policy] of Object.entries(object(profile?.residency))) { + if (policy === 'always' && config.runtimes?.[runtimeId]) residency[runtimeId] = 'always'; + } + return { + routes: Object.fromEntries( + Object.entries(pinned).map(([aliasId, target]) => [aliasId, pinnedModel(config, aliasId, target)]) + ), + residency, + cordon: unique([...(pool.cordon ?? []), ...cordon]).sort() + }; +} + +// `external` is a healthy process LLooM did not start; it is serving all the same. +const SERVING_STATUSES = new Set(['running', 'external']); + +function runtimeRunning(nodeStatuses, nodeId, runtimeId) { + return SERVING_STATUSES.has(nodeStatuses?.[nodeId]?.runtimeManager?.runtimes?.[runtimeId]?.status); +} + +export function currentPoolState(config = {}, { nodeStatuses = {}, variants = [] } = {}) { + const routes = {}; + for (const [aliasId, alias] of Object.entries(object(config.aliases))) { + const model = currentRouteModel(config, alias); + if (model) routes[aliasId] = model; + } + const routed = new Set(Object.values(routes)); + const residency = {}; + const running = []; + for (const variant of variants) { + if (variant.external || variant.source === 'recipe') continue; + if (variant.runtime && variant.managed) { + const runtime = config.runtimes?.[variant.runtime]; + residency[variant.runtime] = residencyPolicy(runtime); + const members = runtime?.placement?.mode === 'distributed' ? (runtime.placement.members ?? []) : []; + const isRunning = members.length + ? members.every((member) => runtimeRunning(nodeStatuses, member.node, member.runtime)) + : variant.nodes.some((nodeId) => runtimeRunning(nodeStatuses, nodeId, variant.runtime)); + if (isRunning) running.push(variant.id); + } else if (variant.remoteRuntime) { + if (runtimeRunning(nodeStatuses, variant.controlNode, variant.remoteRuntime)) running.push(variant.id); + } else if (!variant.managed && routed.has(variant.gatewayModel)) { + // An unmanaged lane that traffic is routed to is assumed to be up. + running.push(variant.id); + } + } + return { routes, residency, running: running.sort() }; +} + +function nodeLinks(config, nodeId, inventory) { + const discovered = (config.cluster?.discovery?.links ?? []).flatMap((link) => + link?.nodeId === nodeId ? [link.peerNodeId] : link?.peerNodeId === nodeId ? [link.nodeId] : [] + ); + return unique([...(inventory?.links ?? []).map((link) => link.peerNodeId), ...discovered]).sort(); +} + +export function plannerNodesFromStatus(config = {}, nodeStatuses = {}, { env = process.env } = {}) { + return Object.entries(clusterNodes(config, env)) + .map(([nodeId, node]) => { + const status = nodeStatuses?.[nodeId] ?? null; + const resources = object(node.resources); + const inventory = status?.inventory ?? (status?.profile ? inventoryFromProfile(status.profile) : null); + const configuredGb = num(resources.memoryGb); + const memoryDomains = inventory?.memoryDomains?.length + ? inventory.memoryDomains + : configuredGb + ? [{ id: 'memory', kind: 'unified', capacityGb: configuredGb }] + : []; + const accelerators = status?.profile?.accelerators ?? resources.accelerators ?? []; + const computeArchs = inventory?.computeArchs?.length + ? inventory.computeArchs + : unique(accelerators.map((name) => ARCH_BY_ACCELERATOR[name])); + const budgetGb = num(resources.memoryBudgetGb); + return compact({ + id: nodeId, + local: node.local === true, + available: status ? status.reachable !== false : true, + platformId: status?.profile?.platformId ?? null, + computeArchs, + accelerators, + memoryDomains: + budgetGb != null && memoryDomains.length === 1 ? [{ ...memoryDomains[0], budgetGb }] : memoryDomains, + maxMemoryUtilization: num(resources.maxMemoryUtilization), + reserveGb: num(resources.reserveMemoryGb), + links: nodeLinks(config, nodeId, inventory), + weights: (inventory?.weights ?? []).filter((entry) => entry.complete).map((entry) => entry.recipeId), + images: inventory?.backends?.dockerImages ?? [], + inventory: inventory ? (inventory.complete ? 'reported' : 'profile') : 'config' + }); + }) + .sort((left, right) => left.id.localeCompare(right.id)); +} + +export function buildPoolPlanInputs({ + config = {}, + nodeStatuses = {}, + metrics = null, + recipes = [], + benchmarks = [], + profile = null, + cordon = [], + routes = {}, + considerInstalls = false, + policy = {}, + env = process.env +} = {}) { + const catalog = buildPoolCatalog({ config, recipes, benchmarks, metrics, nodeStatuses, env }); + const variants = considerInstalls ? catalog.variants : catalog.variants.filter((v) => v.source !== 'recipe'); + return { + nodes: plannerNodesFromStatus(config, nodeStatuses, { env }), + variants, + workloads: poolWorkloadsFromConfig(config, { metrics }), + pins: poolPinsFromConfig(config, { profile, cordon, routes }), + current: currentPoolState(config, { nodeStatuses, variants }), + policy: compact({ + maxMemoryUtilization: num(config.runtimePolicy?.maxMemoryUtilization), + reserveGb: num(config.runtimePolicy?.reserveMemoryGb), + ...object(config.pool?.policy), + ...policy + }), + catalog + }; +} diff --git a/src/pool-cli.mjs b/src/pool-cli.mjs new file mode 100644 index 0000000..854f82e --- /dev/null +++ b/src/pool-cli.mjs @@ -0,0 +1,224 @@ +// `lloom pool plan`: a read-only pool placement plan for the installed config. +// Live node status, inventory and 7-day metrics come from the running gateway; +// without one, the plan uses the config plus this machine's own inventory. +import { loadBenchmarkEvidence } from './benchmarks.mjs'; +import { currentNodeId } from './cluster.mjs'; +import { readProfile } from './config-profiles.mjs'; +import { profileMachine } from './machine-profile.mjs'; +import { modelRootCandidates } from './model-files.mjs'; +import { buildPoolPlanInputs } from './pool-catalog.mjs'; +import { collectNodeInventory } from './pool-inventory.mjs'; +import { planPool } from './pool-planner.mjs'; +import { loadRecipes } from './recipes.mjs'; + +export const POOL_USAGE = + 'Usage: lloom pool plan [--cordon NODE[,NODE]] [--pin ALIAS=MODEL] [--profile NAME] [--consider-installs] ' + + '[--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json]'; + +const METRICS_PERIOD = '7d'; + +function parsePins(values = []) { + const routes = {}; + for (const value of values) { + const index = String(value).indexOf('='); + if (index < 1 || index === value.length - 1) throw new Error(`--pin expects ALIAS=MODEL, got ${value}`); + routes[value.slice(0, index)] = value.slice(index + 1); + } + return routes; +} + +function numberOption(value, name) { + if (value == null) return undefined; + const number = Number(value); + if (!Number.isFinite(number) || number < 0) throw new Error(`${name} must be a non-negative number`); + return number; +} + +export function poolPlanOptions({ + cordon = [], + pins = [], + profile, + considerInstalls, + offline, + headroomGb, + churnWeight, + external +} = {}) { + if (external != null && !['fallback', 'allow', 'never'].includes(external)) { + throw new Error('--external must be fallback, allow, or never'); + } + return { + cordon: cordon + .flatMap((value) => String(value).split(',')) + .map((value) => value.trim()) + .filter(Boolean), + routes: parsePins(pins), + profileName: profile ?? null, + considerInstalls: considerInstalls === true, + offline: offline === true, + policy: Object.fromEntries( + Object.entries({ + headroomGb: numberOption(headroomGb, '--headroom-gb'), + churnWeight: numberOption(churnWeight, '--churn-weight'), + external + }).filter(([, value]) => value !== undefined) + ) + }; +} + +async function liveState(config, gatewayRequest) { + const cluster = await gatewayRequest(config, '/gateway/cluster?inventory=1'); + const metrics = await gatewayRequest(config, `/gateway/metrics?period=${METRICS_PERIOD}`).catch(() => null); + return { + nodeStatuses: cluster?.cluster?.nodes ?? {}, + metrics: metrics ? { ...metrics, period: METRICS_PERIOD } : null + }; +} + +async function localState(config, deps) { + const profile = await deps.profileMachine(); + const inventory = await deps.collectNodeInventory({ + nodeId: currentNodeId(config), + profile, + recipes: await deps.loadRecipes(), + modelRoots: await deps.modelRootCandidates({ config }) + }); + return { nodeStatuses: { [currentNodeId(config)]: { reachable: true, profile, inventory } }, metrics: null }; +} + +export async function createPoolPlanReport(config, options = {}, overrides = {}) { + const deps = { + gatewayRequest: null, + gatewayUrl: null, + profileMachine, + collectNodeInventory, + loadRecipes, + loadBenchmarkEvidence, + modelRootCandidates, + readProfile, + ...overrides + }; + let state = null; + let source = { mode: 'config', gateway: deps.gatewayUrl, metricsPeriod: null, note: null }; + if (!options.offline && deps.gatewayRequest) { + try { + state = await liveState(config, deps.gatewayRequest); + source = { ...source, mode: 'gateway', metricsPeriod: state.metrics ? METRICS_PERIOD : null }; + } catch (error) { + source.note = `gateway unavailable (${error?.message ?? error}); planned from config and this machine only`; + } + } + if (!state) { + state = await localState(config, deps); + source.note ??= 'offline: no running state is known, so every resident lane is listed as a start'; + } + const profile = options.profileName ? await deps.readProfile(config, options.profileName) : null; + const inputs = buildPoolPlanInputs({ + config, + nodeStatuses: state.nodeStatuses, + metrics: state.metrics, + recipes: await deps.loadRecipes(), + benchmarks: await deps.loadBenchmarkEvidence(), + profile, + cordon: options.cordon ?? [], + routes: options.routes ?? {}, + considerInstalls: options.considerInstalls === true, + policy: options.policy ?? {} + }); + const plan = planPool(inputs); + return { + ok: plan.ok, + dryRun: true, + source, + nodes: inputs.nodes.map((node) => ({ + id: node.id, + available: node.available, + cordoned: inputs.pins.cordon.includes(node.id), + inventory: node.inventory, + computeArchs: node.computeArchs + })), + counts: { workloads: inputs.workloads.length, variants: inputs.variants.length }, + pins: inputs.pins, + ...(profile ? { profile: profile.name } : {}), + plan + }; +} + +function gb(value) { + return `${Number(value ?? 0).toFixed(1)} GB`; +} + +function workloadList(workloads) { + if (!workloads.length) return 'pinned'; + const shown = workloads.slice(0, 4).join(', '); + return workloads.length > 4 ? `${shown} (+${workloads.length - 4})` : shown; +} + +export function formatPoolPlanReport(report) { + const { plan } = report; + const lines = ['LLooM pool plan (dry run; nothing was changed)']; + lines.push( + report.source.mode === 'gateway' + ? `Source: gateway ${report.source.gateway ?? ''} live status${report.source.metricsPeriod ? `, ${report.source.metricsPeriod} demand` : ''}`.trim() + : `Source: config only${report.source.note ? ` — ${report.source.note}` : ''}` + ); + if (report.profile) lines.push(`Pinned routes from fleet profile ${report.profile}`); + + lines.push('', 'Nodes'); + for (const node of report.nodes) { + const summary = plan.nodes[node.id]; + const memory = (summary?.domains ?? []) + .map( + (domain) => + `${domain.id} ${gb(domain.residentGb)} resident of ${gb(domain.budgetGb)} (${gb(domain.freeGb)} free)` + ) + .join('; '); + lines.push( + ` ${node.id.padEnd(16)} ${summary?.status ?? 'unknown'}${memory ? ` ${memory}` : ''} [inventory: ${node.inventory}]` + ); + } + + lines.push('', 'Placements'); + const order = { always: 0, preferred: 1, auto: 2 }; + const placements = [...plan.placements].sort( + (a, b) => Number(a.external) - Number(b.external) || order[a.residency] - order[b.residency] + ); + for (const placement of placements) { + const tier = placement.external ? 'external' : placement.residency === 'auto' ? 'on-demand' : placement.residency; + const size = placement.external ? '' : ` ${gb(placement.memoryGbPerNode)}/node`; + const manual = placement.managed || placement.external ? '' : ' (unmanaged)'; + const shared = + placement.models.length > 1 ? ` [runtime ${placement.runtime} serves ${placement.models.length} models]` : ''; + lines.push( + ` ${tier.padEnd(10)} ${placement.placement}${size}${manual}${shared} <- ${workloadList(placement.workloads)}` + ); + } + if (!placements.length) lines.push(' (none)'); + + const { diff } = plan; + const changes = [ + ...diff.routeChanges.map((change) => `route ${change.alias}: ${change.from ?? '(none)'} -> ${change.to}`), + ...diff.start.map((entry) => `start ${entry.placement} as ${entry.residency}`), + ...diff.manual.map((entry) => `${entry.action.padEnd(10)} ${entry.placement} (not managed by LLooM)`), + ...diff.release.map((entry) => `release ${entry.placement} (no planned workload)`), + ...diff.residencyChanges.map((change) => `residency ${change.runtime}: ${change.from} -> ${change.to}`) + ]; + lines.push('', 'Changes from current'); + lines.push(...(changes.length ? changes.map((line) => ` ${line}`) : [' (none)'])); + + if (plan.uncovered.length || plan.pinViolations.length) { + lines.push('', 'Not covered'); + for (const entry of plan.uncovered) { + lines.push(` ${entry.workload}${entry.required ? '' : ' (optional)'}: ${entry.reasons[0] ?? 'no candidate'}`); + } + for (const entry of plan.pinViolations) lines.push(` pinned runtime ${entry.runtime}: ${entry.reasons[0]}`); + } + if (plan.warnings.length) lines.push('', 'Warnings', ...plan.warnings.map((warning) => ` ${warning}`)); + const score = plan.score; + lines.push( + '', + `Score: ${score.uncovered} uncovered, ${score.external} external fallback, ${score.sloMisses} SLO misses, cost ${score.cost.toExponential(2)}`, + 'Applying plans is not available yet; act on this one with `lloom fleet`, `lloom route` and `lloom keep-warm`.' + ); + return lines.join('\n'); +} diff --git a/src/pool-inventory.mjs b/src/pool-inventory.mjs new file mode 100644 index 0000000..0442215 --- /dev/null +++ b/src/pool-inventory.mjs @@ -0,0 +1,165 @@ +// Node inventory for pool planning: what a node could run without new +// downloads or installs. Collection is read-only and every probe is +// injectable; a failing probe yields a partial inventory, never an error. +import { promises as fs } from 'node:fs'; +import { runCommand } from './process-control.mjs'; +import { memoryDomainsForProfile } from './resource-fit.mjs'; +import { modelRootStatusForRecipe } from './model-files.mjs'; + +export const NODE_INVENTORY_VERSION = 1; +export const NODE_INVENTORY_TTL_MS = 300000; + +function round(value, places = 1) { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +function unique(values) { + return [...new Set(values.filter(Boolean))]; +} + +export function computeArchsForProfile(profile = {}) { + const archs = (Array.isArray(profile.devices) ? profile.devices : []) + .map((device) => String(device?.computeCapability ?? '').match(/^(\d+)\.(\d+)/)) + .filter(Boolean) + .map(([, major, minor]) => `sm_${major}${minor}`); + if (profile.isAppleSilicon) archs.push('apple-metal'); + return unique(archs).sort(); +} + +function profileLinks(profile = {}) { + const links = Array.isArray(profile.cluster?.links) ? profile.cluster.links : []; + return links + .filter((link) => link?.peerNodeId) + .map((link) => ({ + peerNodeId: String(link.peerNodeId), + ...(link.localInterface ? { localInterface: String(link.localInterface) } : {}), + ...(link.peerAddress ? { peerAddress: String(link.peerAddress) } : {}) + })); +} + +function capacityDomains(profile) { + return memoryDomainsForProfile(profile).map(({ id, kind, capacityGb }) => ({ id, kind, capacityGb })); +} + +// The planner can work from a machine profile alone (older gateways do not +// report inventory); it just cannot see cached weights, disks or images. +export function inventoryFromProfile(profile = {}) { + return { + version: NODE_INVENTORY_VERSION, + complete: false, + computeArchs: computeArchsForProfile(profile), + memoryDomains: capacityDomains(profile), + links: profileLinks(profile) + }; +} + +async function statfsGb(root) { + const stats = await fs.statfs(root); + return { + freeGb: round((stats.bavail * stats.bsize) / 1024 ** 3), + totalGb: round((stats.blocks * stats.bsize) / 1024 ** 3) + }; +} + +async function commandExists(command) { + const result = await runCommand('/usr/bin/which', [command], { allowFailure: true }); + return result.code === 0 && Boolean(result.stdout.trim()); +} + +async function dockerImages() { + const result = await runCommand( + 'docker', + ['image', 'ls', '--no-trunc', '--format', '{{.Repository}}:{{.Tag}}|{{.Repository}}@{{.Digest}}'], + { allowFailure: true } + ); + if (result.code !== 0) return []; + return result.stdout + .split('\n') + .flatMap((line) => line.trim().split('|')) + .filter((ref) => ref && !ref.includes('')); +} + +function recipeFitsPlatform(recipe, platformId) { + const platforms = recipe?.requirements?.platforms; + return !Array.isArray(platforms) || !platforms.length || platforms.includes(platformId); +} + +const message = (error) => error?.message ?? String(error); + +export async function collectNodeInventory({ nodeId, profile = {}, recipes = [], modelRoots = [], probes = {} } = {}) { + const probe = { + statfs: statfsGb, + modelRootStatus: modelRootStatusForRecipe, + commandExists, + dockerImages, + now: () => new Date().toISOString(), + ...probes + }; + const errors = []; + const disks = []; + for (const root of unique(modelRoots)) { + try { + disks.push({ root, ...(await probe.statfs(root)) }); + } catch { + // A candidate root that does not exist on this node is not an error. + } + } + const compatible = recipes.filter((recipe) => recipe?.id && recipeFitsPlatform(recipe, profile.platformId)); + const weights = []; + for (const recipe of compatible) { + for (const { root } of disks) { + try { + if ((await probe.modelRootStatus(recipe, root))?.complete === true) { + weights.push({ recipeId: recipe.id, root, complete: true }); + break; + } + } catch (error) { + errors.push(`weights ${recipe.id}: ${message(error)}`); + break; + } + } + } + const commands = {}; + for (const command of unique(compatible.flatMap((recipe) => recipe.requirements?.commands ?? [])).sort()) { + if (await probe.commandExists(command).catch(() => false)) commands[command] = true; + } + let images = []; + if (commands.docker) { + try { + images = unique(await probe.dockerImages()).sort(); + } catch (error) { + errors.push(`docker images: ${message(error)}`); + } + } + return { + ...inventoryFromProfile(profile), + complete: true, + nodeId, + collectedAt: probe.now(), + disks, + weights, + backends: { commands, dockerImages: images }, + ...(errors.length ? { errors } : {}) + }; +} + +export function createNodeInventoryCache({ collect, ttlMs = NODE_INVENTORY_TTL_MS, now = Date.now } = {}) { + let cached = null; + let pending = null; + return { + async get() { + if (cached && now() - cached.at < ttlMs) return cached.value; + pending ??= (async () => { + try { + const value = await collect(); + cached = { at: now(), value }; + return value; + } finally { + pending = null; + } + })(); + return pending; + } + }; +} diff --git a/src/pool-planner.mjs b/src/pool-planner.mjs new file mode 100644 index 0000000..bb83b3e --- /dev/null +++ b/src/pool-planner.mjs @@ -0,0 +1,644 @@ +// Pool placement planner. Pure: given node inventories, a variant catalog, +// workloads (aliases with demand and residency intent), pins and the current +// state, it proposes which variants run where, how each alias routes, and +// the diff from what is running now. It never starts, stops or reroutes +// anything; the per-node admission planner (runtime-policy.mjs) remains the +// safety layer that decides whether a start is allowed at request time. +// +// Objective, compared lexicographically: +// 1. cover every required workload and honour every pin +// 2. use external (cloud) members only when nothing local covers a workload +// 3. meet workload latency SLOs +// 4. minimise demand-weighted latency plus the load cost of every change +// Search is a greedy pass followed by single-workload local search. + +import { memoryBudgetGb } from './resource-fit.mjs'; + +export const POOL_PLAN_VERSION = 1; + +export const DEFAULT_POOL_POLICY = Object.freeze({ + maxMemoryUtilization: null, + reserveGb: 8, + headroomGb: 4, + horizonHours: 24, + churnWeight: 10, + routeChangeMs: 1000, + outputTokens: 256, + minDemandPerHour: 0.1, + unknownLatencyMs: 10000, + external: 'fallback', + loadBaseSeconds: 30, + loadSecondsPerGb: 2, + fetchMBps: 100, + imagePullSeconds: 300, + manualActionSeconds: 600, + localSearchPasses: 3 +}); + +const RESIDENCY_RANK = { auto: 0, preferred: 1, always: 2 }; +const EXTERNAL_MODES = new Set(['fallback', 'allow', 'never']); + +function num(value) { + if (value == null || value === '') return null; + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +function round(value, places = 2) { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +function list(value) { + return Array.isArray(value) ? value.filter((item) => item != null && item !== '') : []; +} + +function normalizeArch(value) { + return String(value) + .toLowerCase() + .replace(/^(sm_\d+)[af]$/, '$1'); +} + +function maxResidency(left, right) { + return RESIDENCY_RANK[right] > RESIDENCY_RANK[left] ? right : left; +} + +function normalizeResidency(value, fallback = 'auto') { + return Object.hasOwn(RESIDENCY_RANK, value) ? value : fallback; +} + +export function normalizePoolPolicy(policy = {}) { + const merged = { ...DEFAULT_POOL_POLICY }; + for (const [key, value] of Object.entries(policy ?? {})) { + if (value === undefined) continue; + if (key === 'external') merged.external = EXTERNAL_MODES.has(value) ? value : DEFAULT_POOL_POLICY.external; + else if (key in DEFAULT_POOL_POLICY) merged[key] = value === null ? null : (num(value) ?? merged[key]); + } + return merged; +} + +function domainBudgets(node, policy) { + const headroomGb = num(node.headroomGb) ?? num(policy.headroomGb) ?? 0; + return list(node.memoryDomains).map((domain) => { + const capacityGb = num(domain.capacityGb) ?? 0; + const budgetGb = memoryBudgetGb({ + totalMemoryGb: capacityGb, + memoryBudgetGb: domain.budgetGb, + maxMemoryUtilization: num(node.maxMemoryUtilization) ?? num(policy.maxMemoryUtilization), + reserveMemoryGb: num(node.reserveGb) ?? num(policy.reserveGb) ?? 0 + }); + return { + id: String(domain.id ?? domain.kind ?? 'memory'), + kind: String(domain.kind ?? 'system'), + capacityGb, + budgetGb: round(Math.max(0, budgetGb - headroomGb)) + }; + }); +} + +// Model weights live in unified memory on Apple Silicon and Spark, and in the +// largest device memory otherwise; host-side state lives in system memory. +function acceleratorDomain(domains) { + return ( + domains.find((domain) => domain.kind === 'unified') ?? + [...domains].filter((domain) => domain.kind === 'device').sort((a, b) => b.budgetGb - a.budgetGb)[0] ?? + domains.find((domain) => domain.kind === 'system') ?? + null + ); +} + +function systemDomain(domains) { + return domains.find((domain) => domain.kind === 'system') ?? acceleratorDomain(domains); +} + +function memoryDemands(variant, domains) { + const requirements = variant.requirements ?? {}; + const demands = new Map(); + const add = (domain, gb) => { + if (!domain || !(gb > 0)) return; + demands.set(domain.id, (demands.get(domain.id) ?? 0) + gb); + }; + add(acceleratorDomain(domains), num(requirements.memoryGbPerNode) ?? 0); + add(systemDomain(domains), num(requirements.systemMemoryGbPerNode) ?? 0); + return demands; +} + +function variantMemoryGb(variant) { + return (num(variant.requirements?.memoryGbPerNode) ?? 0) + (num(variant.requirements?.systemMemoryGbPerNode) ?? 0); +} + +function nodeIneligibility(variant, node) { + if (node.cordoned) return 'cordoned'; + if (node.available === false) return 'unavailable'; + const requirements = variant.requirements ?? {}; + const platforms = list(requirements.platforms); + if (platforms.length && !platforms.includes(node.platformId)) return `needs platform ${platforms.join('|')}`; + const archs = list(requirements.computeArchs).map(normalizeArch); + const nodeArchs = new Set(list(node.computeArchs).map(normalizeArch)); + if (archs.length && !archs.some((arch) => nodeArchs.has(arch))) return `needs compute arch ${archs.join('|')}`; + const accelerators = new Set(list(node.accelerators)); + const missing = list(requirements.accelerators).filter((accelerator) => !accelerators.has(accelerator)); + if (missing.length) return `needs accelerator ${missing.join(', ')}`; + if (!list(node.memoryDomains).length) return 'no memory capacity reported'; + return null; +} + +function linked(left, right) { + return list(left.links).includes(right.id) || list(right.links).includes(left.id); +} + +function combinations(items, size) { + if (size === 0) return [[]]; + const result = []; + items.forEach((item, index) => { + for (const rest of combinations(items.slice(index + 1), size - 1)) result.push([item, ...rest]); + }); + return result; +} + +// Candidate node sets for one variant, plus the reasons other sets were ruled +// out. Configured runtimes are fixed to their node(s); recipe variants may run +// on any eligible set, and tensor-parallel sets must be directly linked. +function candidateNodeSets(variant, nodesById) { + const reasons = []; + if (variant.external) return { sets: [[]], reasons }; + if (list(variant.nodes).length) { + const nodes = variant.nodes.map((id) => nodesById.get(id)); + const missing = variant.nodes.filter((id, index) => !nodes[index]); + if (missing.length) return { sets: [], reasons: [`unknown node ${missing.join(', ')}`] }; + const blocked = nodes + .map((node) => [node.id, nodeIneligibility(variant, node)]) + .filter(([, reason]) => reason) + .map(([id, reason]) => `${id} ${reason}`); + return blocked.length ? { sets: [], reasons: blocked } : { sets: [variant.nodes.slice()], reasons }; + } + const size = Math.max(1, Math.floor(num(variant.requirements?.nodeCount) ?? 1)); + const eligible = []; + for (const node of [...nodesById.values()].sort((a, b) => a.id.localeCompare(b.id))) { + const reason = nodeIneligibility(variant, node); + if (reason) reasons.push(`${node.id} ${reason}`); + else eligible.push(node); + } + if (eligible.length < size) { + reasons.push(`needs ${size} eligible node${size === 1 ? '' : 's'}, found ${eligible.length}`); + return { sets: [], reasons }; + } + const sets = combinations(eligible, size) + .filter((set) => set.every((node, index) => set.slice(index + 1).every((other) => linked(node, other)))) + .map((set) => set.map((node) => node.id)); + if (!sets.length) reasons.push(`no directly linked set of ${size} eligible nodes`); + return { sets, reasons }; +} + +// Memory belongs to the serving process, so models that share a runtime (or a +// remote runtime) on the same nodes share one placement and one charge. +function slotKey(variant, nodes) { + if (variant.runtime) return `runtime:${variant.runtime}|${nodes.join('+')}`; + if (variant.remoteRuntime) return `remote:${variant.controlNode ?? ''}/${variant.remoteRuntime}`; + return placementKey(variant, nodes); +} + +function placementKey(variant, nodes) { + return list(variant.nodes).length || variant.external ? variant.id : `${variant.id}@${nodes.join('+')}`; +} + +function latencyMs(variant, workload, policy) { + const performance = variant.performance ?? {}; + const ttft = num(performance.ttftMs); + const tps = num(performance.decodeTps); + const outputTokens = num(workload.outputTokens) ?? policy.outputTokens; + if (ttft != null && tps > 0) return ttft + (Math.max(0, outputTokens - 1) * 1000) / tps; + return num(performance.durationMs) ?? ttft ?? policy.unknownLatencyMs; +} + +function loadSeconds(variant, nodes, nodesById, policy) { + if (variant.external) return 0; + const memoryGb = variantMemoryGb(variant); + let seconds = num(variant.loadSeconds) ?? policy.loadBaseSeconds + memoryGb * policy.loadSecondsPerGb; + if (variant.managed === false) seconds += policy.manualActionSeconds; + // Configured lanes are installed on their nodes; only new placements fetch. + if (list(variant.nodes).length) return seconds; + for (const nodeId of nodes) { + const node = nodesById.get(nodeId); + const recipeId = variant.requirements?.recipeId; + if (recipeId && !list(node?.weights).includes(recipeId)) { + const diskGb = num(variant.requirements?.diskGb) ?? memoryGb; + seconds += (diskGb * 1024) / Math.max(1, policy.fetchMBps); + } + const images = list(variant.requirements?.images); + if (images.some((image) => !list(node?.images).includes(image))) seconds += policy.imagePullSeconds; + } + return seconds; +} + +function emptyState(nodes, policy) { + const usage = new Map(); + const domains = new Map(); + for (const node of nodes) { + const budgets = domainBudgets(node, policy); + domains.set(node.id, budgets); + usage.set(node.id, new Map(budgets.map((domain) => [domain.id, { always: 0, preferred: 0 }]))); + } + return { domains, usage, placements: new Map(), routes: {}, uncovered: [], warnings: [], pinViolations: [] }; +} + +// The largest on-demand placement that must still be admissible next to the +// hard-resident set on one memory domain. +function largestOnDemandGb(state, nodeId, domainId, excludeKey = null) { + let largest = 0; + for (const placement of state.placements.values()) { + if (placement.residency !== 'auto' || placement.slot === excludeKey || placement.variant.external) continue; + if (!placement.nodes.includes(nodeId)) continue; + const gb = memoryDemands(placement.variant, state.domains.get(nodeId) ?? []).get(domainId) ?? 0; + largest = Math.max(largest, gb); + } + return largest; +} + +function fitsAt(variant, nodes, residency, state, existing = null) { + for (const nodeId of nodes) { + const budgets = state.domains.get(nodeId) ?? []; + const usage = state.usage.get(nodeId); + for (const [domainId, gb] of memoryDemands(variant, budgets)) { + const budget = budgets.find((domain) => domain.id === domainId).budgetGb; + const used = usage.get(domainId); + // An upgrade re-counts a placement that is already resident. + const prior = existing && existing.residency !== 'auto' ? gb : 0; + const resident = used.always + used.preferred - prior; + const always = used.always - (existing?.residency === 'always' ? gb : 0); + const onDemand = largestOnDemandGb(state, nodeId, domainId, existing?.slot); + if (residency === 'always' && (resident + gb > budget || always + gb + onDemand > budget)) return false; + if (residency === 'preferred' && resident + gb > budget) return false; + if (residency === 'auto' && always + gb > budget) return false; + } + } + return true; +} + +function charge(variant, nodes, residency, state, sign = 1) { + if (residency === 'auto') return; + for (const nodeId of nodes) { + const budgets = state.domains.get(nodeId) ?? []; + const usage = state.usage.get(nodeId); + for (const [domainId, gb] of memoryDemands(variant, budgets)) usage.get(domainId)[residency] += sign * gb; + } +} + +function place(state, variant, nodes, residency, workloadId = null) { + const slot = slotKey(variant, nodes); + let placement = state.placements.get(slot); + if (!placement) { + placement = { + slot, + key: placementKey(variant, nodes), + variant, + nodes, + residency, + variants: new Map(), + workloads: [] + }; + state.placements.set(slot, placement); + if (!variant.external) charge(variant, nodes, residency, state); + } else if (RESIDENCY_RANK[residency] > RESIDENCY_RANK[placement.residency]) { + if (!variant.external) charge(variant, nodes, placement.residency, state, -1); + placement.residency = residency; + if (!variant.external) charge(variant, nodes, residency, state); + } + placement.variants.set(variant.id, variant); + if (workloadId) placement.workloads.push(workloadId); + return placement; +} + +// Every way a workload could be served right now: an existing placement it +// can share, or a new placement on a node set that fits. +function workloadOptions(workload, candidates, ctx, state) { + const { variantsByModel, nodesById, policy, current } = ctx; + const options = []; + const reasons = []; + const demand = Math.max(num(workload.demandPerHour) ?? 0, policy.minDemandPerHour); + const wanted = normalizeResidency(workload.residency); + const contextWindow = num(workload.slo?.contextWindow); + candidates.forEach((model, preference) => { + const variants = variantsByModel.get(model) ?? []; + if (!variants.length) reasons.push(`${model}: no variant in the pool catalog`); + for (const variant of variants) { + if (variant.external && policy.external === 'never') { + reasons.push(`${model}: external members are disabled`); + continue; + } + if (contextWindow && (num(variant.requirements?.contextWindow) ?? 0) < contextWindow) { + reasons.push(`${variant.id}: context window below ${contextWindow}`); + continue; + } + const latency = latencyMs(variant, workload, policy); + const sloTtft = num(workload.slo?.ttftMs); + const ttft = num(variant.performance?.ttftMs) ?? latency; + const base = { + model, + variant, + preference, + latencyMs: latency, + external: Boolean(variant.external), + sloMiss: sloTtft != null && ttft > sloTtft ? 1 : 0, + cost: + demand * latency * policy.horizonHours + + (current.routes?.[workload.id] != null && current.routes[workload.id] !== model ? policy.routeChangeMs : 0) + }; + const { sets, reasons: setReasons } = candidateNodeSets(variant, nodesById); + reasons.push(...setReasons.map((reason) => `${variant.id}: ${reason}`)); + for (const nodes of sets) { + const existing = state.placements.get(slotKey(variant, nodes)); + const key = existing?.key ?? placementKey(variant, nodes); + const residency = variant.external ? 'auto' : existing ? maxResidency(existing.residency, wanted) : wanted; + let chosen = residency; + if (!variant.external && !fitsAt(variant, nodes, residency, state, existing)) { + // Preferred residency is soft: fall back to on-demand when it fits. + if (residency === 'preferred' && fitsAt(variant, nodes, 'auto', state, existing)) chosen = 'auto'; + else if (existing && fitsAt(variant, nodes, existing.residency, state, existing)) { + chosen = existing.residency; + } else { + reasons.push(`${key}: does not fit as ${residency} on ${nodes.join('+')}`); + continue; + } + } + const isNew = !existing; + const running = current.running?.has(placementKey(variant, nodes)) || current.running?.has(variant.id); + const churnSeconds = isNew && !running ? loadSeconds(variant, nodes, nodesById, policy) : 0; + options.push({ + ...base, + nodes, + key, + residency: chosen, + downgraded: chosen !== residency, + churnSeconds, + cost: base.cost + churnSeconds * 1000 * policy.churnWeight + }); + } + } + }); + return { options, reasons }; +} + +function better(left, right) { + for (const key of ['uncovered', 'external', 'sloMisses']) { + if (left[key] !== right[key]) return left[key] < right[key]; + } + return left.cost < right.cost - Math.max(1e-6, Math.abs(right.cost) * 1e-9); +} + +function optionOrder(left, right, policy) { + if (policy.external === 'fallback' && left.external !== right.external) return left.external ? 1 : -1; + if (left.sloMiss !== right.sloMiss) return left.sloMiss - right.sloMiss; + if (left.cost !== right.cost) return left.cost - right.cost; + return left.preference - right.preference || left.key.localeCompare(right.key); +} + +function workloadOrder(pins) { + return (left, right) => { + const pinned = Number(Boolean(pins.routes?.[right.id])) - Number(Boolean(pins.routes?.[left.id])); + if (pinned) return pinned; + const residency = + RESIDENCY_RANK[normalizeResidency(right.residency)] - RESIDENCY_RANK[normalizeResidency(left.residency)]; + if (residency) return residency; + const required = Number(right.required !== false) - Number(left.required !== false); + if (required) return required; + return (num(right.demandPerHour) ?? 0) - (num(left.demandPerHour) ?? 0) || left.id.localeCompare(right.id); + }; +} + +function greedy(ctx, forced) { + const { workloads, pins, policy, nodes, variants } = ctx; + const state = emptyState(nodes, policy); + let cost = 0; + let external = 0; + let sloMisses = 0; + + for (const [runtimeId, residency] of Object.entries(pins.residency ?? {}).sort(([a], [b]) => a.localeCompare(b))) { + if (residency !== 'always') continue; + const variant = variants.find((candidate) => candidate.runtime === runtimeId && list(candidate.nodes).length); + if (!variant) continue; + const { sets, reasons } = candidateNodeSets(variant, ctx.nodesById); + const key = placementKey(variant, sets[0] ?? []); + if (!sets.length || !fitsAt(variant, sets[0], 'always', state, state.placements.get(slotKey(variant, sets[0])))) { + state.pinViolations.push({ runtime: runtimeId, reasons: reasons.length ? reasons : ['does not fit'] }); + continue; + } + const running = ctx.current.running?.has(key) || ctx.current.running?.has(variant.id); + if (!running) cost += loadSeconds(variant, sets[0], ctx.nodesById, policy) * 1000 * policy.churnWeight; + place(state, variant, sets[0], 'always'); + } + + for (const workload of workloads) { + const pinned = pins.routes?.[workload.id]; + const candidates = pinned ? [pinned] : forced[workload.id] ? [forced[workload.id]] : list(workload.candidates); + const { options, reasons } = workloadOptions(workload, candidates, ctx, state); + options.sort((left, right) => optionOrder(left, right, policy)); + const option = options[0]; + if (!option) { + state.uncovered.push({ + workload: workload.id, + required: workload.required !== false, + ...(pinned ? { pinned } : {}), + reasons: [...new Set(reasons)].slice(0, 12) + }); + continue; + } + if (option.downgraded) { + state.warnings.push( + `${workload.id}: ${option.key} kept ${option.residency} because ${normalizeResidency(workload.residency)} residency does not fit` + ); + } + place(state, option.variant, option.nodes, option.residency, workload.id); + state.routes[workload.id] = { + model: option.model, + variant: option.variant.id, + placement: option.key, + nodes: option.nodes, + residency: option.residency, + external: option.external, + latencyMs: round(option.latencyMs), + reason: pinned ? 'pinned' : option.external ? 'external' : 'planned' + }; + cost += option.cost; + external += option.external && policy.external === 'fallback' ? 1 : 0; + sloMisses += option.sloMiss; + } + + state.score = { + uncovered: state.uncovered.filter((entry) => entry.required).length + state.pinViolations.length, + external, + sloMisses, + cost: round(cost) + }; + return state; +} + +function search(ctx) { + let forced = {}; + let best = greedy(ctx, forced); + for (let pass = 0; pass < ctx.policy.localSearchPasses; pass += 1) { + let improved = false; + for (const workload of ctx.workloads) { + if (ctx.pins.routes?.[workload.id]) continue; + for (const model of list(workload.candidates)) { + if (best.routes[workload.id]?.model === model || forced[workload.id] === model) continue; + const trialForced = { ...forced, [workload.id]: model }; + const trial = greedy(ctx, trialForced); + if (better(trial.score, best.score)) { + best = trial; + forced = trialForced; + improved = true; + } + } + } + if (!improved) break; + } + return best; +} + +function summarizeNodes(ctx, state) { + return Object.fromEntries( + ctx.nodes.map((node) => { + const domains = state.domains.get(node.id).map((domain) => { + const used = state.usage.get(node.id).get(domain.id); + const residentGb = round(used.always + used.preferred); + return { + id: domain.id, + kind: domain.kind, + capacityGb: domain.capacityGb, + budgetGb: domain.budgetGb, + alwaysGb: round(used.always), + residentGb, + largestOnDemandGb: round(largestOnDemandGb(state, node.id, domain.id)), + freeGb: round(domain.budgetGb - residentGb) + }; + }); + return [ + node.id, + { + status: node.cordoned ? 'cordoned' : node.available === false ? 'unavailable' : 'available', + domains, + placements: [...state.placements.values()] + .filter((placement) => placement.nodes.includes(node.id)) + .map((placement) => placement.key) + .sort() + } + ]; + }) + ); +} + +function planDiff(ctx, state, residency) { + const { current } = ctx; + const running = current.running ?? new Set(); + const placements = [...state.placements.values()].filter((placement) => !placement.variant.external); + const planned = new Set(placements.flatMap((placement) => [placement.key, ...placement.variants.keys()])); + const start = []; + const manual = []; + const keep = []; + for (const placement of placements) { + const isRunning = running.has(placement.key) || [...placement.variants.keys()].some((id) => running.has(id)); + if (isRunning) keep.push(placement.key); + else if (placement.residency !== 'auto') { + (placement.variant.managed === false ? manual : start).push({ + action: placement.variant.managed === false ? 'manual-start' : 'start', + placement: placement.key, + nodes: placement.nodes, + residency: placement.residency + }); + } + } + const release = [...running] + .filter((key) => !planned.has(key)) + .sort() + .map((key) => { + const variant = ctx.variants.find((candidate) => candidate.id === key || key.startsWith(`${candidate.id}@`)); + if (variant?.managed === false) { + manual.push({ action: 'manual-stop', placement: key, nodes: variant.nodes ?? [] }); + return null; + } + return { action: 'release', placement: key }; + }) + .filter(Boolean); + const routeChanges = Object.entries(state.routes) + .filter(([alias, route]) => (current.routes?.[alias] ?? null) !== route.model) + .map(([alias, route]) => ({ alias, from: current.routes?.[alias] ?? null, to: route.model })) + .sort((a, b) => a.alias.localeCompare(b.alias)); + const residencyChanges = Object.entries(residency) + .filter(([runtime, policy]) => current.residency?.[runtime] != null && current.residency[runtime] !== policy) + .map(([runtime, policy]) => ({ runtime, from: current.residency[runtime], to: policy })) + .sort((a, b) => a.runtime.localeCompare(b.runtime)); + return { start, manual, release, keep: keep.sort(), routeChanges, residencyChanges }; +} + +export function planPool({ nodes = [], variants = [], workloads = [], pins = {}, current = {}, policy = {} } = {}) { + const normalizedPolicy = normalizePoolPolicy(policy); + const cordoned = new Set(list(pins.cordon)); + const plannerNodes = list(nodes) + .map((node) => ({ ...node, cordoned: node.cordoned === true || cordoned.has(node.id) })) + .sort((a, b) => a.id.localeCompare(b.id)); + const sortedVariants = list(variants) + .map((variant) => ({ managed: true, ...variant })) + .sort((a, b) => a.id.localeCompare(b.id)); + const variantsByModel = new Map(); + for (const variant of sortedVariants) { + if (!variantsByModel.has(variant.gatewayModel)) variantsByModel.set(variant.gatewayModel, []); + variantsByModel.get(variant.gatewayModel).push(variant); + } + const ctx = { + nodes: plannerNodes, + nodesById: new Map(plannerNodes.map((node) => [node.id, node])), + variants: sortedVariants, + variantsByModel, + workloads: [...list(workloads)].sort(workloadOrder(pins)), + pins, + policy: normalizedPolicy, + current: { + routes: current.routes ?? {}, + residency: current.residency ?? {}, + running: new Set(list(current.running instanceof Set ? [...current.running] : current.running)) + } + }; + const state = search(ctx); + + const residency = {}; + for (const variant of sortedVariants) { + if (variant.runtime && variant.managed !== false && !variant.external) residency[variant.runtime] ??= 'auto'; + } + for (const placement of state.placements.values()) { + const runtime = placement.variant.runtime; + if (runtime && placement.variant.managed !== false) { + residency[runtime] = maxResidency(residency[runtime] ?? 'auto', placement.residency); + } + } + + const placements = [...state.placements.values()] + .map((placement) => ({ + placement: placement.key, + variant: placement.variant.id, + models: [...new Set([...placement.variants.values()].map((variant) => variant.gatewayModel))].sort(), + runtime: placement.variant.runtime ?? null, + nodes: placement.nodes, + residency: placement.residency, + managed: placement.variant.managed !== false, + external: Boolean(placement.variant.external), + memoryGbPerNode: variantMemoryGb(placement.variant), + workloads: [...placement.workloads].sort() + })) + .sort((a, b) => a.placement.localeCompare(b.placement)); + + return { + version: POOL_PLAN_VERSION, + ok: state.score.uncovered === 0, + policy: normalizedPolicy, + placements, + routes: Object.fromEntries(Object.entries(state.routes).sort(([a], [b]) => a.localeCompare(b))), + residency: Object.fromEntries(Object.entries(residency).sort(([a], [b]) => a.localeCompare(b))), + uncovered: state.uncovered.sort((a, b) => a.workload.localeCompare(b.workload)), + pinViolations: state.pinViolations, + nodes: summarizeNodes(ctx, state), + diff: planDiff(ctx, state, residency), + score: state.score, + warnings: state.warnings + }; +} diff --git a/src/recipes.mjs b/src/recipes.mjs index 264e7d2..7471329 100644 --- a/src/recipes.mjs +++ b/src/recipes.mjs @@ -84,10 +84,28 @@ export function validateRecipe(recipe, config, { backendIds, checkLocalReference } if (!asArray(recipe.models).length) errors.push(`recipe ${recipe.id ?? '(missing)'} has no models`); + const serving = recipe.requirements ?? {}; + if ( + serving.computeArchs != null && + (!Array.isArray(serving.computeArchs) || + serving.computeArchs.some((arch) => typeof arch !== 'string' || !/^(sm_[0-9]+[af]?|apple-metal)$/.test(arch))) + ) { + errors.push(`recipe ${recipe.id} requirements.computeArchs must list sm_ or apple-metal`); + } + if (serving.tensorParallel != null && !(Number.isInteger(serving.tensorParallel) && serving.tensorParallel >= 1)) { + errors.push(`recipe ${recipe.id} requirements.tensorParallel must be a positive integer`); + } + const modelIds = checkLocalReferences ? configModelIds(config) : new Set(); const runtimeIds = checkLocalReferences ? configRuntimeIds(config) : new Set(); for (const model of asArray(recipe.models)) { if (!model.role) errors.push(`recipe ${recipe.id} has model without role`); + const maxSequences = model.settings?.maxSequences; + if (maxSequences != null && !(Number.isInteger(maxSequences) && maxSequences >= 1)) { + errors.push( + `recipe ${recipe.id} model ${model.role ?? '(missing)'} settings.maxSequences must be a positive integer` + ); + } if (!model.model) errors.push(`recipe ${recipe.id} model ${model.role ?? '(missing)'} is missing model`); if (checkLocalReferences && model.gatewayModel && !modelIds.has(model.gatewayModel)) { errors.push(`recipe ${recipe.id} references unknown gateway model ${model.gatewayModel}`); diff --git a/src/resource-fit.mjs b/src/resource-fit.mjs index 725a941..4cf7816 100644 --- a/src/resource-fit.mjs +++ b/src/resource-fit.mjs @@ -83,6 +83,16 @@ export function memoryDomainsForProfile(profile = {}) { return domains; } +// Host memory LLooM may plan into: an explicit budget, else a utilization +// ceiling, else capacity minus a reserve. Admission and pool planning share it. +export function memoryBudgetGb({ totalMemoryGb, memoryBudgetGb, maxMemoryUtilization, reserveMemoryGb = 8 } = {}) { + const explicit = finiteNumber(memoryBudgetGb); + if (explicit != null) return explicit; + const total = finiteNumber(totalMemoryGb) ?? 0; + const utilization = finiteNumber(maxMemoryUtilization); + return utilization != null ? total * utilization : Math.max(0, total - (finiteNumber(reserveMemoryGb) ?? 0)); +} + export function machineTopologyFingerprint(profile = {}) { const domains = memoryDomainsForProfile(profile) .map(({ availableGb: _availableGb, ...domain }) => domain) diff --git a/src/runtime-policy.mjs b/src/runtime-policy.mjs index 760fc43..20aafb2 100644 --- a/src/runtime-policy.mjs +++ b/src/runtime-policy.mjs @@ -1,6 +1,7 @@ import { assertMaintenanceStartAllowed } from './model-maintenance.mjs'; import os from 'node:os'; import { readHostMemory } from './host-memory.mjs'; +import { memoryBudgetGb as hostMemoryBudgetGb } from './resource-fit.mjs'; import { RuntimeManager } from './runtime-manager.mjs'; import { clusterNodes, currentNodeId, runtimeAuthority, runtimeResourcesByNode } from './cluster.mjs'; @@ -163,11 +164,12 @@ function policyConfig(config, profile = {}) { const totalMemoryGb = numberOrNull(profile.totalMemoryGb) ?? Math.round(os.totalmem() / 1024 / 1024 / 1024); const reserveMemoryGb = numberOrNull(policy.reserveMemoryGb) ?? 8; const maxMemoryUtilization = numberOrNull(policy.maxMemoryUtilization); - const memoryBudgetGb = - numberOrNull(policy.memoryBudgetGb) ?? - (maxMemoryUtilization != null - ? Math.max(0, totalMemoryGb * maxMemoryUtilization) - : Math.max(0, totalMemoryGb - reserveMemoryGb)); + const memoryBudgetGb = hostMemoryBudgetGb({ + totalMemoryGb, + memoryBudgetGb: numberOrNull(policy.memoryBudgetGb), + maxMemoryUtilization, + reserveMemoryGb + }); return { enabled: policy.enabled !== false && policy.memorySafety?.mode !== 'yolo', autoEvict: policy.autoEvict === true, @@ -329,12 +331,13 @@ function clusterRuntimePolicyPlan( const maxMemoryUtilization = numberOrNull( configuredNodes[nodeId].resources?.maxMemoryUtilization ?? policyTemplate.maxMemoryUtilization ); - const memoryBudgetGb = - numberOrNull(configuredNodes[nodeId].resources?.memoryBudgetGb) ?? - numberOrNull(policyTemplate.memoryBudgetGb) ?? - (maxMemoryUtilization != null - ? Math.max(0, totalMemoryGb * maxMemoryUtilization) - : Math.max(0, totalMemoryGb - reserveMemoryGb)); + const memoryBudgetGb = hostMemoryBudgetGb({ + totalMemoryGb, + memoryBudgetGb: + numberOrNull(configuredNodes[nodeId].resources?.memoryBudgetGb) ?? numberOrNull(policyTemplate.memoryBudgetGb), + maxMemoryUtilization, + reserveMemoryGb + }); const loadedMemoryGb = rows .filter((row) => row.loaded) .reduce((sum, row) => sum + numberOrNull(row.resourcesByNode?.[nodeId]?.memoryGb), 0); diff --git a/src/server.mjs b/src/server.mjs index 12ff2f8..5f03ade 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -59,6 +59,8 @@ import { applyOnboarding, createOnboardingPlan } from './onboarding.mjs'; import { applyRecipePack, createRecipePackPlan } from './recipe-pack.mjs'; import { buildRecipeIndexReport } from './recipe-index.mjs'; import { loadRecipes } from './recipes.mjs'; +import { collectNodeInventory, createNodeInventoryCache } from './pool-inventory.mjs'; +import { modelRootCandidates } from './model-files.mjs'; import { createRegistry, UnknownModelError } from './registry.mjs'; import { routeProfileStatus, writeRouteMemberSuspension, writeRouteProfile } from './route-control.mjs'; import { mutateConfigSource } from './config-mutation.mjs'; @@ -1287,14 +1289,15 @@ function restoreMetricMap(entries) { } function createMetricGroup(id = 'all') { - return { totals: emptyMetricBucket(id), models: new Map(), routes: new Map() }; + return { totals: emptyMetricBucket(id), models: new Map(), routes: new Map(), requested: new Map() }; } function restoreMetricGroup(source, id = 'all') { return { totals: restoreMetricBucket(emptyMetricBucket(id), source?.totals), models: restoreMetricMap(source?.models), - routes: restoreMetricMap(source?.routes) + routes: restoreMetricMap(source?.routes), + requested: restoreMetricMap(source?.requested) }; } @@ -1351,6 +1354,9 @@ function serializeMetricGroup(group) { .map((bucket) => finalizeMetricBucket(bucket, true)) .sort((a, b) => a.id.localeCompare(b.id)), routes: [...group.routes.values()] + .map((bucket) => finalizeMetricBucket(bucket, true)) + .sort((a, b) => a.id.localeCompare(b.id)), + requested: [...(group.requested ?? new Map()).values()] .map((bucket) => finalizeMetricBucket(bucket, true)) .sort((a, b) => a.id.localeCompare(b.id)) }; @@ -1363,6 +1369,8 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = const totals = restoreMetricBucket(emptyMetricBucket('all'), initialSnapshot?.totals); const models = restoreMetricMap(initialSnapshot?.models); const routes = restoreMetricMap(initialSnapshot?.routes); + // Demand per requested id (usually an alias), separate from resolved models. + const requested = restoreMetricMap(initialSnapshot?.requested); const days = new Map( (Array.isArray(initialSnapshot?.history?.days) ? initialSnapshot.history.days : []) .filter((entry) => /^\d{4}-\d{2}-\d{2}$/.test(entry?.date || '')) @@ -1508,17 +1516,19 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = apply(totals, entry); if (entry.model) apply(bucketFor(models, entry.model), entry); if (entry.route) apply(bucketFor(routes, entry.route), entry); + if (entry.requestedModel) apply(bucketFor(requested, entry.requestedModel), entry); const dayId = entry.at.slice(0, 10); if (!days.has(dayId)) days.set(dayId, createMetricGroup(dayId)); const day = days.get(dayId); apply(day.totals, entry); if (entry.model) apply(bucketFor(day.models, entry.model), entry); if (entry.route) apply(bucketFor(day.routes, entry.route), entry); + if (entry.requestedModel) apply(bucketFor(day.requested, entry.requestedModel), entry); while (days.size > 400) days.delete([...days.keys()].sort()[0]); }, snapshot({ model, period = 'all' } = {}) { const normalizedPeriod = ['today', '7d', '30d', 'all'].includes(period) ? period : 'all'; - let selectedGroup = { totals, models, routes }; + let selectedGroup = { totals, models, routes, requested }; if (normalizedPeriod !== 'all') { const dayCount = normalizedPeriod === 'today' ? 1 : Number.parseInt(normalizedPeriod, 10); const cutoff = new Date(); @@ -1530,6 +1540,7 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = mergeMetricBucket(selectedGroup.totals, day.totals); mergeMetricMaps(selectedGroup.models, day.models); mergeMetricMaps(selectedGroup.routes, day.routes); + mergeMetricMaps(selectedGroup.requested, day.requested); } } const selectedModel = model ? (selectedGroup.models.get(model) ?? null) : null; @@ -1550,6 +1561,9 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = : [] : [...selectedGroup.models.values()].map(finalizeMetricBucket).sort((a, b) => a.id.localeCompare(b.id)), routes: [...selectedGroup.routes.values()].map(finalizeMetricBucket).sort((a, b) => a.id.localeCompare(b.id)), + requested: [...selectedGroup.requested.values()] + .map(finalizeMetricBucket) + .sort((a, b) => a.id.localeCompare(b.id)), active: [...active.values()].map((entry) => ({ ...entry, elapsedMs: Math.max(0, Date.now() - Date.parse(entry.startedAt)) @@ -1560,7 +1574,7 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = }, persistenceSnapshot() { return { - ...serializeMetricGroup({ totals, models, routes }), + ...serializeMetricGroup({ totals, models, routes, requested }), history: { days: [...days.entries()] .sort(([a], [b]) => a.localeCompare(b)) @@ -1619,6 +1633,7 @@ export function createMetricsPersistence(config, { logger = console } = {}) { totals: latestSnapshot.totals, models: latestSnapshot.models, routes: latestSnapshot.routes, + requested: latestSnapshot.requested, history: latestSnapshot.history } }, @@ -2335,6 +2350,18 @@ export function createLloomServer( telemetry: hostTelemetry, profile: () => machineProfile }); + // Pool inventory is collected on request only and cached; it scans model + // roots and lists container images, which is too slow for the status path. + const nodeInventory = createNodeInventoryCache({ + collect: async () => + collectNodeInventory({ + nodeId: clusterCoordinator.nodeId, + profile: (await machineProfile) ?? {}, + recipes: await loadRecipes().catch(() => []), + modelRoots: await modelRootCandidates({ config: clusterCoordinator.config }) + }) + }); + clusterCoordinator.inventory ??= () => nodeInventory.get(); runtimeManager ??= new RuntimeManager(config, { logger, clusterCoordinator }); if (!runtimeManager.clusterCoordinator) { runtimeManager.clusterCoordinator = clusterCoordinator; @@ -5082,7 +5109,8 @@ export function createLloomServer( sendJson(res, 200, { ok: true, node: await clusterCoordinator.localNodeStatus({ - includeMemoryUsage: url.searchParams.get('memoryUsage') === '1' + includeMemoryUsage: url.searchParams.get('memoryUsage') === '1', + includeInventory: url.searchParams.get('inventory') === '1' }) }); return; @@ -5091,7 +5119,7 @@ export function createLloomServer( if (req.method === 'GET' && url.pathname === '/gateway/cluster') { sendJson(res, 200, { ok: true, - cluster: await clusterCoordinator.status() + cluster: await clusterCoordinator.status({ inventory: url.searchParams.get('inventory') === '1' }) }); return; } diff --git a/test/fixtures/pool/spark-fleet.json b/test/fixtures/pool/spark-fleet.json new file mode 100644 index 0000000..b609bac --- /dev/null +++ b/test/fixtures/pool/spark-fleet.json @@ -0,0 +1,212 @@ +{ + "description": "Planner-level reproduction of a three-Spark fleet on 2026-10-10: spark-node-03 serves embeddings, media and voice; the spark-node-01/02 pair serves SparkQwen RC16 (Atlas, TP2) as an unmanaged lane. Memory figures are the configured runtime estimates; performance figures are representative gateway measurements, not benchmark claims.", + "policy": { "maxMemoryUtilization": 0.95, "headroomGb": 4 }, + "nodes": [ + { + "id": "spark-node-03", + "platformId": "linux-arm64", + "computeArchs": ["sm_121"], + "accelerators": ["cuda", "nvidia-gpu", "blackwell", "gb10"], + "memoryDomains": [{ "id": "unified:0", "kind": "unified", "capacityGb": 128 }], + "links": [] + }, + { + "id": "spark-node-01", + "platformId": "linux-arm64", + "computeArchs": ["sm_121"], + "accelerators": ["cuda", "nvidia-gpu", "blackwell", "gb10"], + "memoryDomains": [{ "id": "unified:0", "kind": "unified", "capacityGb": 128 }], + "links": ["spark-node-02"] + }, + { + "id": "spark-node-02", + "platformId": "linux-arm64", + "computeArchs": ["sm_121"], + "accelerators": ["cuda", "nvidia-gpu", "blackwell", "gb10"], + "memoryDomains": [{ "id": "unified:0", "kind": "unified", "capacityGb": 128 }], + "links": ["spark-node-01"] + } + ], + "variants": [ + { + "id": "qwen3.8-flash-next-atlas@spark-node-01+spark-node-02", + "gatewayModel": "qwen3.8-flash-next-atlas", + "model": "nvidia/Qwen3.8-Flash-Next-NVFP4", + "kind": "chat", + "managed": false, + "nodes": ["spark-node-01", "spark-node-02"], + "requirements": { "nodeCount": 2, "tensorParallel": 2, "computeArchs": ["sm_121"], "memoryGbPerNode": 110, "contextWindow": 262144 }, + "performance": { "ttftMs": 450, "decodeTps": 75, "source": "fixture" } + }, + { + "id": "qwen38-27b-nvfp4@spark-node-03", + "gatewayModel": "qwen38-27b-nvfp4", + "model": "unsloth/Qwen3.8-27B-NVFP4", + "kind": "chat", + "runtime": "qwen38-27b-nvfp4", + "nodes": ["spark-node-03"], + "requirements": { "nodeCount": 1, "computeArchs": ["sm_121"], "memoryGbPerNode": 46, "contextWindow": 262144 }, + "performance": { "ttftMs": 900, "decodeTps": 28, "source": "fixture" } + }, + { + "id": "Qwen/Qwen3-Embedding-4B@spark-node-03", + "gatewayModel": "Qwen/Qwen3-Embedding-4B", + "kind": "embedding", + "runtime": "qwen3-embedding-4b", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 16 }, + "performance": { "durationMs": 60, "source": "fixture" } + }, + { + "id": "hear@spark-node-03", + "gatewayModel": "hear", + "kind": "audio", + "runtime": "hear", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 2 }, + "performance": { "durationMs": 400, "source": "fixture" } + }, + { + "id": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03", + "gatewayModel": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", + "kind": "image", + "runtime": "spark03-qwen-image-21-turbo-nvfp4", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 24 }, + "performance": { "durationMs": 18500, "source": "fixture" } + }, + { + "id": "BennyDaBall/Qwen-Image-2.1-NVFP4@spark-node-03", + "gatewayModel": "BennyDaBall/Qwen-Image-2.1-NVFP4", + "kind": "image", + "runtime": "spark03-qwen-image-21-nvfp4", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 32 }, + "performance": { "durationMs": 110000, "source": "fixture" } + }, + { + "id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03", + "gatewayModel": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "kind": "audio", + "runtime": "spark03-tts-clone", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 10 }, + "performance": { "durationMs": 1500, "source": "fixture" } + }, + { + "id": "openai/whisper-large-v3-turbo@spark-node-03", + "gatewayModel": "openai/whisper-large-v3-turbo", + "kind": "transcription", + "runtime": "spark03-stt", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 6 }, + "performance": { "durationMs": 2000, "source": "fixture" } + }, + { + "id": "Wan-AI/Wan2.2-TI2V-5B-Diffusers@spark-node-03", + "gatewayModel": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "kind": "video", + "runtime": "spark03-video", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 48 }, + "performance": { "durationMs": 240000, "source": "fixture" } + }, + { + "id": "Comfy-Org/YuE2-3B@spark-node-03", + "gatewayModel": "Comfy-Org/YuE2-3B", + "kind": "audio", + "runtime": "spark03-yue2", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 16 }, + "performance": { "durationMs": 90000, "source": "fixture" } + }, + { + "id": "cloud/openrouter/q38fn", + "gatewayModel": "cloud/openrouter/q38fn", + "kind": "chat", + "external": true, + "performance": { "ttftMs": 700, "decodeTps": 120, "source": "fixture" } + } + ], + "cognitionCandidates": ["qwen3.8-flash-next-atlas", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"], + "workloads": [ + { "id": "chat-capable", "cognition": true, "residency": "always", "demandPerHour": 30 }, + { "id": "chat-fast", "cognition": true, "residency": "always", "demandPerHour": 20 }, + { "id": "chat-classifier", "cognition": true, "residency": "always", "demandPerHour": 40 }, + { "id": "multimodal", "cognition": true, "residency": "always", "demandPerHour": 5 }, + { "id": "enntity-presence", "cognition": true, "residency": "always", "demandPerHour": 10 }, + { "id": "omp", "cognition": true, "residency": "always", "demandPerHour": 8 }, + { "id": "ds4f", "cognition": true, "residency": "always", "demandPerHour": 2 }, + { "id": "embeddings", "candidates": ["Qwen/Qwen3-Embedding-4B"], "residency": "always", "demandPerHour": 200 }, + { "id": "hear", "candidates": ["hear"], "residency": "always", "demandPerHour": 20 }, + { "id": "image-fast", "candidates": ["BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", "BennyDaBall/Qwen-Image-2.1-NVFP4"], "residency": "preferred", "demandPerHour": 2 }, + { "id": "image-quality", "candidates": ["BennyDaBall/Qwen-Image-2.1-NVFP4"], "residency": "auto", "demandPerHour": 0.2 }, + { "id": "tts-clone", "candidates": ["Qwen/Qwen3-TTS-12Hz-1.7B-Base"], "residency": "preferred", "demandPerHour": 3 }, + { "id": "stt", "candidates": ["openai/whisper-large-v3-turbo"], "residency": "auto", "demandPerHour": 1 }, + { "id": "video", "candidates": ["Wan-AI/Wan2.2-TI2V-5B-Diffusers"], "residency": "auto", "demandPerHour": 0.1 }, + { "id": "music", "candidates": ["Comfy-Org/YuE2-3B"], "residency": "auto", "demandPerHour": 0.1 } + ], + "pins": { "residency": { "qwen3-embedding-4b": "always", "hear": "always" } }, + "profiles": { + "sparkqwen-pair-rc16": { + "routes": { + "chat-capable": "qwen3.8-flash-next-atlas", + "chat-fast": "qwen3.8-flash-next-atlas", + "chat-classifier": "qwen3.8-flash-next-atlas", + "multimodal": "qwen3.8-flash-next-atlas", + "enntity-presence": "qwen3.8-flash-next-atlas", + "omp": "qwen3.8-flash-next-atlas", + "ds4f": "qwen3.8-flash-next-atlas", + "embeddings": "Qwen/Qwen3-Embedding-4B", + "hear": "hear", + "image-fast": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", + "image-quality": "BennyDaBall/Qwen-Image-2.1-NVFP4", + "tts-clone": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "stt": "openai/whisper-large-v3-turbo", + "video": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "music": "Comfy-Org/YuE2-3B" + }, + "residency": { + "qwen3-embedding-4b": "always", + "hear": "always", + "spark03-tts-clone": "preferred", + "spark03-qwen-image-21-turbo-nvfp4": "preferred", + "qwen38-27b-nvfp4": "auto", + "spark03-qwen-image-21-nvfp4": "auto", + "spark03-stt": "auto", + "spark03-video": "auto", + "spark03-yue2": "auto" + } + }, + "standalone": { + "routes": { + "chat-capable": "qwen38-27b-nvfp4", + "chat-fast": "qwen38-27b-nvfp4", + "chat-classifier": "qwen38-27b-nvfp4", + "multimodal": "qwen38-27b-nvfp4", + "enntity-presence": "qwen38-27b-nvfp4", + "omp": "qwen38-27b-nvfp4", + "ds4f": "qwen38-27b-nvfp4", + "embeddings": "Qwen/Qwen3-Embedding-4B", + "hear": "hear", + "image-fast": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", + "image-quality": "BennyDaBall/Qwen-Image-2.1-NVFP4", + "tts-clone": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "stt": "openai/whisper-large-v3-turbo", + "video": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "music": "Comfy-Org/YuE2-3B" + }, + "residency": { + "qwen3-embedding-4b": "always", + "hear": "always", + "spark03-tts-clone": "preferred", + "spark03-qwen-image-21-turbo-nvfp4": "preferred", + "qwen38-27b-nvfp4": "always", + "spark03-qwen-image-21-nvfp4": "auto", + "spark03-stt": "auto", + "spark03-video": "auto", + "spark03-yue2": "auto" + } + } + } +} diff --git a/test/metrics-persistence.test.mjs b/test/metrics-persistence.test.mjs index 5698bde..f7f83bd 100644 --- a/test/metrics-persistence.test.mjs +++ b/test/metrics-persistence.test.mjs @@ -46,6 +46,21 @@ try { assert.equal(restored.snapshot({ period: 'today' }).totals.inputTokens, 11); assert.equal(restored.snapshot({ period: '7d' }).models[0]?.outputTokens, 7); + // Requested ids (aliases) aggregate separately from resolved models so pool + // planning can see per-alias demand; older history files simply lack them. + assert.equal(restoredSnapshot.requested.find((entry) => entry.id === 'test-model')?.requests, 1); + restored.record({ + route: '/v1/chat/completions', + model: 'test-model', + requestedModel: 'chat-alias', + ok: true, + durationMs: 50 + }); + assert.equal(restored.snapshot().requested.find((entry) => entry.id === 'chat-alias')?.requests, 1); + assert.equal(restored.snapshot({ period: '7d' }).requested.find((entry) => entry.id === 'chat-alias')?.requests, 1); + assert.equal(restored.snapshot().models.find((entry) => entry.id === 'test-model')?.requests, 2); + assert.deepEqual(createMetricsStore({ initialSnapshot: { totals: {} } }).snapshot().requested, []); + // Global throughput is time-weighted and counts concurrent generation once. const RealDate = Date; let clock = RealDate.parse('2026-09-05T12:00:00Z'); diff --git a/test/pool-catalog.test.mjs b/test/pool-catalog.test.mjs new file mode 100644 index 0000000..907d8ba --- /dev/null +++ b/test/pool-catalog.test.mjs @@ -0,0 +1,334 @@ +// Pool catalog and planner-input tests: recipes, installed config, node status +// and metrics become planner variants, workloads, pins and current state. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { + buildPoolCatalog, + buildPoolPlanInputs, + currentPoolState, + plannerNodesFromStatus, + poolPinsFromConfig, + poolWorkloadsFromConfig, + recipeServingRequirements +} from '../src/pool-catalog.mjs'; +import { planPool } from '../src/pool-planner.mjs'; +import { validateRecipe } from '../src/recipes.mjs'; + +const tp2Recipe = JSON.parse( + readFileSync(new URL('../recipes/linux-nvidia-dgx-spark-2x-qwen38-flash-next-vllm.json', import.meta.url), 'utf8') +); + +test('serving requirements are inferred from an existing two-Spark recipe', () => { + const requirements = recipeServingRequirements(tp2Recipe, tp2Recipe.models[0]); + assert.equal(requirements.recipeId, tp2Recipe.id); + assert.equal(requirements.nodeCount, 2); + assert.equal(requirements.tensorParallel, 2); + assert.deepEqual(requirements.computeArchs, ['sm_121']); + assert.deepEqual(requirements.platforms, ['linux-arm64']); + assert.equal(requirements.memoryGbPerNode, 110); + assert.equal(requirements.contextWindow, 262144); + assert.equal(requirements.maxSequences, 8); + assert.equal(requirements.backend, 'docker-vllm'); + assert.equal(requirements.diskGb, 170); + assert.deepEqual(requirements.images, [ + 'vllm/vllm-openai@sha256:df871f170ee7070fbdce162bde08fb616e311570c948a620be0d4b33fe02f87b' + ]); + assert.deepEqual(requirements.inferred.sort(), ['computeArchs', 'maxSequences', 'tensorParallel']); +}); + +test('explicit recipe serving fields override inference', () => { + const recipe = { + id: 'four-way', + backend: { id: 'vllm' }, + requirements: { + platforms: ['linux-x64'], + accelerators: ['cuda'], + computeArchs: ['sm_100'], + tensorParallel: 4, + memoryGb: 300, + cluster: { nodes: 1 } + }, + models: [{ model: 'org/m', gatewayModel: 'm', settings: { maxSequences: 16, contextWindow: 8192 } }] + }; + const requirements = recipeServingRequirements(recipe, recipe.models[0]); + assert.equal(requirements.nodeCount, 1); + assert.equal(requirements.tensorParallel, 4); + assert.deepEqual(requirements.computeArchs, ['sm_100']); + assert.equal(requirements.memoryGbPerNode, 300); + assert.equal(requirements.maxSequences, 16); + assert.deepEqual(requirements.inferred, []); +}); + +test('recipe validation checks the structured serving fields', () => { + const recipe = (requirements, settings = {}) => ({ + schemaVersion: 1, + id: 'r', + name: 'R', + backend: { id: 'vllm' }, + requirements, + models: [{ role: 'default', model: 'org/m', settings }] + }); + const check = (...args) => validateRecipe(recipe(...args), {}, { checkLocalReferences: false }); + assert.deepEqual(check({ computeArchs: ['sm_121a', 'apple-metal'], tensorParallel: 2 }, { maxSequences: 8 }), []); + assert.deepEqual(check({ computeArchs: ['blackwell'] }), [ + 'recipe r requirements.computeArchs must list sm_ or apple-metal' + ]); + assert.deepEqual(check({ tensorParallel: 0 }), ['recipe r requirements.tensorParallel must be a positive integer']); + assert.deepEqual(check({}, { maxSequences: 1.5 }), [ + 'recipe r model default settings.maxSequences must be a positive integer' + ]); +}); + +function fleetConfig() { + return { + cluster: { + nodeId: 'spark-node-03', + nodes: { + 'spark-node-03': { resources: { memoryGb: 128, accelerators: ['cuda', 'gb10'] } }, + 'spark-node-01': { endpoint: 'http://01:8100', resources: { memoryGb: 128, maxMemoryUtilization: 0.9 } }, + 'spark-node-02': { endpoint: 'http://02:8100', resources: { memoryGb: 128 } } + }, + discovery: { links: [{ nodeId: 'spark-node-01', peerNodeId: 'spark-node-02' }] } + }, + runtimePolicy: { maxMemoryUtilization: 0.95 }, + backends: { + q27: { type: 'openai', baseUrl: 'http://127.0.0.1:8213/v1' }, + emb: { type: 'openai', baseUrl: 'http://127.0.0.1:8214/v1' }, + atlas: { type: 'openai', baseUrl: 'http://192.0.2.10:8894/v1' }, + openrouter: { type: 'openai', baseUrl: 'https://openrouter.ai/api/v1' }, + old: { type: 'openai', baseUrl: 'http://127.0.0.1:8299/v1' } + }, + runtimes: { + 'qwen38-27b-nvfp4': { enabled: true, memoryGb: 46, maxActiveRequests: 2, recipe: { id: 'spark-27b' } }, + 'qwen3-embedding-4b': { enabled: true, memoryGb: 16, keepWarm: true }, + retired: { enabled: false, memoryGb: 60 } + }, + models: [ + { id: 'qwen38-27b-nvfp4', kind: 'chat', backend: 'q27', runtime: 'qwen38-27b-nvfp4', contextWindow: 262144 }, + { id: 'Qwen/Qwen3-Embedding-4B', kind: 'embedding', backend: 'emb', runtime: 'qwen3-embedding-4b' }, + { + id: 'qwen3.8-flash-next-atlas', + kind: 'chat', + backend: 'atlas', + contextWindow: 262144, + pool: { nodes: ['spark-node-01', 'spark-node-02'], memoryGbPerNode: 110 } + }, + { id: 'cloud/openrouter/q38fn', kind: 'chat', backend: 'openrouter' }, + { id: 'retired-model', kind: 'chat', backend: 'old', runtime: 'retired' } + ], + aliases: { + 'chat-capable': { + members: ['qwen3.8-flash-next-atlas'], + routeProfiles: { + local: { members: ['qwen38-27b-nvfp4'] }, + cloud: { members: ['q38fn'] }, + pair: { members: ['qwen3.8-flash-next-atlas'] } + }, + activeRoute: 'pair' + }, + q38fn: { members: ['cloud/openrouter/q38fn'] }, + embeddings: { members: ['Qwen/Qwen3-Embedding-4B'] }, + quiet: { members: ['qwen38-27b-nvfp4', 'retired-model'], suspendedMembers: ['retired-model'] } + }, + pool: { + workloads: { quiet: { residency: 'auto', required: false, slo: { ttftMs: 2000 } } }, + cordon: ['spark-node-02'] + } + }; +} + +const sparkProfile = (memoryGb = 119.7) => ({ + platformId: 'linux-arm64', + totalMemoryGb: memoryGb, + accelerators: ['cuda', 'nvidia-gpu', 'blackwell', 'gb10'], + devices: [{ id: 'cuda:0', kind: 'gpu', vendor: 'nvidia', backend: 'cuda', computeCapability: '12.1' }], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: memoryGb }] +}); + +function fleetStatuses() { + return { + 'spark-node-03': { + reachable: true, + profile: sparkProfile(), + inventory: { + computeArchs: ['sm_121'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 119.7 }], + weights: [{ recipeId: 'spark-27b', complete: true }], + backends: { dockerImages: ['vllm/vllm-openai@sha256:abc'] }, + links: [] + }, + runtimeManager: { + runtimes: { + 'qwen3-embedding-4b': { status: 'running', memoryUsage: { residentBytes: 20 * 1024 ** 3 } }, + 'qwen38-27b-nvfp4': { status: 'stopped' }, + 'tts-sidecar': { status: 'external' } + } + } + }, + 'spark-node-01': { reachable: true, profile: sparkProfile() }, + 'spark-node-02': { reachable: false, profile: null } + }; +} + +// Shape of GET /gateway/metrics?period=7d: id-keyed bucket arrays. +const metrics = { + period: '7d', + models: [ + { id: 'qwen38-27b-nvfp4', requests: 50, avgFirstContentMs: 900, decodeTokensPerSecond: 28, avgDurationMs: 9000 }, + { id: 'qwen3.8-flash-next-atlas', requests: 2, avgFirstContentMs: 100, decodeTokensPerSecond: 500 } + ], + routes: [{ id: '/v1/chat/completions', requests: 1764 }], + requested: [ + { id: 'chat-capable', requests: 1680 }, + { id: 'embeddings', requests: 84 }, + { id: 'quiet', requests: 0 } + ] +}; + +test('the catalog groups installed lanes, recipes and evidence by model', () => { + const catalog = buildPoolCatalog({ + config: fleetConfig(), + recipes: [tp2Recipe], + benchmarks: [ + { + id: 'bench-atlas', + recipeId: 'none', + gatewayModel: 'qwen3.8-flash-next-atlas', + metrics: { timeToFirstTokenMs: 450, generationTokPerSec: 75 } + } + ], + metrics, + nodeStatuses: fleetStatuses() + }); + const byId = Object.fromEntries(catalog.variants.map((variant) => [variant.id, variant])); + + const q27 = byId['qwen38-27b-nvfp4@spark-node-03']; + assert.deepEqual(q27.nodes, ['spark-node-03']); + assert.equal(q27.runtime, 'qwen38-27b-nvfp4'); + assert.equal(q27.requirements.memoryGbPerNode, 46); + assert.equal(q27.requirements.recipeId, 'spark-27b'); + assert.equal(q27.requirements.maxSequences, 2); + assert.deepEqual(q27.performance, { ttftMs: 900, decodeTps: 28, durationMs: 9000, source: 'live-metrics' }); + + const embedding = byId['Qwen/Qwen3-Embedding-4B@spark-node-03']; + assert.equal(embedding.requirements.memoryGbPerNode, 20, 'measured footprint above the estimate wins'); + assert.deepEqual(embedding.measured, { memoryGbPerNode: 20, source: 'live-rss' }); + + const atlas = byId['qwen3.8-flash-next-atlas@spark-node-01+spark-node-02']; + assert.equal(atlas.managed, false); + assert.deepEqual(atlas.nodes, ['spark-node-01', 'spark-node-02']); + assert.equal(atlas.requirements.memoryGbPerNode, 110); + assert.deepEqual(atlas.performance, { ttftMs: 450, decodeTps: 75, source: 'benchmark:bench-atlas' }); + + assert.equal(byId['cloud/openrouter/q38fn'].external, true); + assert.ok(!catalog.variants.some((variant) => variant.runtime === 'retired')); + + const recipeVariant = byId[`recipe:${tp2Recipe.id}:qwen3.8-flash-next`]; + assert.equal(recipeVariant.nodes, undefined); + assert.equal(recipeVariant.requirements.nodeCount, 2); + assert.equal(recipeVariant.source, 'recipe'); + + const flashNext = catalog.models.find((model) => model.model === 'nvidia/Qwen3.8-Flash-Next-NVFP4'); + assert.deepEqual( + flashNext.variants.map((variant) => variant.id), + [`recipe:${tp2Recipe.id}:qwen3.8-flash-next`] + ); +}); + +test('workloads come from alias routes, pool overrides and observed demand', () => { + const workloads = Object.fromEntries( + poolWorkloadsFromConfig(fleetConfig(), { metrics }).map((workload) => [workload.id, workload]) + ); + assert.deepEqual(workloads['chat-capable'].candidates, [ + 'qwen3.8-flash-next-atlas', + 'qwen38-27b-nvfp4', + 'cloud/openrouter/q38fn' + ]); + assert.equal(workloads['chat-capable'].demandPerHour, 10); + assert.equal(workloads['chat-capable'].residency, 'always'); + assert.equal(workloads.embeddings.residency, 'always'); + assert.deepEqual(workloads.quiet.candidates, ['qwen38-27b-nvfp4']); + assert.equal(workloads.quiet.residency, 'auto'); + assert.equal(workloads.quiet.required, false); + assert.deepEqual(workloads.quiet.slo, { ttftMs: 2000 }); + + // Gateways that predate per-alias metrics: a model's demand is split across + // the aliases routed to it. + const legacy = Object.fromEntries( + poolWorkloadsFromConfig(fleetConfig(), { + metrics: { period: '7d', models: [{ id: 'qwen38-27b-nvfp4', requests: 336 }] } + }).map((workload) => [workload.id, workload]) + ); + assert.equal(legacy.quiet.demandPerHour, 2); + assert.equal(legacy.embeddings.demandPerHour, 0); + + const offline = Object.fromEntries(poolWorkloadsFromConfig(fleetConfig()).map((workload) => [workload.id, workload])); + assert.equal(offline.embeddings.residency, 'always', 'without metrics the current keep-warm runtime decides'); + assert.equal(offline['chat-capable'].residency, 'auto'); +}); + +test('pins, cordons and current state come from config and node status', () => { + const config = fleetConfig(); + const pins = poolPinsFromConfig(config, { + profile: { routes: { embeddings: 'Qwen/Qwen3-Embedding-4B' } }, + cordon: ['spark-node-01'], + routes: { 'chat-capable': 'qwen38-27b-nvfp4' } + }); + assert.deepEqual(pins.residency, { 'qwen3-embedding-4b': 'always' }); + assert.deepEqual(pins.routes, { + embeddings: 'Qwen/Qwen3-Embedding-4B', + 'chat-capable': 'qwen38-27b-nvfp4' + }); + assert.deepEqual(pins.cordon, ['spark-node-01', 'spark-node-02']); + + const catalog = buildPoolCatalog({ config, nodeStatuses: fleetStatuses() }); + const current = currentPoolState(config, { nodeStatuses: fleetStatuses(), variants: catalog.variants }); + assert.equal(current.routes['chat-capable'], 'qwen3.8-flash-next-atlas'); + assert.equal(current.routes.quiet, 'qwen38-27b-nvfp4'); + assert.deepEqual(current.residency, { 'qwen38-27b-nvfp4': 'auto', 'qwen3-embedding-4b': 'always' }); + assert.deepEqual(current.running.sort(), [ + 'Qwen/Qwen3-Embedding-4B@spark-node-03', + 'qwen3.8-flash-next-atlas@spark-node-01+spark-node-02' + ]); +}); + +test('planner nodes merge inventory, profiles, configured resources and links', () => { + const nodes = Object.fromEntries( + plannerNodesFromStatus(fleetConfig(), fleetStatuses()).map((node) => [node.id, node]) + ); + assert.deepEqual(nodes['spark-node-03'].weights, ['spark-27b']); + assert.deepEqual(nodes['spark-node-03'].images, ['vllm/vllm-openai@sha256:abc']); + assert.deepEqual( + nodes['spark-node-01'].computeArchs, + ['sm_121'], + 'derived from the profile when inventory is absent' + ); + assert.deepEqual(nodes['spark-node-01'].links, ['spark-node-02']); + assert.equal(nodes['spark-node-01'].maxMemoryUtilization, 0.9); + assert.equal(nodes['spark-node-02'].available, false); + assert.deepEqual(nodes['spark-node-02'].memoryDomains, [{ id: 'memory', kind: 'unified', capacityGb: 128 }]); +}); + +test('planner inputs from config produce a coherent plan', () => { + const inputs = buildPoolPlanInputs({ + config: fleetConfig(), + nodeStatuses: fleetStatuses(), + metrics, + recipes: [tp2Recipe] + }); + assert.equal(inputs.policy.maxMemoryUtilization, 0.95); + assert.ok(!inputs.variants.some((variant) => variant.source === 'recipe'), 'installs are opt-in'); + const plan = planPool(inputs); + // spark-node-02 is cordoned and unreachable, so the pair lane cannot serve. + assert.equal(plan.routes['chat-capable'].model, 'qwen38-27b-nvfp4'); + assert.equal(plan.routes.embeddings.model, 'Qwen/Qwen3-Embedding-4B'); + assert.equal(plan.residency['qwen38-27b-nvfp4'], 'always'); + assert.deepEqual(plan.diff.routeChanges, [ + { alias: 'chat-capable', from: 'qwen3.8-flash-next-atlas', to: 'qwen38-27b-nvfp4' } + ]); + assert.deepEqual( + plan.diff.manual.map((entry) => entry.action), + ['manual-stop'] + ); +}); diff --git a/test/pool-cli.test.mjs b/test/pool-cli.test.mjs new file mode 100644 index 0000000..a67f6bc --- /dev/null +++ b/test/pool-cli.test.mjs @@ -0,0 +1,172 @@ +// `lloom pool plan` report tests. Gateway calls and local probes are injected; +// the spawned CLI checks only argument handling and the offline JSON shape. +import { test, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createPoolPlanReport, formatPoolPlanReport, poolPlanOptions } from '../src/pool-cli.mjs'; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const cli = path.join(repoRoot, 'bin', 'lloom.mjs'); +const directories = []; +after(async () => { + await Promise.all(directories.map((dir) => fs.rm(dir, { recursive: true, force: true }))); +}); + +function config() { + return { + cluster: { nodeId: 'head', nodes: { head: {}, pair: { endpoint: 'http://pair:8100' } } }, + runtimePolicy: { maxMemoryUtilization: 0.95 }, + backends: { local: { type: 'openai', baseUrl: 'http://127.0.0.1:8201/v1' } }, + runtimes: { small: { enabled: true, memoryGb: 40 } }, + models: [ + { id: 'small', kind: 'chat', backend: 'local', runtime: 'small' }, + { id: 'big', kind: 'chat', backend: 'local', pool: { nodes: ['pair'], memoryGbPerNode: 100 } } + ], + aliases: { chat: { members: ['big'], routeProfiles: { local: { members: ['small'] } } } } + }; +} + +const spark = (memoryGb = 128) => ({ + reachable: true, + profile: { + platformId: 'linux-arm64', + accelerators: ['cuda', 'gb10'], + devices: [{ id: 'cuda:0', computeCapability: '12.1' }], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: memoryGb }] + } +}); + +const deps = (gatewayRequest) => ({ + gatewayRequest, + gatewayUrl: 'http://127.0.0.1:8100', + loadRecipes: async () => [], + loadBenchmarkEvidence: async () => [], + profileMachine: async () => spark().profile, + collectNodeInventory: async ({ nodeId }) => ({ version: 1, complete: true, nodeId, computeArchs: ['sm_121'] }), + modelRootCandidates: async () => [], + readProfile: async (_config, name) => ({ name, routes: { chat: 'small' }, residency: {} }) +}); + +test('pool plan options parse cordons, pins and policy flags', () => { + assert.deepEqual( + poolPlanOptions({ + cordon: ['a,b', 'c'], + pins: ['chat=small'], + headroomGb: '6', + external: 'never', + offline: true + }), + { + cordon: ['a', 'b', 'c'], + routes: { chat: 'small' }, + profileName: null, + considerInstalls: false, + offline: true, + policy: { headroomGb: 6, external: 'never' } + } + ); + assert.throws(() => poolPlanOptions({ pins: ['chat'] }), /ALIAS=MODEL/); + assert.throws(() => poolPlanOptions({ external: 'sometimes' }), /fallback, allow, or never/); + assert.throws(() => poolPlanOptions({ churnWeight: '-1' }), /non-negative/); +}); + +test('a gateway report plans from live status and metrics', async () => { + const requested = []; + const report = await createPoolPlanReport( + config(), + poolPlanOptions({}), + deps(async (_config, pathname) => { + requested.push(pathname); + if (pathname.startsWith('/gateway/cluster')) { + return { cluster: { nodes: { head: spark(), pair: spark() } } }; + } + return { period: '7d', models: [], routes: [], requested: [{ id: 'chat', requests: 336 }] }; + }) + ); + assert.deepEqual(requested, ['/gateway/cluster?inventory=1', '/gateway/metrics?period=7d']); + assert.equal(report.dryRun, true); + assert.deepEqual(report.source, { + mode: 'gateway', + gateway: 'http://127.0.0.1:8100', + metricsPeriod: '7d', + note: null + }); + assert.equal(report.plan.routes.chat.model, 'big'); + assert.equal(report.plan.routes.chat.residency, 'always', '2 requests per hour makes the lane hot'); + const text = formatPoolPlanReport(report); + assert.match(text, /dry run; nothing was changed/); + assert.match(text, /always +big@pair 100\.0 GB\/node \(unmanaged\) <- chat/); + assert.match(text, /Changes from current\n {2}\(none\)/, 'a routed unmanaged lane is assumed to be serving'); +}); + +test('cordons and profile pins reach the planner', async () => { + const report = await createPoolPlanReport( + config(), + poolPlanOptions({ cordon: ['pair'], profile: 'standalone' }), + deps(async (_config, pathname) => + pathname.startsWith('/gateway/cluster') ? { cluster: { nodes: { head: spark(), pair: spark() } } } : null + ) + ); + assert.equal(report.profile, 'standalone'); + assert.deepEqual(report.pins.routes, { chat: 'small' }); + assert.equal(report.nodes.find((node) => node.id === 'pair').cordoned, true); + assert.equal(report.plan.routes.chat.model, 'small'); + assert.equal(report.plan.routes.chat.reason, 'pinned'); + assert.match(formatPoolPlanReport(report), /route +chat: big -> small/); +}); + +test('an unreachable gateway falls back to config and local inventory', async () => { + const report = await createPoolPlanReport( + config(), + poolPlanOptions({}), + deps(async () => { + throw new Error('connect ECONNREFUSED'); + }) + ); + assert.equal(report.source.mode, 'config'); + assert.match(report.source.note, /ECONNREFUSED/); + assert.deepEqual( + report.nodes.map((node) => [node.id, node.inventory]), + [ + ['head', 'reported'], + ['pair', 'config'] + ] + ); + assert.match(formatPoolPlanReport(report), /Source: config only — gateway unavailable/); +}); + +function runCli(args, env) { + return new Promise((resolve) => { + execFile(process.execPath, [cli, ...args], { env, timeout: 60000 }, (error, stdout, stderr) => + resolve({ code: error ? (error.code ?? 1) : 0, stdout, stderr }) + ); + }); +} + +test('the CLI prints an offline JSON plan and rejects unknown actions', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'lloom-pool-')); + directories.push(dir); + const configPath = path.join(dir, 'config.json'); + const cliConfig = { + ...config(), + cluster: undefined, + models: [config().models[0]], + aliases: { chat: { members: ['small'] } } + }; + await fs.writeFile(configPath, JSON.stringify(cliConfig, null, 2)); + const env = { ...process.env, HOME: dir, LLOOM_MODEL_ROOT: path.join(dir, 'models') }; + const result = await runCli(['pool', 'plan', '--offline', '--json', '--config', configPath], env); + assert.equal(result.code, 0, result.stderr); + const report = JSON.parse(result.stdout); + assert.equal(report.dryRun, true); + assert.equal(report.source.mode, 'config'); + assert.equal(report.plan.routes.chat.model, 'small'); + + const bad = await runCli(['pool', 'apply', '--config', configPath], env); + assert.notEqual(bad.code, 0); + assert.match(bad.stderr, /Usage: lloom pool plan/); +}); diff --git a/test/pool-inventory.test.mjs b/test/pool-inventory.test.mjs new file mode 100644 index 0000000..1170d0b --- /dev/null +++ b/test/pool-inventory.test.mjs @@ -0,0 +1,167 @@ +// Node inventory tests. Every probe (statfs, docker, commands, model files) is +// injected; nothing touches the host's model roots or container runtime. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { ClusterCoordinator } from '../src/cluster.mjs'; +import { + collectNodeInventory, + computeArchsForProfile, + createNodeInventoryCache, + inventoryFromProfile +} from '../src/pool-inventory.mjs'; + +const sparkProfile = { + platformId: 'linux-arm64', + totalMemoryGb: 119.7, + isAppleSilicon: false, + accelerators: ['cuda', 'nvidia-gpu', 'blackwell', 'gb10'], + devices: [{ id: 'cuda:0', kind: 'gpu', vendor: 'nvidia', backend: 'cuda', computeCapability: '12.1' }], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 119.7, deviceIds: ['cuda:0'] }], + cluster: { + provider: 'nvidia-sync', + links: [ + { + nodeId: 'spark-node-01', + peerNodeId: 'spark-node-02', + localInterface: 'enp1s0f0np0', + peerAddress: '10.100.0.2' + }, + { nodeId: 'spark-node-01', peerNodeId: 'spark-node-02', localInterface: 'enp1s0f1np1', peerAddress: '10.100.1.2' } + ] + } +}; + +test('compute architectures are structured from device capabilities', () => { + assert.deepEqual(computeArchsForProfile(sparkProfile), ['sm_121']); + assert.deepEqual(computeArchsForProfile({ isAppleSilicon: true, devices: [] }), ['apple-metal']); + assert.deepEqual(computeArchsForProfile({ devices: [{ computeCapability: '8.9' }, { computeCapability: '8.9' }] }), [ + 'sm_89' + ]); + assert.deepEqual(computeArchsForProfile({ devices: [] }), []); +}); + +test('profile-only inventory carries memory domains and direct links', () => { + const inventory = inventoryFromProfile(sparkProfile); + assert.deepEqual(inventory.computeArchs, ['sm_121']); + assert.deepEqual(inventory.memoryDomains, [{ id: 'unified:0', kind: 'unified', capacityGb: 119.7 }]); + assert.deepEqual(inventory.links, [ + { peerNodeId: 'spark-node-02', localInterface: 'enp1s0f0np0', peerAddress: '10.100.0.2' }, + { peerNodeId: 'spark-node-02', localInterface: 'enp1s0f1np1', peerAddress: '10.100.1.2' } + ]); + assert.equal(inventory.complete, false); +}); + +test('a node inventory reports disk, cached weights, backends and images', async () => { + const recipes = [ + { id: 'spark-a', requirements: { platforms: ['linux-arm64'], commands: ['docker'] } }, + { id: 'spark-b', requirements: { platforms: ['linux-arm64'] } }, + { id: 'mac-only', requirements: { platforms: ['darwin-arm64'], commands: ['mlx_lm.server'] } } + ]; + const checked = []; + const inventory = await collectNodeInventory({ + nodeId: 'spark-node-01', + profile: sparkProfile, + recipes, + modelRoots: ['/models', '/missing'], + probes: { + statfs: async (root) => + root === '/models' ? { freeGb: 812.4, totalGb: 3725.3 } : Promise.reject(new Error('ENOENT')), + modelRootStatus: async (recipe, root) => { + checked.push(`${recipe.id}:${root}`); + return { complete: recipe.id === 'spark-a' }; + }, + commandExists: async (command) => command === 'docker', + dockerImages: async () => ['vllm/vllm-openai@sha256:abc', 'lloom/comfy:cuda'], + now: () => '2026-10-11T00:00:00.000Z' + } + }); + assert.equal(inventory.version, 1); + assert.equal(inventory.nodeId, 'spark-node-01'); + assert.equal(inventory.complete, true); + assert.equal(inventory.collectedAt, '2026-10-11T00:00:00.000Z'); + assert.deepEqual(inventory.computeArchs, ['sm_121']); + assert.deepEqual(inventory.disks, [{ root: '/models', freeGb: 812.4, totalGb: 3725.3 }]); + assert.deepEqual(inventory.weights, [{ recipeId: 'spark-a', root: '/models', complete: true }]); + assert.deepEqual(checked.sort(), ['spark-a:/models', 'spark-b:/models'], 'only platform-compatible recipes'); + assert.deepEqual(inventory.backends, { + commands: { docker: true }, + dockerImages: ['lloom/comfy:cuda', 'vllm/vllm-openai@sha256:abc'] + }); + assert.equal(inventory.links.length, 2); +}); + +test('failing probes degrade to partial inventory instead of throwing', async () => { + const inventory = await collectNodeInventory({ + nodeId: 'mac', + profile: { platformId: 'darwin-arm64', isAppleSilicon: true, totalMemoryGb: 64 }, + recipes: [{ id: 'mac', requirements: { platforms: ['darwin-arm64'] } }], + modelRoots: ['/models'], + probes: { + statfs: async () => ({ freeGb: 10, totalGb: 100 }), + modelRootStatus: async () => { + throw new Error('permission denied'); + }, + commandExists: async () => false, + dockerImages: async () => { + throw new Error('should not run without docker'); + } + } + }); + assert.deepEqual(inventory.computeArchs, ['apple-metal']); + assert.deepEqual(inventory.weights, []); + assert.deepEqual(inventory.backends, { commands: {}, dockerImages: [] }); + assert.deepEqual(inventory.errors, ['weights mac: permission denied']); +}); + +test('the inventory cache reuses results within its TTL and shares in-flight work', async () => { + let now = 0; + let calls = 0; + const cache = createNodeInventoryCache({ + ttlMs: 1000, + now: () => now, + collect: async () => ({ call: ++calls }) + }); + const [first, second] = await Promise.all([cache.get(), cache.get()]); + assert.equal(first.call, 1); + assert.equal(second.call, 1); + now = 999; + assert.equal((await cache.get()).call, 1); + now = 1000; + assert.equal((await cache.get()).call, 2); +}); + +test('gateway node status includes inventory only when asked', async () => { + const requested = []; + const coordinator = new ClusterCoordinator( + { + cluster: { + nodeId: 'head', + nodes: { head: {}, peer: { endpoint: 'http://peer:8100' } } + } + }, + { + env: {}, + profile: () => sparkProfile, + inventory: async () => ({ version: 1, nodeId: 'head' }), + fetchImpl: async (url) => { + requested.push(String(url)); + return new Response(JSON.stringify({ ok: true, node: { id: 'peer', inventory: { version: 1 } } }), { + headers: { 'content-type': 'application/json' } + }); + } + } + ); + assert.equal((await coordinator.localNodeStatus()).inventory, undefined); + assert.deepEqual((await coordinator.localNodeStatus({ includeInventory: true })).inventory, { + version: 1, + nodeId: 'head' + }); + const peer = await coordinator.nodeStatus('peer', { inventory: true }); + assert.deepEqual(peer.inventory, { version: 1 }); + assert.deepEqual(requested, ['http://peer:8100/gateway/node?memoryUsage=1&inventory=1']); + await coordinator.nodeStatus('peer'); + assert.equal(requested.at(-1), 'http://peer:8100/gateway/node?memoryUsage=1'); + const cluster = await coordinator.status({ inventory: true }); + assert.deepEqual(cluster.nodes.head.inventory, { version: 1, nodeId: 'head' }); + assert.deepEqual(cluster.nodes.peer.inventory, { version: 1 }); +}); diff --git a/test/pool-planner.test.mjs b/test/pool-planner.test.mjs new file mode 100644 index 0000000..0f1cba2 --- /dev/null +++ b/test/pool-planner.test.mjs @@ -0,0 +1,428 @@ +// Pool planner tests. Pure planner inputs only: no gateway, runtimes or network. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { planPool } from '../src/pool-planner.mjs'; + +const fleet = JSON.parse(readFileSync(new URL('./fixtures/pool/spark-fleet.json', import.meta.url), 'utf8')); + +function fleetWorkloads() { + return fleet.workloads.map(({ cognition, ...workload }) => ({ + ...workload, + candidates: cognition ? fleet.cognitionCandidates : workload.candidates + })); +} + +function currentFromProfile(name, running) { + const profile = fleet.profiles[name]; + return { routes: profile.routes, residency: profile.residency, running }; +} + +const RC16_RUNNING = [ + 'qwen3.8-flash-next-atlas@spark-node-01+spark-node-02', + 'Qwen/Qwen3-Embedding-4B@spark-node-03', + 'hear@spark-node-03', + 'BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03', + 'Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03' +]; +const STANDALONE_RUNNING = [ + 'qwen38-27b-nvfp4@spark-node-03', + 'Qwen/Qwen3-Embedding-4B@spark-node-03', + 'hear@spark-node-03', + 'BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03', + 'Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03' +]; + +function planFleet({ cordon = [], current, pins = {}, nodes = fleet.nodes, policy = {} } = {}) { + return planPool({ + nodes, + variants: fleet.variants, + workloads: fleetWorkloads(), + pins: { ...fleet.pins, ...pins, cordon }, + current, + policy: { ...fleet.policy, ...policy } + }); +} + +function plannedRoutes(plan) { + return Object.fromEntries(Object.entries(plan.routes).map(([alias, route]) => [alias, route.model])); +} + +function plannedResidency(plan, expected) { + return Object.fromEntries(Object.keys(expected).map((runtime) => [runtime, plan.residency[runtime]])); +} + +test('whole fleet reproduces the sparkqwen-pair-rc16 profile with no changes', () => { + const plan = planFleet({ current: currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING) }); + const profile = fleet.profiles['sparkqwen-pair-rc16']; + assert.equal(plan.ok, true, JSON.stringify(plan.uncovered)); + assert.deepEqual(plannedRoutes(plan), profile.routes); + assert.deepEqual(plannedResidency(plan, profile.residency), profile.residency); + assert.deepEqual(plan.diff.routeChanges, []); + assert.deepEqual(plan.diff.start, []); + assert.deepEqual(plan.diff.manual, []); + assert.deepEqual(plan.diff.release, []); + assert.equal(plan.score.external, 0); +}); + +test('cordoning the 01/02 pair reproduces the standalone profile', () => { + const plan = planFleet({ + cordon: ['spark-node-01', 'spark-node-02'], + current: currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING) + }); + const profile = fleet.profiles.standalone; + assert.equal(plan.ok, true, JSON.stringify(plan.uncovered)); + assert.deepEqual(plannedRoutes(plan), profile.routes); + assert.deepEqual(plannedResidency(plan, profile.residency), profile.residency); + assert.deepEqual( + plan.diff.start.map((entry) => [entry.placement, entry.residency]), + [['qwen38-27b-nvfp4@spark-node-03', 'always']] + ); + assert.deepEqual(plan.diff.manual, [ + { + action: 'manual-stop', + placement: 'qwen3.8-flash-next-atlas@spark-node-01+spark-node-02', + nodes: ['spark-node-01', 'spark-node-02'] + } + ]); + assert.equal(plan.diff.routeChanges.length, 7); + assert.ok(plan.diff.routeChanges.every((change) => change.to === 'qwen38-27b-nvfp4')); + assert.equal(plan.nodes['spark-node-01'].status, 'cordoned'); + assert.equal(plan.score.external, 0, 'local 27B covers cognition before the cloud member'); + const spark03 = plan.nodes['spark-node-03'].domains[0]; + assert.ok(spark03.residentGb <= spark03.budgetGb); + assert.ok(spark03.alwaysGb + spark03.largestOnDemandGb <= spark03.budgetGb); +}); + +test('an unreachable pair is planned around exactly like a cordoned one', () => { + const nodes = fleet.nodes.map((node) => (node.id === 'spark-node-03' ? node : { ...node, available: false })); + const plan = planFleet({ nodes, current: currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING) }); + assert.deepEqual(plannedRoutes(plan), fleet.profiles.standalone.routes); + assert.equal(plan.nodes['spark-node-02'].status, 'unavailable'); +}); + +test('a free pair moves cognition back to SparkQwen as a manual start', () => { + const plan = planFleet({ current: currentFromProfile('standalone', STANDALONE_RUNNING) }); + const profile = fleet.profiles['sparkqwen-pair-rc16']; + assert.deepEqual(plannedRoutes(plan), profile.routes); + assert.equal(plan.residency['qwen38-27b-nvfp4'], 'auto'); + assert.deepEqual( + plan.diff.manual.map((entry) => entry.action + ' ' + entry.placement), + ['manual-start qwen3.8-flash-next-atlas@spark-node-01+spark-node-02'] + ); + assert.deepEqual(plan.diff.release, [{ action: 'release', placement: 'qwen38-27b-nvfp4@spark-node-03' }]); + assert.deepEqual(plan.diff.residencyChanges, [{ runtime: 'qwen38-27b-nvfp4', from: 'always', to: 'auto' }]); +}); + +test('a high churn weight keeps the running lane when the gain is small', () => { + const plan = planFleet({ + current: currentFromProfile('standalone', STANDALONE_RUNNING), + policy: { churnWeight: 100000 } + }); + assert.deepEqual(plannedRoutes(plan), fleet.profiles.standalone.routes); + assert.deepEqual(plan.diff.routeChanges, []); +}); + +test('pinned routes and residency are hard constraints', () => { + const plan = planFleet({ + current: currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING), + pins: { routes: { omp: 'qwen38-27b-nvfp4' } } + }); + assert.equal(plan.routes.omp.model, 'qwen38-27b-nvfp4'); + assert.equal(plan.routes.omp.reason, 'pinned'); + assert.equal(plan.routes['chat-capable'].model, 'qwen3.8-flash-next-atlas'); + assert.equal(plan.residency['qwen3-embedding-4b'], 'always'); + assert.equal(plan.residency.hear, 'always'); + + const impossible = planFleet({ + current: {}, + pins: { routes: { omp: 'qwen3.8-flash-next-atlas' } }, + cordon: ['spark-node-01'] + }); + assert.equal(impossible.ok, false); + assert.equal(impossible.uncovered[0].workload, 'omp'); + assert.equal(impossible.uncovered[0].pinned, 'qwen3.8-flash-next-atlas'); +}); + +const spark = (id, links = []) => ({ + id, + platformId: 'linux-arm64', + computeArchs: ['sm_121'], + accelerators: ['cuda', 'gb10'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 128 }], + links +}); +const mac = { + id: 'studio', + platformId: 'darwin-arm64', + computeArchs: ['apple-metal'], + accelerators: ['metal', 'apple-gpu'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 192 }] +}; +const tp2 = { + id: 'recipe:tp2', + gatewayModel: 'big-moe', + kind: 'chat', + requirements: { nodeCount: 2, tensorParallel: 2, computeArchs: ['sm_121a'], memoryGbPerNode: 100 }, + performance: { ttftMs: 500, decodeTps: 50 } +}; +const one = (workload) => [ + { id: 'chat', candidates: ['big-moe'], residency: 'always', demandPerHour: 10, ...workload } +]; + +test('tensor-parallel variants only span directly linked nodes', () => { + const plan = planPool({ + nodes: [spark('a', ['b']), spark('b'), spark('c')], + variants: [tp2], + workloads: one(), + policy: { maxMemoryUtilization: 0.95 } + }); + assert.deepEqual(plan.routes.chat.nodes, ['a', 'b']); + assert.equal(plan.placements[0].placement, 'recipe:tp2@a+b'); + + const unlinked = planPool({ + nodes: [spark('a'), spark('b'), spark('c')], + variants: [tp2], + workloads: one(), + policy: { maxMemoryUtilization: 0.95 } + }); + assert.equal(unlinked.ok, false); + assert.ok(unlinked.uncovered[0].reasons.some((reason) => reason.includes('no directly linked set of 2'))); +}); + +test('compute architecture must match the variant', () => { + const single = { ...tp2, id: 'recipe:one', requirements: { computeArchs: ['sm_121'], memoryGbPerNode: 60 } }; + const macOnly = planPool({ nodes: [mac], variants: [single], workloads: one() }); + assert.equal(macOnly.ok, false); + assert.ok(macOnly.uncovered[0].reasons.some((reason) => reason.includes('needs compute arch sm_121'))); + const mixed = planPool({ nodes: [mac, spark('s')], variants: [single], workloads: one() }); + assert.deepEqual(mixed.routes.chat.nodes, ['s']); +}); + +test('memory headroom is conservative', () => { + const large = { ...tp2, id: 'recipe:large', requirements: { memoryGbPerNode: 117 } }; + const fits = planPool({ + nodes: [spark('s')], + variants: [large], + workloads: one(), + policy: { maxMemoryUtilization: 0.95, headroomGb: 4 } + }); + assert.equal(fits.ok, true); + const tight = planPool({ + nodes: [spark('s')], + variants: [large], + workloads: one(), + policy: { maxMemoryUtilization: 0.95, headroomGb: 8 } + }); + assert.equal(tight.ok, false); + assert.ok(tight.uncovered[0].reasons.some((reason) => reason.includes('does not fit as always'))); + const defaults = planPool({ nodes: [spark('s')], variants: [large], workloads: one() }); + assert.equal(defaults.ok, false, 'default budget is capacity minus reserve minus headroom'); + assert.equal(defaults.nodes.s.domains[0].budgetGb, 116); +}); + +test('memory is accounted per domain on discrete GPUs', () => { + const pc = { + id: 'pc', + platformId: 'linux-x64', + computeArchs: ['sm_89'], + accelerators: ['cuda'], + memoryDomains: [ + { id: 'system:0', kind: 'system', capacityGb: 64 }, + { id: 'device:cuda:0', kind: 'device', capacityGb: 24 } + ] + }; + const small = { + id: 'small', + gatewayModel: 'small', + requirements: { memoryGbPerNode: 18, systemMemoryGbPerNode: 30 } + }; + const big = { id: 'big', gatewayModel: 'big', requirements: { memoryGbPerNode: 30 } }; + const plan = planPool({ + nodes: [pc], + variants: [small, big], + workloads: [ + { id: 'a', candidates: ['small'], residency: 'always' }, + { id: 'b', candidates: ['big'], residency: 'always' } + ], + policy: { reserveGb: 0, headroomGb: 0 } + }); + assert.equal(plan.routes.a.model, 'small'); + assert.deepEqual( + plan.nodes.pc.domains.map((domain) => [domain.id, domain.residentGb]), + [ + ['system:0', 30], + ['device:cuda:0', 18] + ] + ); + assert.equal(plan.uncovered[0].workload, 'b'); +}); + +test('local search trades a faster model for full coverage', () => { + const fast = { + id: 'fast', + gatewayModel: 'fast', + requirements: { memoryGbPerNode: 80 }, + performance: { durationMs: 100 } + }; + const slow = { + id: 'slow', + gatewayModel: 'slow', + requirements: { memoryGbPerNode: 30 }, + performance: { durationMs: 900 } + }; + const media = { id: 'media', gatewayModel: 'media', requirements: { memoryGbPerNode: 50 } }; + const plan = planPool({ + nodes: [spark('s')], + variants: [fast, slow, media], + workloads: [ + { id: 'chat', candidates: ['fast', 'slow'], residency: 'always', demandPerHour: 100 }, + { id: 'video', candidates: ['media'], residency: 'always', demandPerHour: 1 } + ], + policy: { maxMemoryUtilization: 0.95 } + }); + assert.equal(plan.ok, true); + assert.equal(plan.routes.chat.model, 'slow'); + assert.equal(plan.routes.video.model, 'media'); +}); + +test('external members are a fallback unless allowed or disabled', () => { + const cloud = { id: 'cloud', gatewayModel: 'cloud', external: true, performance: { ttftMs: 100, decodeTps: 500 } }; + const local = { ...tp2, id: 'local', gatewayModel: 'local', requirements: { memoryGbPerNode: 40 } }; + const workloads = [{ id: 'chat', candidates: ['cloud', 'local'], residency: 'always', demandPerHour: 10 }]; + const fallback = planPool({ nodes: [spark('s')], variants: [cloud, local], workloads }); + assert.equal(fallback.routes.chat.model, 'local'); + const allowed = planPool({ nodes: [spark('s')], variants: [cloud, local], workloads, policy: { external: 'allow' } }); + assert.equal(allowed.routes.chat.model, 'cloud'); + const cordoned = planPool({ + nodes: [spark('s')], + variants: [cloud, local], + workloads, + pins: { cordon: ['s'] } + }); + assert.equal(cordoned.routes.chat.model, 'cloud'); + assert.equal(cordoned.score.external, 1); + const never = planPool({ + nodes: [spark('s')], + variants: [cloud, local], + workloads, + pins: { cordon: ['s'] }, + policy: { external: 'never' } + }); + assert.equal(never.ok, false); +}); + +test('workload context and latency SLOs filter and rank variants', () => { + const short = { + ...tp2, + id: 'short', + gatewayModel: 'short', + requirements: { memoryGbPerNode: 20, contextWindow: 32768 } + }; + const long = { + ...tp2, + id: 'long', + gatewayModel: 'long', + requirements: { memoryGbPerNode: 40, contextWindow: 262144 } + }; + const plan = planPool({ + nodes: [spark('s')], + variants: [short, long], + workloads: [{ id: 'chat', candidates: ['short', 'long'], slo: { contextWindow: 131072 } }] + }); + assert.equal(plan.routes.chat.model, 'long'); + + const quick = { + ...tp2, + id: 'quick', + gatewayModel: 'quick', + requirements: { memoryGbPerNode: 20 }, + performance: { ttftMs: 200, decodeTps: 10 } + }; + const steady = { + ...tp2, + id: 'steady', + gatewayModel: 'steady', + requirements: { memoryGbPerNode: 20 }, + performance: { ttftMs: 1500, decodeTps: 200 } + }; + const slo = planPool({ + nodes: [spark('s')], + variants: [quick, steady], + workloads: [{ id: 'voice', candidates: ['steady', 'quick'], slo: { ttftMs: 500 } }] + }); + assert.equal(slo.routes.voice.model, 'quick'); + assert.equal(slo.score.sloMisses, 0); +}); + +test('missing weights and images add load cost to recipe placements', () => { + const recipe = { + ...tp2, + id: 'recipe:fresh', + gatewayModel: 'fresh', + requirements: { memoryGbPerNode: 40, recipeId: 'fresh-recipe', diskGb: 40, images: ['vllm@sha256:1'] } + }; + const cached = { ...spark('cached'), weights: ['fresh-recipe'], images: ['vllm@sha256:1'] }; + const plan = planPool({ + nodes: [spark('bare'), cached], + variants: [recipe], + workloads: [{ id: 'chat', candidates: ['fresh'], residency: 'always' }] + }); + assert.deepEqual(plan.routes.chat.nodes, ['cached']); + assert.deepEqual( + plan.diff.start.map((entry) => entry.placement), + ['recipe:fresh@cached'] + ); +}); + +test('planning is deterministic', () => { + const current = currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING); + assert.deepEqual(planFleet({ current }), planFleet({ current })); +}); + +test('models served by one runtime share a single placement', () => { + const image = { + id: 'image@s', + gatewayModel: 'image', + runtime: 'comfy', + nodes: ['s'], + requirements: { memoryGbPerNode: 60 } + }; + const edit = { + id: 'edit@s', + gatewayModel: 'edit', + runtime: 'comfy', + nodes: ['s'], + requirements: { memoryGbPerNode: 60 } + }; + const chat = { + id: 'chat@s', + gatewayModel: 'chat', + runtime: 'llm', + nodes: ['s'], + requirements: { memoryGbPerNode: 50 } + }; + const plan = planPool({ + nodes: [spark('s')], + variants: [image, edit, chat], + workloads: [ + { id: 'image-fast', candidates: ['image'], residency: 'always' }, + { id: 'image-edit', candidates: ['edit'], residency: 'always' }, + { id: 'chat', candidates: ['chat'], residency: 'always' } + ], + current: { running: ['image@s', 'edit@s'] }, + policy: { maxMemoryUtilization: 0.95 } + }); + assert.equal(plan.ok, true, JSON.stringify(plan.uncovered)); + assert.equal(plan.nodes.s.domains[0].residentGb, 110); + const comfy = plan.placements.find((placement) => placement.runtime === 'comfy'); + assert.deepEqual(comfy.models, ['edit', 'image']); + assert.deepEqual(comfy.workloads, ['image-edit', 'image-fast']); + assert.equal(plan.routes['image-edit'].placement, plan.routes['image-fast'].placement); + assert.deepEqual(plan.diff.release, []); + assert.deepEqual( + plan.diff.start.map((entry) => entry.placement), + ['chat@s'] + ); + assert.equal(plan.residency.comfy, 'always'); +});