From cb4057a158b08ecabb548ab0de90f14e4e6b713c Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 11:37:21 -0700 Subject: [PATCH 1/9] Add a dry-run pool placement planner (lloom pool plan) Treat every cluster node as one pool and propose which model lanes run where, with which residency, and how each alias routes, plus the diff from what runs now. Phase 1 is read-only: nothing is started, stopped or rerouted, and per-node admission stays the safety layer. - Node inventory (/gateway/node?inventory=1, cached): structured compute archs, memory domains, model-root disk, cached recipe weights, commands and Docker images, and NVIDIA Sync fabric links. - Model-centric catalog from managed runtimes, federated targets, unmanaged lanes declared with a model `pool` block, external models and (opt-in) bundled recipes, with measured footprint and live-metric or benchmark performance. - Pure planner: memory per domain with headroom, TP only across directly linked nodes, arch/platform/accelerator matching, pins and cordons as hard constraints; lexicographic objective (coverage, external fallback, SLOs, demand-weighted latency plus load-time churn); greedy placement plus single-workload local search. Models sharing a runtime share one placement. - Per-alias demand: /gateway/metrics adds `requested` buckets. - Recipes may declare requirements.computeArchs, requirements.tensorParallel and settings.maxSequences (schema, validator, example). - Admission and pool planning share one memory-budget helper. Fixture tests reproduce the three-Spark fleet: all nodes give the sparkqwen-pair-rc16 profile; cordoning the pair gives standalone. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + README.md | 1 + bin/lloom.mjs | 24 ++ docs/architecture.md | 2 +- docs/clusters.md | 2 + docs/pool-planning.md | 113 +++++ docs/recipes.md | 4 +- examples/interchange/recipe.v1.json | 5 +- package.json | 4 +- schemas/recipe.v1.schema.json | 19 + src/cluster.mjs | 35 +- src/pool-catalog.mjs | 538 +++++++++++++++++++++++ src/pool-cli.mjs | 224 ++++++++++ src/pool-inventory.mjs | 165 +++++++ src/pool-planner.mjs | 644 ++++++++++++++++++++++++++++ src/recipes.mjs | 18 + src/resource-fit.mjs | 10 + src/runtime-policy.mjs | 25 +- src/server.mjs | 40 +- test/fixtures/pool/spark-fleet.json | 212 +++++++++ test/metrics-persistence.test.mjs | 15 + test/pool-catalog.test.mjs | 334 +++++++++++++++ test/pool-cli.test.mjs | 172 ++++++++ test/pool-inventory.test.mjs | 167 ++++++++ test/pool-planner.test.mjs | 428 ++++++++++++++++++ 25 files changed, 3168 insertions(+), 35 deletions(-) create mode 100644 docs/pool-planning.md create mode 100644 src/pool-catalog.mjs create mode 100644 src/pool-cli.mjs create mode 100644 src/pool-inventory.mjs create mode 100644 src/pool-planner.mjs create mode 100644 test/fixtures/pool/spark-fleet.json create mode 100644 test/pool-catalog.test.mjs create mode 100644 test/pool-cli.test.mjs create mode 100644 test/pool-inventory.test.mjs create mode 100644 test/pool-planner.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 91d88ac..58459c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to LLooM will be documented in this file. The format follows ### Added +- `lloom pool plan`: a dry-run placement plan across every cluster node. It proposes which lanes run where, which residency they get and how each alias routes, and diffs that against what runs now. It covers every required alias first, then minimises demand-weighted latency and the load cost of changes, within per-domain memory headroom, direct-link tensor parallelism, compute architecture and pins. Nodes report a pool inventory through `/gateway/node?inventory=1`. Recipes may declare `requirements.computeArchs`, `requirements.tensorParallel` and `settings.maxSequences`. `/gateway/metrics` adds `requested` buckets, which give per-alias demand. See `docs/pool-planning.md`. + - LLooM Hear CPU audio analysis, with structured estimates, optional dashboard images, and opt-in upstream interpretation. File and URL inputs require operator configuration. - Four standalone NVIDIA Spark audio recipes (Qwen3-TTS 1.7B CustomVoice, Base voice cloning and VoiceDesign, and Whisper large-v3-turbo). Each runs one pinned, integrity-checked checkpoint in its own container from a locally built `spark-audio` image that replaces a host-local prebuilt image. diff --git a/README.md b/README.md index e32c43b..cd07a57 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ lloom route q38fn # inspe lloom route q38fn --suspend-member qwen3.8-flash-next --apply --yes # keep stable traffic on cloud lloom runtime-stop qwen38-flash-next-cluster # stop local TP-2 after suspension lloom route q38fn --resume-member qwen3.8-flash-next --apply --yes # restore preferred-local recovery +lloom pool plan --cordon spark-node-02 # dry-run placement across the cluster lloom integrate omp --apply --yes lloom add-model mlx-community/Qwen3.6-27B-OptiQ-4bit --keep-warm --default --go # Mac: install, download, start lloom remove-model mlx-community/Qwen3.6-27B-OptiQ-4bit # preview safe cleanup diff --git a/bin/lloom.mjs b/bin/lloom.mjs index b599222..00cc0be 100755 --- a/bin/lloom.mjs +++ b/bin/lloom.mjs @@ -36,6 +36,7 @@ import { selectedRecipeIdFromCommunityPlan } from '../src/community-client.mjs'; import { loadConfig } from '../src/config.mjs'; +import { POOL_USAGE, createPoolPlanReport, formatPoolPlanReport, poolPlanOptions } from '../src/pool-cli.mjs'; import { mutateConfigSource } from '../src/config-mutation.mjs'; import { runtimeControlTimeoutMs } from '../src/control-timeout.mjs'; import { createDoctorReport } from '../src/doctor.mjs'; @@ -136,6 +137,7 @@ const COMMAND_REGISTRY = [ { name: 'runtimes', aliases: ['runtime-status'], tier: 'advanced', needsInstalledConfig: true }, { name: 'service', aliases: [], tier: 'advanced', needsInstalledConfig: true }, { name: 'cluster', aliases: ['cluster-status'], tier: 'advanced', needsInstalledConfig: true }, + { name: 'pool', aliases: [], tier: 'advanced', needsInstalledConfig: true }, { name: 'deployment', aliases: ['rollout'], tier: 'advanced', needsInstalledConfig: false }, { name: 'node-agent', aliases: [], tier: 'advanced', needsInstalledConfig: false }, { name: 'runtime-plan', aliases: [], tier: 'advanced', needsInstalledConfig: true }, @@ -228,6 +230,7 @@ Serving and discovery: Backends and runtimes: lloom down lloom cluster [status|doctor|discover] [--apply] [--id NAME] [--api-key-env NAME] [--json] + lloom pool plan [--cordon NODE[,NODE]] [--pin ALIAS=MODEL] [--profile NAME] [--consider-installs] [--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json] lloom deployment plan --plan FILE [--json] lloom deployment apply --plan FILE --nodes FILE [--operation-id ID] --apply --yes [--journal FILE] [--json] lloom deployment status --operation-id ID [--journal FILE] [--json] @@ -370,6 +373,7 @@ function commandLine(command, parts = []) { const INSTALLED_CONFIG_COMMANDS = new Set([ 'add-model', + 'pool', 'bootstrap', 'service', 'cluster', @@ -404,6 +408,7 @@ const INSTALLED_CONFIG_COMMANDS = new Set([ const OPERATIONAL_CONFIG_COMMANDS = new Set([ 'add-model', + 'pool', 'service', 'cluster', 'cluster-status', @@ -2495,6 +2500,25 @@ async function main() { if (!result) throw new Error(`route switch failed through ${gatewayUrlFor(config)}`); console.log(JSON.stringify({ ...result, applied: true }, null, 2)); }, + pool: async ({ args, config }) => { + const action = positional(args)[1] ?? 'plan'; + if (action !== 'plan') throw new Error(POOL_USAGE); + const options = poolPlanOptions({ + cordon: argValues(args, '--cordon'), + pins: argValues(args, '--pin'), + profile: argValue(args, '--profile'), + considerInstalls: hasFlag(args, '--consider-installs'), + offline: hasFlag(args, '--offline'), + headroomGb: argValue(args, '--headroom-gb'), + churnWeight: argValue(args, '--churn-weight'), + external: argValue(args, '--external') + }); + const report = await createPoolPlanReport(config, options, { + gatewayRequest: (target, pathname) => gatewayRequest(target, pathname, { args, timeoutMs: 30000 }), + gatewayUrl: gatewayUrlFor(config) + }); + console.log(hasFlag(args, '--json') ? JSON.stringify(report, null, 2) : formatPoolPlanReport(report)); + }, fleet: async ({ args, config }) => { const action = positional(args)[1] ?? 'list'; const name = positional(args)[2]; diff --git a/docs/architecture.md b/docs/architecture.md index 19e9744..4829b02 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -220,7 +220,7 @@ The gateway expands `voice: "sample-speaker"` into the profile model, on-disk `r `/gateway/doctor` wraps setup status into a product-readiness report. It separates hard `blockers` from non-fatal `warnings`, groups install state into registry, recipe, backend, models, client, runtime, and benchmark phases, and returns ordered next actions that a UI can show directly. -`/gateway/metrics` is the operational feed for dashboards, local tuning, and recipe evidence gathering. Aggregate totals plus 400 days of daily rollups are persisted by default in `~/.lloom/metrics-history.json` with atomic replacement and restored when the gateway starts, while active connections, rolling windows, and the bounded recent-request list remain process-local. `period=today|7d|30d|all` selects the durable window, and the topology HUD exposes the same choices. The persistence file contains aggregate counts and timing only—never prompts, responses, raw headers, callers, or client IPs. Before the first persistence-aware deployment, `node scripts/capture-live-metrics.mjs` can capture the old gateway's in-memory snapshot so its counters become the initial durable baseline. +`/gateway/metrics` is the operational feed for dashboards, local tuning, and recipe evidence gathering. Aggregate totals plus 400 days of daily rollups are persisted by default in `~/.lloom/metrics-history.json` with atomic replacement and restored when the gateway starts, while active connections, rolling windows, and the bounded recent-request list remain process-local. Buckets are kept per resolved model, per HTTP route, and per requested id (`requested`, usually an alias), which is the demand signal for [pool planning](pool-planning.md). `period=today|7d|30d|all` selects the durable window, and the topology HUD exposes the same choices. The persistence file contains aggregate counts and timing only—never prompts, responses, raw headers, callers, or client IPs. Before the first persistence-aware deployment, `node scripts/capture-live-metrics.mjs` can capture the old gateway's in-memory snapshot so its counters become the initial durable baseline. Model-facing routes record requested model ID, resolved model ID, upstream model ID, kind, backend, runtime, status, stream flag, duration, first-content latency, last-content latency, response bytes, and normalized usage when the upstream reports it. Each completed streaming decode measurement follows the Spark Arena / `llama-benchy` timing convention: `(generated tokens - 1) / (last content token time - first content token time)`. The dashboard header shows `totals.activeDecodeTokensPerSecond`: completed decode tokens divided by the union of their first-to-last-content generation intervals within the selected period. Overlapping streams contribute tokens together but count elapsed generation time once; idle gaps and prefill are excluded. It updates as requests complete and remains visible during idle or buffered work. Daily windows select requests by completion date, as with token totals. Merged generation intervals are persisted but omitted from the HTTP metrics response. Legacy history without interval timing contributes its summed generation duration as an estimate; `activeDecodeRateEstimated` flags this or estimated token counts, and the HUD prefixes the rate with `~`. Model topology cards show instantaneous output rates; the model inspector shows `decodeTokensPerSecond`, the rolling mean of the latest ten measured streaming request rates in the selected period. LLooM uses reported output tokens when available and otherwise estimates them from observed output characters. Embedding responses have no generative token count, so their output contribution is estimated from observed response bytes for cumulative volume only. `estimatedDecodeSamples` makes estimated decode samples explicit. Non-streaming generative responses are excluded from decode speed because LLooM cannot observe their token-by-token decode interval; their tokens and end-to-end duration remain available in the other aggregate fields. Connection telemetry derives a short caller label from explicit client-name headers, recognized User-Agent families, or the SDK language header, but never stores raw headers or client IPs; unknown callers retain `conn_N`. Client disconnects before completion are recorded as status `499` so local timeouts do not look like backend crashes. The endpoint aggregates durable totals by model and route while retaining live-only rolling and recent windows. diff --git a/docs/clusters.md b/docs/clusters.md index a941081..38f29e6 100644 --- a/docs/clusters.md +++ b/docs/clusters.md @@ -256,6 +256,8 @@ before contacting any node. The live dashboard renders node cards between the main LLooM chassis and model cards. Each node card reports architecture, accelerator type, CPU, RAM, GPU compute, GPU memory, temperature, and power when the host can report them. Unsupported fields render as unavailable rather than inventing values; Apple Silicon still reports its profile and shared-memory pressure even when per-GPU utilization is unavailable. Select a node for its complete profile. Replicated models connect to every target node and distributed models connect to every member node. +`GET /gateway/node?inventory=1` and `GET /gateway/cluster?inventory=1` add each node's pool inventory (compute architectures, model-root disk, cached recipe weights, installed images and fabric links). `lloom pool plan` uses it to propose placements across the whole cluster; see [`pool-planning.md`](pool-planning.md). + Before sending model traffic, use `lloom cluster doctor --runtime ID` to verify the required nodes and serving endpoint. Whole-topology doctor reports optional offline nodes and missing memory telemetry as warnings; they do not make unrelated runtimes unavailable. Then start a small replicated lane, verify requests alternate under equal load, and only then canary the distributed engine. ## A note on example identifiers diff --git a/docs/pool-planning.md b/docs/pool-planning.md new file mode 100644 index 0000000..78ab5c7 --- /dev/null +++ b/docs/pool-planning.md @@ -0,0 +1,113 @@ +# Pool planning + +`lloom pool plan` treats every node in the cluster as one pool and proposes which model lanes run where, how each alias routes, and what would change from today. It is a dry run: it never starts, stops or reroutes anything, and there is no apply step yet. Act on a plan with `lloom fleet`, `lloom route`, `lloom keep-warm` and `lloom suspend`/`resume`. + +```bash +lloom pool plan # live status + 7-day demand from the gateway +lloom pool plan --cordon spark-node-01,spark-node-02 # what if the pair is taken? +lloom pool plan --profile standalone # keep a fleet profile's routes fixed +lloom pool plan --pin omp=qwen38-27b-nvfp4 # keep one alias fixed +lloom pool plan --consider-installs # also place bundled recipes not installed yet +lloom pool plan --offline --json # config + this machine only +``` + +The planner sits above the per-node admission planner (`runtime-policy.mjs`), and does not replace it. Admission still decides at request time whether a start fits in live memory. The pool plan decides which lanes should be resident and where. + +## Inputs + +**Nodes.** Each node comes from `GET /gateway/cluster?inventory=1`. `GET /gateway/node?inventory=1` adds the node's inventory, which is collected on request and cached for five minutes. The inventory contains: + +- structured compute architectures (`sm_121` for GB10, `apple-metal` for Apple Silicon); +- memory domains; +- free disk per model root; +- recipes whose weights are already complete on disk; +- installed commands and Docker images; +- direct fabric links to other nodes, from NVIDIA Sync discovery. + +A gateway that predates inventory still reports its machine profile, and the planner works from that, without weights or images. A node that does not answer is planned around as unavailable. Without a running gateway, the plan uses the config's node resources plus this machine's own inventory. + +**Variants (the catalog).** A variant is one way to serve a gateway model. Variants are grouped by underlying model, and come from four sources: + +- **Managed runtimes:** each runtime is fixed to its configured `node` or `placement`. Its memory per node is the configured estimate, raised to the measured resident size when that is larger. +- **Federated targets:** a node's proxy model with a `remoteRuntime`. +- **Unmanaged lanes:** declared with a `pool` block on the model, for example a hand-run TP2 server on the pair: + + ```json + { "id": "qwen3.8-flash-next-atlas", "backend": "atlas", "pool": { "nodes": ["spark-node-01", "spark-node-02"], "memoryGbPerNode": 110 } } + ``` + + LLooM cannot start or stop these, so the plan lists them as manual actions. +- **External models:** cloud and other models with no node. + +With `--consider-installs`, bundled recipes add variants that may go on any eligible node set. + +Each variant has structured requirements: node count, tensor-parallel degree, compute architectures, platform, backend, context window and scheduler sequences. Recipes can state these with `requirements.computeArchs`, `requirements.tensorParallel` and `models[].settings.maxSequences`. When they are absent, they are inferred from launch settings (`CUTE_DSL_ARCH`, `TORCH_CUDA_ARCH_LIST`, `--tensor-parallel-size`, `MAX_NUM_SEQS`) or from the accelerators and cluster node count. Performance comes from live gateway metrics when a model has at least five requests in the window, otherwise from the best matching benchmark result. + +**Workloads.** Every alias is a workload. Its candidates are its current members plus the members of all its route profiles, expanded through nested aliases, with suspended members left out. Demand is the alias's request rate over the last seven days, from the `requested` buckets in `/gateway/metrics`. A gateway that predates those buckets reports demand per resolved model only, so each model's demand is split evenly across the aliases routed to it. + +Residency is the larger of two values: + +- **Demand-based:** at least 1 request/hour means `always`; at least 0.125 means `preferred`; otherwise `auto`. +- **Current-target:** the residency of the runtime the alias routes to now. + +Override any of this per alias in config: + +```json +"pool": { + "cordon": ["spark-node-02"], + "pins": { "routes": { "omp": "qwen38-27b-nvfp4" } }, + "workloads": { + "embeddings": { "residency": "always" }, + "video": { "required": false, "slo": { "ttftMs": 2000, "contextWindow": 131072 } } + }, + "policy": { "headroomGb": 4, "churnWeight": 10, "external": "fallback" } +} +``` + +**Pins are hard constraints.** A pin is any of the following: + +- a runtime's `node` or `placement`; +- `keepWarm` (residency `always`); +- `pool.pins.routes` and `--pin`; +- the routes and `always` residency of a `--profile`. + +The currently active routes are not pins; they are what the diff is measured against. To keep a hand-chosen route, pin it. + +## Constraints and objective + +- **Memory per domain.** Unified memory on Spark and Apple Silicon. Device memory, plus system memory for host-side state, on discrete GPUs. + - The budget follows admission: `memoryBudgetGb`, else capacity × `maxMemoryUtilization`, else capacity minus `reserveMemoryGb`. Pool planning then subtracts `headroomGb` (default 4 GB), because exhausting unified memory hangs a Spark. + - `always` and `preferred` lanes must fit together. Every on-demand lane must fit next to the `always` set. + - A `preferred` lane that does not fit becomes on demand, with a warning. +- **Tensor parallel only across directly linked nodes.** Every pair in a multi-node set must share a reported fabric link. +- **Compute architecture, platform and accelerators** must match the node. +- **Cordoned and unavailable nodes** host nothing. + +Plans are compared in this order: + +1. Every required workload and pin covered. +2. Fewest workloads on external models, which are a fallback by default. `--external allow` lets them compete on latency; `never` excludes them. +3. Fewest latency SLO misses. +4. Lowest cost. Cost is demand × latency over a 24-hour horizon, plus each change's load time × `churnWeight`. + +Latency is time to first token plus decode time for 256 output tokens, or the measured request duration for media. Load time is estimated from the variant's memory, or taken from its `loadSeconds`. Missing weights add download time, a missing image adds pull time, and an unmanaged lane adds a fixed manual-action time. + +The search places workloads greedily, pinned and hot ones first, sharing a placement wherever several aliases can use the same lane. Local search then retries each workload with each alternative candidate and keeps any change that makes the whole plan better. For example, local search can move one alias to a smaller model so that another required alias still fits, or move all cognition aliases to a faster lane together once one of them pays its load cost. + +## Output + +The text output lists: + +- each node's memory budget and resident set; +- every placement by tier, with the aliases it serves; +- the changes from now: + - route changes; + - lanes to start; + - manual starts and stops for unmanaged lanes; + - lanes no workload needs any more (`release`); + - residency changes; +- uncovered workloads, with reasons. + +`--json` returns the full report. It includes the plan's profile-shaped `routes` and `residency`, `placements`, per-node memory, the `diff`, and the `score`. + +On a three-Spark fleet, the planner's fixture tests (`test/fixtures/pool/spark-fleet.json`) reproduce the hand-written fleet profiles. With every node present, it gives the `sparkqwen-pair-rc16` routes and residency. With the 01/02 pair cordoned or offline, it gives `standalone`: cognition moves to the 27B on spark-node-03, which becomes `always`, and the SparkQwen lane gets a manual stop. diff --git a/docs/recipes.md b/docs/recipes.md index f614043..2cdcde8 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -73,7 +73,7 @@ limits, permissions, tests, and measurement limitations. Recipes are JSON documents with these top-level sections: -- `requirements`: platform, memory, disk, and command prerequisites. +- `requirements`: platform, memory, disk, and command prerequisites. Optional `computeArchs` (for example `["sm_121"]`) and `tensorParallel` state what [pool planning](pool-planning.md) otherwise infers from launch settings. - `backend`: the runtime family being configured. - `setup.steps`: ordered checks, downloads, commands, and tuning operations. - `models`: role-to-artifact mappings plus gateway model/runtime IDs. @@ -89,7 +89,7 @@ Model entries should be specific enough for LLooM to create the local runtime wi - `backendConfig`: optional stable backend config ID when the recipe should not use an auto-generated ID. - `input` and `output`: modalities such as `text`, `image`, `video`, `audio`, `embedding`, or `scores`. - `capabilities`: the API contract the model supports, for example `responses`, `anthropic-messages`, `tools`, `reasoning`, `vision`, `mtp`, or `long-context`. Media capabilities select the model kind and therefore the serving route: `image-generation`/`image-editing` give `image`, `video-generation` gives `video`, `audio-generation`/`music-generation`/`audio-music-generation` give `audio_generation`, and `audio-speech`/`tts` give `audio_speech`. A music capability wins over a speech capability when a recipe declares both. -- `settings`: runtime knobs consumed during config materialization, including `contextWindow`, `maxOutputTokens`, `maxActiveRequests`, `profile`, `draftDepth`, `reasoning`, `preserveThinking`, `batchingPreset`, `memoryGb`, `startupTimeoutMs`, `priority`, `keepWarm`, and session-cache fields. `keepWarm` is the single hard residency pin and is valid only for a managed internal runtime. `priority` orders only unpinned eviction candidates; lower values are evicted first. +- `settings`: runtime knobs consumed during config materialization, including `contextWindow`, `maxOutputTokens`, `maxActiveRequests`, `maxSequences` (backend scheduler sequences, reported by pool planning), `profile`, `draftDepth`, `reasoning`, `preserveThinking`, `batchingPreset`, `memoryGb`, `startupTimeoutMs`, `priority`, `keepWarm`, and session-cache fields. `keepWarm` is the single hard residency pin and is valid only for a managed internal runtime. `priority` orders only unpinned eviction candidates; lower values are evicted first. - `settings.runtime`: optional explicit launcher data for recipes that need backend-specific command lines or bootstrap-only managed containers. LLooM templates `command`, `args`, `env`, `bootstrap`, `healthPath` or `healthUrl`, `warmup`, `watchdog`, and session-cache hints with variables such as `${modelRoot}`, `${modelPath}`, `${modelId}`, `${port}`, `${contextWindow}`, `${maxOutputTokens}`, `${maxActiveRequests}`, `${runtimeId}`, and `${sessionCacheDir}`. If it is absent, LLooM uses the built-in defaults for known backends. - `settings.placement`: optional cluster placement. `{"mode":"replicated"}` materializes one local target on every configured node (or the `nodes` subset) plus leader-side LLooM proxy targets for remote replicas; distributed logical runtimes use ordered delegated `members` with per-node resource estimates. See [`clusters.md`](clusters.md). - `setDefault`: optional explicit instruction to select that model as the installed default for its output modality. This also applies during additive recipe updates, so use it only when the recipe intentionally owns the default choice. diff --git a/examples/interchange/recipe.v1.json b/examples/interchange/recipe.v1.json index ead31f1..6f95167 100644 --- a/examples/interchange/recipe.v1.json +++ b/examples/interchange/recipe.v1.json @@ -32,7 +32,9 @@ "memoryGb": 48, "diskGb": 45, "commands": ["mtplx"], - "accelerators": ["apple-gpu"] + "accelerators": ["apple-gpu"], + "computeArchs": ["apple-metal"], + "tensorParallel": 1 }, "backend": { "id": "mtplx", @@ -66,6 +68,7 @@ "contextWindow": 262144, "maxOutputTokens": 32768, "draftDepth": 2, + "maxSequences": 8, "maxActiveRequests": 8, "interactiveReservedSlots": 1, "interactiveReservedQueueSlots": 1, diff --git a/package.json b/package.json index 1fe3596..329499b 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "check": "npm run check:cluster && npm run check:js && npm run check:entity-stagger && npm run check:python && npm run check:shell && npm run check:spark-deploy && npm run check:workers", "check:cluster": "node --check src/cluster.mjs && node --check test/cluster.test.mjs && node --check test/cluster-availability.test.mjs && node --check test/cluster-readiness.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs", "check:entity-stagger": "node --check scripts/entity-stagger-benchmark.mjs && node --check test/entity-stagger-benchmark.test.mjs", - "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/request-scheduler.mjs && node --check src/rate-limit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/deployment-fence.mjs && node --check test/deployment-fence.test.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/standby.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check test/spark-audio-recipes.test.mjs && node --check src/runtime-residency.mjs && node --check test/ltx-pipelines.test.mjs && node --check test/ltx-pipelines-recipe.test.mjs && node --check src/video-contracts.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs && node --check src/runtime-policy-config.mjs && node --check test/runtime-policy-config.test.mjs && node --check src/audio-providers.mjs && node --check src/protocol/openrouter-provider.mjs && node --check test/audio-providers.test.mjs && node --check test/openrouter-provider.test.mjs && node --check backends/atlas-sparkglm/verify-pins.mjs && node --check test/atlas-sparkglm-recipe.test.mjs && node --check src/head-preparation.mjs && node --check src/head-transfer.mjs && node --check src/service-control.mjs && node --check src/gateway-relay.mjs && node --check src/client-retarget.mjs && node --check src/head-promotion.mjs && node --check src/head-promotion-transfer.mjs && node --check src/gateway-client.mjs && node --check test/cli-diagnostics.test.mjs && node --check src/service-install.mjs && node --check test/service-install.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check src/node-release-agent.mjs && node --check src/node-gateway-adapter.mjs && node --check src/node-agent-cli.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs && node --check test/node-release-agent.test.mjs && node --check test/node-gateway-adapter.test.mjs && node --check test/node-gateway-adapter-e2e.test.mjs && node --check test/deployment-bundle.test.mjs && node --check test/node-agent-cli.test.mjs && node --check test/ssh-deployment-transport.test.mjs && node --check scripts/build-deployment-bundle.mjs", + "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/request-scheduler.mjs && node --check src/rate-limit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/deployment-fence.mjs && node --check test/deployment-fence.test.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/standby.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check test/spark-audio-recipes.test.mjs && node --check src/runtime-residency.mjs && node --check test/ltx-pipelines.test.mjs && node --check test/ltx-pipelines-recipe.test.mjs && node --check src/video-contracts.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs && node --check src/pool-planner.mjs && node --check src/pool-catalog.mjs && node --check src/pool-inventory.mjs && node --check src/pool-cli.mjs && node --check test/pool-planner.test.mjs && node --check test/pool-catalog.test.mjs && node --check test/pool-inventory.test.mjs && node --check test/pool-cli.test.mjs && node --check src/runtime-policy-config.mjs && node --check test/runtime-policy-config.test.mjs && node --check src/audio-providers.mjs && node --check src/protocol/openrouter-provider.mjs && node --check test/audio-providers.test.mjs && node --check test/openrouter-provider.test.mjs && node --check backends/atlas-sparkglm/verify-pins.mjs && node --check test/atlas-sparkglm-recipe.test.mjs && node --check src/head-preparation.mjs && node --check src/head-transfer.mjs && node --check src/service-control.mjs && node --check src/gateway-relay.mjs && node --check src/client-retarget.mjs && node --check src/head-promotion.mjs && node --check src/head-promotion-transfer.mjs && node --check src/gateway-client.mjs && node --check test/cli-diagnostics.test.mjs && node --check src/service-install.mjs && node --check test/service-install.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check src/node-release-agent.mjs && node --check src/node-gateway-adapter.mjs && node --check src/node-agent-cli.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs && node --check test/node-release-agent.test.mjs && node --check test/node-gateway-adapter.test.mjs && node --check test/node-gateway-adapter-e2e.test.mjs && node --check test/deployment-bundle.test.mjs && node --check test/node-agent-cli.test.mjs && node --check test/ssh-deployment-transport.test.mjs && node --check scripts/build-deployment-bundle.mjs", "check:python": "python3 -m py_compile backends/dspark-vllm/apply-patch-pack.py backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/vision_exp/*.py backends/qwen38-vllm/apply-ple-fp8-patch.py backends/qwen38-sglang/*.py backends/glm53-exl3/*.py backends/mlx-audio/lloom_audio_server.py backends/hear/lloom_hear_server.py backends/chatterbox/lloom_chatterbox_server.py patches/apply_mtplx_longctx_fix.py backends/comfyui-media/bridge/*.py backends/comfyui-media/graphs/*.py backends/comfyui-media/build/*.py backends/comfyui-media/install.py backends/qwen-image-diffusers/*.py backends/ace-step-diffusers/*.py backends/ace-step-diffusers/tests/*.py backends/hear/test/test_hear_security.py backends/spark-audio/install.py backends/spark-audio/lloom_audio_cuda_server.py backends/spark-audio/test_lloom_audio_cuda_server.py backends/ltx-pipelines/*.py src/darwin-memory-usage.py", "check:shell": "bash -n backends/dspark-vllm/entrypoint.sh backends/dspark-vllm/runtime/build-hardened-image.sh backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/apply-runtime.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.sh backends/qwen38-vllm/entrypoint.sh backends/qwen38-sglang/entrypoint.sh backends/glm53-exl3/entrypoint.sh backends/mlx-audio/install.sh backends/hear/install.sh backends/chatterbox/install.sh clients/examples/lloom-claude clients/examples/lloom-codex clients/examples/lloom-hermes clients/examples/lloom-opencode clients/examples/lloom-zero scripts/remote-install-spark.sh scripts/remote-stage-spark-worker.sh backends/atlas-sparkglm/install.sh backends/atlas-sparkglm/convert-overlay.sh", "check:spark-deploy": "node --check test/spark-deploy.test.mjs && node test/spark-deploy.test.mjs", @@ -73,7 +73,7 @@ "community:check": "node scripts/check-community-deploy.mjs", "package:check": "node scripts/check-package.mjs", "test:adoption": "node test/resource-fit.test.mjs && node test/model-acquisition.test.mjs && node test/dashboard-status.test.mjs", - "test:unit": "node --test test/deployment-fence.test.mjs test/cluster-deployment.test.mjs test/cluster-deployment-cli.test.mjs test/cluster-deployment-e2e.test.mjs test/node-release-agent.test.mjs test/node-gateway-adapter.test.mjs test/node-gateway-adapter-e2e.test.mjs test/deployment-bundle.test.mjs test/node-agent-cli.test.mjs test/ssh-deployment-transport.test.mjs && node test/runtime-health-ownership.test.mjs && node test/long-prefill-transport.test.mjs && node test/sparkglm-managed.test.mjs && node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/stream-metrics.test.mjs && node test/responses-codex.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/config-profiles.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-policy-config.test.mjs && node --test test/request-scheduler.test.mjs test/adaptive-priority.test.mjs test/rate-limit.test.mjs test/runtime-queue.test.mjs test/foreground-priority.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node --test test/standby.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && node test/hear-recipe.test.mjs && npm run test:adoption && node --test test/performance-sampler.test.mjs && node --test test/audio-generations.test.mjs && node test/video-providers.test.mjs && node --test test/web-functions.test.mjs && node --test test/config-mutation.test.mjs && node --test test/model-maintenance.test.mjs && node --test test/model-maintenance-control.test.mjs && node --test test/route-maintenance-concurrency.test.mjs && node --test test/model-maintenance-integration.test.mjs && npm run test:workers && node test/comfyui-media.test.mjs && node test/qwen-image-diffusers.test.mjs && node test/ace-step-diffusers.test.mjs && node test/runtime-residency.test.mjs && node test/spark-audio-recipes.test.mjs && node --test test/video-contracts.test.mjs && node test/ltx-pipelines-recipe.test.mjs && npm run test:ux && npm run test:memory-safety && node --test test/dashboard-memory.test.mjs test/runtime-memory-usage.test.mjs && node --test test/speech-stream.test.mjs && node --test test/audio-providers.test.mjs && node test/openrouter-provider.test.mjs && node test/atlas-sparkglm-recipe.test.mjs && node --test test/head-preparation.test.mjs test/head-transfer.test.mjs test/service-control.test.mjs && node --test test/gateway-relay.test.mjs test/client-retarget.test.mjs && node --test test/head-profile-acceptance.test.mjs test/head-promotion.test.mjs test/head-promotion-transfer.test.mjs test/head-transfer-acceptance.test.mjs && node --test test/cli-diagnostics.test.mjs && node --test test/service-install.test.mjs", + "test:unit": "node --test test/deployment-fence.test.mjs test/cluster-deployment.test.mjs test/cluster-deployment-cli.test.mjs test/cluster-deployment-e2e.test.mjs test/node-release-agent.test.mjs test/node-gateway-adapter.test.mjs test/node-gateway-adapter-e2e.test.mjs test/deployment-bundle.test.mjs test/node-agent-cli.test.mjs test/ssh-deployment-transport.test.mjs && node test/runtime-health-ownership.test.mjs && node test/long-prefill-transport.test.mjs && node test/sparkglm-managed.test.mjs && node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/stream-metrics.test.mjs && node test/responses-codex.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/config-profiles.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-policy-config.test.mjs && node --test test/pool-planner.test.mjs test/pool-catalog.test.mjs test/pool-inventory.test.mjs test/pool-cli.test.mjs && node --test test/request-scheduler.test.mjs test/adaptive-priority.test.mjs test/rate-limit.test.mjs test/runtime-queue.test.mjs test/foreground-priority.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node --test test/standby.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && node test/hear-recipe.test.mjs && npm run test:adoption && node --test test/performance-sampler.test.mjs && node --test test/audio-generations.test.mjs && node test/video-providers.test.mjs && node --test test/web-functions.test.mjs && node --test test/config-mutation.test.mjs && node --test test/model-maintenance.test.mjs && node --test test/model-maintenance-control.test.mjs && node --test test/route-maintenance-concurrency.test.mjs && node --test test/model-maintenance-integration.test.mjs && npm run test:workers && node test/comfyui-media.test.mjs && node test/qwen-image-diffusers.test.mjs && node test/ace-step-diffusers.test.mjs && node test/runtime-residency.test.mjs && node test/spark-audio-recipes.test.mjs && node --test test/video-contracts.test.mjs && node test/ltx-pipelines-recipe.test.mjs && npm run test:ux && npm run test:memory-safety && node --test test/dashboard-memory.test.mjs test/runtime-memory-usage.test.mjs && node --test test/speech-stream.test.mjs && node --test test/audio-providers.test.mjs && node test/openrouter-provider.test.mjs && node test/atlas-sparkglm-recipe.test.mjs && node --test test/head-preparation.test.mjs test/head-transfer.test.mjs test/service-control.test.mjs && node --test test/gateway-relay.test.mjs test/client-retarget.test.mjs && node --test test/head-profile-acceptance.test.mjs test/head-promotion.test.mjs test/head-promotion-transfer.test.mjs test/head-transfer-acceptance.test.mjs && node --test test/cli-diagnostics.test.mjs && node --test test/service-install.test.mjs", "test:cluster": "node test/cluster.test.mjs && node --test test/cluster-availability.test.mjs test/cluster-readiness.test.mjs", "test": "npm run test:cluster && npm run test:unit && npm run test:entity-stagger && npm run smoke && npm run community:check", "test:entity-stagger": "node test/entity-stagger-benchmark.test.mjs", diff --git a/schemas/recipe.v1.schema.json b/schemas/recipe.v1.schema.json index cbdf183..06b97ad 100644 --- a/schemas/recipe.v1.schema.json +++ b/schemas/recipe.v1.schema.json @@ -125,6 +125,20 @@ "type": "string" }, "uniqueItems": true + }, + "computeArchs": { + "type": "array", + "description": "Compute architectures the lane's kernels are built for, such as sm_121 or apple-metal. Pool planning only places the lane on nodes that report one of them; when absent, it is inferred from launch settings or accelerators.", + "items": { + "type": "string", + "pattern": "^(sm_[0-9]+[af]?|apple-metal)$" + }, + "uniqueItems": true + }, + "tensorParallel": { + "type": "integer", + "minimum": 1, + "description": "Tensor-parallel degree across the lane's GPUs. Multi-node lanes are only placed on directly linked nodes; when absent, it is inferred from launch arguments or the cluster node count." } } }, @@ -344,6 +358,11 @@ "type": "integer", "minimum": 0 }, + "maxSequences": { + "type": "integer", + "minimum": 1, + "description": "Concurrent sequences the backend scheduler is configured for. Pool planning reports it with the lane; when absent, it is inferred from launch arguments or maxActiveRequests." + }, "contextWindow": { "type": "integer", "minimum": 1 diff --git a/src/cluster.mjs b/src/cluster.mjs index 8038946..dd5bb45 100644 --- a/src/cluster.mjs +++ b/src/cluster.mjs @@ -978,7 +978,8 @@ export class ClusterCoordinator { logger = console, telemetry = null, profile = null, - models = null + models = null, + inventory = null } = {} ) { this.config = config; @@ -988,6 +989,7 @@ export class ClusterCoordinator { this.telemetry = telemetry; this.profile = profile; this.models = models; + this.inventory = inventory; this.runtimeManager = null; this.targetCursor = new Map(); this.targetLoads = new Map(); @@ -1062,9 +1064,15 @@ export class ClusterCoordinator { } } - async localNodeStatus({ runtimeStatus = null, includeMemoryUsage = false } = {}) { + async localNodeStatus({ runtimeStatus = null, includeMemoryUsage = false, includeInventory = false } = {}) { const profile = typeof this.profile === 'function' ? await this.profile() : this.profile; const models = typeof this.models === 'function' ? await this.models() : this.models; + const inventory = + includeInventory && this.inventory + ? typeof this.inventory === 'function' + ? await this.inventory() + : this.inventory + : undefined; return { id: this.nodeId, name: this.nodes[this.nodeId]?.name ?? this.nodeId, @@ -1086,20 +1094,22 @@ export class ClusterCoordinator { runtimeStatus ?? (this.runtimeManager ? await this.runtimeManager.status({ localOnly: true, includeMemoryUsage }) - : { runtimes: {} }) + : { runtimes: {} }), + ...(inventory ? { inventory } : {}) }; } - async nodeStatus(nodeId, { refresh = false } = {}) { - if (this.isLocalNode(nodeId)) return this.localNodeStatus(); + async nodeStatus(nodeId, { refresh = false, inventory = false } = {}) { + if (this.isLocalNode(nodeId)) return this.localNodeStatus({ includeInventory: inventory }); const node = this.nodes[nodeId]; const ttlMs = Math.max(0, Number(this.config.cluster?.statusCacheMs ?? 1000)); - const cached = this.nodeCache.get(nodeId); + const cacheKey = inventory ? `${nodeId}\u0000inventory` : nodeId; + const cached = this.nodeCache.get(cacheKey); if (!refresh && cached && Date.now() - cached.at < ttlMs) return cached.value; if (!refresh && cached?.pending) return cached.pending; const pending = (async () => { try { - const result = await this.requestNode(nodeId, '/gateway/node?memoryUsage=1'); + const result = await this.requestNode(nodeId, `/gateway/node?memoryUsage=1${inventory ? '&inventory=1' : ''}`); const claimedNodeId = result?.node?.id ?? null; if (claimedNodeId !== nodeId) { return { @@ -1155,19 +1165,19 @@ export class ClusterCoordinator { }; } })(); - this.nodeCache.set(nodeId, { at: cached?.at ?? 0, value: cached?.value, pending }); + this.nodeCache.set(cacheKey, { at: cached?.at ?? 0, value: cached?.value, pending }); const value = await pending; - this.nodeCache.set(nodeId, { at: Date.now(), value, pending: null }); + this.nodeCache.set(cacheKey, { at: Date.now(), value, pending: null }); return value; } - async status({ localRuntimeStatus = null } = {}) { + async status({ localRuntimeStatus = null, inventory = false } = {}) { const entries = await Promise.all( Object.keys(this.nodes).map(async (id) => [ id, this.isLocalNode(id) && localRuntimeStatus - ? await this.localNodeStatus({ runtimeStatus: localRuntimeStatus }) - : await this.nodeStatus(id) + ? await this.localNodeStatus({ runtimeStatus: localRuntimeStatus, includeInventory: inventory }) + : await this.nodeStatus(id, { inventory }) ]) ); const scopes = Object.keys(this.config.runtimes ?? {}) @@ -1336,6 +1346,7 @@ export class ClusterCoordinator { signal }); this.nodeCache.delete(nodeId); + this.nodeCache.delete(`${nodeId}\u0000inventory`); return result; } diff --git a/src/pool-catalog.mjs b/src/pool-catalog.mjs new file mode 100644 index 0000000..291df8d --- /dev/null +++ b/src/pool-catalog.mjs @@ -0,0 +1,538 @@ +// Pool catalog and planner inputs. Turns recipes, the installed config, node +// status and gateway metrics into the plain data pool-planner.mjs consumes: +// a model-centric variant catalog, node inventories, workloads, pins and the +// current state. Read-only: nothing here contacts nodes or changes config. +import { aliasRoutableMemberIds, expandedAliasMemberIds } from './alias-resolution.mjs'; +import { benchmarkRankingEligible, benchmarkScore } from './benchmarks.mjs'; +import { clusterNodes, modelTargets, runtimeResourcesByNode } from './cluster.mjs'; +import { resolveRouteTarget } from './config-profiles.mjs'; +import { inventoryFromProfile } from './pool-inventory.mjs'; +import { residencyPolicy } from './runtime-preferences.mjs'; + +export const POOL_CATALOG_VERSION = 1; +const LIVE_METRIC_MIN_REQUESTS = 5; +const PERIOD_HOURS = { today: 24, '7d': 168, '30d': 720 }; +const RESIDENCY_RANK = { auto: 0, preferred: 1, always: 2 }; +const ARCH_BY_ACCELERATOR = { gb10: 'sm_121', 'dgx-spark': 'sm_121' }; + +function object(value) { + return value && typeof value === 'object' && !Array.isArray(value) ? value : {}; +} + +function num(value) { + if (value == null || value === '') return null; + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +function round(value, places = 1) { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +function unique(values) { + return [...new Set(values.filter((value) => value != null && value !== ''))]; +} + +function compact(value) { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry != null)); +} + +function archFromCapability(value) { + const match = String(value).match(/(\d+)\.(\d+)/); + return match ? `sm_${match[1]}${match[2]}` : null; +} + +function normalizeArch(value) { + return String(value) + .toLowerCase() + .replace(/^(sm_\d+)[af]$/, '$1'); +} + +// Launch settings carry facts the recipe does not yet state structurally: +// container images, tensor-parallel size, scheduler sequences and target arch. +function launchSignals(settings) { + const signals = { images: [], env: {}, flags: {} }; + const visit = (value, key) => { + if (typeof value === 'string') { + if (key === 'image') signals.images.push(value); + const env = value.match(/^([A-Z][A-Z0-9_]*)=(.*)$/); + if (env) signals.env[env[1]] ??= env[2]; + return; + } + if (Array.isArray(value)) { + value.forEach((item, index) => { + if (typeof item === 'string' && item.startsWith('--') && typeof value[index + 1] === 'string') { + signals.flags[item] ??= value[index + 1]; + } + visit(item); + }); + return; + } + if (value && typeof value === 'object') { + for (const [childKey, child] of Object.entries(value)) visit(child, childKey); + } + }; + visit(settings); + return signals; +} + +export function recipeServingRequirements(recipe, model = {}) { + const requirements = object(recipe?.requirements); + const settings = object(model.settings); + const signals = launchSignals(settings); + const inferred = []; + const infer = (field, value) => { + if (value != null && !(Array.isArray(value) && !value.length)) inferred.push(field); + return value; + }; + const nodeCount = Math.max(1, Math.floor(num(requirements.cluster?.nodes ?? requirements.cluster?.minNodes) ?? 1)); + const explicitArchs = Array.isArray(requirements.computeArchs) ? requirements.computeArchs : null; + const signalArchs = unique([ + signals.env.CUTE_DSL_ARCH ? normalizeArch(signals.env.CUTE_DSL_ARCH) : null, + ...String(signals.env.TORCH_CUDA_ARCH_LIST ?? '') + .split(/[;,\s]+/) + .map(archFromCapability) + ]); + const acceleratorArchs = unique((requirements.accelerators ?? []).map((name) => ARCH_BY_ACCELERATOR[name])); + const explicitTp = num(requirements.tensorParallel ?? settings.tensorParallel); + const signalTp = num( + signals.flags['--tensor-parallel-size'] ?? signals.env.TENSOR_PARALLEL_SIZE ?? signals.env.TP_SIZE + ); + const explicitSequences = num(settings.maxSequences); + const signalSequences = num(signals.flags['--max-num-seqs'] ?? signals.env.MAX_NUM_SEQS); + const placementMemory = Math.max( + 0, + ...(settings.placement?.members ?? []).map((member) => num(member?.resources?.memoryGb) ?? 0) + ); + return { + recipeId: recipe?.id ?? null, + nodeCount, + tensorParallel: explicitTp ?? infer('tensorParallel', signalTp ?? nodeCount), + computeArchs: explicitArchs + ? explicitArchs.map(normalizeArch) + : (infer('computeArchs', signalArchs.length ? signalArchs : acceleratorArchs) ?? []), + platforms: requirements.platforms ?? [], + accelerators: requirements.accelerators ?? [], + backend: recipe?.backend?.id ?? null, + contextWindow: num(settings.contextWindow ?? model.contextWindow), + maxSequences: explicitSequences ?? infer('maxSequences', signalSequences ?? num(settings.maxActiveRequests)), + memoryGbPerNode: num(settings.memoryGb) ?? (placementMemory || num(requirements.memoryGb)), + diskGb: num(requirements.diskGb), + images: unique(signals.images).sort(), + inferred + }; +} + +// /gateway/metrics returns id-keyed bucket arrays; accept a plain map too. +function metricBucket(buckets, id) { + if (Array.isArray(buckets)) return buckets.find((bucket) => bucket?.id === id) ?? null; + return object(buckets)[id] ?? null; +} + +function livePerformance(metrics, gatewayModel) { + const bucket = metricBucket(metrics?.models, gatewayModel); + if (!bucket || (num(bucket.requests) ?? 0) < LIVE_METRIC_MIN_REQUESTS) return null; + return compact({ + ttftMs: num(bucket.avgFirstContentMs), + decodeTps: num(bucket.decodeTokensPerSecond) ?? num(bucket.activeDecodeTokensPerSecond), + durationMs: num(bucket.avgDurationMs) || null, + source: 'live-metrics' + }); +} + +function benchmarkPerformance(benchmarks, { gatewayModel, recipeId, model }) { + const best = benchmarks + .filter((result) => benchmarkRankingEligible(result)) + .filter( + (result) => + (result.gatewayModel && result.gatewayModel === gatewayModel) || + (recipeId && result.recipeId === recipeId && (!model || result.model === model)) + ) + .sort((left, right) => benchmarkScore(right) - benchmarkScore(left))[0]; + if (!best) return null; + const metrics = object(best.metrics); + return compact({ + ttftMs: num(metrics.timeToFirstTokenMs), + decodeTps: num(metrics.generationTokPerSec), + durationMs: num(metrics.generationSeconds) != null ? num(metrics.generationSeconds) * 1000 : null, + source: `benchmark:${best.id}` + }); +} + +function measuredMemoryGb(nodeStatuses, nodes, runtimeIds) { + let largest = 0; + for (const nodeId of nodes) { + const runtimes = object(nodeStatuses?.[nodeId]?.runtimeManager?.runtimes); + for (const runtimeId of runtimeIds) { + const bytes = num(runtimes[runtimeId]?.memoryUsage?.residentBytes); + if (bytes) largest = Math.max(largest, bytes / 1024 ** 3); + } + } + return largest ? round(largest) : null; +} + +function memberRuntimeIds(runtime, runtimeId) { + const members = runtime?.placement?.mode === 'distributed' ? (runtime.placement.members ?? []) : []; + return members.length ? members.map((member) => member.runtime).filter(Boolean) : [runtimeId]; +} + +function configVariants(config, { nodeStatuses, env }) { + const variants = []; + for (const model of config.models ?? []) { + if (!model?.id) continue; + const base = { + gatewayModel: model.id, + model: model.pool?.model ?? model.upstreamModel ?? model.id, + kind: model.kind ?? 'chat', + capabilities: model.capabilities ?? [], + source: 'config' + }; + if (model.pool?.nodes?.length) { + const nodes = [...model.pool.nodes].sort(); + variants.push({ + ...base, + id: `${model.id}@${nodes.join('+')}`, + managed: model.pool.managed === true, + nodes, + requirements: compact({ + nodeCount: nodes.length, + memoryGbPerNode: num(model.pool.memoryGbPerNode), + contextWindow: num(model.contextWindow) + }) + }); + continue; + } + const runtime = model.runtime ? config.runtimes?.[model.runtime] : null; + if (model.runtime) { + if (!runtime || runtime.enabled === false) continue; + const resources = runtimeResourcesByNode(runtime, config, env); + const nodes = Object.keys(resources).sort(); + if (!nodes.length) continue; + const configuredGb = Math.max(0, ...nodes.map((nodeId) => resources[nodeId].memoryGb)); + const measuredGb = measuredMemoryGb(nodeStatuses, nodes, memberRuntimeIds(runtime, model.runtime)); + variants.push({ + ...base, + id: `${model.id}@${nodes.join('+')}`, + runtime: model.runtime, + managed: runtime.managed !== false && runtime.management !== 'external', + nodes, + requirements: compact({ + nodeCount: nodes.length, + memoryGbPerNode: Math.max(configuredGb, measuredGb ?? 0) || null, + contextWindow: num(model.contextWindow ?? runtime.contextWindow), + maxSequences: num(runtime.maxActiveRequests), + recipeId: runtime.recipe?.id ?? null, + images: runtime.bootstrap?.image ? [runtime.bootstrap.image] : null + }), + ...(measuredGb ? { measured: { memoryGbPerNode: measuredGb, source: 'live-rss' } } : {}) + }); + continue; + } + const federated = modelTargets(model).filter((target) => target.node && target.remoteRuntime); + if (federated.length) { + for (const target of federated) { + const remote = object(nodeStatuses?.[target.node]?.runtimeManager?.runtimes?.[target.remoteRuntime]); + const members = remote.placement?.mode === 'distributed' ? (remote.placement.members ?? []) : []; + const nodes = unique(members.length ? members.map((member) => member.node) : [target.node]).sort(); + const memberMemory = members.map((member) => num(member.resources?.memoryGb) ?? 0); + variants.push({ + ...base, + id: `${model.id}@${nodes.join('+')}`, + remoteRuntime: target.remoteRuntime, + controlNode: target.node, + managed: true, + nodes, + requirements: compact({ + nodeCount: nodes.length, + memoryGbPerNode: Math.max(num(remote.memoryGb) ?? 0, ...memberMemory) || null, + contextWindow: num(model.contextWindow) + }) + }); + } + continue; + } + variants.push({ ...base, id: model.id, external: true, managed: false, requirements: {} }); + } + return variants; +} + +function recipeVariants(recipes) { + return recipes.flatMap((recipe) => + (recipe.models ?? []).map((model) => { + const gatewayModel = model.gatewayModel ?? model.model; + return { + id: `recipe:${recipe.id}:${gatewayModel}`, + gatewayModel, + model: model.model, + kind: model.kind ?? (model.capabilities?.includes('chat') ? 'chat' : (recipe.kind ?? 'chat')), + capabilities: model.capabilities ?? recipe.capabilities ?? [], + source: 'recipe', + managed: true, + requirements: recipeServingRequirements(recipe, model) + }; + }) + ); +} + +export function buildPoolCatalog({ + config = {}, + recipes = [], + benchmarks = [], + metrics = null, + nodeStatuses = {}, + env = process.env +} = {}) { + const variants = [...configVariants(config, { nodeStatuses, env }), ...recipeVariants(recipes)] + .map((variant) => { + const performance = + livePerformance(metrics, variant.gatewayModel) ?? + benchmarkPerformance(benchmarks, { + gatewayModel: variant.gatewayModel, + recipeId: variant.requirements?.recipeId, + model: variant.source === 'recipe' ? variant.model : null + }); + return performance ? { ...variant, performance } : variant; + }) + .sort((left, right) => left.id.localeCompare(right.id)); + const groups = new Map(); + for (const variant of variants) { + if (!groups.has(variant.model)) { + groups.set(variant.model, { + model: variant.model, + kind: variant.kind, + capabilities: variant.capabilities, + variants: [] + }); + } + groups.get(variant.model).variants.push(variant); + } + return { + version: POOL_CATALOG_VERSION, + variants, + models: [...groups.values()].sort((left, right) => left.model.localeCompare(right.model)) + }; +} + +function aliasLeaves(config, ids) { + const aliases = object(config.aliases); + const modelIds = new Set((config.models ?? []).map((model) => model.id)); + return unique( + ids.flatMap((id) => { + if (!Object.hasOwn(aliases, id)) return modelIds.has(id) ? [id] : []; + try { + return expandedAliasMemberIds(id, aliases, modelIds).filter((leaf) => modelIds.has(leaf)); + } catch { + return []; + } + }) + ); +} + +function currentRouteModel(config, alias) { + return aliasLeaves(config, aliasRoutableMemberIds(alias))[0] ?? null; +} + +function maxResidency(left, right) { + return RESIDENCY_RANK[right] > RESIDENCY_RANK[left] ? right : left; +} + +export function poolWorkloadsFromConfig(config = {}, { metrics = null, periodHours } = {}) { + const pool = object(config.pool); + const policy = object(pool.policy); + const hours = num(periodHours) ?? PERIOD_HOURS[metrics?.period] ?? null; + const alwaysAt = num(policy.alwaysAtRequestsPerHour) ?? 1; + const preferredAt = num(policy.preferredAtRequestsPerHour) ?? 0.125; + // Older gateways only aggregate by resolved model; split each model's demand + // evenly across the aliases currently routed to it. + const perAlias = Array.isArray(metrics?.requested); + const routedTo = new Map(); + for (const alias of Object.values(object(config.aliases))) { + const model = currentRouteModel(config, alias); + if (model) routedTo.set(model, (routedTo.get(model) ?? 0) + 1); + } + const requestsFor = (aliasId, alias) => { + if (perAlias) return num(metricBucket(metrics.requested, aliasId)?.requests) ?? 0; + const model = currentRouteModel(config, alias); + // A model with no bucket in the window had no traffic. + return model ? (num(metricBucket(metrics.models, model)?.requests) ?? 0) / routedTo.get(model) : 0; + }; + const workloads = []; + for (const [aliasId, alias] of Object.entries(object(config.aliases))) { + const profileMembers = Object.values(object(alias?.routeProfiles)).flatMap((profile) => + Array.isArray(profile?.members) ? profile.members : [] + ); + const candidates = aliasLeaves(config, [...aliasRoutableMemberIds(alias), ...profileMembers]); + if (!candidates.length) continue; + const override = object(object(pool.workloads)[aliasId]); + const requests = metrics ? requestsFor(aliasId, alias) : null; + const demandPerHour = hours && requests != null ? round(requests / hours, 3) : null; + const demandResidency = + demandPerHour == null + ? 'auto' + : demandPerHour >= alwaysAt + ? 'always' + : demandPerHour >= preferredAt + ? 'preferred' + : 'auto'; + const target = (config.models ?? []).find((model) => model.id === currentRouteModel(config, alias)); + const targetRuntime = target?.runtime ? config.runtimes?.[target.runtime] : null; + const currentResidency = targetRuntime ? residencyPolicy(targetRuntime) : 'auto'; + workloads.push( + compact({ + id: aliasId, + candidates, + residency: + RESIDENCY_RANK[override.residency] != null + ? override.residency + : maxResidency(demandResidency, currentResidency), + demandPerHour, + required: override.required === false ? false : true, + outputTokens: num(override.outputTokens), + slo: override.slo && typeof override.slo === 'object' ? override.slo : null + }) + ); + } + return workloads.sort((left, right) => left.id.localeCompare(right.id)); +} + +function pinnedModel(config, aliasId, target) { + const alias = object(config.aliases)[aliasId]; + if (!alias) return target; + try { + return aliasLeaves(config, resolveRouteTarget(alias, target).members)[0] ?? target; + } catch { + return target; + } +} + +export function poolPinsFromConfig(config = {}, { profile = null, cordon = [], routes = {} } = {}) { + const pool = object(config.pool); + const pinned = { ...object(object(pool.pins).routes), ...object(profile?.routes), ...object(routes) }; + const residency = {}; + for (const [runtimeId, runtime] of Object.entries(object(config.runtimes))) { + if (runtime?.enabled === false) continue; + if (residencyPolicy(runtime) === 'always') residency[runtimeId] = 'always'; + } + for (const [runtimeId, policy] of Object.entries(object(profile?.residency))) { + if (policy === 'always' && config.runtimes?.[runtimeId]) residency[runtimeId] = 'always'; + } + return { + routes: Object.fromEntries( + Object.entries(pinned).map(([aliasId, target]) => [aliasId, pinnedModel(config, aliasId, target)]) + ), + residency, + cordon: unique([...(pool.cordon ?? []), ...cordon]).sort() + }; +} + +// `external` is a healthy process LLooM did not start; it is serving all the same. +const SERVING_STATUSES = new Set(['running', 'external']); + +function runtimeRunning(nodeStatuses, nodeId, runtimeId) { + return SERVING_STATUSES.has(nodeStatuses?.[nodeId]?.runtimeManager?.runtimes?.[runtimeId]?.status); +} + +export function currentPoolState(config = {}, { nodeStatuses = {}, variants = [] } = {}) { + const routes = {}; + for (const [aliasId, alias] of Object.entries(object(config.aliases))) { + const model = currentRouteModel(config, alias); + if (model) routes[aliasId] = model; + } + const routed = new Set(Object.values(routes)); + const residency = {}; + const running = []; + for (const variant of variants) { + if (variant.external || variant.source === 'recipe') continue; + if (variant.runtime && variant.managed) { + const runtime = config.runtimes?.[variant.runtime]; + residency[variant.runtime] = residencyPolicy(runtime); + const members = runtime?.placement?.mode === 'distributed' ? (runtime.placement.members ?? []) : []; + const isRunning = members.length + ? members.every((member) => runtimeRunning(nodeStatuses, member.node, member.runtime)) + : variant.nodes.some((nodeId) => runtimeRunning(nodeStatuses, nodeId, variant.runtime)); + if (isRunning) running.push(variant.id); + } else if (variant.remoteRuntime) { + if (runtimeRunning(nodeStatuses, variant.controlNode, variant.remoteRuntime)) running.push(variant.id); + } else if (!variant.managed && routed.has(variant.gatewayModel)) { + // An unmanaged lane that traffic is routed to is assumed to be up. + running.push(variant.id); + } + } + return { routes, residency, running: running.sort() }; +} + +function nodeLinks(config, nodeId, inventory) { + const discovered = (config.cluster?.discovery?.links ?? []).flatMap((link) => + link?.nodeId === nodeId ? [link.peerNodeId] : link?.peerNodeId === nodeId ? [link.nodeId] : [] + ); + return unique([...(inventory?.links ?? []).map((link) => link.peerNodeId), ...discovered]).sort(); +} + +export function plannerNodesFromStatus(config = {}, nodeStatuses = {}, { env = process.env } = {}) { + return Object.entries(clusterNodes(config, env)) + .map(([nodeId, node]) => { + const status = nodeStatuses?.[nodeId] ?? null; + const resources = object(node.resources); + const inventory = status?.inventory ?? (status?.profile ? inventoryFromProfile(status.profile) : null); + const configuredGb = num(resources.memoryGb); + const memoryDomains = inventory?.memoryDomains?.length + ? inventory.memoryDomains + : configuredGb + ? [{ id: 'memory', kind: 'unified', capacityGb: configuredGb }] + : []; + const accelerators = status?.profile?.accelerators ?? resources.accelerators ?? []; + const computeArchs = inventory?.computeArchs?.length + ? inventory.computeArchs + : unique(accelerators.map((name) => ARCH_BY_ACCELERATOR[name])); + const budgetGb = num(resources.memoryBudgetGb); + return compact({ + id: nodeId, + local: node.local === true, + available: status ? status.reachable !== false : true, + platformId: status?.profile?.platformId ?? null, + computeArchs, + accelerators, + memoryDomains: + budgetGb != null && memoryDomains.length === 1 ? [{ ...memoryDomains[0], budgetGb }] : memoryDomains, + maxMemoryUtilization: num(resources.maxMemoryUtilization), + reserveGb: num(resources.reserveMemoryGb), + links: nodeLinks(config, nodeId, inventory), + weights: (inventory?.weights ?? []).filter((entry) => entry.complete).map((entry) => entry.recipeId), + images: inventory?.backends?.dockerImages ?? [], + inventory: inventory ? (inventory.complete ? 'reported' : 'profile') : 'config' + }); + }) + .sort((left, right) => left.id.localeCompare(right.id)); +} + +export function buildPoolPlanInputs({ + config = {}, + nodeStatuses = {}, + metrics = null, + recipes = [], + benchmarks = [], + profile = null, + cordon = [], + routes = {}, + considerInstalls = false, + policy = {}, + env = process.env +} = {}) { + const catalog = buildPoolCatalog({ config, recipes, benchmarks, metrics, nodeStatuses, env }); + const variants = considerInstalls ? catalog.variants : catalog.variants.filter((v) => v.source !== 'recipe'); + return { + nodes: plannerNodesFromStatus(config, nodeStatuses, { env }), + variants, + workloads: poolWorkloadsFromConfig(config, { metrics }), + pins: poolPinsFromConfig(config, { profile, cordon, routes }), + current: currentPoolState(config, { nodeStatuses, variants }), + policy: compact({ + maxMemoryUtilization: num(config.runtimePolicy?.maxMemoryUtilization), + reserveGb: num(config.runtimePolicy?.reserveMemoryGb), + ...object(config.pool?.policy), + ...policy + }), + catalog + }; +} diff --git a/src/pool-cli.mjs b/src/pool-cli.mjs new file mode 100644 index 0000000..854f82e --- /dev/null +++ b/src/pool-cli.mjs @@ -0,0 +1,224 @@ +// `lloom pool plan`: a read-only pool placement plan for the installed config. +// Live node status, inventory and 7-day metrics come from the running gateway; +// without one, the plan uses the config plus this machine's own inventory. +import { loadBenchmarkEvidence } from './benchmarks.mjs'; +import { currentNodeId } from './cluster.mjs'; +import { readProfile } from './config-profiles.mjs'; +import { profileMachine } from './machine-profile.mjs'; +import { modelRootCandidates } from './model-files.mjs'; +import { buildPoolPlanInputs } from './pool-catalog.mjs'; +import { collectNodeInventory } from './pool-inventory.mjs'; +import { planPool } from './pool-planner.mjs'; +import { loadRecipes } from './recipes.mjs'; + +export const POOL_USAGE = + 'Usage: lloom pool plan [--cordon NODE[,NODE]] [--pin ALIAS=MODEL] [--profile NAME] [--consider-installs] ' + + '[--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json]'; + +const METRICS_PERIOD = '7d'; + +function parsePins(values = []) { + const routes = {}; + for (const value of values) { + const index = String(value).indexOf('='); + if (index < 1 || index === value.length - 1) throw new Error(`--pin expects ALIAS=MODEL, got ${value}`); + routes[value.slice(0, index)] = value.slice(index + 1); + } + return routes; +} + +function numberOption(value, name) { + if (value == null) return undefined; + const number = Number(value); + if (!Number.isFinite(number) || number < 0) throw new Error(`${name} must be a non-negative number`); + return number; +} + +export function poolPlanOptions({ + cordon = [], + pins = [], + profile, + considerInstalls, + offline, + headroomGb, + churnWeight, + external +} = {}) { + if (external != null && !['fallback', 'allow', 'never'].includes(external)) { + throw new Error('--external must be fallback, allow, or never'); + } + return { + cordon: cordon + .flatMap((value) => String(value).split(',')) + .map((value) => value.trim()) + .filter(Boolean), + routes: parsePins(pins), + profileName: profile ?? null, + considerInstalls: considerInstalls === true, + offline: offline === true, + policy: Object.fromEntries( + Object.entries({ + headroomGb: numberOption(headroomGb, '--headroom-gb'), + churnWeight: numberOption(churnWeight, '--churn-weight'), + external + }).filter(([, value]) => value !== undefined) + ) + }; +} + +async function liveState(config, gatewayRequest) { + const cluster = await gatewayRequest(config, '/gateway/cluster?inventory=1'); + const metrics = await gatewayRequest(config, `/gateway/metrics?period=${METRICS_PERIOD}`).catch(() => null); + return { + nodeStatuses: cluster?.cluster?.nodes ?? {}, + metrics: metrics ? { ...metrics, period: METRICS_PERIOD } : null + }; +} + +async function localState(config, deps) { + const profile = await deps.profileMachine(); + const inventory = await deps.collectNodeInventory({ + nodeId: currentNodeId(config), + profile, + recipes: await deps.loadRecipes(), + modelRoots: await deps.modelRootCandidates({ config }) + }); + return { nodeStatuses: { [currentNodeId(config)]: { reachable: true, profile, inventory } }, metrics: null }; +} + +export async function createPoolPlanReport(config, options = {}, overrides = {}) { + const deps = { + gatewayRequest: null, + gatewayUrl: null, + profileMachine, + collectNodeInventory, + loadRecipes, + loadBenchmarkEvidence, + modelRootCandidates, + readProfile, + ...overrides + }; + let state = null; + let source = { mode: 'config', gateway: deps.gatewayUrl, metricsPeriod: null, note: null }; + if (!options.offline && deps.gatewayRequest) { + try { + state = await liveState(config, deps.gatewayRequest); + source = { ...source, mode: 'gateway', metricsPeriod: state.metrics ? METRICS_PERIOD : null }; + } catch (error) { + source.note = `gateway unavailable (${error?.message ?? error}); planned from config and this machine only`; + } + } + if (!state) { + state = await localState(config, deps); + source.note ??= 'offline: no running state is known, so every resident lane is listed as a start'; + } + const profile = options.profileName ? await deps.readProfile(config, options.profileName) : null; + const inputs = buildPoolPlanInputs({ + config, + nodeStatuses: state.nodeStatuses, + metrics: state.metrics, + recipes: await deps.loadRecipes(), + benchmarks: await deps.loadBenchmarkEvidence(), + profile, + cordon: options.cordon ?? [], + routes: options.routes ?? {}, + considerInstalls: options.considerInstalls === true, + policy: options.policy ?? {} + }); + const plan = planPool(inputs); + return { + ok: plan.ok, + dryRun: true, + source, + nodes: inputs.nodes.map((node) => ({ + id: node.id, + available: node.available, + cordoned: inputs.pins.cordon.includes(node.id), + inventory: node.inventory, + computeArchs: node.computeArchs + })), + counts: { workloads: inputs.workloads.length, variants: inputs.variants.length }, + pins: inputs.pins, + ...(profile ? { profile: profile.name } : {}), + plan + }; +} + +function gb(value) { + return `${Number(value ?? 0).toFixed(1)} GB`; +} + +function workloadList(workloads) { + if (!workloads.length) return 'pinned'; + const shown = workloads.slice(0, 4).join(', '); + return workloads.length > 4 ? `${shown} (+${workloads.length - 4})` : shown; +} + +export function formatPoolPlanReport(report) { + const { plan } = report; + const lines = ['LLooM pool plan (dry run; nothing was changed)']; + lines.push( + report.source.mode === 'gateway' + ? `Source: gateway ${report.source.gateway ?? ''} live status${report.source.metricsPeriod ? `, ${report.source.metricsPeriod} demand` : ''}`.trim() + : `Source: config only${report.source.note ? ` — ${report.source.note}` : ''}` + ); + if (report.profile) lines.push(`Pinned routes from fleet profile ${report.profile}`); + + lines.push('', 'Nodes'); + for (const node of report.nodes) { + const summary = plan.nodes[node.id]; + const memory = (summary?.domains ?? []) + .map( + (domain) => + `${domain.id} ${gb(domain.residentGb)} resident of ${gb(domain.budgetGb)} (${gb(domain.freeGb)} free)` + ) + .join('; '); + lines.push( + ` ${node.id.padEnd(16)} ${summary?.status ?? 'unknown'}${memory ? ` ${memory}` : ''} [inventory: ${node.inventory}]` + ); + } + + lines.push('', 'Placements'); + const order = { always: 0, preferred: 1, auto: 2 }; + const placements = [...plan.placements].sort( + (a, b) => Number(a.external) - Number(b.external) || order[a.residency] - order[b.residency] + ); + for (const placement of placements) { + const tier = placement.external ? 'external' : placement.residency === 'auto' ? 'on-demand' : placement.residency; + const size = placement.external ? '' : ` ${gb(placement.memoryGbPerNode)}/node`; + const manual = placement.managed || placement.external ? '' : ' (unmanaged)'; + const shared = + placement.models.length > 1 ? ` [runtime ${placement.runtime} serves ${placement.models.length} models]` : ''; + lines.push( + ` ${tier.padEnd(10)} ${placement.placement}${size}${manual}${shared} <- ${workloadList(placement.workloads)}` + ); + } + if (!placements.length) lines.push(' (none)'); + + const { diff } = plan; + const changes = [ + ...diff.routeChanges.map((change) => `route ${change.alias}: ${change.from ?? '(none)'} -> ${change.to}`), + ...diff.start.map((entry) => `start ${entry.placement} as ${entry.residency}`), + ...diff.manual.map((entry) => `${entry.action.padEnd(10)} ${entry.placement} (not managed by LLooM)`), + ...diff.release.map((entry) => `release ${entry.placement} (no planned workload)`), + ...diff.residencyChanges.map((change) => `residency ${change.runtime}: ${change.from} -> ${change.to}`) + ]; + lines.push('', 'Changes from current'); + lines.push(...(changes.length ? changes.map((line) => ` ${line}`) : [' (none)'])); + + if (plan.uncovered.length || plan.pinViolations.length) { + lines.push('', 'Not covered'); + for (const entry of plan.uncovered) { + lines.push(` ${entry.workload}${entry.required ? '' : ' (optional)'}: ${entry.reasons[0] ?? 'no candidate'}`); + } + for (const entry of plan.pinViolations) lines.push(` pinned runtime ${entry.runtime}: ${entry.reasons[0]}`); + } + if (plan.warnings.length) lines.push('', 'Warnings', ...plan.warnings.map((warning) => ` ${warning}`)); + const score = plan.score; + lines.push( + '', + `Score: ${score.uncovered} uncovered, ${score.external} external fallback, ${score.sloMisses} SLO misses, cost ${score.cost.toExponential(2)}`, + 'Applying plans is not available yet; act on this one with `lloom fleet`, `lloom route` and `lloom keep-warm`.' + ); + return lines.join('\n'); +} diff --git a/src/pool-inventory.mjs b/src/pool-inventory.mjs new file mode 100644 index 0000000..0442215 --- /dev/null +++ b/src/pool-inventory.mjs @@ -0,0 +1,165 @@ +// Node inventory for pool planning: what a node could run without new +// downloads or installs. Collection is read-only and every probe is +// injectable; a failing probe yields a partial inventory, never an error. +import { promises as fs } from 'node:fs'; +import { runCommand } from './process-control.mjs'; +import { memoryDomainsForProfile } from './resource-fit.mjs'; +import { modelRootStatusForRecipe } from './model-files.mjs'; + +export const NODE_INVENTORY_VERSION = 1; +export const NODE_INVENTORY_TTL_MS = 300000; + +function round(value, places = 1) { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +function unique(values) { + return [...new Set(values.filter(Boolean))]; +} + +export function computeArchsForProfile(profile = {}) { + const archs = (Array.isArray(profile.devices) ? profile.devices : []) + .map((device) => String(device?.computeCapability ?? '').match(/^(\d+)\.(\d+)/)) + .filter(Boolean) + .map(([, major, minor]) => `sm_${major}${minor}`); + if (profile.isAppleSilicon) archs.push('apple-metal'); + return unique(archs).sort(); +} + +function profileLinks(profile = {}) { + const links = Array.isArray(profile.cluster?.links) ? profile.cluster.links : []; + return links + .filter((link) => link?.peerNodeId) + .map((link) => ({ + peerNodeId: String(link.peerNodeId), + ...(link.localInterface ? { localInterface: String(link.localInterface) } : {}), + ...(link.peerAddress ? { peerAddress: String(link.peerAddress) } : {}) + })); +} + +function capacityDomains(profile) { + return memoryDomainsForProfile(profile).map(({ id, kind, capacityGb }) => ({ id, kind, capacityGb })); +} + +// The planner can work from a machine profile alone (older gateways do not +// report inventory); it just cannot see cached weights, disks or images. +export function inventoryFromProfile(profile = {}) { + return { + version: NODE_INVENTORY_VERSION, + complete: false, + computeArchs: computeArchsForProfile(profile), + memoryDomains: capacityDomains(profile), + links: profileLinks(profile) + }; +} + +async function statfsGb(root) { + const stats = await fs.statfs(root); + return { + freeGb: round((stats.bavail * stats.bsize) / 1024 ** 3), + totalGb: round((stats.blocks * stats.bsize) / 1024 ** 3) + }; +} + +async function commandExists(command) { + const result = await runCommand('/usr/bin/which', [command], { allowFailure: true }); + return result.code === 0 && Boolean(result.stdout.trim()); +} + +async function dockerImages() { + const result = await runCommand( + 'docker', + ['image', 'ls', '--no-trunc', '--format', '{{.Repository}}:{{.Tag}}|{{.Repository}}@{{.Digest}}'], + { allowFailure: true } + ); + if (result.code !== 0) return []; + return result.stdout + .split('\n') + .flatMap((line) => line.trim().split('|')) + .filter((ref) => ref && !ref.includes('')); +} + +function recipeFitsPlatform(recipe, platformId) { + const platforms = recipe?.requirements?.platforms; + return !Array.isArray(platforms) || !platforms.length || platforms.includes(platformId); +} + +const message = (error) => error?.message ?? String(error); + +export async function collectNodeInventory({ nodeId, profile = {}, recipes = [], modelRoots = [], probes = {} } = {}) { + const probe = { + statfs: statfsGb, + modelRootStatus: modelRootStatusForRecipe, + commandExists, + dockerImages, + now: () => new Date().toISOString(), + ...probes + }; + const errors = []; + const disks = []; + for (const root of unique(modelRoots)) { + try { + disks.push({ root, ...(await probe.statfs(root)) }); + } catch { + // A candidate root that does not exist on this node is not an error. + } + } + const compatible = recipes.filter((recipe) => recipe?.id && recipeFitsPlatform(recipe, profile.platformId)); + const weights = []; + for (const recipe of compatible) { + for (const { root } of disks) { + try { + if ((await probe.modelRootStatus(recipe, root))?.complete === true) { + weights.push({ recipeId: recipe.id, root, complete: true }); + break; + } + } catch (error) { + errors.push(`weights ${recipe.id}: ${message(error)}`); + break; + } + } + } + const commands = {}; + for (const command of unique(compatible.flatMap((recipe) => recipe.requirements?.commands ?? [])).sort()) { + if (await probe.commandExists(command).catch(() => false)) commands[command] = true; + } + let images = []; + if (commands.docker) { + try { + images = unique(await probe.dockerImages()).sort(); + } catch (error) { + errors.push(`docker images: ${message(error)}`); + } + } + return { + ...inventoryFromProfile(profile), + complete: true, + nodeId, + collectedAt: probe.now(), + disks, + weights, + backends: { commands, dockerImages: images }, + ...(errors.length ? { errors } : {}) + }; +} + +export function createNodeInventoryCache({ collect, ttlMs = NODE_INVENTORY_TTL_MS, now = Date.now } = {}) { + let cached = null; + let pending = null; + return { + async get() { + if (cached && now() - cached.at < ttlMs) return cached.value; + pending ??= (async () => { + try { + const value = await collect(); + cached = { at: now(), value }; + return value; + } finally { + pending = null; + } + })(); + return pending; + } + }; +} diff --git a/src/pool-planner.mjs b/src/pool-planner.mjs new file mode 100644 index 0000000..bb83b3e --- /dev/null +++ b/src/pool-planner.mjs @@ -0,0 +1,644 @@ +// Pool placement planner. Pure: given node inventories, a variant catalog, +// workloads (aliases with demand and residency intent), pins and the current +// state, it proposes which variants run where, how each alias routes, and +// the diff from what is running now. It never starts, stops or reroutes +// anything; the per-node admission planner (runtime-policy.mjs) remains the +// safety layer that decides whether a start is allowed at request time. +// +// Objective, compared lexicographically: +// 1. cover every required workload and honour every pin +// 2. use external (cloud) members only when nothing local covers a workload +// 3. meet workload latency SLOs +// 4. minimise demand-weighted latency plus the load cost of every change +// Search is a greedy pass followed by single-workload local search. + +import { memoryBudgetGb } from './resource-fit.mjs'; + +export const POOL_PLAN_VERSION = 1; + +export const DEFAULT_POOL_POLICY = Object.freeze({ + maxMemoryUtilization: null, + reserveGb: 8, + headroomGb: 4, + horizonHours: 24, + churnWeight: 10, + routeChangeMs: 1000, + outputTokens: 256, + minDemandPerHour: 0.1, + unknownLatencyMs: 10000, + external: 'fallback', + loadBaseSeconds: 30, + loadSecondsPerGb: 2, + fetchMBps: 100, + imagePullSeconds: 300, + manualActionSeconds: 600, + localSearchPasses: 3 +}); + +const RESIDENCY_RANK = { auto: 0, preferred: 1, always: 2 }; +const EXTERNAL_MODES = new Set(['fallback', 'allow', 'never']); + +function num(value) { + if (value == null || value === '') return null; + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +function round(value, places = 2) { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +function list(value) { + return Array.isArray(value) ? value.filter((item) => item != null && item !== '') : []; +} + +function normalizeArch(value) { + return String(value) + .toLowerCase() + .replace(/^(sm_\d+)[af]$/, '$1'); +} + +function maxResidency(left, right) { + return RESIDENCY_RANK[right] > RESIDENCY_RANK[left] ? right : left; +} + +function normalizeResidency(value, fallback = 'auto') { + return Object.hasOwn(RESIDENCY_RANK, value) ? value : fallback; +} + +export function normalizePoolPolicy(policy = {}) { + const merged = { ...DEFAULT_POOL_POLICY }; + for (const [key, value] of Object.entries(policy ?? {})) { + if (value === undefined) continue; + if (key === 'external') merged.external = EXTERNAL_MODES.has(value) ? value : DEFAULT_POOL_POLICY.external; + else if (key in DEFAULT_POOL_POLICY) merged[key] = value === null ? null : (num(value) ?? merged[key]); + } + return merged; +} + +function domainBudgets(node, policy) { + const headroomGb = num(node.headroomGb) ?? num(policy.headroomGb) ?? 0; + return list(node.memoryDomains).map((domain) => { + const capacityGb = num(domain.capacityGb) ?? 0; + const budgetGb = memoryBudgetGb({ + totalMemoryGb: capacityGb, + memoryBudgetGb: domain.budgetGb, + maxMemoryUtilization: num(node.maxMemoryUtilization) ?? num(policy.maxMemoryUtilization), + reserveMemoryGb: num(node.reserveGb) ?? num(policy.reserveGb) ?? 0 + }); + return { + id: String(domain.id ?? domain.kind ?? 'memory'), + kind: String(domain.kind ?? 'system'), + capacityGb, + budgetGb: round(Math.max(0, budgetGb - headroomGb)) + }; + }); +} + +// Model weights live in unified memory on Apple Silicon and Spark, and in the +// largest device memory otherwise; host-side state lives in system memory. +function acceleratorDomain(domains) { + return ( + domains.find((domain) => domain.kind === 'unified') ?? + [...domains].filter((domain) => domain.kind === 'device').sort((a, b) => b.budgetGb - a.budgetGb)[0] ?? + domains.find((domain) => domain.kind === 'system') ?? + null + ); +} + +function systemDomain(domains) { + return domains.find((domain) => domain.kind === 'system') ?? acceleratorDomain(domains); +} + +function memoryDemands(variant, domains) { + const requirements = variant.requirements ?? {}; + const demands = new Map(); + const add = (domain, gb) => { + if (!domain || !(gb > 0)) return; + demands.set(domain.id, (demands.get(domain.id) ?? 0) + gb); + }; + add(acceleratorDomain(domains), num(requirements.memoryGbPerNode) ?? 0); + add(systemDomain(domains), num(requirements.systemMemoryGbPerNode) ?? 0); + return demands; +} + +function variantMemoryGb(variant) { + return (num(variant.requirements?.memoryGbPerNode) ?? 0) + (num(variant.requirements?.systemMemoryGbPerNode) ?? 0); +} + +function nodeIneligibility(variant, node) { + if (node.cordoned) return 'cordoned'; + if (node.available === false) return 'unavailable'; + const requirements = variant.requirements ?? {}; + const platforms = list(requirements.platforms); + if (platforms.length && !platforms.includes(node.platformId)) return `needs platform ${platforms.join('|')}`; + const archs = list(requirements.computeArchs).map(normalizeArch); + const nodeArchs = new Set(list(node.computeArchs).map(normalizeArch)); + if (archs.length && !archs.some((arch) => nodeArchs.has(arch))) return `needs compute arch ${archs.join('|')}`; + const accelerators = new Set(list(node.accelerators)); + const missing = list(requirements.accelerators).filter((accelerator) => !accelerators.has(accelerator)); + if (missing.length) return `needs accelerator ${missing.join(', ')}`; + if (!list(node.memoryDomains).length) return 'no memory capacity reported'; + return null; +} + +function linked(left, right) { + return list(left.links).includes(right.id) || list(right.links).includes(left.id); +} + +function combinations(items, size) { + if (size === 0) return [[]]; + const result = []; + items.forEach((item, index) => { + for (const rest of combinations(items.slice(index + 1), size - 1)) result.push([item, ...rest]); + }); + return result; +} + +// Candidate node sets for one variant, plus the reasons other sets were ruled +// out. Configured runtimes are fixed to their node(s); recipe variants may run +// on any eligible set, and tensor-parallel sets must be directly linked. +function candidateNodeSets(variant, nodesById) { + const reasons = []; + if (variant.external) return { sets: [[]], reasons }; + if (list(variant.nodes).length) { + const nodes = variant.nodes.map((id) => nodesById.get(id)); + const missing = variant.nodes.filter((id, index) => !nodes[index]); + if (missing.length) return { sets: [], reasons: [`unknown node ${missing.join(', ')}`] }; + const blocked = nodes + .map((node) => [node.id, nodeIneligibility(variant, node)]) + .filter(([, reason]) => reason) + .map(([id, reason]) => `${id} ${reason}`); + return blocked.length ? { sets: [], reasons: blocked } : { sets: [variant.nodes.slice()], reasons }; + } + const size = Math.max(1, Math.floor(num(variant.requirements?.nodeCount) ?? 1)); + const eligible = []; + for (const node of [...nodesById.values()].sort((a, b) => a.id.localeCompare(b.id))) { + const reason = nodeIneligibility(variant, node); + if (reason) reasons.push(`${node.id} ${reason}`); + else eligible.push(node); + } + if (eligible.length < size) { + reasons.push(`needs ${size} eligible node${size === 1 ? '' : 's'}, found ${eligible.length}`); + return { sets: [], reasons }; + } + const sets = combinations(eligible, size) + .filter((set) => set.every((node, index) => set.slice(index + 1).every((other) => linked(node, other)))) + .map((set) => set.map((node) => node.id)); + if (!sets.length) reasons.push(`no directly linked set of ${size} eligible nodes`); + return { sets, reasons }; +} + +// Memory belongs to the serving process, so models that share a runtime (or a +// remote runtime) on the same nodes share one placement and one charge. +function slotKey(variant, nodes) { + if (variant.runtime) return `runtime:${variant.runtime}|${nodes.join('+')}`; + if (variant.remoteRuntime) return `remote:${variant.controlNode ?? ''}/${variant.remoteRuntime}`; + return placementKey(variant, nodes); +} + +function placementKey(variant, nodes) { + return list(variant.nodes).length || variant.external ? variant.id : `${variant.id}@${nodes.join('+')}`; +} + +function latencyMs(variant, workload, policy) { + const performance = variant.performance ?? {}; + const ttft = num(performance.ttftMs); + const tps = num(performance.decodeTps); + const outputTokens = num(workload.outputTokens) ?? policy.outputTokens; + if (ttft != null && tps > 0) return ttft + (Math.max(0, outputTokens - 1) * 1000) / tps; + return num(performance.durationMs) ?? ttft ?? policy.unknownLatencyMs; +} + +function loadSeconds(variant, nodes, nodesById, policy) { + if (variant.external) return 0; + const memoryGb = variantMemoryGb(variant); + let seconds = num(variant.loadSeconds) ?? policy.loadBaseSeconds + memoryGb * policy.loadSecondsPerGb; + if (variant.managed === false) seconds += policy.manualActionSeconds; + // Configured lanes are installed on their nodes; only new placements fetch. + if (list(variant.nodes).length) return seconds; + for (const nodeId of nodes) { + const node = nodesById.get(nodeId); + const recipeId = variant.requirements?.recipeId; + if (recipeId && !list(node?.weights).includes(recipeId)) { + const diskGb = num(variant.requirements?.diskGb) ?? memoryGb; + seconds += (diskGb * 1024) / Math.max(1, policy.fetchMBps); + } + const images = list(variant.requirements?.images); + if (images.some((image) => !list(node?.images).includes(image))) seconds += policy.imagePullSeconds; + } + return seconds; +} + +function emptyState(nodes, policy) { + const usage = new Map(); + const domains = new Map(); + for (const node of nodes) { + const budgets = domainBudgets(node, policy); + domains.set(node.id, budgets); + usage.set(node.id, new Map(budgets.map((domain) => [domain.id, { always: 0, preferred: 0 }]))); + } + return { domains, usage, placements: new Map(), routes: {}, uncovered: [], warnings: [], pinViolations: [] }; +} + +// The largest on-demand placement that must still be admissible next to the +// hard-resident set on one memory domain. +function largestOnDemandGb(state, nodeId, domainId, excludeKey = null) { + let largest = 0; + for (const placement of state.placements.values()) { + if (placement.residency !== 'auto' || placement.slot === excludeKey || placement.variant.external) continue; + if (!placement.nodes.includes(nodeId)) continue; + const gb = memoryDemands(placement.variant, state.domains.get(nodeId) ?? []).get(domainId) ?? 0; + largest = Math.max(largest, gb); + } + return largest; +} + +function fitsAt(variant, nodes, residency, state, existing = null) { + for (const nodeId of nodes) { + const budgets = state.domains.get(nodeId) ?? []; + const usage = state.usage.get(nodeId); + for (const [domainId, gb] of memoryDemands(variant, budgets)) { + const budget = budgets.find((domain) => domain.id === domainId).budgetGb; + const used = usage.get(domainId); + // An upgrade re-counts a placement that is already resident. + const prior = existing && existing.residency !== 'auto' ? gb : 0; + const resident = used.always + used.preferred - prior; + const always = used.always - (existing?.residency === 'always' ? gb : 0); + const onDemand = largestOnDemandGb(state, nodeId, domainId, existing?.slot); + if (residency === 'always' && (resident + gb > budget || always + gb + onDemand > budget)) return false; + if (residency === 'preferred' && resident + gb > budget) return false; + if (residency === 'auto' && always + gb > budget) return false; + } + } + return true; +} + +function charge(variant, nodes, residency, state, sign = 1) { + if (residency === 'auto') return; + for (const nodeId of nodes) { + const budgets = state.domains.get(nodeId) ?? []; + const usage = state.usage.get(nodeId); + for (const [domainId, gb] of memoryDemands(variant, budgets)) usage.get(domainId)[residency] += sign * gb; + } +} + +function place(state, variant, nodes, residency, workloadId = null) { + const slot = slotKey(variant, nodes); + let placement = state.placements.get(slot); + if (!placement) { + placement = { + slot, + key: placementKey(variant, nodes), + variant, + nodes, + residency, + variants: new Map(), + workloads: [] + }; + state.placements.set(slot, placement); + if (!variant.external) charge(variant, nodes, residency, state); + } else if (RESIDENCY_RANK[residency] > RESIDENCY_RANK[placement.residency]) { + if (!variant.external) charge(variant, nodes, placement.residency, state, -1); + placement.residency = residency; + if (!variant.external) charge(variant, nodes, residency, state); + } + placement.variants.set(variant.id, variant); + if (workloadId) placement.workloads.push(workloadId); + return placement; +} + +// Every way a workload could be served right now: an existing placement it +// can share, or a new placement on a node set that fits. +function workloadOptions(workload, candidates, ctx, state) { + const { variantsByModel, nodesById, policy, current } = ctx; + const options = []; + const reasons = []; + const demand = Math.max(num(workload.demandPerHour) ?? 0, policy.minDemandPerHour); + const wanted = normalizeResidency(workload.residency); + const contextWindow = num(workload.slo?.contextWindow); + candidates.forEach((model, preference) => { + const variants = variantsByModel.get(model) ?? []; + if (!variants.length) reasons.push(`${model}: no variant in the pool catalog`); + for (const variant of variants) { + if (variant.external && policy.external === 'never') { + reasons.push(`${model}: external members are disabled`); + continue; + } + if (contextWindow && (num(variant.requirements?.contextWindow) ?? 0) < contextWindow) { + reasons.push(`${variant.id}: context window below ${contextWindow}`); + continue; + } + const latency = latencyMs(variant, workload, policy); + const sloTtft = num(workload.slo?.ttftMs); + const ttft = num(variant.performance?.ttftMs) ?? latency; + const base = { + model, + variant, + preference, + latencyMs: latency, + external: Boolean(variant.external), + sloMiss: sloTtft != null && ttft > sloTtft ? 1 : 0, + cost: + demand * latency * policy.horizonHours + + (current.routes?.[workload.id] != null && current.routes[workload.id] !== model ? policy.routeChangeMs : 0) + }; + const { sets, reasons: setReasons } = candidateNodeSets(variant, nodesById); + reasons.push(...setReasons.map((reason) => `${variant.id}: ${reason}`)); + for (const nodes of sets) { + const existing = state.placements.get(slotKey(variant, nodes)); + const key = existing?.key ?? placementKey(variant, nodes); + const residency = variant.external ? 'auto' : existing ? maxResidency(existing.residency, wanted) : wanted; + let chosen = residency; + if (!variant.external && !fitsAt(variant, nodes, residency, state, existing)) { + // Preferred residency is soft: fall back to on-demand when it fits. + if (residency === 'preferred' && fitsAt(variant, nodes, 'auto', state, existing)) chosen = 'auto'; + else if (existing && fitsAt(variant, nodes, existing.residency, state, existing)) { + chosen = existing.residency; + } else { + reasons.push(`${key}: does not fit as ${residency} on ${nodes.join('+')}`); + continue; + } + } + const isNew = !existing; + const running = current.running?.has(placementKey(variant, nodes)) || current.running?.has(variant.id); + const churnSeconds = isNew && !running ? loadSeconds(variant, nodes, nodesById, policy) : 0; + options.push({ + ...base, + nodes, + key, + residency: chosen, + downgraded: chosen !== residency, + churnSeconds, + cost: base.cost + churnSeconds * 1000 * policy.churnWeight + }); + } + } + }); + return { options, reasons }; +} + +function better(left, right) { + for (const key of ['uncovered', 'external', 'sloMisses']) { + if (left[key] !== right[key]) return left[key] < right[key]; + } + return left.cost < right.cost - Math.max(1e-6, Math.abs(right.cost) * 1e-9); +} + +function optionOrder(left, right, policy) { + if (policy.external === 'fallback' && left.external !== right.external) return left.external ? 1 : -1; + if (left.sloMiss !== right.sloMiss) return left.sloMiss - right.sloMiss; + if (left.cost !== right.cost) return left.cost - right.cost; + return left.preference - right.preference || left.key.localeCompare(right.key); +} + +function workloadOrder(pins) { + return (left, right) => { + const pinned = Number(Boolean(pins.routes?.[right.id])) - Number(Boolean(pins.routes?.[left.id])); + if (pinned) return pinned; + const residency = + RESIDENCY_RANK[normalizeResidency(right.residency)] - RESIDENCY_RANK[normalizeResidency(left.residency)]; + if (residency) return residency; + const required = Number(right.required !== false) - Number(left.required !== false); + if (required) return required; + return (num(right.demandPerHour) ?? 0) - (num(left.demandPerHour) ?? 0) || left.id.localeCompare(right.id); + }; +} + +function greedy(ctx, forced) { + const { workloads, pins, policy, nodes, variants } = ctx; + const state = emptyState(nodes, policy); + let cost = 0; + let external = 0; + let sloMisses = 0; + + for (const [runtimeId, residency] of Object.entries(pins.residency ?? {}).sort(([a], [b]) => a.localeCompare(b))) { + if (residency !== 'always') continue; + const variant = variants.find((candidate) => candidate.runtime === runtimeId && list(candidate.nodes).length); + if (!variant) continue; + const { sets, reasons } = candidateNodeSets(variant, ctx.nodesById); + const key = placementKey(variant, sets[0] ?? []); + if (!sets.length || !fitsAt(variant, sets[0], 'always', state, state.placements.get(slotKey(variant, sets[0])))) { + state.pinViolations.push({ runtime: runtimeId, reasons: reasons.length ? reasons : ['does not fit'] }); + continue; + } + const running = ctx.current.running?.has(key) || ctx.current.running?.has(variant.id); + if (!running) cost += loadSeconds(variant, sets[0], ctx.nodesById, policy) * 1000 * policy.churnWeight; + place(state, variant, sets[0], 'always'); + } + + for (const workload of workloads) { + const pinned = pins.routes?.[workload.id]; + const candidates = pinned ? [pinned] : forced[workload.id] ? [forced[workload.id]] : list(workload.candidates); + const { options, reasons } = workloadOptions(workload, candidates, ctx, state); + options.sort((left, right) => optionOrder(left, right, policy)); + const option = options[0]; + if (!option) { + state.uncovered.push({ + workload: workload.id, + required: workload.required !== false, + ...(pinned ? { pinned } : {}), + reasons: [...new Set(reasons)].slice(0, 12) + }); + continue; + } + if (option.downgraded) { + state.warnings.push( + `${workload.id}: ${option.key} kept ${option.residency} because ${normalizeResidency(workload.residency)} residency does not fit` + ); + } + place(state, option.variant, option.nodes, option.residency, workload.id); + state.routes[workload.id] = { + model: option.model, + variant: option.variant.id, + placement: option.key, + nodes: option.nodes, + residency: option.residency, + external: option.external, + latencyMs: round(option.latencyMs), + reason: pinned ? 'pinned' : option.external ? 'external' : 'planned' + }; + cost += option.cost; + external += option.external && policy.external === 'fallback' ? 1 : 0; + sloMisses += option.sloMiss; + } + + state.score = { + uncovered: state.uncovered.filter((entry) => entry.required).length + state.pinViolations.length, + external, + sloMisses, + cost: round(cost) + }; + return state; +} + +function search(ctx) { + let forced = {}; + let best = greedy(ctx, forced); + for (let pass = 0; pass < ctx.policy.localSearchPasses; pass += 1) { + let improved = false; + for (const workload of ctx.workloads) { + if (ctx.pins.routes?.[workload.id]) continue; + for (const model of list(workload.candidates)) { + if (best.routes[workload.id]?.model === model || forced[workload.id] === model) continue; + const trialForced = { ...forced, [workload.id]: model }; + const trial = greedy(ctx, trialForced); + if (better(trial.score, best.score)) { + best = trial; + forced = trialForced; + improved = true; + } + } + } + if (!improved) break; + } + return best; +} + +function summarizeNodes(ctx, state) { + return Object.fromEntries( + ctx.nodes.map((node) => { + const domains = state.domains.get(node.id).map((domain) => { + const used = state.usage.get(node.id).get(domain.id); + const residentGb = round(used.always + used.preferred); + return { + id: domain.id, + kind: domain.kind, + capacityGb: domain.capacityGb, + budgetGb: domain.budgetGb, + alwaysGb: round(used.always), + residentGb, + largestOnDemandGb: round(largestOnDemandGb(state, node.id, domain.id)), + freeGb: round(domain.budgetGb - residentGb) + }; + }); + return [ + node.id, + { + status: node.cordoned ? 'cordoned' : node.available === false ? 'unavailable' : 'available', + domains, + placements: [...state.placements.values()] + .filter((placement) => placement.nodes.includes(node.id)) + .map((placement) => placement.key) + .sort() + } + ]; + }) + ); +} + +function planDiff(ctx, state, residency) { + const { current } = ctx; + const running = current.running ?? new Set(); + const placements = [...state.placements.values()].filter((placement) => !placement.variant.external); + const planned = new Set(placements.flatMap((placement) => [placement.key, ...placement.variants.keys()])); + const start = []; + const manual = []; + const keep = []; + for (const placement of placements) { + const isRunning = running.has(placement.key) || [...placement.variants.keys()].some((id) => running.has(id)); + if (isRunning) keep.push(placement.key); + else if (placement.residency !== 'auto') { + (placement.variant.managed === false ? manual : start).push({ + action: placement.variant.managed === false ? 'manual-start' : 'start', + placement: placement.key, + nodes: placement.nodes, + residency: placement.residency + }); + } + } + const release = [...running] + .filter((key) => !planned.has(key)) + .sort() + .map((key) => { + const variant = ctx.variants.find((candidate) => candidate.id === key || key.startsWith(`${candidate.id}@`)); + if (variant?.managed === false) { + manual.push({ action: 'manual-stop', placement: key, nodes: variant.nodes ?? [] }); + return null; + } + return { action: 'release', placement: key }; + }) + .filter(Boolean); + const routeChanges = Object.entries(state.routes) + .filter(([alias, route]) => (current.routes?.[alias] ?? null) !== route.model) + .map(([alias, route]) => ({ alias, from: current.routes?.[alias] ?? null, to: route.model })) + .sort((a, b) => a.alias.localeCompare(b.alias)); + const residencyChanges = Object.entries(residency) + .filter(([runtime, policy]) => current.residency?.[runtime] != null && current.residency[runtime] !== policy) + .map(([runtime, policy]) => ({ runtime, from: current.residency[runtime], to: policy })) + .sort((a, b) => a.runtime.localeCompare(b.runtime)); + return { start, manual, release, keep: keep.sort(), routeChanges, residencyChanges }; +} + +export function planPool({ nodes = [], variants = [], workloads = [], pins = {}, current = {}, policy = {} } = {}) { + const normalizedPolicy = normalizePoolPolicy(policy); + const cordoned = new Set(list(pins.cordon)); + const plannerNodes = list(nodes) + .map((node) => ({ ...node, cordoned: node.cordoned === true || cordoned.has(node.id) })) + .sort((a, b) => a.id.localeCompare(b.id)); + const sortedVariants = list(variants) + .map((variant) => ({ managed: true, ...variant })) + .sort((a, b) => a.id.localeCompare(b.id)); + const variantsByModel = new Map(); + for (const variant of sortedVariants) { + if (!variantsByModel.has(variant.gatewayModel)) variantsByModel.set(variant.gatewayModel, []); + variantsByModel.get(variant.gatewayModel).push(variant); + } + const ctx = { + nodes: plannerNodes, + nodesById: new Map(plannerNodes.map((node) => [node.id, node])), + variants: sortedVariants, + variantsByModel, + workloads: [...list(workloads)].sort(workloadOrder(pins)), + pins, + policy: normalizedPolicy, + current: { + routes: current.routes ?? {}, + residency: current.residency ?? {}, + running: new Set(list(current.running instanceof Set ? [...current.running] : current.running)) + } + }; + const state = search(ctx); + + const residency = {}; + for (const variant of sortedVariants) { + if (variant.runtime && variant.managed !== false && !variant.external) residency[variant.runtime] ??= 'auto'; + } + for (const placement of state.placements.values()) { + const runtime = placement.variant.runtime; + if (runtime && placement.variant.managed !== false) { + residency[runtime] = maxResidency(residency[runtime] ?? 'auto', placement.residency); + } + } + + const placements = [...state.placements.values()] + .map((placement) => ({ + placement: placement.key, + variant: placement.variant.id, + models: [...new Set([...placement.variants.values()].map((variant) => variant.gatewayModel))].sort(), + runtime: placement.variant.runtime ?? null, + nodes: placement.nodes, + residency: placement.residency, + managed: placement.variant.managed !== false, + external: Boolean(placement.variant.external), + memoryGbPerNode: variantMemoryGb(placement.variant), + workloads: [...placement.workloads].sort() + })) + .sort((a, b) => a.placement.localeCompare(b.placement)); + + return { + version: POOL_PLAN_VERSION, + ok: state.score.uncovered === 0, + policy: normalizedPolicy, + placements, + routes: Object.fromEntries(Object.entries(state.routes).sort(([a], [b]) => a.localeCompare(b))), + residency: Object.fromEntries(Object.entries(residency).sort(([a], [b]) => a.localeCompare(b))), + uncovered: state.uncovered.sort((a, b) => a.workload.localeCompare(b.workload)), + pinViolations: state.pinViolations, + nodes: summarizeNodes(ctx, state), + diff: planDiff(ctx, state, residency), + score: state.score, + warnings: state.warnings + }; +} diff --git a/src/recipes.mjs b/src/recipes.mjs index 264e7d2..7471329 100644 --- a/src/recipes.mjs +++ b/src/recipes.mjs @@ -84,10 +84,28 @@ export function validateRecipe(recipe, config, { backendIds, checkLocalReference } if (!asArray(recipe.models).length) errors.push(`recipe ${recipe.id ?? '(missing)'} has no models`); + const serving = recipe.requirements ?? {}; + if ( + serving.computeArchs != null && + (!Array.isArray(serving.computeArchs) || + serving.computeArchs.some((arch) => typeof arch !== 'string' || !/^(sm_[0-9]+[af]?|apple-metal)$/.test(arch))) + ) { + errors.push(`recipe ${recipe.id} requirements.computeArchs must list sm_ or apple-metal`); + } + if (serving.tensorParallel != null && !(Number.isInteger(serving.tensorParallel) && serving.tensorParallel >= 1)) { + errors.push(`recipe ${recipe.id} requirements.tensorParallel must be a positive integer`); + } + const modelIds = checkLocalReferences ? configModelIds(config) : new Set(); const runtimeIds = checkLocalReferences ? configRuntimeIds(config) : new Set(); for (const model of asArray(recipe.models)) { if (!model.role) errors.push(`recipe ${recipe.id} has model without role`); + const maxSequences = model.settings?.maxSequences; + if (maxSequences != null && !(Number.isInteger(maxSequences) && maxSequences >= 1)) { + errors.push( + `recipe ${recipe.id} model ${model.role ?? '(missing)'} settings.maxSequences must be a positive integer` + ); + } if (!model.model) errors.push(`recipe ${recipe.id} model ${model.role ?? '(missing)'} is missing model`); if (checkLocalReferences && model.gatewayModel && !modelIds.has(model.gatewayModel)) { errors.push(`recipe ${recipe.id} references unknown gateway model ${model.gatewayModel}`); diff --git a/src/resource-fit.mjs b/src/resource-fit.mjs index 725a941..4cf7816 100644 --- a/src/resource-fit.mjs +++ b/src/resource-fit.mjs @@ -83,6 +83,16 @@ export function memoryDomainsForProfile(profile = {}) { return domains; } +// Host memory LLooM may plan into: an explicit budget, else a utilization +// ceiling, else capacity minus a reserve. Admission and pool planning share it. +export function memoryBudgetGb({ totalMemoryGb, memoryBudgetGb, maxMemoryUtilization, reserveMemoryGb = 8 } = {}) { + const explicit = finiteNumber(memoryBudgetGb); + if (explicit != null) return explicit; + const total = finiteNumber(totalMemoryGb) ?? 0; + const utilization = finiteNumber(maxMemoryUtilization); + return utilization != null ? total * utilization : Math.max(0, total - (finiteNumber(reserveMemoryGb) ?? 0)); +} + export function machineTopologyFingerprint(profile = {}) { const domains = memoryDomainsForProfile(profile) .map(({ availableGb: _availableGb, ...domain }) => domain) diff --git a/src/runtime-policy.mjs b/src/runtime-policy.mjs index 760fc43..20aafb2 100644 --- a/src/runtime-policy.mjs +++ b/src/runtime-policy.mjs @@ -1,6 +1,7 @@ import { assertMaintenanceStartAllowed } from './model-maintenance.mjs'; import os from 'node:os'; import { readHostMemory } from './host-memory.mjs'; +import { memoryBudgetGb as hostMemoryBudgetGb } from './resource-fit.mjs'; import { RuntimeManager } from './runtime-manager.mjs'; import { clusterNodes, currentNodeId, runtimeAuthority, runtimeResourcesByNode } from './cluster.mjs'; @@ -163,11 +164,12 @@ function policyConfig(config, profile = {}) { const totalMemoryGb = numberOrNull(profile.totalMemoryGb) ?? Math.round(os.totalmem() / 1024 / 1024 / 1024); const reserveMemoryGb = numberOrNull(policy.reserveMemoryGb) ?? 8; const maxMemoryUtilization = numberOrNull(policy.maxMemoryUtilization); - const memoryBudgetGb = - numberOrNull(policy.memoryBudgetGb) ?? - (maxMemoryUtilization != null - ? Math.max(0, totalMemoryGb * maxMemoryUtilization) - : Math.max(0, totalMemoryGb - reserveMemoryGb)); + const memoryBudgetGb = hostMemoryBudgetGb({ + totalMemoryGb, + memoryBudgetGb: numberOrNull(policy.memoryBudgetGb), + maxMemoryUtilization, + reserveMemoryGb + }); return { enabled: policy.enabled !== false && policy.memorySafety?.mode !== 'yolo', autoEvict: policy.autoEvict === true, @@ -329,12 +331,13 @@ function clusterRuntimePolicyPlan( const maxMemoryUtilization = numberOrNull( configuredNodes[nodeId].resources?.maxMemoryUtilization ?? policyTemplate.maxMemoryUtilization ); - const memoryBudgetGb = - numberOrNull(configuredNodes[nodeId].resources?.memoryBudgetGb) ?? - numberOrNull(policyTemplate.memoryBudgetGb) ?? - (maxMemoryUtilization != null - ? Math.max(0, totalMemoryGb * maxMemoryUtilization) - : Math.max(0, totalMemoryGb - reserveMemoryGb)); + const memoryBudgetGb = hostMemoryBudgetGb({ + totalMemoryGb, + memoryBudgetGb: + numberOrNull(configuredNodes[nodeId].resources?.memoryBudgetGb) ?? numberOrNull(policyTemplate.memoryBudgetGb), + maxMemoryUtilization, + reserveMemoryGb + }); const loadedMemoryGb = rows .filter((row) => row.loaded) .reduce((sum, row) => sum + numberOrNull(row.resourcesByNode?.[nodeId]?.memoryGb), 0); diff --git a/src/server.mjs b/src/server.mjs index 12ff2f8..5f03ade 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -59,6 +59,8 @@ import { applyOnboarding, createOnboardingPlan } from './onboarding.mjs'; import { applyRecipePack, createRecipePackPlan } from './recipe-pack.mjs'; import { buildRecipeIndexReport } from './recipe-index.mjs'; import { loadRecipes } from './recipes.mjs'; +import { collectNodeInventory, createNodeInventoryCache } from './pool-inventory.mjs'; +import { modelRootCandidates } from './model-files.mjs'; import { createRegistry, UnknownModelError } from './registry.mjs'; import { routeProfileStatus, writeRouteMemberSuspension, writeRouteProfile } from './route-control.mjs'; import { mutateConfigSource } from './config-mutation.mjs'; @@ -1287,14 +1289,15 @@ function restoreMetricMap(entries) { } function createMetricGroup(id = 'all') { - return { totals: emptyMetricBucket(id), models: new Map(), routes: new Map() }; + return { totals: emptyMetricBucket(id), models: new Map(), routes: new Map(), requested: new Map() }; } function restoreMetricGroup(source, id = 'all') { return { totals: restoreMetricBucket(emptyMetricBucket(id), source?.totals), models: restoreMetricMap(source?.models), - routes: restoreMetricMap(source?.routes) + routes: restoreMetricMap(source?.routes), + requested: restoreMetricMap(source?.requested) }; } @@ -1351,6 +1354,9 @@ function serializeMetricGroup(group) { .map((bucket) => finalizeMetricBucket(bucket, true)) .sort((a, b) => a.id.localeCompare(b.id)), routes: [...group.routes.values()] + .map((bucket) => finalizeMetricBucket(bucket, true)) + .sort((a, b) => a.id.localeCompare(b.id)), + requested: [...(group.requested ?? new Map()).values()] .map((bucket) => finalizeMetricBucket(bucket, true)) .sort((a, b) => a.id.localeCompare(b.id)) }; @@ -1363,6 +1369,8 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = const totals = restoreMetricBucket(emptyMetricBucket('all'), initialSnapshot?.totals); const models = restoreMetricMap(initialSnapshot?.models); const routes = restoreMetricMap(initialSnapshot?.routes); + // Demand per requested id (usually an alias), separate from resolved models. + const requested = restoreMetricMap(initialSnapshot?.requested); const days = new Map( (Array.isArray(initialSnapshot?.history?.days) ? initialSnapshot.history.days : []) .filter((entry) => /^\d{4}-\d{2}-\d{2}$/.test(entry?.date || '')) @@ -1508,17 +1516,19 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = apply(totals, entry); if (entry.model) apply(bucketFor(models, entry.model), entry); if (entry.route) apply(bucketFor(routes, entry.route), entry); + if (entry.requestedModel) apply(bucketFor(requested, entry.requestedModel), entry); const dayId = entry.at.slice(0, 10); if (!days.has(dayId)) days.set(dayId, createMetricGroup(dayId)); const day = days.get(dayId); apply(day.totals, entry); if (entry.model) apply(bucketFor(day.models, entry.model), entry); if (entry.route) apply(bucketFor(day.routes, entry.route), entry); + if (entry.requestedModel) apply(bucketFor(day.requested, entry.requestedModel), entry); while (days.size > 400) days.delete([...days.keys()].sort()[0]); }, snapshot({ model, period = 'all' } = {}) { const normalizedPeriod = ['today', '7d', '30d', 'all'].includes(period) ? period : 'all'; - let selectedGroup = { totals, models, routes }; + let selectedGroup = { totals, models, routes, requested }; if (normalizedPeriod !== 'all') { const dayCount = normalizedPeriod === 'today' ? 1 : Number.parseInt(normalizedPeriod, 10); const cutoff = new Date(); @@ -1530,6 +1540,7 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = mergeMetricBucket(selectedGroup.totals, day.totals); mergeMetricMaps(selectedGroup.models, day.models); mergeMetricMaps(selectedGroup.routes, day.routes); + mergeMetricMaps(selectedGroup.requested, day.requested); } } const selectedModel = model ? (selectedGroup.models.get(model) ?? null) : null; @@ -1550,6 +1561,9 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = : [] : [...selectedGroup.models.values()].map(finalizeMetricBucket).sort((a, b) => a.id.localeCompare(b.id)), routes: [...selectedGroup.routes.values()].map(finalizeMetricBucket).sort((a, b) => a.id.localeCompare(b.id)), + requested: [...selectedGroup.requested.values()] + .map(finalizeMetricBucket) + .sort((a, b) => a.id.localeCompare(b.id)), active: [...active.values()].map((entry) => ({ ...entry, elapsedMs: Math.max(0, Date.now() - Date.parse(entry.startedAt)) @@ -1560,7 +1574,7 @@ export function createMetricsStore({ maxRecent = 200, initialSnapshot = null } = }, persistenceSnapshot() { return { - ...serializeMetricGroup({ totals, models, routes }), + ...serializeMetricGroup({ totals, models, routes, requested }), history: { days: [...days.entries()] .sort(([a], [b]) => a.localeCompare(b)) @@ -1619,6 +1633,7 @@ export function createMetricsPersistence(config, { logger = console } = {}) { totals: latestSnapshot.totals, models: latestSnapshot.models, routes: latestSnapshot.routes, + requested: latestSnapshot.requested, history: latestSnapshot.history } }, @@ -2335,6 +2350,18 @@ export function createLloomServer( telemetry: hostTelemetry, profile: () => machineProfile }); + // Pool inventory is collected on request only and cached; it scans model + // roots and lists container images, which is too slow for the status path. + const nodeInventory = createNodeInventoryCache({ + collect: async () => + collectNodeInventory({ + nodeId: clusterCoordinator.nodeId, + profile: (await machineProfile) ?? {}, + recipes: await loadRecipes().catch(() => []), + modelRoots: await modelRootCandidates({ config: clusterCoordinator.config }) + }) + }); + clusterCoordinator.inventory ??= () => nodeInventory.get(); runtimeManager ??= new RuntimeManager(config, { logger, clusterCoordinator }); if (!runtimeManager.clusterCoordinator) { runtimeManager.clusterCoordinator = clusterCoordinator; @@ -5082,7 +5109,8 @@ export function createLloomServer( sendJson(res, 200, { ok: true, node: await clusterCoordinator.localNodeStatus({ - includeMemoryUsage: url.searchParams.get('memoryUsage') === '1' + includeMemoryUsage: url.searchParams.get('memoryUsage') === '1', + includeInventory: url.searchParams.get('inventory') === '1' }) }); return; @@ -5091,7 +5119,7 @@ export function createLloomServer( if (req.method === 'GET' && url.pathname === '/gateway/cluster') { sendJson(res, 200, { ok: true, - cluster: await clusterCoordinator.status() + cluster: await clusterCoordinator.status({ inventory: url.searchParams.get('inventory') === '1' }) }); return; } diff --git a/test/fixtures/pool/spark-fleet.json b/test/fixtures/pool/spark-fleet.json new file mode 100644 index 0000000..b609bac --- /dev/null +++ b/test/fixtures/pool/spark-fleet.json @@ -0,0 +1,212 @@ +{ + "description": "Planner-level reproduction of a three-Spark fleet on 2026-10-10: spark-node-03 serves embeddings, media and voice; the spark-node-01/02 pair serves SparkQwen RC16 (Atlas, TP2) as an unmanaged lane. Memory figures are the configured runtime estimates; performance figures are representative gateway measurements, not benchmark claims.", + "policy": { "maxMemoryUtilization": 0.95, "headroomGb": 4 }, + "nodes": [ + { + "id": "spark-node-03", + "platformId": "linux-arm64", + "computeArchs": ["sm_121"], + "accelerators": ["cuda", "nvidia-gpu", "blackwell", "gb10"], + "memoryDomains": [{ "id": "unified:0", "kind": "unified", "capacityGb": 128 }], + "links": [] + }, + { + "id": "spark-node-01", + "platformId": "linux-arm64", + "computeArchs": ["sm_121"], + "accelerators": ["cuda", "nvidia-gpu", "blackwell", "gb10"], + "memoryDomains": [{ "id": "unified:0", "kind": "unified", "capacityGb": 128 }], + "links": ["spark-node-02"] + }, + { + "id": "spark-node-02", + "platformId": "linux-arm64", + "computeArchs": ["sm_121"], + "accelerators": ["cuda", "nvidia-gpu", "blackwell", "gb10"], + "memoryDomains": [{ "id": "unified:0", "kind": "unified", "capacityGb": 128 }], + "links": ["spark-node-01"] + } + ], + "variants": [ + { + "id": "qwen3.8-flash-next-atlas@spark-node-01+spark-node-02", + "gatewayModel": "qwen3.8-flash-next-atlas", + "model": "nvidia/Qwen3.8-Flash-Next-NVFP4", + "kind": "chat", + "managed": false, + "nodes": ["spark-node-01", "spark-node-02"], + "requirements": { "nodeCount": 2, "tensorParallel": 2, "computeArchs": ["sm_121"], "memoryGbPerNode": 110, "contextWindow": 262144 }, + "performance": { "ttftMs": 450, "decodeTps": 75, "source": "fixture" } + }, + { + "id": "qwen38-27b-nvfp4@spark-node-03", + "gatewayModel": "qwen38-27b-nvfp4", + "model": "unsloth/Qwen3.8-27B-NVFP4", + "kind": "chat", + "runtime": "qwen38-27b-nvfp4", + "nodes": ["spark-node-03"], + "requirements": { "nodeCount": 1, "computeArchs": ["sm_121"], "memoryGbPerNode": 46, "contextWindow": 262144 }, + "performance": { "ttftMs": 900, "decodeTps": 28, "source": "fixture" } + }, + { + "id": "Qwen/Qwen3-Embedding-4B@spark-node-03", + "gatewayModel": "Qwen/Qwen3-Embedding-4B", + "kind": "embedding", + "runtime": "qwen3-embedding-4b", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 16 }, + "performance": { "durationMs": 60, "source": "fixture" } + }, + { + "id": "hear@spark-node-03", + "gatewayModel": "hear", + "kind": "audio", + "runtime": "hear", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 2 }, + "performance": { "durationMs": 400, "source": "fixture" } + }, + { + "id": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03", + "gatewayModel": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", + "kind": "image", + "runtime": "spark03-qwen-image-21-turbo-nvfp4", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 24 }, + "performance": { "durationMs": 18500, "source": "fixture" } + }, + { + "id": "BennyDaBall/Qwen-Image-2.1-NVFP4@spark-node-03", + "gatewayModel": "BennyDaBall/Qwen-Image-2.1-NVFP4", + "kind": "image", + "runtime": "spark03-qwen-image-21-nvfp4", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 32 }, + "performance": { "durationMs": 110000, "source": "fixture" } + }, + { + "id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03", + "gatewayModel": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "kind": "audio", + "runtime": "spark03-tts-clone", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 10 }, + "performance": { "durationMs": 1500, "source": "fixture" } + }, + { + "id": "openai/whisper-large-v3-turbo@spark-node-03", + "gatewayModel": "openai/whisper-large-v3-turbo", + "kind": "transcription", + "runtime": "spark03-stt", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 6 }, + "performance": { "durationMs": 2000, "source": "fixture" } + }, + { + "id": "Wan-AI/Wan2.2-TI2V-5B-Diffusers@spark-node-03", + "gatewayModel": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "kind": "video", + "runtime": "spark03-video", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 48 }, + "performance": { "durationMs": 240000, "source": "fixture" } + }, + { + "id": "Comfy-Org/YuE2-3B@spark-node-03", + "gatewayModel": "Comfy-Org/YuE2-3B", + "kind": "audio", + "runtime": "spark03-yue2", + "nodes": ["spark-node-03"], + "requirements": { "memoryGbPerNode": 16 }, + "performance": { "durationMs": 90000, "source": "fixture" } + }, + { + "id": "cloud/openrouter/q38fn", + "gatewayModel": "cloud/openrouter/q38fn", + "kind": "chat", + "external": true, + "performance": { "ttftMs": 700, "decodeTps": 120, "source": "fixture" } + } + ], + "cognitionCandidates": ["qwen3.8-flash-next-atlas", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"], + "workloads": [ + { "id": "chat-capable", "cognition": true, "residency": "always", "demandPerHour": 30 }, + { "id": "chat-fast", "cognition": true, "residency": "always", "demandPerHour": 20 }, + { "id": "chat-classifier", "cognition": true, "residency": "always", "demandPerHour": 40 }, + { "id": "multimodal", "cognition": true, "residency": "always", "demandPerHour": 5 }, + { "id": "enntity-presence", "cognition": true, "residency": "always", "demandPerHour": 10 }, + { "id": "omp", "cognition": true, "residency": "always", "demandPerHour": 8 }, + { "id": "ds4f", "cognition": true, "residency": "always", "demandPerHour": 2 }, + { "id": "embeddings", "candidates": ["Qwen/Qwen3-Embedding-4B"], "residency": "always", "demandPerHour": 200 }, + { "id": "hear", "candidates": ["hear"], "residency": "always", "demandPerHour": 20 }, + { "id": "image-fast", "candidates": ["BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", "BennyDaBall/Qwen-Image-2.1-NVFP4"], "residency": "preferred", "demandPerHour": 2 }, + { "id": "image-quality", "candidates": ["BennyDaBall/Qwen-Image-2.1-NVFP4"], "residency": "auto", "demandPerHour": 0.2 }, + { "id": "tts-clone", "candidates": ["Qwen/Qwen3-TTS-12Hz-1.7B-Base"], "residency": "preferred", "demandPerHour": 3 }, + { "id": "stt", "candidates": ["openai/whisper-large-v3-turbo"], "residency": "auto", "demandPerHour": 1 }, + { "id": "video", "candidates": ["Wan-AI/Wan2.2-TI2V-5B-Diffusers"], "residency": "auto", "demandPerHour": 0.1 }, + { "id": "music", "candidates": ["Comfy-Org/YuE2-3B"], "residency": "auto", "demandPerHour": 0.1 } + ], + "pins": { "residency": { "qwen3-embedding-4b": "always", "hear": "always" } }, + "profiles": { + "sparkqwen-pair-rc16": { + "routes": { + "chat-capable": "qwen3.8-flash-next-atlas", + "chat-fast": "qwen3.8-flash-next-atlas", + "chat-classifier": "qwen3.8-flash-next-atlas", + "multimodal": "qwen3.8-flash-next-atlas", + "enntity-presence": "qwen3.8-flash-next-atlas", + "omp": "qwen3.8-flash-next-atlas", + "ds4f": "qwen3.8-flash-next-atlas", + "embeddings": "Qwen/Qwen3-Embedding-4B", + "hear": "hear", + "image-fast": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", + "image-quality": "BennyDaBall/Qwen-Image-2.1-NVFP4", + "tts-clone": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "stt": "openai/whisper-large-v3-turbo", + "video": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "music": "Comfy-Org/YuE2-3B" + }, + "residency": { + "qwen3-embedding-4b": "always", + "hear": "always", + "spark03-tts-clone": "preferred", + "spark03-qwen-image-21-turbo-nvfp4": "preferred", + "qwen38-27b-nvfp4": "auto", + "spark03-qwen-image-21-nvfp4": "auto", + "spark03-stt": "auto", + "spark03-video": "auto", + "spark03-yue2": "auto" + } + }, + "standalone": { + "routes": { + "chat-capable": "qwen38-27b-nvfp4", + "chat-fast": "qwen38-27b-nvfp4", + "chat-classifier": "qwen38-27b-nvfp4", + "multimodal": "qwen38-27b-nvfp4", + "enntity-presence": "qwen38-27b-nvfp4", + "omp": "qwen38-27b-nvfp4", + "ds4f": "qwen38-27b-nvfp4", + "embeddings": "Qwen/Qwen3-Embedding-4B", + "hear": "hear", + "image-fast": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", + "image-quality": "BennyDaBall/Qwen-Image-2.1-NVFP4", + "tts-clone": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "stt": "openai/whisper-large-v3-turbo", + "video": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "music": "Comfy-Org/YuE2-3B" + }, + "residency": { + "qwen3-embedding-4b": "always", + "hear": "always", + "spark03-tts-clone": "preferred", + "spark03-qwen-image-21-turbo-nvfp4": "preferred", + "qwen38-27b-nvfp4": "always", + "spark03-qwen-image-21-nvfp4": "auto", + "spark03-stt": "auto", + "spark03-video": "auto", + "spark03-yue2": "auto" + } + } + } +} diff --git a/test/metrics-persistence.test.mjs b/test/metrics-persistence.test.mjs index 5698bde..f7f83bd 100644 --- a/test/metrics-persistence.test.mjs +++ b/test/metrics-persistence.test.mjs @@ -46,6 +46,21 @@ try { assert.equal(restored.snapshot({ period: 'today' }).totals.inputTokens, 11); assert.equal(restored.snapshot({ period: '7d' }).models[0]?.outputTokens, 7); + // Requested ids (aliases) aggregate separately from resolved models so pool + // planning can see per-alias demand; older history files simply lack them. + assert.equal(restoredSnapshot.requested.find((entry) => entry.id === 'test-model')?.requests, 1); + restored.record({ + route: '/v1/chat/completions', + model: 'test-model', + requestedModel: 'chat-alias', + ok: true, + durationMs: 50 + }); + assert.equal(restored.snapshot().requested.find((entry) => entry.id === 'chat-alias')?.requests, 1); + assert.equal(restored.snapshot({ period: '7d' }).requested.find((entry) => entry.id === 'chat-alias')?.requests, 1); + assert.equal(restored.snapshot().models.find((entry) => entry.id === 'test-model')?.requests, 2); + assert.deepEqual(createMetricsStore({ initialSnapshot: { totals: {} } }).snapshot().requested, []); + // Global throughput is time-weighted and counts concurrent generation once. const RealDate = Date; let clock = RealDate.parse('2026-09-05T12:00:00Z'); diff --git a/test/pool-catalog.test.mjs b/test/pool-catalog.test.mjs new file mode 100644 index 0000000..907d8ba --- /dev/null +++ b/test/pool-catalog.test.mjs @@ -0,0 +1,334 @@ +// Pool catalog and planner-input tests: recipes, installed config, node status +// and metrics become planner variants, workloads, pins and current state. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { + buildPoolCatalog, + buildPoolPlanInputs, + currentPoolState, + plannerNodesFromStatus, + poolPinsFromConfig, + poolWorkloadsFromConfig, + recipeServingRequirements +} from '../src/pool-catalog.mjs'; +import { planPool } from '../src/pool-planner.mjs'; +import { validateRecipe } from '../src/recipes.mjs'; + +const tp2Recipe = JSON.parse( + readFileSync(new URL('../recipes/linux-nvidia-dgx-spark-2x-qwen38-flash-next-vllm.json', import.meta.url), 'utf8') +); + +test('serving requirements are inferred from an existing two-Spark recipe', () => { + const requirements = recipeServingRequirements(tp2Recipe, tp2Recipe.models[0]); + assert.equal(requirements.recipeId, tp2Recipe.id); + assert.equal(requirements.nodeCount, 2); + assert.equal(requirements.tensorParallel, 2); + assert.deepEqual(requirements.computeArchs, ['sm_121']); + assert.deepEqual(requirements.platforms, ['linux-arm64']); + assert.equal(requirements.memoryGbPerNode, 110); + assert.equal(requirements.contextWindow, 262144); + assert.equal(requirements.maxSequences, 8); + assert.equal(requirements.backend, 'docker-vllm'); + assert.equal(requirements.diskGb, 170); + assert.deepEqual(requirements.images, [ + 'vllm/vllm-openai@sha256:df871f170ee7070fbdce162bde08fb616e311570c948a620be0d4b33fe02f87b' + ]); + assert.deepEqual(requirements.inferred.sort(), ['computeArchs', 'maxSequences', 'tensorParallel']); +}); + +test('explicit recipe serving fields override inference', () => { + const recipe = { + id: 'four-way', + backend: { id: 'vllm' }, + requirements: { + platforms: ['linux-x64'], + accelerators: ['cuda'], + computeArchs: ['sm_100'], + tensorParallel: 4, + memoryGb: 300, + cluster: { nodes: 1 } + }, + models: [{ model: 'org/m', gatewayModel: 'm', settings: { maxSequences: 16, contextWindow: 8192 } }] + }; + const requirements = recipeServingRequirements(recipe, recipe.models[0]); + assert.equal(requirements.nodeCount, 1); + assert.equal(requirements.tensorParallel, 4); + assert.deepEqual(requirements.computeArchs, ['sm_100']); + assert.equal(requirements.memoryGbPerNode, 300); + assert.equal(requirements.maxSequences, 16); + assert.deepEqual(requirements.inferred, []); +}); + +test('recipe validation checks the structured serving fields', () => { + const recipe = (requirements, settings = {}) => ({ + schemaVersion: 1, + id: 'r', + name: 'R', + backend: { id: 'vllm' }, + requirements, + models: [{ role: 'default', model: 'org/m', settings }] + }); + const check = (...args) => validateRecipe(recipe(...args), {}, { checkLocalReferences: false }); + assert.deepEqual(check({ computeArchs: ['sm_121a', 'apple-metal'], tensorParallel: 2 }, { maxSequences: 8 }), []); + assert.deepEqual(check({ computeArchs: ['blackwell'] }), [ + 'recipe r requirements.computeArchs must list sm_ or apple-metal' + ]); + assert.deepEqual(check({ tensorParallel: 0 }), ['recipe r requirements.tensorParallel must be a positive integer']); + assert.deepEqual(check({}, { maxSequences: 1.5 }), [ + 'recipe r model default settings.maxSequences must be a positive integer' + ]); +}); + +function fleetConfig() { + return { + cluster: { + nodeId: 'spark-node-03', + nodes: { + 'spark-node-03': { resources: { memoryGb: 128, accelerators: ['cuda', 'gb10'] } }, + 'spark-node-01': { endpoint: 'http://01:8100', resources: { memoryGb: 128, maxMemoryUtilization: 0.9 } }, + 'spark-node-02': { endpoint: 'http://02:8100', resources: { memoryGb: 128 } } + }, + discovery: { links: [{ nodeId: 'spark-node-01', peerNodeId: 'spark-node-02' }] } + }, + runtimePolicy: { maxMemoryUtilization: 0.95 }, + backends: { + q27: { type: 'openai', baseUrl: 'http://127.0.0.1:8213/v1' }, + emb: { type: 'openai', baseUrl: 'http://127.0.0.1:8214/v1' }, + atlas: { type: 'openai', baseUrl: 'http://192.0.2.10:8894/v1' }, + openrouter: { type: 'openai', baseUrl: 'https://openrouter.ai/api/v1' }, + old: { type: 'openai', baseUrl: 'http://127.0.0.1:8299/v1' } + }, + runtimes: { + 'qwen38-27b-nvfp4': { enabled: true, memoryGb: 46, maxActiveRequests: 2, recipe: { id: 'spark-27b' } }, + 'qwen3-embedding-4b': { enabled: true, memoryGb: 16, keepWarm: true }, + retired: { enabled: false, memoryGb: 60 } + }, + models: [ + { id: 'qwen38-27b-nvfp4', kind: 'chat', backend: 'q27', runtime: 'qwen38-27b-nvfp4', contextWindow: 262144 }, + { id: 'Qwen/Qwen3-Embedding-4B', kind: 'embedding', backend: 'emb', runtime: 'qwen3-embedding-4b' }, + { + id: 'qwen3.8-flash-next-atlas', + kind: 'chat', + backend: 'atlas', + contextWindow: 262144, + pool: { nodes: ['spark-node-01', 'spark-node-02'], memoryGbPerNode: 110 } + }, + { id: 'cloud/openrouter/q38fn', kind: 'chat', backend: 'openrouter' }, + { id: 'retired-model', kind: 'chat', backend: 'old', runtime: 'retired' } + ], + aliases: { + 'chat-capable': { + members: ['qwen3.8-flash-next-atlas'], + routeProfiles: { + local: { members: ['qwen38-27b-nvfp4'] }, + cloud: { members: ['q38fn'] }, + pair: { members: ['qwen3.8-flash-next-atlas'] } + }, + activeRoute: 'pair' + }, + q38fn: { members: ['cloud/openrouter/q38fn'] }, + embeddings: { members: ['Qwen/Qwen3-Embedding-4B'] }, + quiet: { members: ['qwen38-27b-nvfp4', 'retired-model'], suspendedMembers: ['retired-model'] } + }, + pool: { + workloads: { quiet: { residency: 'auto', required: false, slo: { ttftMs: 2000 } } }, + cordon: ['spark-node-02'] + } + }; +} + +const sparkProfile = (memoryGb = 119.7) => ({ + platformId: 'linux-arm64', + totalMemoryGb: memoryGb, + accelerators: ['cuda', 'nvidia-gpu', 'blackwell', 'gb10'], + devices: [{ id: 'cuda:0', kind: 'gpu', vendor: 'nvidia', backend: 'cuda', computeCapability: '12.1' }], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: memoryGb }] +}); + +function fleetStatuses() { + return { + 'spark-node-03': { + reachable: true, + profile: sparkProfile(), + inventory: { + computeArchs: ['sm_121'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 119.7 }], + weights: [{ recipeId: 'spark-27b', complete: true }], + backends: { dockerImages: ['vllm/vllm-openai@sha256:abc'] }, + links: [] + }, + runtimeManager: { + runtimes: { + 'qwen3-embedding-4b': { status: 'running', memoryUsage: { residentBytes: 20 * 1024 ** 3 } }, + 'qwen38-27b-nvfp4': { status: 'stopped' }, + 'tts-sidecar': { status: 'external' } + } + } + }, + 'spark-node-01': { reachable: true, profile: sparkProfile() }, + 'spark-node-02': { reachable: false, profile: null } + }; +} + +// Shape of GET /gateway/metrics?period=7d: id-keyed bucket arrays. +const metrics = { + period: '7d', + models: [ + { id: 'qwen38-27b-nvfp4', requests: 50, avgFirstContentMs: 900, decodeTokensPerSecond: 28, avgDurationMs: 9000 }, + { id: 'qwen3.8-flash-next-atlas', requests: 2, avgFirstContentMs: 100, decodeTokensPerSecond: 500 } + ], + routes: [{ id: '/v1/chat/completions', requests: 1764 }], + requested: [ + { id: 'chat-capable', requests: 1680 }, + { id: 'embeddings', requests: 84 }, + { id: 'quiet', requests: 0 } + ] +}; + +test('the catalog groups installed lanes, recipes and evidence by model', () => { + const catalog = buildPoolCatalog({ + config: fleetConfig(), + recipes: [tp2Recipe], + benchmarks: [ + { + id: 'bench-atlas', + recipeId: 'none', + gatewayModel: 'qwen3.8-flash-next-atlas', + metrics: { timeToFirstTokenMs: 450, generationTokPerSec: 75 } + } + ], + metrics, + nodeStatuses: fleetStatuses() + }); + const byId = Object.fromEntries(catalog.variants.map((variant) => [variant.id, variant])); + + const q27 = byId['qwen38-27b-nvfp4@spark-node-03']; + assert.deepEqual(q27.nodes, ['spark-node-03']); + assert.equal(q27.runtime, 'qwen38-27b-nvfp4'); + assert.equal(q27.requirements.memoryGbPerNode, 46); + assert.equal(q27.requirements.recipeId, 'spark-27b'); + assert.equal(q27.requirements.maxSequences, 2); + assert.deepEqual(q27.performance, { ttftMs: 900, decodeTps: 28, durationMs: 9000, source: 'live-metrics' }); + + const embedding = byId['Qwen/Qwen3-Embedding-4B@spark-node-03']; + assert.equal(embedding.requirements.memoryGbPerNode, 20, 'measured footprint above the estimate wins'); + assert.deepEqual(embedding.measured, { memoryGbPerNode: 20, source: 'live-rss' }); + + const atlas = byId['qwen3.8-flash-next-atlas@spark-node-01+spark-node-02']; + assert.equal(atlas.managed, false); + assert.deepEqual(atlas.nodes, ['spark-node-01', 'spark-node-02']); + assert.equal(atlas.requirements.memoryGbPerNode, 110); + assert.deepEqual(atlas.performance, { ttftMs: 450, decodeTps: 75, source: 'benchmark:bench-atlas' }); + + assert.equal(byId['cloud/openrouter/q38fn'].external, true); + assert.ok(!catalog.variants.some((variant) => variant.runtime === 'retired')); + + const recipeVariant = byId[`recipe:${tp2Recipe.id}:qwen3.8-flash-next`]; + assert.equal(recipeVariant.nodes, undefined); + assert.equal(recipeVariant.requirements.nodeCount, 2); + assert.equal(recipeVariant.source, 'recipe'); + + const flashNext = catalog.models.find((model) => model.model === 'nvidia/Qwen3.8-Flash-Next-NVFP4'); + assert.deepEqual( + flashNext.variants.map((variant) => variant.id), + [`recipe:${tp2Recipe.id}:qwen3.8-flash-next`] + ); +}); + +test('workloads come from alias routes, pool overrides and observed demand', () => { + const workloads = Object.fromEntries( + poolWorkloadsFromConfig(fleetConfig(), { metrics }).map((workload) => [workload.id, workload]) + ); + assert.deepEqual(workloads['chat-capable'].candidates, [ + 'qwen3.8-flash-next-atlas', + 'qwen38-27b-nvfp4', + 'cloud/openrouter/q38fn' + ]); + assert.equal(workloads['chat-capable'].demandPerHour, 10); + assert.equal(workloads['chat-capable'].residency, 'always'); + assert.equal(workloads.embeddings.residency, 'always'); + assert.deepEqual(workloads.quiet.candidates, ['qwen38-27b-nvfp4']); + assert.equal(workloads.quiet.residency, 'auto'); + assert.equal(workloads.quiet.required, false); + assert.deepEqual(workloads.quiet.slo, { ttftMs: 2000 }); + + // Gateways that predate per-alias metrics: a model's demand is split across + // the aliases routed to it. + const legacy = Object.fromEntries( + poolWorkloadsFromConfig(fleetConfig(), { + metrics: { period: '7d', models: [{ id: 'qwen38-27b-nvfp4', requests: 336 }] } + }).map((workload) => [workload.id, workload]) + ); + assert.equal(legacy.quiet.demandPerHour, 2); + assert.equal(legacy.embeddings.demandPerHour, 0); + + const offline = Object.fromEntries(poolWorkloadsFromConfig(fleetConfig()).map((workload) => [workload.id, workload])); + assert.equal(offline.embeddings.residency, 'always', 'without metrics the current keep-warm runtime decides'); + assert.equal(offline['chat-capable'].residency, 'auto'); +}); + +test('pins, cordons and current state come from config and node status', () => { + const config = fleetConfig(); + const pins = poolPinsFromConfig(config, { + profile: { routes: { embeddings: 'Qwen/Qwen3-Embedding-4B' } }, + cordon: ['spark-node-01'], + routes: { 'chat-capable': 'qwen38-27b-nvfp4' } + }); + assert.deepEqual(pins.residency, { 'qwen3-embedding-4b': 'always' }); + assert.deepEqual(pins.routes, { + embeddings: 'Qwen/Qwen3-Embedding-4B', + 'chat-capable': 'qwen38-27b-nvfp4' + }); + assert.deepEqual(pins.cordon, ['spark-node-01', 'spark-node-02']); + + const catalog = buildPoolCatalog({ config, nodeStatuses: fleetStatuses() }); + const current = currentPoolState(config, { nodeStatuses: fleetStatuses(), variants: catalog.variants }); + assert.equal(current.routes['chat-capable'], 'qwen3.8-flash-next-atlas'); + assert.equal(current.routes.quiet, 'qwen38-27b-nvfp4'); + assert.deepEqual(current.residency, { 'qwen38-27b-nvfp4': 'auto', 'qwen3-embedding-4b': 'always' }); + assert.deepEqual(current.running.sort(), [ + 'Qwen/Qwen3-Embedding-4B@spark-node-03', + 'qwen3.8-flash-next-atlas@spark-node-01+spark-node-02' + ]); +}); + +test('planner nodes merge inventory, profiles, configured resources and links', () => { + const nodes = Object.fromEntries( + plannerNodesFromStatus(fleetConfig(), fleetStatuses()).map((node) => [node.id, node]) + ); + assert.deepEqual(nodes['spark-node-03'].weights, ['spark-27b']); + assert.deepEqual(nodes['spark-node-03'].images, ['vllm/vllm-openai@sha256:abc']); + assert.deepEqual( + nodes['spark-node-01'].computeArchs, + ['sm_121'], + 'derived from the profile when inventory is absent' + ); + assert.deepEqual(nodes['spark-node-01'].links, ['spark-node-02']); + assert.equal(nodes['spark-node-01'].maxMemoryUtilization, 0.9); + assert.equal(nodes['spark-node-02'].available, false); + assert.deepEqual(nodes['spark-node-02'].memoryDomains, [{ id: 'memory', kind: 'unified', capacityGb: 128 }]); +}); + +test('planner inputs from config produce a coherent plan', () => { + const inputs = buildPoolPlanInputs({ + config: fleetConfig(), + nodeStatuses: fleetStatuses(), + metrics, + recipes: [tp2Recipe] + }); + assert.equal(inputs.policy.maxMemoryUtilization, 0.95); + assert.ok(!inputs.variants.some((variant) => variant.source === 'recipe'), 'installs are opt-in'); + const plan = planPool(inputs); + // spark-node-02 is cordoned and unreachable, so the pair lane cannot serve. + assert.equal(plan.routes['chat-capable'].model, 'qwen38-27b-nvfp4'); + assert.equal(plan.routes.embeddings.model, 'Qwen/Qwen3-Embedding-4B'); + assert.equal(plan.residency['qwen38-27b-nvfp4'], 'always'); + assert.deepEqual(plan.diff.routeChanges, [ + { alias: 'chat-capable', from: 'qwen3.8-flash-next-atlas', to: 'qwen38-27b-nvfp4' } + ]); + assert.deepEqual( + plan.diff.manual.map((entry) => entry.action), + ['manual-stop'] + ); +}); diff --git a/test/pool-cli.test.mjs b/test/pool-cli.test.mjs new file mode 100644 index 0000000..a67f6bc --- /dev/null +++ b/test/pool-cli.test.mjs @@ -0,0 +1,172 @@ +// `lloom pool plan` report tests. Gateway calls and local probes are injected; +// the spawned CLI checks only argument handling and the offline JSON shape. +import { test, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createPoolPlanReport, formatPoolPlanReport, poolPlanOptions } from '../src/pool-cli.mjs'; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const cli = path.join(repoRoot, 'bin', 'lloom.mjs'); +const directories = []; +after(async () => { + await Promise.all(directories.map((dir) => fs.rm(dir, { recursive: true, force: true }))); +}); + +function config() { + return { + cluster: { nodeId: 'head', nodes: { head: {}, pair: { endpoint: 'http://pair:8100' } } }, + runtimePolicy: { maxMemoryUtilization: 0.95 }, + backends: { local: { type: 'openai', baseUrl: 'http://127.0.0.1:8201/v1' } }, + runtimes: { small: { enabled: true, memoryGb: 40 } }, + models: [ + { id: 'small', kind: 'chat', backend: 'local', runtime: 'small' }, + { id: 'big', kind: 'chat', backend: 'local', pool: { nodes: ['pair'], memoryGbPerNode: 100 } } + ], + aliases: { chat: { members: ['big'], routeProfiles: { local: { members: ['small'] } } } } + }; +} + +const spark = (memoryGb = 128) => ({ + reachable: true, + profile: { + platformId: 'linux-arm64', + accelerators: ['cuda', 'gb10'], + devices: [{ id: 'cuda:0', computeCapability: '12.1' }], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: memoryGb }] + } +}); + +const deps = (gatewayRequest) => ({ + gatewayRequest, + gatewayUrl: 'http://127.0.0.1:8100', + loadRecipes: async () => [], + loadBenchmarkEvidence: async () => [], + profileMachine: async () => spark().profile, + collectNodeInventory: async ({ nodeId }) => ({ version: 1, complete: true, nodeId, computeArchs: ['sm_121'] }), + modelRootCandidates: async () => [], + readProfile: async (_config, name) => ({ name, routes: { chat: 'small' }, residency: {} }) +}); + +test('pool plan options parse cordons, pins and policy flags', () => { + assert.deepEqual( + poolPlanOptions({ + cordon: ['a,b', 'c'], + pins: ['chat=small'], + headroomGb: '6', + external: 'never', + offline: true + }), + { + cordon: ['a', 'b', 'c'], + routes: { chat: 'small' }, + profileName: null, + considerInstalls: false, + offline: true, + policy: { headroomGb: 6, external: 'never' } + } + ); + assert.throws(() => poolPlanOptions({ pins: ['chat'] }), /ALIAS=MODEL/); + assert.throws(() => poolPlanOptions({ external: 'sometimes' }), /fallback, allow, or never/); + assert.throws(() => poolPlanOptions({ churnWeight: '-1' }), /non-negative/); +}); + +test('a gateway report plans from live status and metrics', async () => { + const requested = []; + const report = await createPoolPlanReport( + config(), + poolPlanOptions({}), + deps(async (_config, pathname) => { + requested.push(pathname); + if (pathname.startsWith('/gateway/cluster')) { + return { cluster: { nodes: { head: spark(), pair: spark() } } }; + } + return { period: '7d', models: [], routes: [], requested: [{ id: 'chat', requests: 336 }] }; + }) + ); + assert.deepEqual(requested, ['/gateway/cluster?inventory=1', '/gateway/metrics?period=7d']); + assert.equal(report.dryRun, true); + assert.deepEqual(report.source, { + mode: 'gateway', + gateway: 'http://127.0.0.1:8100', + metricsPeriod: '7d', + note: null + }); + assert.equal(report.plan.routes.chat.model, 'big'); + assert.equal(report.plan.routes.chat.residency, 'always', '2 requests per hour makes the lane hot'); + const text = formatPoolPlanReport(report); + assert.match(text, /dry run; nothing was changed/); + assert.match(text, /always +big@pair 100\.0 GB\/node \(unmanaged\) <- chat/); + assert.match(text, /Changes from current\n {2}\(none\)/, 'a routed unmanaged lane is assumed to be serving'); +}); + +test('cordons and profile pins reach the planner', async () => { + const report = await createPoolPlanReport( + config(), + poolPlanOptions({ cordon: ['pair'], profile: 'standalone' }), + deps(async (_config, pathname) => + pathname.startsWith('/gateway/cluster') ? { cluster: { nodes: { head: spark(), pair: spark() } } } : null + ) + ); + assert.equal(report.profile, 'standalone'); + assert.deepEqual(report.pins.routes, { chat: 'small' }); + assert.equal(report.nodes.find((node) => node.id === 'pair').cordoned, true); + assert.equal(report.plan.routes.chat.model, 'small'); + assert.equal(report.plan.routes.chat.reason, 'pinned'); + assert.match(formatPoolPlanReport(report), /route +chat: big -> small/); +}); + +test('an unreachable gateway falls back to config and local inventory', async () => { + const report = await createPoolPlanReport( + config(), + poolPlanOptions({}), + deps(async () => { + throw new Error('connect ECONNREFUSED'); + }) + ); + assert.equal(report.source.mode, 'config'); + assert.match(report.source.note, /ECONNREFUSED/); + assert.deepEqual( + report.nodes.map((node) => [node.id, node.inventory]), + [ + ['head', 'reported'], + ['pair', 'config'] + ] + ); + assert.match(formatPoolPlanReport(report), /Source: config only — gateway unavailable/); +}); + +function runCli(args, env) { + return new Promise((resolve) => { + execFile(process.execPath, [cli, ...args], { env, timeout: 60000 }, (error, stdout, stderr) => + resolve({ code: error ? (error.code ?? 1) : 0, stdout, stderr }) + ); + }); +} + +test('the CLI prints an offline JSON plan and rejects unknown actions', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'lloom-pool-')); + directories.push(dir); + const configPath = path.join(dir, 'config.json'); + const cliConfig = { + ...config(), + cluster: undefined, + models: [config().models[0]], + aliases: { chat: { members: ['small'] } } + }; + await fs.writeFile(configPath, JSON.stringify(cliConfig, null, 2)); + const env = { ...process.env, HOME: dir, LLOOM_MODEL_ROOT: path.join(dir, 'models') }; + const result = await runCli(['pool', 'plan', '--offline', '--json', '--config', configPath], env); + assert.equal(result.code, 0, result.stderr); + const report = JSON.parse(result.stdout); + assert.equal(report.dryRun, true); + assert.equal(report.source.mode, 'config'); + assert.equal(report.plan.routes.chat.model, 'small'); + + const bad = await runCli(['pool', 'apply', '--config', configPath], env); + assert.notEqual(bad.code, 0); + assert.match(bad.stderr, /Usage: lloom pool plan/); +}); diff --git a/test/pool-inventory.test.mjs b/test/pool-inventory.test.mjs new file mode 100644 index 0000000..1170d0b --- /dev/null +++ b/test/pool-inventory.test.mjs @@ -0,0 +1,167 @@ +// Node inventory tests. Every probe (statfs, docker, commands, model files) is +// injected; nothing touches the host's model roots or container runtime. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { ClusterCoordinator } from '../src/cluster.mjs'; +import { + collectNodeInventory, + computeArchsForProfile, + createNodeInventoryCache, + inventoryFromProfile +} from '../src/pool-inventory.mjs'; + +const sparkProfile = { + platformId: 'linux-arm64', + totalMemoryGb: 119.7, + isAppleSilicon: false, + accelerators: ['cuda', 'nvidia-gpu', 'blackwell', 'gb10'], + devices: [{ id: 'cuda:0', kind: 'gpu', vendor: 'nvidia', backend: 'cuda', computeCapability: '12.1' }], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 119.7, deviceIds: ['cuda:0'] }], + cluster: { + provider: 'nvidia-sync', + links: [ + { + nodeId: 'spark-node-01', + peerNodeId: 'spark-node-02', + localInterface: 'enp1s0f0np0', + peerAddress: '10.100.0.2' + }, + { nodeId: 'spark-node-01', peerNodeId: 'spark-node-02', localInterface: 'enp1s0f1np1', peerAddress: '10.100.1.2' } + ] + } +}; + +test('compute architectures are structured from device capabilities', () => { + assert.deepEqual(computeArchsForProfile(sparkProfile), ['sm_121']); + assert.deepEqual(computeArchsForProfile({ isAppleSilicon: true, devices: [] }), ['apple-metal']); + assert.deepEqual(computeArchsForProfile({ devices: [{ computeCapability: '8.9' }, { computeCapability: '8.9' }] }), [ + 'sm_89' + ]); + assert.deepEqual(computeArchsForProfile({ devices: [] }), []); +}); + +test('profile-only inventory carries memory domains and direct links', () => { + const inventory = inventoryFromProfile(sparkProfile); + assert.deepEqual(inventory.computeArchs, ['sm_121']); + assert.deepEqual(inventory.memoryDomains, [{ id: 'unified:0', kind: 'unified', capacityGb: 119.7 }]); + assert.deepEqual(inventory.links, [ + { peerNodeId: 'spark-node-02', localInterface: 'enp1s0f0np0', peerAddress: '10.100.0.2' }, + { peerNodeId: 'spark-node-02', localInterface: 'enp1s0f1np1', peerAddress: '10.100.1.2' } + ]); + assert.equal(inventory.complete, false); +}); + +test('a node inventory reports disk, cached weights, backends and images', async () => { + const recipes = [ + { id: 'spark-a', requirements: { platforms: ['linux-arm64'], commands: ['docker'] } }, + { id: 'spark-b', requirements: { platforms: ['linux-arm64'] } }, + { id: 'mac-only', requirements: { platforms: ['darwin-arm64'], commands: ['mlx_lm.server'] } } + ]; + const checked = []; + const inventory = await collectNodeInventory({ + nodeId: 'spark-node-01', + profile: sparkProfile, + recipes, + modelRoots: ['/models', '/missing'], + probes: { + statfs: async (root) => + root === '/models' ? { freeGb: 812.4, totalGb: 3725.3 } : Promise.reject(new Error('ENOENT')), + modelRootStatus: async (recipe, root) => { + checked.push(`${recipe.id}:${root}`); + return { complete: recipe.id === 'spark-a' }; + }, + commandExists: async (command) => command === 'docker', + dockerImages: async () => ['vllm/vllm-openai@sha256:abc', 'lloom/comfy:cuda'], + now: () => '2026-10-11T00:00:00.000Z' + } + }); + assert.equal(inventory.version, 1); + assert.equal(inventory.nodeId, 'spark-node-01'); + assert.equal(inventory.complete, true); + assert.equal(inventory.collectedAt, '2026-10-11T00:00:00.000Z'); + assert.deepEqual(inventory.computeArchs, ['sm_121']); + assert.deepEqual(inventory.disks, [{ root: '/models', freeGb: 812.4, totalGb: 3725.3 }]); + assert.deepEqual(inventory.weights, [{ recipeId: 'spark-a', root: '/models', complete: true }]); + assert.deepEqual(checked.sort(), ['spark-a:/models', 'spark-b:/models'], 'only platform-compatible recipes'); + assert.deepEqual(inventory.backends, { + commands: { docker: true }, + dockerImages: ['lloom/comfy:cuda', 'vllm/vllm-openai@sha256:abc'] + }); + assert.equal(inventory.links.length, 2); +}); + +test('failing probes degrade to partial inventory instead of throwing', async () => { + const inventory = await collectNodeInventory({ + nodeId: 'mac', + profile: { platformId: 'darwin-arm64', isAppleSilicon: true, totalMemoryGb: 64 }, + recipes: [{ id: 'mac', requirements: { platforms: ['darwin-arm64'] } }], + modelRoots: ['/models'], + probes: { + statfs: async () => ({ freeGb: 10, totalGb: 100 }), + modelRootStatus: async () => { + throw new Error('permission denied'); + }, + commandExists: async () => false, + dockerImages: async () => { + throw new Error('should not run without docker'); + } + } + }); + assert.deepEqual(inventory.computeArchs, ['apple-metal']); + assert.deepEqual(inventory.weights, []); + assert.deepEqual(inventory.backends, { commands: {}, dockerImages: [] }); + assert.deepEqual(inventory.errors, ['weights mac: permission denied']); +}); + +test('the inventory cache reuses results within its TTL and shares in-flight work', async () => { + let now = 0; + let calls = 0; + const cache = createNodeInventoryCache({ + ttlMs: 1000, + now: () => now, + collect: async () => ({ call: ++calls }) + }); + const [first, second] = await Promise.all([cache.get(), cache.get()]); + assert.equal(first.call, 1); + assert.equal(second.call, 1); + now = 999; + assert.equal((await cache.get()).call, 1); + now = 1000; + assert.equal((await cache.get()).call, 2); +}); + +test('gateway node status includes inventory only when asked', async () => { + const requested = []; + const coordinator = new ClusterCoordinator( + { + cluster: { + nodeId: 'head', + nodes: { head: {}, peer: { endpoint: 'http://peer:8100' } } + } + }, + { + env: {}, + profile: () => sparkProfile, + inventory: async () => ({ version: 1, nodeId: 'head' }), + fetchImpl: async (url) => { + requested.push(String(url)); + return new Response(JSON.stringify({ ok: true, node: { id: 'peer', inventory: { version: 1 } } }), { + headers: { 'content-type': 'application/json' } + }); + } + } + ); + assert.equal((await coordinator.localNodeStatus()).inventory, undefined); + assert.deepEqual((await coordinator.localNodeStatus({ includeInventory: true })).inventory, { + version: 1, + nodeId: 'head' + }); + const peer = await coordinator.nodeStatus('peer', { inventory: true }); + assert.deepEqual(peer.inventory, { version: 1 }); + assert.deepEqual(requested, ['http://peer:8100/gateway/node?memoryUsage=1&inventory=1']); + await coordinator.nodeStatus('peer'); + assert.equal(requested.at(-1), 'http://peer:8100/gateway/node?memoryUsage=1'); + const cluster = await coordinator.status({ inventory: true }); + assert.deepEqual(cluster.nodes.head.inventory, { version: 1, nodeId: 'head' }); + assert.deepEqual(cluster.nodes.peer.inventory, { version: 1 }); +}); diff --git a/test/pool-planner.test.mjs b/test/pool-planner.test.mjs new file mode 100644 index 0000000..0f1cba2 --- /dev/null +++ b/test/pool-planner.test.mjs @@ -0,0 +1,428 @@ +// Pool planner tests. Pure planner inputs only: no gateway, runtimes or network. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { planPool } from '../src/pool-planner.mjs'; + +const fleet = JSON.parse(readFileSync(new URL('./fixtures/pool/spark-fleet.json', import.meta.url), 'utf8')); + +function fleetWorkloads() { + return fleet.workloads.map(({ cognition, ...workload }) => ({ + ...workload, + candidates: cognition ? fleet.cognitionCandidates : workload.candidates + })); +} + +function currentFromProfile(name, running) { + const profile = fleet.profiles[name]; + return { routes: profile.routes, residency: profile.residency, running }; +} + +const RC16_RUNNING = [ + 'qwen3.8-flash-next-atlas@spark-node-01+spark-node-02', + 'Qwen/Qwen3-Embedding-4B@spark-node-03', + 'hear@spark-node-03', + 'BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03', + 'Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03' +]; +const STANDALONE_RUNNING = [ + 'qwen38-27b-nvfp4@spark-node-03', + 'Qwen/Qwen3-Embedding-4B@spark-node-03', + 'hear@spark-node-03', + 'BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03', + 'Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03' +]; + +function planFleet({ cordon = [], current, pins = {}, nodes = fleet.nodes, policy = {} } = {}) { + return planPool({ + nodes, + variants: fleet.variants, + workloads: fleetWorkloads(), + pins: { ...fleet.pins, ...pins, cordon }, + current, + policy: { ...fleet.policy, ...policy } + }); +} + +function plannedRoutes(plan) { + return Object.fromEntries(Object.entries(plan.routes).map(([alias, route]) => [alias, route.model])); +} + +function plannedResidency(plan, expected) { + return Object.fromEntries(Object.keys(expected).map((runtime) => [runtime, plan.residency[runtime]])); +} + +test('whole fleet reproduces the sparkqwen-pair-rc16 profile with no changes', () => { + const plan = planFleet({ current: currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING) }); + const profile = fleet.profiles['sparkqwen-pair-rc16']; + assert.equal(plan.ok, true, JSON.stringify(plan.uncovered)); + assert.deepEqual(plannedRoutes(plan), profile.routes); + assert.deepEqual(plannedResidency(plan, profile.residency), profile.residency); + assert.deepEqual(plan.diff.routeChanges, []); + assert.deepEqual(plan.diff.start, []); + assert.deepEqual(plan.diff.manual, []); + assert.deepEqual(plan.diff.release, []); + assert.equal(plan.score.external, 0); +}); + +test('cordoning the 01/02 pair reproduces the standalone profile', () => { + const plan = planFleet({ + cordon: ['spark-node-01', 'spark-node-02'], + current: currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING) + }); + const profile = fleet.profiles.standalone; + assert.equal(plan.ok, true, JSON.stringify(plan.uncovered)); + assert.deepEqual(plannedRoutes(plan), profile.routes); + assert.deepEqual(plannedResidency(plan, profile.residency), profile.residency); + assert.deepEqual( + plan.diff.start.map((entry) => [entry.placement, entry.residency]), + [['qwen38-27b-nvfp4@spark-node-03', 'always']] + ); + assert.deepEqual(plan.diff.manual, [ + { + action: 'manual-stop', + placement: 'qwen3.8-flash-next-atlas@spark-node-01+spark-node-02', + nodes: ['spark-node-01', 'spark-node-02'] + } + ]); + assert.equal(plan.diff.routeChanges.length, 7); + assert.ok(plan.diff.routeChanges.every((change) => change.to === 'qwen38-27b-nvfp4')); + assert.equal(plan.nodes['spark-node-01'].status, 'cordoned'); + assert.equal(plan.score.external, 0, 'local 27B covers cognition before the cloud member'); + const spark03 = plan.nodes['spark-node-03'].domains[0]; + assert.ok(spark03.residentGb <= spark03.budgetGb); + assert.ok(spark03.alwaysGb + spark03.largestOnDemandGb <= spark03.budgetGb); +}); + +test('an unreachable pair is planned around exactly like a cordoned one', () => { + const nodes = fleet.nodes.map((node) => (node.id === 'spark-node-03' ? node : { ...node, available: false })); + const plan = planFleet({ nodes, current: currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING) }); + assert.deepEqual(plannedRoutes(plan), fleet.profiles.standalone.routes); + assert.equal(plan.nodes['spark-node-02'].status, 'unavailable'); +}); + +test('a free pair moves cognition back to SparkQwen as a manual start', () => { + const plan = planFleet({ current: currentFromProfile('standalone', STANDALONE_RUNNING) }); + const profile = fleet.profiles['sparkqwen-pair-rc16']; + assert.deepEqual(plannedRoutes(plan), profile.routes); + assert.equal(plan.residency['qwen38-27b-nvfp4'], 'auto'); + assert.deepEqual( + plan.diff.manual.map((entry) => entry.action + ' ' + entry.placement), + ['manual-start qwen3.8-flash-next-atlas@spark-node-01+spark-node-02'] + ); + assert.deepEqual(plan.diff.release, [{ action: 'release', placement: 'qwen38-27b-nvfp4@spark-node-03' }]); + assert.deepEqual(plan.diff.residencyChanges, [{ runtime: 'qwen38-27b-nvfp4', from: 'always', to: 'auto' }]); +}); + +test('a high churn weight keeps the running lane when the gain is small', () => { + const plan = planFleet({ + current: currentFromProfile('standalone', STANDALONE_RUNNING), + policy: { churnWeight: 100000 } + }); + assert.deepEqual(plannedRoutes(plan), fleet.profiles.standalone.routes); + assert.deepEqual(plan.diff.routeChanges, []); +}); + +test('pinned routes and residency are hard constraints', () => { + const plan = planFleet({ + current: currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING), + pins: { routes: { omp: 'qwen38-27b-nvfp4' } } + }); + assert.equal(plan.routes.omp.model, 'qwen38-27b-nvfp4'); + assert.equal(plan.routes.omp.reason, 'pinned'); + assert.equal(plan.routes['chat-capable'].model, 'qwen3.8-flash-next-atlas'); + assert.equal(plan.residency['qwen3-embedding-4b'], 'always'); + assert.equal(plan.residency.hear, 'always'); + + const impossible = planFleet({ + current: {}, + pins: { routes: { omp: 'qwen3.8-flash-next-atlas' } }, + cordon: ['spark-node-01'] + }); + assert.equal(impossible.ok, false); + assert.equal(impossible.uncovered[0].workload, 'omp'); + assert.equal(impossible.uncovered[0].pinned, 'qwen3.8-flash-next-atlas'); +}); + +const spark = (id, links = []) => ({ + id, + platformId: 'linux-arm64', + computeArchs: ['sm_121'], + accelerators: ['cuda', 'gb10'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 128 }], + links +}); +const mac = { + id: 'studio', + platformId: 'darwin-arm64', + computeArchs: ['apple-metal'], + accelerators: ['metal', 'apple-gpu'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 192 }] +}; +const tp2 = { + id: 'recipe:tp2', + gatewayModel: 'big-moe', + kind: 'chat', + requirements: { nodeCount: 2, tensorParallel: 2, computeArchs: ['sm_121a'], memoryGbPerNode: 100 }, + performance: { ttftMs: 500, decodeTps: 50 } +}; +const one = (workload) => [ + { id: 'chat', candidates: ['big-moe'], residency: 'always', demandPerHour: 10, ...workload } +]; + +test('tensor-parallel variants only span directly linked nodes', () => { + const plan = planPool({ + nodes: [spark('a', ['b']), spark('b'), spark('c')], + variants: [tp2], + workloads: one(), + policy: { maxMemoryUtilization: 0.95 } + }); + assert.deepEqual(plan.routes.chat.nodes, ['a', 'b']); + assert.equal(plan.placements[0].placement, 'recipe:tp2@a+b'); + + const unlinked = planPool({ + nodes: [spark('a'), spark('b'), spark('c')], + variants: [tp2], + workloads: one(), + policy: { maxMemoryUtilization: 0.95 } + }); + assert.equal(unlinked.ok, false); + assert.ok(unlinked.uncovered[0].reasons.some((reason) => reason.includes('no directly linked set of 2'))); +}); + +test('compute architecture must match the variant', () => { + const single = { ...tp2, id: 'recipe:one', requirements: { computeArchs: ['sm_121'], memoryGbPerNode: 60 } }; + const macOnly = planPool({ nodes: [mac], variants: [single], workloads: one() }); + assert.equal(macOnly.ok, false); + assert.ok(macOnly.uncovered[0].reasons.some((reason) => reason.includes('needs compute arch sm_121'))); + const mixed = planPool({ nodes: [mac, spark('s')], variants: [single], workloads: one() }); + assert.deepEqual(mixed.routes.chat.nodes, ['s']); +}); + +test('memory headroom is conservative', () => { + const large = { ...tp2, id: 'recipe:large', requirements: { memoryGbPerNode: 117 } }; + const fits = planPool({ + nodes: [spark('s')], + variants: [large], + workloads: one(), + policy: { maxMemoryUtilization: 0.95, headroomGb: 4 } + }); + assert.equal(fits.ok, true); + const tight = planPool({ + nodes: [spark('s')], + variants: [large], + workloads: one(), + policy: { maxMemoryUtilization: 0.95, headroomGb: 8 } + }); + assert.equal(tight.ok, false); + assert.ok(tight.uncovered[0].reasons.some((reason) => reason.includes('does not fit as always'))); + const defaults = planPool({ nodes: [spark('s')], variants: [large], workloads: one() }); + assert.equal(defaults.ok, false, 'default budget is capacity minus reserve minus headroom'); + assert.equal(defaults.nodes.s.domains[0].budgetGb, 116); +}); + +test('memory is accounted per domain on discrete GPUs', () => { + const pc = { + id: 'pc', + platformId: 'linux-x64', + computeArchs: ['sm_89'], + accelerators: ['cuda'], + memoryDomains: [ + { id: 'system:0', kind: 'system', capacityGb: 64 }, + { id: 'device:cuda:0', kind: 'device', capacityGb: 24 } + ] + }; + const small = { + id: 'small', + gatewayModel: 'small', + requirements: { memoryGbPerNode: 18, systemMemoryGbPerNode: 30 } + }; + const big = { id: 'big', gatewayModel: 'big', requirements: { memoryGbPerNode: 30 } }; + const plan = planPool({ + nodes: [pc], + variants: [small, big], + workloads: [ + { id: 'a', candidates: ['small'], residency: 'always' }, + { id: 'b', candidates: ['big'], residency: 'always' } + ], + policy: { reserveGb: 0, headroomGb: 0 } + }); + assert.equal(plan.routes.a.model, 'small'); + assert.deepEqual( + plan.nodes.pc.domains.map((domain) => [domain.id, domain.residentGb]), + [ + ['system:0', 30], + ['device:cuda:0', 18] + ] + ); + assert.equal(plan.uncovered[0].workload, 'b'); +}); + +test('local search trades a faster model for full coverage', () => { + const fast = { + id: 'fast', + gatewayModel: 'fast', + requirements: { memoryGbPerNode: 80 }, + performance: { durationMs: 100 } + }; + const slow = { + id: 'slow', + gatewayModel: 'slow', + requirements: { memoryGbPerNode: 30 }, + performance: { durationMs: 900 } + }; + const media = { id: 'media', gatewayModel: 'media', requirements: { memoryGbPerNode: 50 } }; + const plan = planPool({ + nodes: [spark('s')], + variants: [fast, slow, media], + workloads: [ + { id: 'chat', candidates: ['fast', 'slow'], residency: 'always', demandPerHour: 100 }, + { id: 'video', candidates: ['media'], residency: 'always', demandPerHour: 1 } + ], + policy: { maxMemoryUtilization: 0.95 } + }); + assert.equal(plan.ok, true); + assert.equal(plan.routes.chat.model, 'slow'); + assert.equal(plan.routes.video.model, 'media'); +}); + +test('external members are a fallback unless allowed or disabled', () => { + const cloud = { id: 'cloud', gatewayModel: 'cloud', external: true, performance: { ttftMs: 100, decodeTps: 500 } }; + const local = { ...tp2, id: 'local', gatewayModel: 'local', requirements: { memoryGbPerNode: 40 } }; + const workloads = [{ id: 'chat', candidates: ['cloud', 'local'], residency: 'always', demandPerHour: 10 }]; + const fallback = planPool({ nodes: [spark('s')], variants: [cloud, local], workloads }); + assert.equal(fallback.routes.chat.model, 'local'); + const allowed = planPool({ nodes: [spark('s')], variants: [cloud, local], workloads, policy: { external: 'allow' } }); + assert.equal(allowed.routes.chat.model, 'cloud'); + const cordoned = planPool({ + nodes: [spark('s')], + variants: [cloud, local], + workloads, + pins: { cordon: ['s'] } + }); + assert.equal(cordoned.routes.chat.model, 'cloud'); + assert.equal(cordoned.score.external, 1); + const never = planPool({ + nodes: [spark('s')], + variants: [cloud, local], + workloads, + pins: { cordon: ['s'] }, + policy: { external: 'never' } + }); + assert.equal(never.ok, false); +}); + +test('workload context and latency SLOs filter and rank variants', () => { + const short = { + ...tp2, + id: 'short', + gatewayModel: 'short', + requirements: { memoryGbPerNode: 20, contextWindow: 32768 } + }; + const long = { + ...tp2, + id: 'long', + gatewayModel: 'long', + requirements: { memoryGbPerNode: 40, contextWindow: 262144 } + }; + const plan = planPool({ + nodes: [spark('s')], + variants: [short, long], + workloads: [{ id: 'chat', candidates: ['short', 'long'], slo: { contextWindow: 131072 } }] + }); + assert.equal(plan.routes.chat.model, 'long'); + + const quick = { + ...tp2, + id: 'quick', + gatewayModel: 'quick', + requirements: { memoryGbPerNode: 20 }, + performance: { ttftMs: 200, decodeTps: 10 } + }; + const steady = { + ...tp2, + id: 'steady', + gatewayModel: 'steady', + requirements: { memoryGbPerNode: 20 }, + performance: { ttftMs: 1500, decodeTps: 200 } + }; + const slo = planPool({ + nodes: [spark('s')], + variants: [quick, steady], + workloads: [{ id: 'voice', candidates: ['steady', 'quick'], slo: { ttftMs: 500 } }] + }); + assert.equal(slo.routes.voice.model, 'quick'); + assert.equal(slo.score.sloMisses, 0); +}); + +test('missing weights and images add load cost to recipe placements', () => { + const recipe = { + ...tp2, + id: 'recipe:fresh', + gatewayModel: 'fresh', + requirements: { memoryGbPerNode: 40, recipeId: 'fresh-recipe', diskGb: 40, images: ['vllm@sha256:1'] } + }; + const cached = { ...spark('cached'), weights: ['fresh-recipe'], images: ['vllm@sha256:1'] }; + const plan = planPool({ + nodes: [spark('bare'), cached], + variants: [recipe], + workloads: [{ id: 'chat', candidates: ['fresh'], residency: 'always' }] + }); + assert.deepEqual(plan.routes.chat.nodes, ['cached']); + assert.deepEqual( + plan.diff.start.map((entry) => entry.placement), + ['recipe:fresh@cached'] + ); +}); + +test('planning is deterministic', () => { + const current = currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING); + assert.deepEqual(planFleet({ current }), planFleet({ current })); +}); + +test('models served by one runtime share a single placement', () => { + const image = { + id: 'image@s', + gatewayModel: 'image', + runtime: 'comfy', + nodes: ['s'], + requirements: { memoryGbPerNode: 60 } + }; + const edit = { + id: 'edit@s', + gatewayModel: 'edit', + runtime: 'comfy', + nodes: ['s'], + requirements: { memoryGbPerNode: 60 } + }; + const chat = { + id: 'chat@s', + gatewayModel: 'chat', + runtime: 'llm', + nodes: ['s'], + requirements: { memoryGbPerNode: 50 } + }; + const plan = planPool({ + nodes: [spark('s')], + variants: [image, edit, chat], + workloads: [ + { id: 'image-fast', candidates: ['image'], residency: 'always' }, + { id: 'image-edit', candidates: ['edit'], residency: 'always' }, + { id: 'chat', candidates: ['chat'], residency: 'always' } + ], + current: { running: ['image@s', 'edit@s'] }, + policy: { maxMemoryUtilization: 0.95 } + }); + assert.equal(plan.ok, true, JSON.stringify(plan.uncovered)); + assert.equal(plan.nodes.s.domains[0].residentGb, 110); + const comfy = plan.placements.find((placement) => placement.runtime === 'comfy'); + assert.deepEqual(comfy.models, ['edit', 'image']); + assert.deepEqual(comfy.workloads, ['image-edit', 'image-fast']); + assert.equal(plan.routes['image-edit'].placement, plan.routes['image-fast'].placement); + assert.deepEqual(plan.diff.release, []); + assert.deepEqual( + plan.diff.start.map((entry) => entry.placement), + ['chat@s'] + ); + assert.equal(plan.residency.comfy, 'always'); +}); From 8847cb48996069c00b111cc44995733e68280963 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 12:21:57 -0700 Subject: [PATCH 2/9] Pool head: reconciler, controller and a deterministic fleet simulator - pool-reconciler: orders a plan into make-before-break steps (prefetch, start, route, drain+stop), swaps through interim members, residency downgrades before stops; dependency-aware executor skips dependents of a failed step so a lane that never starts never strands its aliases. - pool-controller: off/advise/auto loop with urgency (coverage restored), stable-plan confirmation, cooldowns, an hourly load budget, slower hysteresis for residency-only changes, and failure backoff. - pool-sim: discrete-event fleet model (requests, admission + LRU eviction, keep-warm restore, memory guard aborts, node hangs/failures, operator actions) driving the real planner/reconciler/controller; scenarios for pair taken, pair crash, a Mac joining, a low memory estimate and noisy demand, compared against today's behaviour (off). - planner: budgets capped at the runtime memory guard ceiling (0.9 by default) so plans never rely on loads the guard would abort; variants can exclude nodes after failed starts. Co-Authored-By: Claude Opus 5.5 --- package.json | 4 +- src/pool-catalog.mjs | 5 + src/pool-controller.mjs | 221 ++++ src/pool-planner.mjs | 17 +- src/pool-reconciler.mjs | 406 ++++++++ src/pool-sim-cli.mjs | 150 +++ src/pool-sim.mjs | 950 ++++++++++++++++++ test/fixtures/pool/scenarios/demand-flap.json | 120 +++ test/fixtures/pool/scenarios/mac-joins.json | 86 ++ .../pool/scenarios/memory-estimate-low.json | 13 + test/fixtures/pool/scenarios/pair-crash.json | 30 + test/fixtures/pool/scenarios/pair-taken.json | 61 ++ .../pool/scenarios/spark-fleet-base.json | 535 ++++++++++ test/pool-controller.test.mjs | 197 ++++ test/pool-planner.test.mjs | 46 +- test/pool-reconciler.test.mjs | 260 +++++ test/pool-sim.test.mjs | 199 ++++ 17 files changed, 3291 insertions(+), 9 deletions(-) create mode 100644 src/pool-controller.mjs create mode 100644 src/pool-reconciler.mjs create mode 100644 src/pool-sim-cli.mjs create mode 100644 src/pool-sim.mjs create mode 100644 test/fixtures/pool/scenarios/demand-flap.json create mode 100644 test/fixtures/pool/scenarios/mac-joins.json create mode 100644 test/fixtures/pool/scenarios/memory-estimate-low.json create mode 100644 test/fixtures/pool/scenarios/pair-crash.json create mode 100644 test/fixtures/pool/scenarios/pair-taken.json create mode 100644 test/fixtures/pool/scenarios/spark-fleet-base.json create mode 100644 test/pool-controller.test.mjs create mode 100644 test/pool-reconciler.test.mjs create mode 100644 test/pool-sim.test.mjs diff --git a/package.json b/package.json index 329499b..e98d9b3 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "check": "npm run check:cluster && npm run check:js && npm run check:entity-stagger && npm run check:python && npm run check:shell && npm run check:spark-deploy && npm run check:workers", "check:cluster": "node --check src/cluster.mjs && node --check test/cluster.test.mjs && node --check test/cluster-availability.test.mjs && node --check test/cluster-readiness.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs", "check:entity-stagger": "node --check scripts/entity-stagger-benchmark.mjs && node --check test/entity-stagger-benchmark.test.mjs", - "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/request-scheduler.mjs && node --check src/rate-limit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/deployment-fence.mjs && node --check test/deployment-fence.test.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/standby.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check test/spark-audio-recipes.test.mjs && node --check src/runtime-residency.mjs && node --check test/ltx-pipelines.test.mjs && node --check test/ltx-pipelines-recipe.test.mjs && node --check src/video-contracts.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs && node --check src/pool-planner.mjs && node --check src/pool-catalog.mjs && node --check src/pool-inventory.mjs && node --check src/pool-cli.mjs && node --check test/pool-planner.test.mjs && node --check test/pool-catalog.test.mjs && node --check test/pool-inventory.test.mjs && node --check test/pool-cli.test.mjs && node --check src/runtime-policy-config.mjs && node --check test/runtime-policy-config.test.mjs && node --check src/audio-providers.mjs && node --check src/protocol/openrouter-provider.mjs && node --check test/audio-providers.test.mjs && node --check test/openrouter-provider.test.mjs && node --check backends/atlas-sparkglm/verify-pins.mjs && node --check test/atlas-sparkglm-recipe.test.mjs && node --check src/head-preparation.mjs && node --check src/head-transfer.mjs && node --check src/service-control.mjs && node --check src/gateway-relay.mjs && node --check src/client-retarget.mjs && node --check src/head-promotion.mjs && node --check src/head-promotion-transfer.mjs && node --check src/gateway-client.mjs && node --check test/cli-diagnostics.test.mjs && node --check src/service-install.mjs && node --check test/service-install.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check src/node-release-agent.mjs && node --check src/node-gateway-adapter.mjs && node --check src/node-agent-cli.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs && node --check test/node-release-agent.test.mjs && node --check test/node-gateway-adapter.test.mjs && node --check test/node-gateway-adapter-e2e.test.mjs && node --check test/deployment-bundle.test.mjs && node --check test/node-agent-cli.test.mjs && node --check test/ssh-deployment-transport.test.mjs && node --check scripts/build-deployment-bundle.mjs", + "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/request-scheduler.mjs && node --check src/rate-limit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/deployment-fence.mjs && node --check test/deployment-fence.test.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/standby.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check test/spark-audio-recipes.test.mjs && node --check src/runtime-residency.mjs && node --check test/ltx-pipelines.test.mjs && node --check test/ltx-pipelines-recipe.test.mjs && node --check src/video-contracts.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs && node --check src/pool-planner.mjs && node --check src/pool-catalog.mjs && node --check src/pool-inventory.mjs && node --check src/pool-cli.mjs && node --check test/pool-planner.test.mjs && node --check test/pool-catalog.test.mjs && node --check test/pool-inventory.test.mjs && node --check test/pool-cli.test.mjs && node --check src/pool-reconciler.mjs && node --check src/pool-controller.mjs && node --check src/pool-sim.mjs && node --check src/pool-sim-cli.mjs && node --check test/pool-reconciler.test.mjs && node --check test/pool-controller.test.mjs && node --check test/pool-sim.test.mjs && node --check src/runtime-policy-config.mjs && node --check test/runtime-policy-config.test.mjs && node --check src/audio-providers.mjs && node --check src/protocol/openrouter-provider.mjs && node --check test/audio-providers.test.mjs && node --check test/openrouter-provider.test.mjs && node --check backends/atlas-sparkglm/verify-pins.mjs && node --check test/atlas-sparkglm-recipe.test.mjs && node --check src/head-preparation.mjs && node --check src/head-transfer.mjs && node --check src/service-control.mjs && node --check src/gateway-relay.mjs && node --check src/client-retarget.mjs && node --check src/head-promotion.mjs && node --check src/head-promotion-transfer.mjs && node --check src/gateway-client.mjs && node --check test/cli-diagnostics.test.mjs && node --check src/service-install.mjs && node --check test/service-install.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check src/node-release-agent.mjs && node --check src/node-gateway-adapter.mjs && node --check src/node-agent-cli.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs && node --check test/node-release-agent.test.mjs && node --check test/node-gateway-adapter.test.mjs && node --check test/node-gateway-adapter-e2e.test.mjs && node --check test/deployment-bundle.test.mjs && node --check test/node-agent-cli.test.mjs && node --check test/ssh-deployment-transport.test.mjs && node --check scripts/build-deployment-bundle.mjs", "check:python": "python3 -m py_compile backends/dspark-vllm/apply-patch-pack.py backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/vision_exp/*.py backends/qwen38-vllm/apply-ple-fp8-patch.py backends/qwen38-sglang/*.py backends/glm53-exl3/*.py backends/mlx-audio/lloom_audio_server.py backends/hear/lloom_hear_server.py backends/chatterbox/lloom_chatterbox_server.py patches/apply_mtplx_longctx_fix.py backends/comfyui-media/bridge/*.py backends/comfyui-media/graphs/*.py backends/comfyui-media/build/*.py backends/comfyui-media/install.py backends/qwen-image-diffusers/*.py backends/ace-step-diffusers/*.py backends/ace-step-diffusers/tests/*.py backends/hear/test/test_hear_security.py backends/spark-audio/install.py backends/spark-audio/lloom_audio_cuda_server.py backends/spark-audio/test_lloom_audio_cuda_server.py backends/ltx-pipelines/*.py src/darwin-memory-usage.py", "check:shell": "bash -n backends/dspark-vllm/entrypoint.sh backends/dspark-vllm/runtime/build-hardened-image.sh backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/apply-runtime.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.sh backends/qwen38-vllm/entrypoint.sh backends/qwen38-sglang/entrypoint.sh backends/glm53-exl3/entrypoint.sh backends/mlx-audio/install.sh backends/hear/install.sh backends/chatterbox/install.sh clients/examples/lloom-claude clients/examples/lloom-codex clients/examples/lloom-hermes clients/examples/lloom-opencode clients/examples/lloom-zero scripts/remote-install-spark.sh scripts/remote-stage-spark-worker.sh backends/atlas-sparkglm/install.sh backends/atlas-sparkglm/convert-overlay.sh", "check:spark-deploy": "node --check test/spark-deploy.test.mjs && node test/spark-deploy.test.mjs", @@ -73,7 +73,7 @@ "community:check": "node scripts/check-community-deploy.mjs", "package:check": "node scripts/check-package.mjs", "test:adoption": "node test/resource-fit.test.mjs && node test/model-acquisition.test.mjs && node test/dashboard-status.test.mjs", - "test:unit": "node --test test/deployment-fence.test.mjs test/cluster-deployment.test.mjs test/cluster-deployment-cli.test.mjs test/cluster-deployment-e2e.test.mjs test/node-release-agent.test.mjs test/node-gateway-adapter.test.mjs test/node-gateway-adapter-e2e.test.mjs test/deployment-bundle.test.mjs test/node-agent-cli.test.mjs test/ssh-deployment-transport.test.mjs && node test/runtime-health-ownership.test.mjs && node test/long-prefill-transport.test.mjs && node test/sparkglm-managed.test.mjs && node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/stream-metrics.test.mjs && node test/responses-codex.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/config-profiles.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-policy-config.test.mjs && node --test test/pool-planner.test.mjs test/pool-catalog.test.mjs test/pool-inventory.test.mjs test/pool-cli.test.mjs && node --test test/request-scheduler.test.mjs test/adaptive-priority.test.mjs test/rate-limit.test.mjs test/runtime-queue.test.mjs test/foreground-priority.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node --test test/standby.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && node test/hear-recipe.test.mjs && npm run test:adoption && node --test test/performance-sampler.test.mjs && node --test test/audio-generations.test.mjs && node test/video-providers.test.mjs && node --test test/web-functions.test.mjs && node --test test/config-mutation.test.mjs && node --test test/model-maintenance.test.mjs && node --test test/model-maintenance-control.test.mjs && node --test test/route-maintenance-concurrency.test.mjs && node --test test/model-maintenance-integration.test.mjs && npm run test:workers && node test/comfyui-media.test.mjs && node test/qwen-image-diffusers.test.mjs && node test/ace-step-diffusers.test.mjs && node test/runtime-residency.test.mjs && node test/spark-audio-recipes.test.mjs && node --test test/video-contracts.test.mjs && node test/ltx-pipelines-recipe.test.mjs && npm run test:ux && npm run test:memory-safety && node --test test/dashboard-memory.test.mjs test/runtime-memory-usage.test.mjs && node --test test/speech-stream.test.mjs && node --test test/audio-providers.test.mjs && node test/openrouter-provider.test.mjs && node test/atlas-sparkglm-recipe.test.mjs && node --test test/head-preparation.test.mjs test/head-transfer.test.mjs test/service-control.test.mjs && node --test test/gateway-relay.test.mjs test/client-retarget.test.mjs && node --test test/head-profile-acceptance.test.mjs test/head-promotion.test.mjs test/head-promotion-transfer.test.mjs test/head-transfer-acceptance.test.mjs && node --test test/cli-diagnostics.test.mjs && node --test test/service-install.test.mjs", + "test:unit": "node --test test/deployment-fence.test.mjs test/cluster-deployment.test.mjs test/cluster-deployment-cli.test.mjs test/cluster-deployment-e2e.test.mjs test/node-release-agent.test.mjs test/node-gateway-adapter.test.mjs test/node-gateway-adapter-e2e.test.mjs test/deployment-bundle.test.mjs test/node-agent-cli.test.mjs test/ssh-deployment-transport.test.mjs && node test/runtime-health-ownership.test.mjs && node test/long-prefill-transport.test.mjs && node test/sparkglm-managed.test.mjs && node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/stream-metrics.test.mjs && node test/responses-codex.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/config-profiles.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-policy-config.test.mjs && node --test test/pool-planner.test.mjs test/pool-catalog.test.mjs test/pool-inventory.test.mjs test/pool-cli.test.mjs test/pool-reconciler.test.mjs test/pool-controller.test.mjs test/pool-sim.test.mjs && node --test test/request-scheduler.test.mjs test/adaptive-priority.test.mjs test/rate-limit.test.mjs test/runtime-queue.test.mjs test/foreground-priority.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node --test test/standby.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && node test/hear-recipe.test.mjs && npm run test:adoption && node --test test/performance-sampler.test.mjs && node --test test/audio-generations.test.mjs && node test/video-providers.test.mjs && node --test test/web-functions.test.mjs && node --test test/config-mutation.test.mjs && node --test test/model-maintenance.test.mjs && node --test test/model-maintenance-control.test.mjs && node --test test/route-maintenance-concurrency.test.mjs && node --test test/model-maintenance-integration.test.mjs && npm run test:workers && node test/comfyui-media.test.mjs && node test/qwen-image-diffusers.test.mjs && node test/ace-step-diffusers.test.mjs && node test/runtime-residency.test.mjs && node test/spark-audio-recipes.test.mjs && node --test test/video-contracts.test.mjs && node test/ltx-pipelines-recipe.test.mjs && npm run test:ux && npm run test:memory-safety && node --test test/dashboard-memory.test.mjs test/runtime-memory-usage.test.mjs && node --test test/speech-stream.test.mjs && node --test test/audio-providers.test.mjs && node test/openrouter-provider.test.mjs && node test/atlas-sparkglm-recipe.test.mjs && node --test test/head-preparation.test.mjs test/head-transfer.test.mjs test/service-control.test.mjs && node --test test/gateway-relay.test.mjs test/client-retarget.test.mjs && node --test test/head-profile-acceptance.test.mjs test/head-promotion.test.mjs test/head-promotion-transfer.test.mjs test/head-transfer-acceptance.test.mjs && node --test test/cli-diagnostics.test.mjs && node --test test/service-install.test.mjs", "test:cluster": "node test/cluster.test.mjs && node --test test/cluster-availability.test.mjs test/cluster-readiness.test.mjs", "test": "npm run test:cluster && npm run test:unit && npm run test:entity-stagger && npm run smoke && npm run community:check", "test:entity-stagger": "node test/entity-stagger-benchmark.test.mjs", diff --git a/src/pool-catalog.mjs b/src/pool-catalog.mjs index 291df8d..bcaf598 100644 --- a/src/pool-catalog.mjs +++ b/src/pool-catalog.mjs @@ -530,6 +530,11 @@ export function buildPoolPlanInputs({ policy: compact({ maxMemoryUtilization: num(config.runtimePolicy?.maxMemoryUtilization), reserveGb: num(config.runtimePolicy?.reserveMemoryGb), + // "yolo" turns the load guard off, so it no longer caps the budget. + memorySafetyUtilization: + config.runtimePolicy?.memorySafety?.mode === 'yolo' + ? 1 + : num(config.runtimePolicy?.memorySafety?.maxMemoryUtilization), ...object(config.pool?.policy), ...policy }), diff --git a/src/pool-controller.mjs b/src/pool-controller.mjs new file mode 100644 index 0000000..83e0151 --- /dev/null +++ b/src/pool-controller.mjs @@ -0,0 +1,221 @@ +// Pool controller: the loop that plans the pool, decides whether a plan is +// worth acting on, and (in auto mode) reconciles the pool to it. +// +// off never plans +// advise plans and records proposals; a person applies them +// auto applies proposals through the actuator +// +// Moving a model is expensive (minutes of load time), so a non-urgent change +// must be stable across `confirmTicks` consecutive plans, must clear a minimum +// gain, waits out a cooldown after the last change, and stays inside an +// hourly load budget. A plan that restores coverage (a node went away, a lane +// died) is urgent and skips those guards. A lane that fails to start is +// excluded from planning for `failureBackoffMs`. + +import { planPool } from './pool-planner.mjs'; +import { executeReconcile, planReconcile } from './pool-reconciler.mjs'; + +export const POOL_CONTROLLER_MODES = Object.freeze(['off', 'advise', 'auto']); + +export const DEFAULT_CONTROLLER_POLICY = Object.freeze({ + mode: 'advise', + confirmTicks: 2, + cooldownMs: 15 * 60 * 1000, + // Residency-only changes follow noisy demand; they need a longer, steadier signal. + maintenanceConfirmTicks: 10, + maintenanceCooldownMs: 60 * 60 * 1000, + minGain: 0.1, + maxLoadGbPerHour: 400, + failureBackoffMs: 30 * 60 * 1000, + concurrency: 4, + historyLimit: 50 +}); + +function num(value) { + if (value == null || value === '') return null; + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +export function normalizeControllerPolicy(policy = {}) { + const merged = { ...DEFAULT_CONTROLLER_POLICY }; + for (const [key, value] of Object.entries(policy ?? {})) { + if (value === undefined || !(key in DEFAULT_CONTROLLER_POLICY)) continue; + if (key === 'mode') merged.mode = POOL_CONTROLLER_MODES.includes(value) ? value : DEFAULT_CONTROLLER_POLICY.mode; + else merged[key] = num(value) ?? merged[key]; + } + return merged; +} + +// What the plan changes, independent of scores: two plans with the same +// signature would make the same moves. +export function planSignature(plan) { + const routes = Object.entries(plan.routes ?? {}) + .map(([alias, route]) => `${alias}=${route.model}`) + .sort(); + const placements = (plan.placements ?? []) + .filter((placement) => !placement.external && placement.residency !== 'auto') + .map((placement) => `${placement.placement}:${placement.residency}`) + .sort(); + return JSON.stringify([routes, placements]); +} + +function excludeFailures(variants, failures, now) { + return variants.map((variant) => { + const excluded = [...failures.entries()] + .filter(([, entry]) => entry.until > now && entry.variant === variant.id) + .flatMap(([, entry]) => entry.nodes); + if (!excluded.length) return variant; + if ((variant.nodes ?? []).length) return { ...variant, excludeNodes: variant.nodes }; + return { ...variant, excludeNodes: [...new Set([...(variant.excludeNodes ?? []), ...excluded])] }; + }); +} + +// The score of keeping today's routes: the plan with every viable current +// route pinned. If keeping them leaves a required workload uncovered, the new +// plan is urgent. +function stayPlan(inputs) { + const routes = { ...(inputs.current?.routes ?? {}) }; + const viable = Object.fromEntries( + Object.entries(routes).filter(([alias]) => (inputs.workloads ?? []).some((workload) => workload.id === alias)) + ); + return planPool({ ...inputs, pins: { ...inputs.pins, routes: { ...viable, ...(inputs.pins?.routes ?? {}) } } }); +} + +export function decide({ plan, stay, reconcile, state, policy, now }) { + if (reconcile.empty) return { act: false, reason: 'no changes' }; + const restoresCoverage = + plan.score.uncovered < stay.score.uncovered || + (stay.pinViolations?.length ?? 0) > (plan.pinViolations?.length ?? 0); + if (restoresCoverage) return { act: true, urgent: true, reason: 'restores coverage' }; + if (plan.score.uncovered > stay.score.uncovered) return { act: false, reason: 'plan covers less than today' }; + const quality = plan.score.sloMisses < stay.score.sloMisses || plan.score.external < stay.score.external; + const gain = stay.score.cost > 0 ? (stay.score.cost - plan.score.cost) / stay.score.cost : 0; + // Without route moves the plan only follows demand (residency, warm lanes, + // releases); a longer stable signal and cooldown are its hysteresis, not a cost gain. + const maintenance = + !quality && !reconcile.steps.some((step) => step.type === 'route' || step.type === 'interim-route'); + const confirmTicks = maintenance ? policy.maintenanceConfirmTicks : policy.confirmTicks; + if (state.stableTicks < confirmTicks) { + return { act: false, reason: `waiting for a stable plan (${state.stableTicks}/${confirmTicks})` }; + } + if (!quality && !maintenance && gain < policy.minGain) { + return { act: false, reason: `gain ${(gain * 100).toFixed(1)}% below ${(policy.minGain * 100).toFixed(0)}%` }; + } + const cooldownMs = maintenance ? policy.maintenanceCooldownMs : policy.cooldownMs; + if (state.lastActionAt != null && now - state.lastActionAt < cooldownMs) { + return { act: false, reason: 'cooldown after the last change' }; + } + const loadGb = reconcile.steps + .filter((step) => step.type === 'start') + .reduce((sum, step) => sum + (num(step.memoryGbPerNode) ?? 0) * (step.nodes?.length ?? 1), 0); + const recentGb = state.loads.filter((entry) => now - entry.at < 3600000).reduce((sum, entry) => sum + entry.gb, 0); + if (recentGb + loadGb > policy.maxLoadGbPerHour) { + return { + act: false, + reason: `hourly load budget (${Math.round(recentGb + loadGb)} GB > ${policy.maxLoadGbPerHour} GB)` + }; + } + const reason = quality + ? 'improves SLOs or local coverage' + : maintenance + ? 'residency follows demand' + : `gain ${(gain * 100).toFixed(1)}%`; + return { act: true, urgent: false, reason }; +} + +/** + * observe(): Promise<{ nodes, variants, workloads, pins, policy, current }> + * current = { routes, residency, running: [placement keys], + * placements: [{ key, models, runtime, nodes, memoryGbPerNode, managed, state }], + * missingWeights: { placementKey: true } } + * actuator: see executeReconcile(). + */ +export function createPoolController({ observe, actuator, policy = {}, now = Date.now, onEvent = () => {} } = {}) { + const controllerPolicy = normalizeControllerPolicy(policy); + const state = { + signature: null, + stableTicks: 0, + lastActionAt: null, + loads: [], + failures: new Map(), + executing: null, + history: [] + }; + const record = (entry) => { + state.history.push(entry); + if (state.history.length > controllerPolicy.historyLimit) state.history.shift(); + onEvent(entry); + return entry; + }; + + async function tick() { + const at = now(); + if (controllerPolicy.mode === 'off') return record({ at, action: 'off' }); + if (state.executing) return record({ at, action: 'busy' }); + const observed = await observe(); + const inputs = { ...observed, variants: excludeFailures(observed.variants ?? [], state.failures, at) }; + const plan = planPool(inputs); + const signature = planSignature(plan); + state.stableTicks = signature === state.signature ? state.stableTicks + 1 : 1; + state.signature = signature; + const stay = stayPlan(inputs); + const reconcile = planReconcile({ + plan, + current: inputs.current, + workloads: inputs.workloads, + variants: inputs.variants, + nodes: inputs.nodes + }); + const decision = decide({ plan, stay, reconcile, state, policy: controllerPolicy, now: at }); + const entry = { + at, + action: decision.act ? (controllerPolicy.mode === 'auto' ? 'apply' : 'propose') : 'hold', + reason: decision.reason, + urgent: Boolean(decision.urgent), + score: plan.score, + stayScore: stay.score, + steps: reconcile.steps, + interruptions: reconcile.interruptions, + blocked: reconcile.blocked, + plan + }; + if (!decision.act || controllerPolicy.mode !== 'auto') return record(entry); + record(entry); + state.lastActionAt = at; + state.stableTicks = 0; + for (const step of reconcile.steps.filter((candidate) => candidate.type === 'start')) { + state.loads.push({ at, gb: (num(step.memoryGbPerNode) ?? 0) * (step.nodes?.length ?? 1) }); + } + state.loads = state.loads.filter((load) => at - load.at < 3600000); + state.executing = executeReconcile(reconcile.steps, actuator, { + concurrency: controllerPolicy.concurrency, + onStep: ({ step, status, error }) => { + onEvent({ at: now(), action: 'step', step: step.id, status, ...(error ? { error } : {}) }); + if (status === 'failed' && (step.type === 'start' || step.type === 'manual-start')) { + const placement = plan.placements.find((entry) => entry.placement === step.placement); + state.failures.set(step.placement, { + variant: placement?.variant ?? step.placement, + nodes: step.nodes ?? [], + until: now() + controllerPolicy.failureBackoffMs + }); + } + } + }).then((result) => { + state.executing = null; + record({ at: now(), action: 'applied', ok: result.ok, errors: result.errors }); + return result; + }); + return entry; + } + + return { + policy: controllerPolicy, + tick, + get executing() { + return state.executing; + }, + history: () => [...state.history], + failures: () => Object.fromEntries(state.failures) + }; +} diff --git a/src/pool-planner.mjs b/src/pool-planner.mjs index bb83b3e..c653972 100644 --- a/src/pool-planner.mjs +++ b/src/pool-planner.mjs @@ -18,6 +18,7 @@ export const POOL_PLAN_VERSION = 1; export const DEFAULT_POOL_POLICY = Object.freeze({ maxMemoryUtilization: null, + memorySafetyUtilization: 0.9, reserveGb: 8, headroomGb: 4, horizonHours: 24, @@ -77,16 +78,29 @@ export function normalizePoolPolicy(policy = {}) { return merged; } +// The runtime memory safety guard aborts a load once host memory use reaches +// min(memorySafety.maxMemoryUtilization, runtimePolicy.maxMemoryUtilization), +// 0.9 by default (runtime-memory-safety.mjs). Admission may allow more, so a +// plan that only fits under the admission budget would be aborted mid-load. +function memorySafetyCeiling(node, policy) { + const safety = num(policy.memorySafetyUtilization); + if (safety == null) return null; + const utilization = num(node.maxMemoryUtilization) ?? num(policy.maxMemoryUtilization) ?? safety; + return Math.min(safety, utilization); +} + function domainBudgets(node, policy) { const headroomGb = num(node.headroomGb) ?? num(policy.headroomGb) ?? 0; + const ceiling = memorySafetyCeiling(node, policy); return list(node.memoryDomains).map((domain) => { const capacityGb = num(domain.capacityGb) ?? 0; - const budgetGb = memoryBudgetGb({ + let budgetGb = memoryBudgetGb({ totalMemoryGb: capacityGb, memoryBudgetGb: domain.budgetGb, maxMemoryUtilization: num(node.maxMemoryUtilization) ?? num(policy.maxMemoryUtilization), reserveMemoryGb: num(node.reserveGb) ?? num(policy.reserveGb) ?? 0 }); + if (ceiling != null && domain.kind !== 'device') budgetGb = Math.min(budgetGb, capacityGb * ceiling); return { id: String(domain.id ?? domain.kind ?? 'memory'), kind: String(domain.kind ?? 'system'), @@ -130,6 +144,7 @@ function variantMemoryGb(variant) { function nodeIneligibility(variant, node) { if (node.cordoned) return 'cordoned'; if (node.available === false) return 'unavailable'; + if (list(variant.excludeNodes).includes(node.id)) return 'excluded after a failed start'; const requirements = variant.requirements ?? {}; const platforms = list(requirements.platforms); if (platforms.length && !platforms.includes(node.platformId)) return `needs platform ${platforms.join('|')}`; diff --git a/src/pool-reconciler.mjs b/src/pool-reconciler.mjs new file mode 100644 index 0000000..147b145 --- /dev/null +++ b/src/pool-reconciler.mjs @@ -0,0 +1,406 @@ +// Pool reconciler. Pure: turns a pool plan (pool-planner.mjs) and the observed +// state into ordered steps that move the pool to the plan without dropping a +// route: +// +// prefetch weights -> start new lanes -> flip routes -> drain + stop old lanes +// +// A start that only fits once an old lane is gone is a swap: the old lane's +// aliases move to an interim member that is already serving (or are reported +// as interrupted), the old lane drains and stops, the new lane starts, and the +// aliases flip to it. Residency downgrades come before stops, so a keep-warm +// tier never restarts a lane that is being retired; upgrades come after starts. +// +// executeReconcile() runs the steps against an actuator. A step whose +// dependency failed is skipped, so a lane that fails to start never strands +// its aliases: the old lane keeps serving because its stop depended on the +// route flip, which depended on the start. + +export const POOL_RECONCILE_VERSION = 1; + +const RESIDENCY_RANK = { auto: 0, preferred: 1, always: 2 }; + +function list(value) { + return Array.isArray(value) ? value.filter((item) => item != null && item !== '') : []; +} + +function num(value) { + if (value == null || value === '') return null; + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +// Weights live in unified memory, else the largest device domain; this mirrors +// the planner's accelerator domain and is the one that decides whether lanes fit. +function nodeBudgetGb(planNode) { + const domains = list(planNode?.domains); + const domain = + domains.find((entry) => entry.kind === 'unified') ?? + [...domains].filter((entry) => entry.kind === 'device').sort((a, b) => b.budgetGb - a.budgetGb)[0] ?? + domains.find((entry) => entry.kind === 'system'); + return num(domain?.budgetGb) ?? Infinity; +} + +function routeChanges(plan, current) { + return Object.entries(plan.routes ?? {}) + .filter(([alias, route]) => (current.routes?.[alias] ?? null) !== route.model) + .map(([alias, route]) => ({ alias, from: current.routes?.[alias] ?? null, to: route.model })); +} + +/** + * Order the work that moves `current` to `plan`. + * + * current = { + * placements: [{ key, models: [gatewayModel], runtime, nodes, memoryGbPerNode, managed, state }], + * routes: { alias: model }, residency: { runtime: tier }, missingWeights?: { placementKey: true } + * } + * workloads: planner workloads (for interim members during a swap). + * variants, nodes: planner inputs; a start on a node whose inventory lacks the + * variant's recipe weights gets a prefetch step first. + */ +export function planReconcile({ plan, current = {}, workloads = [], variants = [], nodes = [] } = {}) { + const steps = []; + const interruptions = []; + const blocked = []; + const ids = new Map(); + const add = (step, after = []) => { + const id = `${step.type}:${step.placement ?? step.alias ?? step.runtime}`; + const unique = ids.has(id) ? `${id}#${ids.get(id) + 1}` : id; + ids.set(id, (ids.get(id) ?? 0) + 1); + steps.push({ id: unique, ...step, after: [...new Set(after.filter(Boolean))] }); + return unique; + }; + + const running = new Map( + list(current.placements) + .filter((placement) => placement.state !== 'stopping') + .map((placement) => [placement.key, { ...placement, models: list(placement.models) }]) + ); + const planned = list(plan.placements).filter((placement) => !placement.external); + const plannedKeys = new Set(planned.map((placement) => placement.placement)); + const plannedByModel = new Map(); + for (const placement of planned) { + for (const model of placement.models) if (!plannedByModel.has(model)) plannedByModel.set(model, placement); + } + const externalModels = new Set([ + ...list(plan.placements) + .filter((placement) => placement.external) + .flatMap((placement) => placement.models), + ...list(variants) + .filter((variant) => variant.external) + .map((variant) => variant.gatewayModel) + ]); + const budgets = new Map(Object.entries(plan.nodes ?? {}).map(([id, node]) => [id, nodeBudgetGb(node)])); + const usage = new Map([...budgets.keys()].map((id) => [id, 0])); + for (const placement of running.values()) { + for (const node of placement.nodes) usage.set(node, (usage.get(node) ?? 0) + (num(placement.memoryGbPerNode) ?? 0)); + } + + // Where each model is (or will be) ready, and which step makes it ready. + const readyStep = new Map(); + for (const placement of running.values()) { + if (placement.state === 'ready' || placement.state == null) { + for (const model of placement.models) readyStep.set(model, null); + } + } + const releaseStep = new Map(); + const routes = { ...(current.routes ?? {}) }; + const routeStep = new Map(); + + // Residency downgrades first: a lane leaving keep-warm must not be restarted + // by the residency reconciler while it drains. + const residencyDown = []; + const residencyUp = []; + for (const [runtime, tier] of Object.entries(plan.residency ?? {})) { + const from = current.residency?.[runtime]; + if (from == null || from === tier) continue; + (RESIDENCY_RANK[tier] < RESIDENCY_RANK[from] ? residencyDown : residencyUp).push({ runtime, from, to: tier }); + } + const downgradeSteps = residencyDown + .sort((a, b) => a.runtime.localeCompare(b.runtime)) + .map((change) => add({ type: 'residency', ...change })); + + const pendingStarts = planned + .filter((placement) => placement.residency !== 'auto' && !running.has(placement.placement)) + .sort( + (a, b) => RESIDENCY_RANK[b.residency] - RESIDENCY_RANK[a.residency] || a.placement.localeCompare(b.placement) + ); + let pendingRoutes = routeChanges(plan, current).sort((a, b) => a.alias.localeCompare(b.alias)); + const startStep = new Map(); + const prefetchStep = new Map(); + const nodeWeights = new Map(list(nodes).map((node) => [node.id, new Set(list(node.weights))])); + const missingWeights = (placement) => { + if (current.missingWeights?.[placement.placement]) return true; + const recipeId = list(variants).find((variant) => variant.id === placement.variant)?.requirements?.recipeId; + return ( + Boolean(recipeId) && placement.nodes.some((node) => nodeWeights.has(node) && !nodeWeights.get(node).has(recipeId)) + ); + }; + for (const placement of pendingStarts) { + if (missingWeights(placement)) { + prefetchStep.set( + placement.placement, + add({ type: 'prefetch', placement: placement.placement, nodes: placement.nodes }) + ); + } + } + + const aliasesOn = (placement) => + Object.entries(routes) + .filter(([, model]) => placement.models.includes(model)) + .map(([alias]) => alias); + const fits = (placement, freed = new Map()) => + placement.nodes.every( + (node) => + (usage.get(node) ?? 0) - (freed.get(node) ?? 0) + (num(placement.memoryGbPerNode) ?? 0) <= + (budgets.get(node) ?? Infinity) + 1e-9 + ); + const startAfter = (placement) => [ + prefetchStep.get(placement.placement), + ...downgradeSteps, + ...placement.nodes.flatMap((node) => + [...releaseStep.values()].filter((entry) => entry.nodes.includes(node)).map((entry) => entry.id) + ) + ]; + + const release = (placement, after = []) => { + const id = add( + { + type: placement.managed === false ? 'manual-stop' : 'stop', + placement: placement.key, + nodes: placement.nodes, + drain: true + }, + [...after, ...downgradeSteps] + ); + releaseStep.set(placement.key, { id, nodes: placement.nodes }); + for (const node of placement.nodes) usage.set(node, (usage.get(node) ?? 0) - (num(placement.memoryGbPerNode) ?? 0)); + for (const model of placement.models) { + if (readyStep.get(model) === null) readyStep.delete(model); + } + running.delete(placement.key); + return id; + }; + + const routeTo = (alias, to, type = 'route') => { + const target = plannedByModel.get(to); + const after = [readyStep.get(to), ...downgradeSteps]; + const id = add( + { type, alias, from: routes[alias] ?? null, to, ...(target ? { placement: target.placement } : {}) }, + after + ); + routes[alias] = to; + routeStep.set(alias, id); + return id; + }; + + for (let guard = 0; guard < 1000; guard += 1) { + let progress = false; + + // Flip routes whose target is serving, will be served on demand, or is external. + pendingRoutes = pendingRoutes.filter(({ alias, to }) => { + const target = plannedByModel.get(to); + const onDemand = target && target.residency === 'auto' && !running.has(target.placement); + if (!readyStep.has(to) && !externalModels.has(to) && !onDemand) return true; + routeTo(alias, to); + progress = true; + return false; + }); + + // Retire lanes that are not planned and no longer routed to. + for (const placement of [...running.values()].sort((a, b) => a.key.localeCompare(b.key))) { + if (plannedKeys.has(placement.key) || aliasesOn(placement).length) continue; + const after = Object.entries(routes) + .filter(([alias]) => routeStep.has(alias)) + .filter(([alias]) => placement.models.includes(current.routes?.[alias])) + .map(([alias]) => routeStep.get(alias)); + release(placement, after); + progress = true; + } + + // Start lanes that fit next to everything still running (make before break). + for (const placement of [...pendingStarts]) { + if (startStep.has(placement.placement) || !fits(placement)) continue; + const id = add( + { + type: placement.managed === false ? 'manual-start' : 'start', + placement: placement.placement, + nodes: placement.nodes, + runtime: placement.runtime, + residency: placement.residency, + memoryGbPerNode: placement.memoryGbPerNode + }, + startAfter(placement) + ); + startStep.set(placement.placement, id); + for (const node of placement.nodes) + usage.set(node, (usage.get(node) ?? 0) + (num(placement.memoryGbPerNode) ?? 0)); + for (const model of placement.models) readyStep.set(model, id); + running.set(placement.placement, { key: placement.placement, ...placement, state: 'starting' }); + pendingStarts.splice(pendingStarts.indexOf(placement), 1); + progress = true; + } + if (progress) continue; + if (!pendingStarts.length && !pendingRoutes.length) break; + + // Nothing moves: free memory for the first blocked start. On-demand lanes + // go first (they simply cold start later), then unplanned lanes, whose + // aliases take an interim member while the swap happens. + const blockedStart = pendingStarts[0]; + if (!blockedStart) { + for (const change of pendingRoutes) + blocked.push({ alias: change.alias, to: change.to, reason: 'target never starts' }); + break; + } + const victims = [...running.values()] + .filter( + (placement) => + placement.state !== 'starting' && placement.nodes.some((node) => blockedStart.nodes.includes(node)) + ) + .filter((placement) => { + const plannedPlacement = planned.find((entry) => entry.placement === placement.key); + return !plannedPlacement || plannedPlacement.residency === 'auto'; + }) + .sort((a, b) => { + const aPlanned = plannedKeys.has(a.key) ? 0 : 1; + const bPlanned = plannedKeys.has(b.key) ? 0 : 1; + return ( + aPlanned - bPlanned || + (num(b.memoryGbPerNode) ?? 0) - (num(a.memoryGbPerNode) ?? 0) || + a.key.localeCompare(b.key) + ); + }); + const freed = new Map(); + const chosen = []; + for (const victim of victims) { + chosen.push(victim); + for (const node of victim.nodes) freed.set(node, (freed.get(node) ?? 0) + (num(victim.memoryGbPerNode) ?? 0)); + if (fits(blockedStart, freed)) break; + } + if (!chosen.length || !fits(blockedStart, freed)) { + blocked.push({ + placement: blockedStart.placement, + reason: 'does not fit even after releasing every movable lane' + }); + pendingStarts.shift(); + continue; + } + for (const victim of chosen) { + const interimAfter = []; + for (const alias of aliasesOn(victim)) { + const workload = workloads.find((entry) => entry.id === alias); + const interim = list(workload?.candidates).find( + (model) => + !victim.models.includes(model) && + (externalModels.has(model) || + [...running.values()].some( + (placement) => + placement.state !== 'starting' && + placement.key !== victim.key && + !chosen.includes(placement) && + placement.models.includes(model) + )) + ); + if (interim) interimAfter.push(routeTo(alias, interim, 'interim-route')); + else interruptions.push({ alias, placement: victim.key, until: blockedStart.placement }); + } + release(victim, interimAfter); + } + } + + for (const change of residencyUp.sort((a, b) => a.runtime.localeCompare(b.runtime))) { + const placement = planned.find((entry) => entry.runtime === change.runtime); + add({ type: 'residency', ...change }, [placement && startStep.get(placement.placement)]); + } + + return { + version: POOL_RECONCILE_VERSION, + steps, + interruptions, + blocked, + manual: steps.filter((step) => step.type.startsWith('manual-')), + empty: steps.length === 0 + }; +} + +/** + * Run reconcile steps against an actuator, honouring dependencies. + * + * actuator: { prefetch(step), start(step), stop(step), route(step), residency(step), manual(step) } + * Every method returns a promise that settles when the step is done (a start + * when the lane is ready, a stop when it has drained). Manual steps resolve + * when the actuator observes the operator's change or rejects on timeout. + */ +export async function executeReconcile(steps, actuator, { concurrency = 4, onStep = () => {} } = {}) { + const status = new Map(steps.map((step) => [step.id, 'pending'])); + const errors = {}; + const byId = new Map(steps.map((step) => [step.id, step])); + const running = new Set(); + const method = (type) => + ({ + prefetch: 'prefetch', + start: 'start', + stop: 'stop', + route: 'route', + 'interim-route': 'route', + residency: 'residency', + 'manual-start': 'manual', + 'manual-stop': 'manual' + })[type]; + + return new Promise((resolve) => { + const finish = () => + resolve({ + ok: [...status.values()].every((value) => value === 'done'), + status: Object.fromEntries(status), + errors + }); + const pump = () => { + for (const step of steps) { + if (status.get(step.id) !== 'pending') continue; + const deps = step.after.map((id) => status.get(id)); + if (deps.some((value) => value === 'failed' || value === 'skipped')) { + status.set(step.id, 'skipped'); + onStep({ step, status: 'skipped' }); + continue; + } + if (deps.some((value) => value !== 'done' && value !== undefined)) continue; + if (running.size >= concurrency) break; + status.set(step.id, 'running'); + running.add(step.id); + onStep({ step, status: 'running' }); + Promise.resolve() + .then(() => actuator[method(step.type)](step)) + .then( + () => { + status.set(step.id, 'done'); + onStep({ step, status: 'done' }); + }, + (error) => { + status.set(step.id, 'failed'); + errors[step.id] = String(error?.message ?? error); + onStep({ step, status: 'failed', error: errors[step.id] }); + } + ) + .finally(() => { + running.delete(step.id); + pump(); + }); + } + // Skips can cascade without any step running; settle once nothing is left. + if (!running.size) { + const stillPending = [...status.entries()].some(([id, value]) => value === 'pending' && byId.get(id)); + if (stillPending) { + const progressed = steps.some( + (step) => + status.get(step.id) === 'pending' && + step.after.every((id) => ['done', 'failed', 'skipped', undefined].includes(status.get(id))) + ); + if (progressed) return pump(); + for (const [id, value] of status) if (value === 'pending') status.set(id, 'skipped'); + } + finish(); + } + }; + pump(); + }); +} diff --git a/src/pool-sim-cli.mjs b/src/pool-sim-cli.mjs new file mode 100644 index 0000000..1b09484 --- /dev/null +++ b/src/pool-sim-cli.mjs @@ -0,0 +1,150 @@ +// `lloom pool simulate`: run a fleet scenario through the pool planner, +// reconciler and controller in a deterministic simulation. Nothing on the +// installed gateway or any node is read or changed. +import { readFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { comparePoolSimulation } from './pool-sim.mjs'; + +export const POOL_SIMULATE_USAGE = + 'Usage: lloom pool simulate [--mode off|advise|auto|compare] [--hours N] [--seed N] [--json]'; + +function isObject(value) { + return value != null && typeof value === 'object' && !Array.isArray(value); +} + +function mergeValue(base, overlay) { + if (Array.isArray(base) && Array.isArray(overlay) && overlay.every((item) => isObject(item) && item.id != null)) { + const merged = base.map((item) => ({ ...item })); + for (const item of overlay) { + const index = merged.findIndex((existing) => existing.id === item.id); + if (item.remove === true) { + if (index >= 0) merged.splice(index, 1); + } else if (index >= 0) merged[index] = mergeValue(merged[index], item); + else merged.push(item); + } + return merged; + } + if (isObject(base) && isObject(overlay)) { + const merged = { ...base }; + for (const [key, value] of Object.entries(overlay)) + merged[key] = key in base ? mergeValue(base[key], value) : value; + return merged; + } + return overlay; +} + +// A scenario may name a `base` file (relative to itself). Objects merge; arrays +// of objects with ids merge by id (`"remove": true` drops one); anything else +// in the scenario replaces the base value. +export async function loadPoolScenario(path, seen = new Set()) { + const file = resolve(path); + if (seen.has(file)) throw new Error(`scenario base cycle at ${file}`); + seen.add(file); + const scenario = JSON.parse(await readFile(file, 'utf8')); + if (!scenario.base) return scenario; + const { base, ...overlay } = scenario; + const parent = await loadPoolScenario(resolve(dirname(file), base), seen); + const { description: _description, ...inherited } = parent; + return mergeValue(inherited, overlay); +} + +export function poolSimulateOptions({ mode = 'compare', hours, seed } = {}) { + if (!['off', 'advise', 'auto', 'compare'].includes(mode)) + throw new Error('--mode must be off, advise, auto, or compare'); + const number = (value, name) => { + if (value == null) return undefined; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) throw new Error(`${name} must be a positive number`); + return parsed; + }; + return { + modes: mode === 'compare' ? ['off', 'auto'] : [mode], + durationHours: number(hours, '--hours'), + seed: number(seed, '--seed') + }; +} + +export async function runPoolSimulateCommand(path, options = {}) { + const scenario = await loadPoolScenario(path); + const overrides = Object.fromEntries( + Object.entries({ durationHours: options.durationHours, seed: options.seed }).filter( + ([, value]) => value !== undefined + ) + ); + return comparePoolSimulation({ ...scenario, ...overrides }, { modes: options.modes ?? ['off', 'auto'] }); +} + +function pct(value) { + return `${(value * 100).toFixed(1)}%`; +} + +function seconds(ms) { + return ms >= 10000 ? `${(ms / 1000).toFixed(0)}s` : `${(ms / 1000).toFixed(2)}s`; +} + +export function formatPoolSimulation(reports) { + const modes = Object.keys(reports); + const first = reports[modes[0]]; + const lines = [`Pool simulation: ${first.scenario} (${first.durationHours} h, simulated)`, '']; + const rows = [ + ['requests', (r) => String(r.requests.total)], + ['failed', (r) => String(r.requests.failed)], + ['external', (r) => String(r.requests.external)], + ['cold waits', (r) => String(r.requests.coldWaits)], + ['SLO attainment', (r) => pct(r.requests.sloAttainment)], + ['TTFT p50', (r) => seconds(r.requests.ttftP50Ms)], + ['TTFT p95', (r) => seconds(r.requests.ttftP95Ms)], + ['lane loads', (r) => `${r.lanes.loads} (${r.lanes.loadGb} GB)`], + ['evictions', (r) => String(r.lanes.evictions)], + ['admission refusals', (r) => String(r.lanes.admissionRefusals)], + ['memory guard aborts', (r) => String(r.lanes.guardAborts)], + ['memory violations', (r) => String(r.safety.memoryViolations.length)], + [ + 'controller applies', + (r) => (r.mode === 'off' ? '-' : `${r.controller.applies} (${r.controller.urgentApplies} urgent)`) + ], + ['step failures', (r) => (r.mode === 'off' ? '-' : String(r.controller.stepFailures))] + ]; + const width = Math.max(...rows.map(([label]) => label.length)) + 2; + lines.push(`${''.padEnd(width)}${modes.map((mode) => mode.padStart(16)).join('')}`); + for (const [label, value] of rows) { + lines.push(`${label.padEnd(width)}${modes.map((mode) => value(reports[mode]).padStart(16)).join('')}`); + } + for (const mode of modes) { + const report = reports[mode]; + const worst = Object.entries(report.workloads) + .filter(([, stats]) => stats.requests && stats.sloAttainment < 1) + .sort(([, a], [, b]) => a.sloAttainment - b.sloAttainment) + .slice(0, 5); + if (worst.length) { + lines.push('', `${mode}: lowest SLO attainment`); + for (const [id, stats] of worst) { + const reasons = Object.entries(stats.failReasons) + .map(([reason, count]) => `${count} ${reason}`) + .join(', '); + lines.push( + ` ${id}: ${pct(stats.sloAttainment)} of ${stats.requests}, p95 ${seconds(stats.ttftP95Ms)}${reasons ? `, failed: ${reasons}` : ''}` + ); + } + } + const decisions = report.timeline.filter( + (entry) => entry.kind === 'apply' || entry.kind === 'propose' || entry.kind.startsWith('event:') + ); + if (decisions.length && mode !== 'off') { + lines.push('', `${mode}: events and decisions`); + for (const entry of decisions.slice(0, 20)) { + const detail = entry.kind.startsWith('event:') + ? (entry.nodes ?? []).join(', ') + : `${entry.reason}; ${entry.steps.length} step${entry.steps.length === 1 ? '' : 's'}`; + lines.push(` ${String(entry.atMinutes).padStart(7)} min ${entry.kind} ${detail}`); + } + } + const failedChecks = report.checks.filter((check) => !check.ok); + if (report.checks.length) { + lines.push('', `${mode}: ${report.checks.length - failedChecks.length}/${report.checks.length} expectations met`); + for (const check of failedChecks) + lines.push(` FAIL ${check.path} = ${JSON.stringify(check.value)}, expected ${check.expected}`); + } + } + return `${lines.join('\n')}\n`; +} diff --git a/src/pool-sim.mjs b/src/pool-sim.mjs new file mode 100644 index 0000000..f3b1f86 --- /dev/null +++ b/src/pool-sim.mjs @@ -0,0 +1,950 @@ +// Pool simulator: a deterministic discrete-event model of a LLooM fleet, used +// to test pool planning and reconciliation without touching a live gateway. +// +// The simulator owns the "truth" (real memory footprints, load times, request +// service times, failures) and drives the real planner, reconciler and +// controller through a simulated actuator. It also models what LLooM already +// does without a pool controller, so `off` is today's behaviour: +// +// - requests go to the alias's routed member, else the first ready candidate, +// else a cold start that per-node admission allows, else an external member; +// - admission evicts idle on-demand lanes (then idle preferred lanes) LRU and +// never evicts keep-warm lanes; +// - keep-warm and preferred lanes are restored on their configured nodes; +// - the memory safety guard aborts a managed load that would push the host +// past `memoryGuard.maxMemoryUtilization` (default 0.9) of physical memory. +// +// A lane whose true footprint overflows a node's physical memory (an +// unguarded load, or the guard turned off) is a memory violation: the node +// hangs, as a Spark does on unified-memory exhaustion, and is unavailable for +// `hangMinutes`. Plans should never cause one. + +import { createPoolController } from './pool-controller.mjs'; +import { memoryBudgetGb } from './resource-fit.mjs'; + +export const POOL_SIM_VERSION = 1; + +const HOUR = 3600000; +const MINUTE = 60000; +const RESIDENCY_RANK = { auto: 0, preferred: 1, always: 2 }; + +function list(value) { + return Array.isArray(value) ? value.filter((item) => item != null && item !== '') : []; +} + +function num(value) { + if (value == null || value === '') return null; + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +function round(value, places = 3) { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +function mulberry32(seed) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function percentile(values, p) { + if (!values.length) return null; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))]; +} + +class EventQueue { + constructor() { + this.heap = []; + this.seq = 0; + } + push(at, fn) { + const item = { at, seq: this.seq++, fn }; + const heap = this.heap; + heap.push(item); + let index = heap.length - 1; + while (index > 0) { + const parent = (index - 1) >> 1; + if (this.less(heap[parent], item)) break; + heap[index] = heap[parent]; + index = parent; + } + heap[index] = item; + } + less(a, b) { + return a.at < b.at || (a.at === b.at && a.seq < b.seq); + } + pop() { + const heap = this.heap; + const top = heap[0]; + const last = heap.pop(); + if (heap.length) { + let index = 0; + for (;;) { + const left = index * 2 + 1; + const right = left + 1; + let next = left < heap.length && this.less(heap[left], last) ? left : -1; + if (right < heap.length && this.less(heap[right], next < 0 ? last : heap[next])) next = right; + if (next < 0) break; + heap[index] = heap[next]; + index = next; + } + heap[index] = last; + } + return top; + } + get size() { + return this.heap.length; + } +} + +const flush = () => new Promise((resolve) => setImmediate(resolve)); + +function placementKey(variant, nodes) { + return list(variant.nodes).length || variant.external ? variant.id : `${variant.id}@${nodes.join('+')}`; +} + +function demandRate(workload, at) { + const demand = workload.demand ?? {}; + let rate = num(demand.perHour) ?? num(workload.demandPerHour) ?? 0; + const diurnal = demand.diurnal; + if (diurnal) { + const hour = (at / HOUR) % 24; + const amplitude = num(diurnal.amplitude) ?? 0; + rate *= 1 + amplitude * Math.cos(((hour - (num(diurnal.peakHour) ?? 14)) / 24) * 2 * Math.PI); + } + for (const change of list(demand.changes)) { + if (at >= change.atHours * HOUR) rate = num(change.perHour) ?? rate; + } + for (const burst of list(demand.bursts)) { + const start = burst.atHours * HOUR; + if (at >= start && at < start + burst.hours * HOUR) rate *= num(burst.multiplier) ?? 1; + } + return Math.max(0, rate); +} + +function maxDemandRate(workload) { + const demand = workload.demand ?? {}; + let rate = num(demand.perHour) ?? num(workload.demandPerHour) ?? 0; + for (const change of list(demand.changes)) rate = Math.max(rate, num(change.perHour) ?? 0); + rate *= 1 + Math.abs(num(demand.diurnal?.amplitude) ?? 0); + const burst = Math.max(1, ...list(demand.bursts).map((entry) => num(entry.multiplier) ?? 1)); + return rate * burst; +} + +/** + * Run one scenario in one controller mode. Returns a report; never throws for + * scenario-level failures (they show up as failed requests and violations). + */ +export async function runPoolSimulation(scenario, { mode = 'auto', controller: controllerOverrides = {} } = {}) { + const random = mulberry32(num(scenario.seed) ?? 1); + const queue = new EventQueue(); + const durationMs = (num(scenario.durationHours) ?? 24) * HOUR; + const tickMs = (num(scenario.tickMinutes) ?? 1) * MINUTE; + const policy = scenario.policy ?? {}; + const queueTimeoutMs = (num(scenario.queueTimeoutSeconds) ?? 600) * 1000; + const drainTimeoutMs = (num(scenario.drainTimeoutSeconds) ?? 600) * 1000; + const hangMs = (num(scenario.hangMinutes) ?? 30) * MINUTE; + const demandWindowMs = (num(scenario.demandWindowHours) ?? 24) * HOUR; + const operator = scenario.operator ?? {}; + const memoryGuard = scenario.memoryGuard ?? {}; + const guardUtilization = num(memoryGuard.maxMemoryUtilization) ?? 0.9; + const osGb = num(scenario.osMemoryGb) ?? 4; + let now = 0; + + const report = { + version: POOL_SIM_VERSION, + scenario: scenario.name ?? 'scenario', + mode, + durationHours: durationMs / HOUR, + requests: { total: 0, served: 0, failed: 0, external: 0, coldWaits: 0, sloMet: 0 }, + workloads: {}, + lanes: { loads: 0, loadGb: 0, stops: 0, startFailures: 0, guardAborts: 0, admissionRefusals: 0, evictions: 0 }, + safety: { memoryViolations: [], nodeHangs: 0 }, + controller: { + ticks: 0, + proposals: 0, + applies: 0, + urgentApplies: 0, + steps: 0, + stepFailures: 0, + interruptions: 0, + firstProposal: null, + decisions: [] + }, + timeline: [] + }; + const ttfts = {}; + const timeline = (kind, detail = {}) => { + if (report.timeline.length < (num(scenario.timelineLimit) ?? 400)) { + report.timeline.push({ atMinutes: round(now / MINUTE, 2), kind, ...detail }); + } + }; + + // --- fleet state ----------------------------------------------------------- + const nodes = new Map( + list(scenario.nodes).map((node) => [ + node.id, + { + spec: node, + joined: node.joinAtHours == null, + available: node.available !== false, + cordoned: node.cordoned === true, + hungUntil: 0, + weights: new Set(list(node.weights)), + images: new Set(list(node.images)) + } + ]) + ); + const variants = list(scenario.variants).map((variant) => ({ managed: true, ...variant })); + const variantById = new Map(variants.map((variant) => [variant.id, variant])); + const variantsForModel = (model) => variants.filter((variant) => variant.gatewayModel === model); + const workloads = list(scenario.workloads); + const routes = { ...(scenario.initial?.routes ?? {}) }; + const residency = { ...(scenario.initial?.residency ?? {}) }; + const lanes = new Map(); + const learnedMemory = new Map(); + const served = new Map(); + const arrivals = new Map(workloads.map((workload) => [workload.id, []])); + const flexiblePlacements = new Map(); + + const truth = (variant) => variant.sim ?? {}; + const memoryTruth = (variant) => + num(truth(variant).memoryGbPerNode) ?? num(variant.requirements?.memoryGbPerNode) ?? 0; + const memoryEstimate = (variant) => + Math.max(num(variant.requirements?.memoryGbPerNode) ?? 0, learnedMemory.get(variant.id) ?? 0); + const nodeUsable = (id) => { + const node = nodes.get(id); + return Boolean(node?.joined && node.available && !node.cordoned && node.hungUntil <= now); + }; + const capacity = (id) => { + const domain = list(nodes.get(id)?.spec.memoryDomains)[0]; + return num(domain?.capacityGb) ?? 0; + }; + const budget = (id) => + memoryBudgetGb({ + totalMemoryGb: capacity(id), + maxMemoryUtilization: num(nodes.get(id)?.spec.maxMemoryUtilization) ?? num(policy.maxMemoryUtilization), + reserveMemoryGb: num(policy.reserveGb) ?? 8 + }); + const lanesOn = (id) => [...lanes.values()].filter((lane) => lane.nodes.includes(id)); + const usedTruth = (id) => lanesOn(id).reduce((sum, lane) => sum + lane.memoryGb, 0); + const laneModels = (variant, laneNodes) => + new Set( + variants + .filter( + (other) => + other.id === variant.id || + (variant.runtime && + other.runtime === variant.runtime && + list(other.nodes).join('+') === laneNodes.join('+')) + ) + .map((other) => other.gatewayModel) + ); + const tierOf = (lane) => residency[lane.variant.runtime] ?? 'auto'; + + // --- lanes ----------------------------------------------------------------- + function newLane(variant, laneNodes, { state, reason }) { + const sim = truth(variant); + return { + key: placementKey(variant, laneNodes), + variant, + model: variant.gatewayModel, + models: laneModels(variant, laneNodes), + nodes: laneNodes, + memoryGb: memoryTruth(variant), + state, + maxSequences: + num(sim.maxSequences) ?? num(variant.requirements?.maxSequences) ?? (variant.kind === 'chat' ? 4 : 1), + active: new Set(), + queue: [], + waiters: [], + onReady: [], + onRemoved: [], + lastUsedAt: now, + reason + }; + } + + function serveNext(lane) { + while (lane.state !== 'loading' && lane.queue.length && lane.active.size < lane.maxSequences) { + const request = lane.queue.shift(); + if (request.done) continue; + begin(lane, request); + } + if (lane.state === 'draining' && !lane.active.size && !lane.queue.length) removeLane(lane, 'drained'); + } + + function begin(lane, request) { + const sim = truth(lane.variant); + const performance = lane.variant.performance ?? {}; + const ttft = num(sim.ttftMs) ?? num(performance.ttftMs); + const tps = num(sim.decodeTps) ?? num(performance.decodeTps); + let ttftMs; + let totalMs; + if (ttft != null && tps > 0) { + const penalty = num(sim.concurrencyPenalty) ?? 0.15; + const perSeq = tps / (1 + penalty * lane.active.size); + ttftMs = ttft; + totalMs = ttft + (Math.max(0, (request.outputTokens ?? 256) - 1) * 1000) / perSeq; + } else { + totalMs = num(sim.durationMs) ?? num(performance.durationMs) ?? 1000; + ttftMs = totalMs; + } + lane.active.add(request); + lane.lastUsedAt = now; + request.lane = lane; + request.started = true; + request.serviceTtftMs = ttftMs; + const startedAt = now; + queue.push(now + totalMs, () => { + if (request.done || !lane.active.has(request)) return; + lane.active.delete(request); + lane.lastUsedAt = now; + complete(request, { ttftMs: startedAt - request.arrival + ttftMs, model: lane.model ?? request.model, lane }); + serveNext(lane); + }); + } + + function removeLane(lane, reason) { + if (!lanes.has(lane.key) || lanes.get(lane.key) !== lane) return; + lanes.delete(lane.key); + report.lanes.stops += reason === 'evicted' || reason === 'drained' || reason === 'stopped' ? 1 : 0; + timeline('lane-stop', { lane: lane.key, reason }); + for (const resolve of lane.onRemoved.splice(0)) resolve(); + } + + function killLane(lane, reason) { + const active = [...lane.active]; + const pending = [...lane.queue, ...lane.waiters]; + lane.active.clear(); + lane.queue = []; + lane.waiters = []; + // In-flight requests fail; requests not started yet fail over to another member. + for (const request of active) fail(request, reason); + for (const request of pending) dispatch(request, { avoid: lane.key }); + for (const { reject } of lane.onReady.splice(0)) reject(new Error(reason)); + removeLane(lane, reason); + } + + // Per-node admission: the estimate must fit next to what is resident now, + // evicting idle on-demand lanes first, then idle preferred lanes. Keep-warm + // lanes, loading lanes and busy lanes are never evicted. + function admit(variant, laneNodes, { evict = true, protect = 0 } = {}) { + const need = memoryEstimate(variant); + const evictions = []; + for (const id of laneNodes) { + let used = + usedTruth(id) - evictions.filter((lane) => lane.nodes.includes(id)).reduce((s, l) => s + l.memoryGb, 0); + if (used + need <= budget(id)) continue; + if (!evict) return null; + const idle = lanesOn(id) + .filter( + (lane) => !evictions.includes(lane) && lane.state === 'ready' && !lane.active.size && !lane.queue.length + ) + .filter((lane) => RESIDENCY_RANK[tierOf(lane)] <= protect) + .sort((a, b) => RESIDENCY_RANK[tierOf(a)] - RESIDENCY_RANK[tierOf(b)] || a.lastUsedAt - b.lastUsedAt); + for (const lane of idle) { + if (used + need <= budget(id)) break; + evictions.push(lane); + used -= lane.memoryGb; + } + if (used + need > budget(id)) return null; + } + return evictions; + } + + function startLane(variant, laneNodes, { reason, protect = 1, fetch = true, background = false } = {}) { + const key = placementKey(variant, laneNodes); + const existing = lanes.get(key); + if (existing && existing.state !== 'draining') return existing; + if (!laneNodes.every(nodeUsable)) return null; + const evictions = admit(variant, laneNodes, { protect }); + if (!evictions) { + // The residency reconciler retries every 30 s; only count refusals someone waits on. + if (!background) { + report.lanes.admissionRefusals += 1; + timeline('admission-refused', { lane: key, reason }); + } + return null; + } + for (const lane of evictions) { + report.lanes.evictions += 1; + removeLane(lane, 'evicted'); + } + const sim = truth(variant); + const memoryGb = memoryTruth(variant); + let seconds = num(sim.loadSeconds) ?? num(variant.loadSeconds) ?? 30 + memoryGb * 2; + const recipeId = variant.requirements?.recipeId; + if (fetch && recipeId) { + for (const id of laneNodes) { + if (!nodes.get(id).weights.has(recipeId)) { + const diskGb = num(variant.requirements?.diskGb) ?? memoryGb; + seconds += (diskGb * 1024) / (num(nodes.get(id).spec.fetchMBps) ?? num(policy.fetchMBps) ?? 100); + nodes.get(id).weights.add(recipeId); + } + } + } + const lane = newLane(variant, laneNodes, { state: 'loading', reason }); + lanes.set(key, lane); + report.lanes.loads += 1; + report.lanes.loadGb += memoryGb * laneNodes.length; + timeline('lane-load', { lane: key, reason, seconds: Math.round(seconds) }); + // The memory safety guard watches managed loads and aborts one that pushes + // the host past its limit, before the node can hang. + const guarded = variant.managed !== false && memoryGuard.enabled !== false; + const overGuard = laneNodes.find((id) => usedTruth(id) + osGb > capacity(id) * guardUtilization); + if (guarded && overGuard) { + queue.push(now + seconds * 500, () => { + if (lanes.get(key) !== lane || lane.state !== 'loading') return; + report.lanes.guardAborts += 1; + report.lanes.startFailures += 1; + timeline('memory-guard-abort', { lane: key, node: overGuard, usedGb: round(usedTruth(overGuard) + osGb, 1) }); + killLane(lane, 'memory safety abort'); + }); + } else { + for (const id of laneNodes) checkMemory(id, key); + } + queue.push(now + seconds * 1000, () => { + if (lanes.get(key) !== lane || lane.state !== 'loading') return; + const failures = list(sim.failsOn); + const failRate = num(sim.startFailRate) ?? 0; + if (failures.some((id) => laneNodes.includes(id)) || (failRate && random() < failRate)) { + report.lanes.startFailures += 1; + timeline('lane-failed', { lane: key }); + killLane(lane, 'start failed'); + return; + } + lane.state = 'ready'; + lane.lastUsedAt = now; + // A ready lane's resident size is measured; the catalog learns it. + learnedMemory.set(variant.id, Math.max(learnedMemory.get(variant.id) ?? 0, lane.memoryGb)); + for (const { resolve } of lane.onReady.splice(0)) resolve(lane); + const waiters = lane.waiters.splice(0); + for (const request of waiters) if (!request.done) lane.queue.push(request); + serveNext(lane); + }); + return lane; + } + + function checkMemory(id, cause) { + const used = usedTruth(id); + if (used + osGb <= capacity(id)) return; + report.safety.memoryViolations.push({ + atMinutes: round(now / MINUTE, 2), + node: id, + usedGb: round(used, 1), + capacityGb: capacity(id), + cause + }); + report.safety.nodeHangs += 1; + timeline('memory-violation', { node: id, usedGb: round(used, 1) }); + hangNode(id); + } + + function hangNode(id) { + const node = nodes.get(id); + node.hungUntil = now + hangMs; + for (const lane of lanesOn(id)) killLane(lane, `node ${id} down`); + queue.push(node.hungUntil, () => timeline('node-recovered', { node: id })); + } + + // --- requests -------------------------------------------------------------- + function complete(request, { ttftMs, model, lane = null, external = false }) { + if (request.done) return; + request.done = true; + const workload = request.workload; + const stats = report.workloads[workload.id]; + report.requests.served += 1; + stats.served += 1; + if (external) { + report.requests.external += 1; + stats.external += 1; + } + const slo = num(workload.slo?.ttftMs) ?? num(workload.slo?.durationMs); + if (slo == null || ttftMs <= slo) { + report.requests.sloMet += 1; + stats.sloMet += 1; + } + ttfts[workload.id].push(ttftMs); + if (lane) { + const perf = served.get(model) ?? []; + perf.push(request.serviceTtftMs ?? ttftMs); + if (perf.length > 50) perf.shift(); + served.set(model, perf); + } + } + + function fail(request, reason) { + if (request.done) return; + request.done = true; + report.requests.failed += 1; + report.workloads[request.workload.id].failed += 1; + const reasons = report.workloads[request.workload.id].failReasons; + reasons[reason] = (reasons[reason] ?? 0) + 1; + } + + function readyLane(model, avoid) { + return [...lanes.values()] + .filter((lane) => lane.state === 'ready' && lane.models.has(model) && lane.key !== avoid) + .sort( + (a, b) => (a.active.size + a.queue.length) / a.maxSequences - (b.active.size + b.queue.length) / b.maxSequences + )[0]; + } + + function knownPlacement(variant) { + if (list(variant.nodes).length) return variant.nodes; + return flexiblePlacements.get(variant.id) ?? null; + } + + function dispatch(request, { avoid = null } = {}) { + if (request.done) return; + const workload = request.workload; + const order = [...new Set([routes[workload.id], ...list(workload.candidates)].filter(Boolean))]; + for (const model of order) { + const lane = readyLane(model, avoid); + if (lane) { + request.model = model; + lane.queue.push(request); + serveNext(lane); + return; + } + } + // The routed member is loading: wait for it. + const loading = [...lanes.values()].find( + (lane) => lane.state === 'loading' && lane.models.has(routes[workload.id]) && lane.key !== avoid + ); + if (loading) { + request.coldWait = true; + loading.waiters.push(request); + return; + } + for (const model of order) { + for (const variant of variantsForModel(model)) { + if (variant.external) { + const performance = variant.performance ?? {}; + const ttft = num(performance.ttftMs) ?? num(performance.durationMs) ?? 1000; + request.started = true; + queue.push(now + ttft, () => complete(request, { ttftMs: now - request.arrival, model, external: true })); + return; + } + if (variant.managed === false) continue; + const laneNodes = knownPlacement(variant); + if (!laneNodes) continue; + const lane = startLane(variant, laneNodes, { reason: `demand:${workload.id}`, protect: 1 }); + if (lane && lane.key !== avoid) { + request.coldWait = true; + report.requests.coldWaits += 1; + if (lane.state === 'ready') lane.queue.push(request); + else lane.waiters.push(request); + serveNext(lane); + return; + } + } + } + fail(request, 'no member available'); + } + + function scheduleArrival(workload) { + const maxRate = maxDemandRate(workload); + if (!(maxRate > 0)) return; + let at = now; + for (;;) { + at += (-Math.log(1 - random()) / maxRate) * HOUR; + if (at >= durationMs) return; + if (random() * maxRate <= demandRate(workload, at)) break; + } + queue.push(at, () => { + const request = { workload, arrival: now, outputTokens: num(workload.outputTokens) ?? 256 }; + report.requests.total += 1; + report.workloads[workload.id].requests += 1; + arrivals.get(workload.id).push(now); + queue.push(now + queueTimeoutMs, () => { + if (!request.done && !request.started) { + fail(request, 'queue timeout'); + } + }); + dispatch(request); + scheduleArrival(workload); + }); + } + + // --- what LLooM already does: keep-warm and preferred restoration ----------- + function residencyReconcile() { + for (const variant of variants) { + if (variant.managed === false || variant.external || !variant.runtime) continue; + const tier = residency[variant.runtime] ?? 'auto'; + if (tier === 'auto') continue; + const laneNodes = knownPlacement(variant); + if (!laneNodes || !laneNodes.every(nodeUsable)) continue; + const key = placementKey(variant, laneNodes); + if (lanes.has(key)) continue; + // Keep-warm may evict on-demand and preferred lanes; preferred only on-demand ones. + startLane(variant, laneNodes, { + reason: `residency:${tier}`, + protect: tier === 'always' ? 1 : 0, + background: true + }); + } + } + + // --- the controller's view and hands ------------------------------------------ + function observedVariants() { + return variants.map((variant) => { + const samples = served.get(variant.gatewayModel) ?? []; + const learned = learnedMemory.get(variant.id); + let next = variant; + if (learned != null && learned > (num(variant.requirements?.memoryGbPerNode) ?? 0)) { + next = { ...next, requirements: { ...next.requirements, memoryGbPerNode: learned } }; + } + if (samples.length >= 5 && !variant.external && num(variant.performance?.ttftMs) != null) { + const mean = samples.reduce((sum, value) => sum + value, 0) / samples.length; + next = { ...next, performance: { ...next.performance, ttftMs: Math.round(mean), source: 'observed' } }; + } + const { sim: _sim, ...plannerVariant } = next; + return plannerVariant; + }); + } + + function observedWorkloads() { + const alwaysAt = num(policy.alwaysAtRequestsPerHour) ?? 1; + const preferredAt = num(policy.preferredAtRequestsPerHour) ?? 0.125; + return workloads.map((workload) => { + const times = arrivals.get(workload.id); + while (times.length && times[0] < now - demandWindowMs) times.shift(); + const prior = num(workload.demandPerHour) ?? num(workload.demand?.perHour) ?? 0; + const priorShare = Math.max(0, demandWindowMs - now) / demandWindowMs; + const demandPerHour = round(prior * priorShare + times.length / (demandWindowMs / HOUR), 3); + let tier = workload.residency; + if (tier == null || tier === 'demand') { + tier = demandPerHour >= alwaysAt ? 'always' : demandPerHour >= preferredAt ? 'preferred' : 'auto'; + } + const { demand: _demand, ...plannerWorkload } = workload; + return { ...plannerWorkload, residency: tier, demandPerHour }; + }); + } + + function observe() { + const plannerNodes = [...nodes.values()] + .filter((node) => node.joined) + .map((node) => ({ + ...node.spec, + available: node.available && node.hungUntil <= now, + cordoned: node.cordoned, + weights: [...node.weights], + images: [...node.images] + })); + const placements = [...lanes.values()] + .filter((lane) => lane.state !== 'draining') + .map((lane) => ({ + key: lane.key, + models: [...lane.models], + runtime: lane.variant.runtime ?? null, + nodes: lane.nodes, + memoryGbPerNode: lane.memoryGb, + managed: lane.variant.managed !== false, + state: lane.state === 'loading' ? 'starting' : 'ready' + })); + return { + nodes: plannerNodes, + variants: observedVariants(), + workloads: observedWorkloads(), + pins: scenario.pins ?? {}, + policy, + current: { + routes: { ...routes }, + residency: { ...residency }, + running: placements.map((placement) => placement.key), + placements + } + }; + } + + const waitFor = (fn) => new Promise((resolve, reject) => fn(resolve, reject)); + const variantForPlacement = (key) => + variantById.get(key) ?? variants.find((variant) => key.startsWith(`${variant.id}@`)); + + const actuator = { + prefetch: (step) => + waitFor((resolve) => { + const variant = variantForPlacement(step.placement); + const recipeId = variant?.requirements?.recipeId; + let seconds = 0; + for (const id of step.nodes ?? []) { + if (recipeId && !nodes.get(id)?.weights.has(recipeId)) { + const diskGb = num(variant.requirements?.diskGb) ?? memoryTruth(variant); + seconds = Math.max( + seconds, + (diskGb * 1024) / (num(nodes.get(id).spec.fetchMBps) ?? num(policy.fetchMBps) ?? 100) + ); + } + } + queue.push(now + seconds * 1000, () => { + for (const id of step.nodes ?? []) if (recipeId) nodes.get(id)?.weights.add(recipeId); + resolve(); + }); + }), + start: (step) => + waitFor((resolve, reject) => { + const variant = variantForPlacement(step.placement); + if (!variant) return reject(new Error(`unknown placement ${step.placement}`)); + if (!list(variant.nodes).length) flexiblePlacements.set(variant.id, step.nodes); + const lane = startLane(variant, step.nodes, { reason: 'controller', protect: 0 }); + if (!lane) return reject(new Error('admission refused')); + if (lane.state === 'ready') return resolve(); + lane.onReady.push({ resolve: () => resolve(), reject }); + }), + stop: (step) => + waitFor((resolve) => { + const lane = lanes.get(step.placement); + if (!lane) return resolve(); + lane.onRemoved.push(resolve); + if (lane.state === 'loading') return killLane(lane, 'stopped'); + lane.state = 'draining'; + timeline('lane-drain', { lane: lane.key }); + // Requests already waiting on this lane go back through routing. + const requeue = lane.queue.splice(0); + for (const request of requeue) dispatch(request, { avoid: lane.key }); + serveNext(lane); + queue.push(now + drainTimeoutMs, () => { + if (lanes.get(lane.key) === lane) killLane(lane, 'drain timeout'); + }); + }), + route: async (step) => { + routes[step.alias] = step.to; + timeline(step.type, { alias: step.alias, to: step.to }); + }, + residency: async (step) => { + residency[step.runtime] = step.to; + }, + manual: (step) => + waitFor((resolve, reject) => { + timeline('manual-requested', { action: step.type, lane: step.placement }); + const delay = num(operator.manualDelayMinutes); + if (delay == null) { + queue.push(now + (num(operator.manualTimeoutMinutes) ?? 60) * MINUTE, () => + reject(new Error('operator did not act')) + ); + return; + } + queue.push(now + delay * MINUTE, () => { + if (step.type === 'manual-stop') { + const lane = lanes.get(step.placement); + if (lane) { + lane.onRemoved.push(resolve); + killLane(lane, 'stopped'); + } else resolve(); + return; + } + const variant = variantForPlacement(step.placement); + const lane = variant && startLane(variant, step.nodes, { reason: 'operator', protect: 0 }); + if (!lane) return reject(new Error('operator start refused')); + if (lane.state === 'ready') resolve(); + else lane.onReady.push({ resolve: () => resolve(), reject }); + }); + }) + }; + + const controller = + mode === 'off' + ? null + : createPoolController({ + observe: async () => observe(), + actuator, + now: () => now, + policy: { + ...(scenario.controller ?? {}), + ...controllerOverrides, + mode: mode === 'advise' ? 'advise' : 'auto' + }, + onEvent: (entry) => { + if (entry.action === 'apply' || entry.action === 'propose') { + if (entry.action === 'propose') report.controller.proposals += 1; + else report.controller.applies += 1; + if (entry.urgent && entry.action === 'apply') report.controller.urgentApplies += 1; + report.controller.interruptions += entry.interruptions.length; + report.controller.firstProposal ??= round(entry.at / MINUTE, 2); + if (report.controller.decisions.length < 100) { + report.controller.decisions.push({ + atMinutes: round(entry.at / MINUTE, 2), + action: entry.action, + reason: entry.reason, + urgent: entry.urgent, + steps: entry.steps, + interruptions: entry.interruptions, + blocked: entry.blocked + }); + } + timeline(entry.action, { + reason: entry.reason, + urgent: entry.urgent, + steps: entry.steps.map((step) => step.id), + ...(entry.blocked.length ? { blocked: entry.blocked } : {}) + }); + } else if (entry.action === 'step') { + if (entry.status === 'done') report.controller.steps += 1; + if (entry.status === 'failed') { + report.controller.stepFailures += 1; + timeline('step-failed', { step: entry.step, error: entry.error }); + } + } + } + }); + + // --- initial state and scenario events --------------------------------------- + for (const workload of workloads) { + report.workloads[workload.id] = { requests: 0, served: 0, failed: 0, external: 0, sloMet: 0, failReasons: {} }; + ttfts[workload.id] = []; + } + for (const entry of list(scenario.initial?.running)) { + const variant = variantById.get(typeof entry === 'string' ? entry : entry.variant); + if (!variant) throw new Error(`initial.running: unknown variant ${JSON.stringify(entry)}`); + const laneNodes = list(variant.nodes).length ? variant.nodes : list(entry.nodes); + if (!list(variant.nodes).length) flexiblePlacements.set(variant.id, laneNodes); + const lane = newLane(variant, laneNodes, { state: 'ready', reason: 'initial' }); + lanes.set(lane.key, lane); + learnedMemory.set(variant.id, Math.max(learnedMemory.get(variant.id) ?? 0, lane.memoryGb)); + } + + const applyEvent = (event) => { + const ids = list(event.nodes ?? (event.node ? [event.node] : [])); + timeline(`event:${event.type}`, { + ...(ids.length ? { nodes: ids } : {}), + ...(event.note ? { note: event.note } : {}) + }); + switch (event.type) { + case 'cordon': + for (const id of ids) nodes.get(id).cordoned = true; + break; + case 'uncordon': + for (const id of ids) nodes.get(id).cordoned = false; + break; + case 'fail': + for (const id of ids) { + nodes.get(id).available = false; + for (const lane of lanesOn(id)) killLane(lane, `node ${id} failed`); + } + break; + case 'recover': + for (const id of ids) nodes.get(id).available = true; + break; + case 'join': + for (const id of ids) nodes.get(id).joined = true; + break; + case 'leave': + for (const id of ids) { + nodes.get(id).joined = false; + for (const lane of lanesOn(id)) killLane(lane, `node ${id} left`); + } + break; + case 'stop-lane': { + const lane = lanes.get(event.placement); + if (lane) killLane(lane, 'operator stop'); + break; + } + case 'start-lane': { + const variant = variantForPlacement(event.placement); + if (variant) + startLane(variant, list(variant.nodes).length ? variant.nodes : list(event.nodes), { + reason: 'operator', + protect: 0 + }); + break; + } + case 'apply-profile': + Object.assign(routes, event.routes ?? {}); + Object.assign(residency, event.residency ?? {}); + break; + default: + throw new Error(`unknown scenario event ${event.type}`); + } + }; + + // `modes` limits an event to some runs, e.g. an operator's manual fleet switch + // that only happens when there is no pool controller. In advise mode nothing + // is applied for the operator, so their manual (`off`) events still happen. + const eventMode = mode === 'advise' ? 'off' : mode; + for (const event of list(scenario.events)) { + if (event.modes && !event.modes.includes(eventMode)) continue; + queue.push(event.atHours * HOUR, () => applyEvent(event)); + } + for (const node of nodes.values()) { + if (node.spec.joinAtHours != null) + queue.push(node.spec.joinAtHours * HOUR, () => applyEvent({ type: 'join', nodes: [node.spec.id] })); + } + for (const workload of workloads) scheduleArrival(workload); + for (let at = 0; at < durationMs; at += 30000) queue.push(at, residencyReconcile); + if (controller) { + for (let at = tickMs; at < durationMs; at += tickMs) { + queue.push(at, async () => { + report.controller.ticks += 1; + await controller.tick(); + }); + } + } + + // --- run --------------------------------------------------------------------- + while (queue.size) { + const item = queue.pop(); + if (item.at > durationMs) break; + now = item.at; + await item.fn(); + await flush(); + } + + // --- summary ------------------------------------------------------------------- + const all = Object.values(ttfts).flat(); + report.requests.sloAttainment = report.requests.total ? round(report.requests.sloMet / report.requests.total) : 1; + report.requests.ttftP50Ms = Math.round(percentile(all, 50) ?? 0); + report.requests.ttftP95Ms = Math.round(percentile(all, 95) ?? 0); + for (const [id, stats] of Object.entries(report.workloads)) { + stats.sloAttainment = stats.requests ? round(stats.sloMet / stats.requests) : 1; + stats.ttftP50Ms = Math.round(percentile(ttfts[id], 50) ?? 0); + stats.ttftP95Ms = Math.round(percentile(ttfts[id], 95) ?? 0); + } + report.lanes.loadGb = round(report.lanes.loadGb, 1); + report.final = { + routes: { ...routes }, + residency: { ...residency }, + lanes: [...lanes.values()].map((lane) => lane.key).sort() + }; + report.checks = checkExpectations(report, scenario.expect?.[mode] ?? scenario.expect?.all ?? {}); + report.ok = report.checks.every((check) => check.ok); + return report; +} + +function pathValue(object, path) { + return String(path) + .split('.') + .reduce((value, key) => (value == null ? undefined : value[key]), object); +} + +// expect: { "safety.memoryViolations.length": { "max": 0 }, "final.routes.chat": { "equals": "m" } } +export function checkExpectations(report, expect = {}) { + return Object.entries(expect).map(([path, rule]) => { + const value = pathValue(report, path); + const failures = []; + if (rule.min != null && !(value >= rule.min)) failures.push(`>= ${rule.min}`); + if (rule.max != null && !(value <= rule.max)) failures.push(`<= ${rule.max}`); + if (Object.hasOwn(rule, 'equals') && JSON.stringify(value) !== JSON.stringify(rule.equals)) { + failures.push(`== ${JSON.stringify(rule.equals)}`); + } + if (rule.includes != null && !(Array.isArray(value) && value.includes(rule.includes))) { + failures.push(`includes ${JSON.stringify(rule.includes)}`); + } + return { path, value, ok: failures.length === 0, ...(failures.length ? { expected: failures.join(', ') } : {}) }; + }); +} + +export async function comparePoolSimulation(scenario, { modes = ['off', 'auto'], controller } = {}) { + const reports = {}; + for (const mode of modes) reports[mode] = await runPoolSimulation(scenario, { mode, controller }); + return reports; +} diff --git a/test/fixtures/pool/scenarios/demand-flap.json b/test/fixtures/pool/scenarios/demand-flap.json new file mode 100644 index 0000000..b83931d --- /dev/null +++ b/test/fixtures/pool/scenarios/demand-flap.json @@ -0,0 +1,120 @@ +{ + "base": "spark-fleet-base.json", + "name": "demand-flap", + "description": "Image and music demand swings hard every 30 minutes around the residency thresholds, with a 1-hour demand window. The controller must not chase it: residency-only changes are limited to one an hour and must not cause extra lane loads.", + "durationHours": 12, + "demandWindowHours": 1, + "workloads": [ + { + "id": "image-quality", + "residency": "demand", + "demand": { + "perHour": 0.1, + "diurnal": null, + "bursts": [ + { + "atHours": 1, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 2, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 3, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 4, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 5, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 6, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 7, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 8, + "hours": 0.5, + "multiplier": 40 + } + ] + } + }, + { + "id": "music", + "residency": "demand", + "demand": { + "perHour": 0.1, + "diurnal": null, + "bursts": [ + { + "atHours": 1.5, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 2.5, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 3.5, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 4.5, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 5.5, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 6.5, + "hours": 0.5, + "multiplier": 40 + }, + { + "atHours": 7.5, + "hours": 0.5, + "multiplier": 40 + } + ] + } + } + ], + "expect": { + "auto": { + "safety.memoryViolations.length": { + "max": 0 + }, + "controller.applies": { + "max": 12 + }, + "lanes.loads": { + "max": 6 + }, + "controller.stepFailures": { + "max": 0 + } + } + } +} diff --git a/test/fixtures/pool/scenarios/mac-joins.json b/test/fixtures/pool/scenarios/mac-joins.json new file mode 100644 index 0000000..f814dc9 --- /dev/null +++ b/test/fixtures/pool/scenarios/mac-joins.json @@ -0,0 +1,86 @@ +{ + "base": "spark-fleet-base.json", + "name": "mac-joins", + "description": "Only spark-node-03 is in the pool, serving cognition from the 27B (the standalone profile). A Mac Studio with no model weights joins at hour 1. The pool has an MLX build of Qwen3.8 Flash Next for Apple Silicon. The controller should prefetch the weights, start the lane on the Mac, move cognition, and release the 27B on 03, without dropping requests.", + "durationHours": 6, + "nodes": [ + { "id": "spark-node-01", "remove": true }, + { "id": "spark-node-02", "remove": true }, + { + "id": "mac-studio", + "platformId": "darwin-arm64", + "computeArchs": ["apple-metal"], + "accelerators": ["metal", "apple-gpu"], + "memoryDomains": [{ "id": "unified:0", "kind": "unified", "capacityGb": 128 }], + "links": [], + "weights": [], + "fetchMBps": 120, + "joinAtHours": 1 + } + ], + "variants": [ + { "id": "qwen3.8-flash-next-atlas@spark-node-01+spark-node-02", "remove": true }, + { + "id": "qwen3.8-flash-next-mlx", + "gatewayModel": "qwen3.8-flash-next-mlx", + "model": "mlx-community/Qwen3.8-Flash-Next-4bit", + "kind": "chat", + "runtime": "qwen38fn-mlx", + "requirements": { + "nodeCount": 1, + "platforms": ["darwin-arm64"], + "computeArchs": ["apple-metal"], + "memoryGbPerNode": 58, + "diskGb": 52, + "contextWindow": 262144, + "recipeId": "macos-qwen38fn-mlx" + }, + "performance": { "ttftMs": 600, "decodeTps": 45, "source": "fixture" }, + "sim": { "loadSeconds": 60, "maxSequences": 4 } + } + ], + "workloads": [ + { "id": "chat-capable", "candidates": ["qwen3.8-flash-next-mlx", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"] }, + { "id": "chat-fast", "candidates": ["qwen3.8-flash-next-mlx", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"] }, + { "id": "chat-classifier", "candidates": ["qwen3.8-flash-next-mlx", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"] }, + { "id": "multimodal", "candidates": ["qwen3.8-flash-next-mlx", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"] }, + { + "id": "enntity-presence", + "candidates": ["qwen3.8-flash-next-mlx", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"] + }, + { "id": "omp", "candidates": ["qwen3.8-flash-next-mlx", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"] }, + { "id": "ds4f", "candidates": ["qwen3.8-flash-next-mlx", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"] } + ], + "initial": { + "running": [ + "qwen38-27b-nvfp4@spark-node-03", + "Qwen/Qwen3-Embedding-4B@spark-node-03", + "hear@spark-node-03", + "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03", + "Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03" + ], + "routes": { + "chat-capable": "qwen38-27b-nvfp4", + "chat-fast": "qwen38-27b-nvfp4", + "chat-classifier": "qwen38-27b-nvfp4", + "multimodal": "qwen38-27b-nvfp4", + "enntity-presence": "qwen38-27b-nvfp4", + "omp": "qwen38-27b-nvfp4", + "ds4f": "qwen38-27b-nvfp4" + }, + "residency": { "qwen38-27b-nvfp4": "always" } + }, + "expect": { + "auto": { + "safety.memoryViolations.length": { "max": 0 }, + "final.routes.chat-capable": { "equals": "qwen3.8-flash-next-mlx" }, + "final.lanes": { "includes": "qwen3.8-flash-next-mlx@mac-studio" }, + "final.residency.qwen38-27b-nvfp4": { "equals": "auto" }, + "requests.failed": { "max": 0 }, + "controller.stepFailures": { "max": 0 } + }, + "off": { + "final.routes.chat-capable": { "equals": "qwen38-27b-nvfp4" } + } + } +} diff --git a/test/fixtures/pool/scenarios/memory-estimate-low.json b/test/fixtures/pool/scenarios/memory-estimate-low.json new file mode 100644 index 0000000..eed11ae --- /dev/null +++ b/test/fixtures/pool/scenarios/memory-estimate-low.json @@ -0,0 +1,13 @@ +{ + "base": "spark-fleet-base.json", + "name": "memory-estimate-low", + "description": "The video lane's configured memory estimate (48 GB) is well below what it really uses (74 GB). Video requests arrive while spark-node-03 also holds its resident set. The memory guard must abort the load before the node runs out of unified memory; nothing may hang.", + "durationHours": 6, + "variants": [ + { "id": "Wan-AI/Wan2.2-TI2V-5B-Diffusers@spark-node-03", "sim": { "memoryGbPerNode": 74, "loadSeconds": 180 } } + ], + "workloads": [{ "id": "video", "demand": { "perHour": 2, "diurnal": null } }], + "expect": { + "all": { "safety.memoryViolations.length": { "max": 0 } } + } +} diff --git a/test/fixtures/pool/scenarios/pair-crash.json b/test/fixtures/pool/scenarios/pair-crash.json new file mode 100644 index 0000000..cfc66cb --- /dev/null +++ b/test/fixtures/pool/scenarios/pair-crash.json @@ -0,0 +1,30 @@ +{ + "base": "spark-fleet-base.json", + "name": "pair-crash", + "description": "spark-node-01 crashes at hour 3 with nobody watching; the TP2 lane dies with it. The node comes back at hour 7 and the operator restarts the unmanaged pair lane 30 minutes later. Without a controller (off), cognition fails over to an on-demand 27B that admission can evict for media; with the controller (auto), the 27B becomes keep-warm and the routes move.", + "durationHours": 12, + "operator": { "manualDelayMinutes": 30 }, + "workloads": [ + { "id": "image-quality", "demand": { "perHour": 3, "diurnal": null } }, + { "id": "video", "demand": { "perHour": 0.5, "diurnal": null } } + ], + "events": [ + { "atHours": 3, "type": "fail", "nodes": ["spark-node-01"] }, + { "atHours": 7, "type": "recover", "nodes": ["spark-node-01"] }, + { + "atHours": 7.5, + "type": "start-lane", + "modes": ["off"], + "placement": "qwen3.8-flash-next-atlas@spark-node-01+spark-node-02" + } + ], + "expect": { + "auto": { + "safety.memoryViolations.length": { "max": 0 }, + "controller.urgentApplies": { "min": 1 }, + "final.routes.chat-capable": { "equals": "qwen3.8-flash-next-atlas" }, + "controller.stepFailures": { "max": 0 } + }, + "off": { "safety.memoryViolations.length": { "max": 0 } } + } +} diff --git a/test/fixtures/pool/scenarios/pair-taken.json b/test/fixtures/pool/scenarios/pair-taken.json new file mode 100644 index 0000000..a47b1ca --- /dev/null +++ b/test/fixtures/pool/scenarios/pair-taken.json @@ -0,0 +1,61 @@ +{ + "base": "spark-fleet-base.json", + "name": "pair-taken", + "description": "The operator takes the 01/02 pair for other work at hour 2 and gives it back at hour 8. Without a controller (off), the operator also switches fleet profiles by hand at the same moments, as on 2026-10-10. With the controller (auto), the operator only cordons and uncordons.", + "durationHours": 12, + "operator": { "manualDelayMinutes": 5 }, + "events": [ + { "atHours": 2, "type": "cordon", "nodes": ["spark-node-01", "spark-node-02"] }, + { "atHours": 2, "type": "stop-lane", "placement": "qwen3.8-flash-next-atlas@spark-node-01+spark-node-02" }, + { + "atHours": 2, + "type": "apply-profile", + "modes": ["off"], + "note": "lloom fleet use standalone", + "routes": { + "chat-capable": "qwen38-27b-nvfp4", + "chat-fast": "qwen38-27b-nvfp4", + "chat-classifier": "qwen38-27b-nvfp4", + "multimodal": "qwen38-27b-nvfp4", + "enntity-presence": "qwen38-27b-nvfp4", + "omp": "qwen38-27b-nvfp4", + "ds4f": "qwen38-27b-nvfp4" + }, + "residency": { "qwen38-27b-nvfp4": "always" } + }, + { "atHours": 8, "type": "uncordon", "nodes": ["spark-node-01", "spark-node-02"] }, + { + "atHours": 8, + "type": "start-lane", + "modes": ["off"], + "placement": "qwen3.8-flash-next-atlas@spark-node-01+spark-node-02" + }, + { + "atHours": 8.25, + "type": "apply-profile", + "modes": ["off"], + "note": "lloom fleet use sparkqwen-pair-rc16", + "routes": { + "chat-capable": "qwen3.8-flash-next-atlas", + "chat-fast": "qwen3.8-flash-next-atlas", + "chat-classifier": "qwen3.8-flash-next-atlas", + "multimodal": "qwen3.8-flash-next-atlas", + "enntity-presence": "qwen3.8-flash-next-atlas", + "omp": "qwen3.8-flash-next-atlas", + "ds4f": "qwen3.8-flash-next-atlas" + }, + "residency": { "qwen38-27b-nvfp4": "auto" } + } + ], + "expect": { + "auto": { + "safety.memoryViolations.length": { "max": 0 }, + "final.routes.chat-capable": { "equals": "qwen3.8-flash-next-atlas" }, + "final.residency.qwen38-27b-nvfp4": { "equals": "auto" }, + "controller.stepFailures": { "max": 0 } + }, + "off": { + "safety.memoryViolations.length": { "max": 0 } + } + } +} diff --git a/test/fixtures/pool/scenarios/spark-fleet-base.json b/test/fixtures/pool/scenarios/spark-fleet-base.json new file mode 100644 index 0000000..83cffc6 --- /dev/null +++ b/test/fixtures/pool/scenarios/spark-fleet-base.json @@ -0,0 +1,535 @@ +{ + "description": "Simulation base for the three-Spark fleet (spark-fleet.json): spark-node-03 serves embeddings, media and voice; the 01/02 pair serves SparkQwen RC16 as an unmanaged TP2 lane. Starts in the sparkqwen-pair-rc16 profile. Load times and concurrency are representative, not measured claims.", + "seed": 7, + "durationHours": 12, + "tickMinutes": 1, + "policy": { + "maxMemoryUtilization": 0.95, + "headroomGb": 4 + }, + "controller": { + "confirmTicks": 2, + "cooldownMs": 900000, + "minGain": 0.1 + }, + "nodes": [ + { + "id": "spark-node-03", + "platformId": "linux-arm64", + "computeArchs": ["sm_121"], + "accelerators": ["cuda", "nvidia-gpu", "blackwell", "gb10"], + "memoryDomains": [ + { + "id": "unified:0", + "kind": "unified", + "capacityGb": 128 + } + ], + "links": [] + }, + { + "id": "spark-node-01", + "platformId": "linux-arm64", + "computeArchs": ["sm_121"], + "accelerators": ["cuda", "nvidia-gpu", "blackwell", "gb10"], + "memoryDomains": [ + { + "id": "unified:0", + "kind": "unified", + "capacityGb": 128 + } + ], + "links": ["spark-node-02"] + }, + { + "id": "spark-node-02", + "platformId": "linux-arm64", + "computeArchs": ["sm_121"], + "accelerators": ["cuda", "nvidia-gpu", "blackwell", "gb10"], + "memoryDomains": [ + { + "id": "unified:0", + "kind": "unified", + "capacityGb": 128 + } + ], + "links": ["spark-node-01"] + } + ], + "variants": [ + { + "id": "qwen3.8-flash-next-atlas@spark-node-01+spark-node-02", + "gatewayModel": "qwen3.8-flash-next-atlas", + "model": "nvidia/Qwen3.8-Flash-Next-NVFP4", + "kind": "chat", + "managed": false, + "nodes": ["spark-node-01", "spark-node-02"], + "requirements": { + "nodeCount": 2, + "tensorParallel": 2, + "computeArchs": ["sm_121"], + "memoryGbPerNode": 110, + "contextWindow": 262144 + }, + "performance": { + "ttftMs": 450, + "decodeTps": 75, + "source": "fixture" + }, + "sim": { + "loadSeconds": 420, + "maxSequences": 8 + } + }, + { + "id": "qwen38-27b-nvfp4@spark-node-03", + "gatewayModel": "qwen38-27b-nvfp4", + "model": "unsloth/Qwen3.8-27B-NVFP4", + "kind": "chat", + "runtime": "qwen38-27b-nvfp4", + "nodes": ["spark-node-03"], + "requirements": { + "nodeCount": 1, + "computeArchs": ["sm_121"], + "memoryGbPerNode": 46, + "contextWindow": 262144 + }, + "performance": { + "ttftMs": 900, + "decodeTps": 28, + "source": "fixture" + }, + "sim": { + "loadSeconds": 150, + "maxSequences": 2 + } + }, + { + "id": "Qwen/Qwen3-Embedding-4B@spark-node-03", + "gatewayModel": "Qwen/Qwen3-Embedding-4B", + "kind": "embedding", + "runtime": "qwen3-embedding-4b", + "nodes": ["spark-node-03"], + "requirements": { + "memoryGbPerNode": 16 + }, + "performance": { + "durationMs": 60, + "source": "fixture" + }, + "sim": { + "maxSequences": 32, + "loadSeconds": 40 + } + }, + { + "id": "hear@spark-node-03", + "gatewayModel": "hear", + "kind": "audio", + "runtime": "hear", + "nodes": ["spark-node-03"], + "requirements": { + "memoryGbPerNode": 2 + }, + "performance": { + "durationMs": 400, + "source": "fixture" + }, + "sim": { + "maxSequences": 4, + "loadSeconds": 20 + } + }, + { + "id": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03", + "gatewayModel": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", + "kind": "image", + "runtime": "spark03-qwen-image-21-turbo-nvfp4", + "nodes": ["spark-node-03"], + "requirements": { + "memoryGbPerNode": 24 + }, + "performance": { + "durationMs": 18500, + "source": "fixture" + }, + "sim": { + "loadSeconds": 90 + } + }, + { + "id": "BennyDaBall/Qwen-Image-2.1-NVFP4@spark-node-03", + "gatewayModel": "BennyDaBall/Qwen-Image-2.1-NVFP4", + "kind": "image", + "runtime": "spark03-qwen-image-21-nvfp4", + "nodes": ["spark-node-03"], + "requirements": { + "memoryGbPerNode": 32 + }, + "performance": { + "durationMs": 110000, + "source": "fixture" + }, + "sim": { + "loadSeconds": 110 + } + }, + { + "id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03", + "gatewayModel": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "kind": "audio", + "runtime": "spark03-tts-clone", + "nodes": ["spark-node-03"], + "requirements": { + "memoryGbPerNode": 10 + }, + "performance": { + "durationMs": 1500, + "source": "fixture" + }, + "sim": { + "maxSequences": 2, + "loadSeconds": 45 + } + }, + { + "id": "openai/whisper-large-v3-turbo@spark-node-03", + "gatewayModel": "openai/whisper-large-v3-turbo", + "kind": "transcription", + "runtime": "spark03-stt", + "nodes": ["spark-node-03"], + "requirements": { + "memoryGbPerNode": 6 + }, + "performance": { + "durationMs": 2000, + "source": "fixture" + }, + "sim": { + "maxSequences": 2, + "loadSeconds": 30 + } + }, + { + "id": "Wan-AI/Wan2.2-TI2V-5B-Diffusers@spark-node-03", + "gatewayModel": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "kind": "video", + "runtime": "spark03-video", + "nodes": ["spark-node-03"], + "requirements": { + "memoryGbPerNode": 48 + }, + "performance": { + "durationMs": 240000, + "source": "fixture" + }, + "sim": { + "loadSeconds": 180 + } + }, + { + "id": "Comfy-Org/YuE2-3B@spark-node-03", + "gatewayModel": "Comfy-Org/YuE2-3B", + "kind": "audio", + "runtime": "spark03-yue2", + "nodes": ["spark-node-03"], + "requirements": { + "memoryGbPerNode": 16 + }, + "performance": { + "durationMs": 90000, + "source": "fixture" + }, + "sim": { + "loadSeconds": 90 + } + }, + { + "id": "cloud/openrouter/q38fn", + "gatewayModel": "cloud/openrouter/q38fn", + "kind": "chat", + "external": true, + "performance": { + "ttftMs": 700, + "decodeTps": 120, + "source": "fixture" + } + } + ], + "workloads": [ + { + "id": "chat-capable", + "residency": "always", + "candidates": ["qwen3.8-flash-next-atlas", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"], + "demandPerHour": 30, + "demand": { + "perHour": 30, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 3000 + } + }, + { + "id": "chat-fast", + "residency": "always", + "candidates": ["qwen3.8-flash-next-atlas", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"], + "demandPerHour": 20, + "demand": { + "perHour": 20, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 3000 + } + }, + { + "id": "chat-classifier", + "residency": "always", + "candidates": ["qwen3.8-flash-next-atlas", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"], + "demandPerHour": 40, + "demand": { + "perHour": 40, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 3000 + } + }, + { + "id": "multimodal", + "residency": "always", + "candidates": ["qwen3.8-flash-next-atlas", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"], + "demandPerHour": 5, + "demand": { + "perHour": 5, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 3000 + } + }, + { + "id": "enntity-presence", + "residency": "always", + "candidates": ["qwen3.8-flash-next-atlas", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"], + "demandPerHour": 10, + "demand": { + "perHour": 10, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 3000 + } + }, + { + "id": "omp", + "residency": "always", + "candidates": ["qwen3.8-flash-next-atlas", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"], + "demandPerHour": 8, + "demand": { + "perHour": 8, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 3000 + } + }, + { + "id": "ds4f", + "residency": "always", + "candidates": ["qwen3.8-flash-next-atlas", "qwen38-27b-nvfp4", "cloud/openrouter/q38fn"], + "demandPerHour": 2, + "demand": { + "perHour": 2, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 3000 + } + }, + { + "id": "embeddings", + "candidates": ["Qwen/Qwen3-Embedding-4B"], + "residency": "always", + "demandPerHour": 200, + "demand": { + "perHour": 200, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 500 + } + }, + { + "id": "hear", + "candidates": ["hear"], + "residency": "always", + "demandPerHour": 20, + "demand": { + "perHour": 20, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 2000 + } + }, + { + "id": "image-fast", + "candidates": ["BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", "BennyDaBall/Qwen-Image-2.1-NVFP4"], + "residency": "preferred", + "demandPerHour": 2, + "demand": { + "perHour": 2, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 60000 + } + }, + { + "id": "image-quality", + "candidates": ["BennyDaBall/Qwen-Image-2.1-NVFP4"], + "residency": "auto", + "demandPerHour": 0.2, + "demand": { + "perHour": 0.2, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + } + }, + { + "id": "tts-clone", + "candidates": ["Qwen/Qwen3-TTS-12Hz-1.7B-Base"], + "residency": "preferred", + "demandPerHour": 3, + "demand": { + "perHour": 3, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 5000 + } + }, + { + "id": "stt", + "candidates": ["openai/whisper-large-v3-turbo"], + "residency": "auto", + "demandPerHour": 1, + "demand": { + "perHour": 1, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + }, + "slo": { + "ttftMs": 10000 + } + }, + { + "id": "video", + "candidates": ["Wan-AI/Wan2.2-TI2V-5B-Diffusers"], + "residency": "auto", + "demandPerHour": 0.1, + "demand": { + "perHour": 0.1, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + } + }, + { + "id": "music", + "candidates": ["Comfy-Org/YuE2-3B"], + "residency": "auto", + "demandPerHour": 0.1, + "demand": { + "perHour": 0.1, + "diurnal": { + "amplitude": 0.5, + "peakHour": 15 + } + } + } + ], + "pins": { + "residency": { + "qwen3-embedding-4b": "always", + "hear": "always" + } + }, + "initial": { + "running": [ + "qwen3.8-flash-next-atlas@spark-node-01+spark-node-02", + "Qwen/Qwen3-Embedding-4B@spark-node-03", + "hear@spark-node-03", + "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03", + "Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03" + ], + "routes": { + "chat-capable": "qwen3.8-flash-next-atlas", + "chat-fast": "qwen3.8-flash-next-atlas", + "chat-classifier": "qwen3.8-flash-next-atlas", + "multimodal": "qwen3.8-flash-next-atlas", + "enntity-presence": "qwen3.8-flash-next-atlas", + "omp": "qwen3.8-flash-next-atlas", + "ds4f": "qwen3.8-flash-next-atlas", + "embeddings": "Qwen/Qwen3-Embedding-4B", + "hear": "hear", + "image-fast": "BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4", + "image-quality": "BennyDaBall/Qwen-Image-2.1-NVFP4", + "tts-clone": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "stt": "openai/whisper-large-v3-turbo", + "video": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "music": "Comfy-Org/YuE2-3B" + }, + "residency": { + "qwen3-embedding-4b": "always", + "hear": "always", + "spark03-tts-clone": "preferred", + "spark03-qwen-image-21-turbo-nvfp4": "preferred", + "qwen38-27b-nvfp4": "auto", + "spark03-qwen-image-21-nvfp4": "auto", + "spark03-stt": "auto", + "spark03-video": "auto", + "spark03-yue2": "auto" + } + } +} diff --git a/test/pool-controller.test.mjs b/test/pool-controller.test.mjs new file mode 100644 index 0000000..71bdab5 --- /dev/null +++ b/test/pool-controller.test.mjs @@ -0,0 +1,197 @@ +// Pool controller tests: modes, urgency, hysteresis and failure backoff, with +// a fake observer and actuator. No gateway, runtimes or timers. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { createPoolController, decide, normalizeControllerPolicy } from '../src/pool-controller.mjs'; + +const fleet = JSON.parse(readFileSync(new URL('./fixtures/pool/spark-fleet.json', import.meta.url), 'utf8')); + +const RC16_RUNNING = [ + 'qwen3.8-flash-next-atlas@spark-node-01+spark-node-02', + 'Qwen/Qwen3-Embedding-4B@spark-node-03', + 'hear@spark-node-03', + 'BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03', + 'Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03' +]; + +// A tiny in-memory fleet the fake actuator mutates, so successive ticks see +// the effect of the previous apply. +function fakeFleet({ cordon = [], failStarts = [] } = {}) { + const profile = fleet.profiles['sparkqwen-pair-rc16']; + const state = { + routes: { ...profile.routes }, + residency: { ...profile.residency }, + running: new Set(RC16_RUNNING), + cordon: new Set(cordon), + calls: [] + }; + const variant = (key) => fleet.variants.find((candidate) => candidate.id === key); + const observe = async () => ({ + nodes: fleet.nodes.map((node) => ({ ...node, cordoned: state.cordon.has(node.id) })), + variants: fleet.variants, + workloads: fleet.workloads.map(({ cognition, ...workload }) => ({ + ...workload, + candidates: cognition ? fleet.cognitionCandidates : workload.candidates + })), + pins: fleet.pins, + policy: { ...fleet.policy, memorySafetyUtilization: null }, + current: { + routes: { ...state.routes }, + residency: { ...state.residency }, + running: [...state.running], + placements: [...state.running].map((key) => ({ + key, + models: [variant(key).gatewayModel], + runtime: variant(key).runtime ?? null, + nodes: variant(key).nodes, + memoryGbPerNode: variant(key).requirements.memoryGbPerNode, + managed: variant(key).managed !== false, + state: 'ready' + })) + } + }); + const actuator = { + prefetch: async () => {}, + start: async (step) => { + state.calls.push(step.id); + if (failStarts.includes(step.placement)) throw new Error('admission refused'); + state.running.add(step.placement); + }, + stop: async (step) => { + state.calls.push(step.id); + state.running.delete(step.placement); + }, + route: async (step) => { + state.calls.push(step.id); + state.routes[step.alias] = step.to; + }, + residency: async (step) => { + state.calls.push(step.id); + state.residency[step.runtime] = step.to; + }, + manual: async (step) => { + state.calls.push(step.id); + if (step.type === 'manual-stop') state.running.delete(step.placement); + else state.running.add(step.placement); + } + }; + return { state, observe, actuator }; +} + +test('policy normalisation keeps known keys and valid modes only', () => { + const policy = normalizeControllerPolicy({ mode: 'yolo', confirmTicks: '3', unknown: 1 }); + assert.equal(policy.mode, 'advise'); + assert.equal(policy.confirmTicks, 3); + assert.equal('unknown' in policy, false); +}); + +test('off mode never observes or acts', async () => { + let observed = 0; + const controller = createPoolController({ observe: async () => observed++, actuator: {}, policy: { mode: 'off' } }); + const entry = await controller.tick(); + assert.equal(entry.action, 'off'); + assert.equal(observed, 0); +}); + +test('a fleet that matches its plan holds', async () => { + const { observe, actuator, state } = fakeFleet(); + const controller = createPoolController({ observe, actuator, policy: { mode: 'auto' } }); + const entry = await controller.tick(); + assert.equal(entry.action, 'hold'); + assert.equal(entry.reason, 'no changes'); + assert.deepEqual(state.calls, []); +}); + +test('losing the pair is urgent: auto applies on the first tick', async () => { + const { observe, actuator, state } = fakeFleet({ cordon: ['spark-node-01', 'spark-node-02'] }); + const controller = createPoolController({ observe, actuator, policy: { mode: 'auto', confirmTicks: 5 } }); + const entry = await controller.tick(); + assert.equal(entry.action, 'apply'); + assert.equal(entry.urgent, true); + await controller.executing; + assert.equal(state.routes['chat-capable'], 'qwen38-27b-nvfp4'); + assert.equal(state.residency['qwen38-27b-nvfp4'], 'always'); + assert.ok(!state.running.has('qwen3.8-flash-next-atlas@spark-node-01+spark-node-02')); + const next = await controller.tick(); + assert.equal(next.reason, 'no changes', 'the pool settles'); +}); + +test('advise mode proposes the same change and never touches the actuator', async () => { + const { observe, actuator, state } = fakeFleet({ cordon: ['spark-node-01', 'spark-node-02'] }); + const controller = createPoolController({ observe, actuator, policy: { mode: 'advise' } }); + const entry = await controller.tick(); + assert.equal(entry.action, 'propose'); + assert.ok(entry.steps.length > 0); + assert.deepEqual(state.calls, []); +}); + +test('a non-urgent improvement waits for a stable plan and then for the cooldown', async () => { + // Standalone with the pair free: moving back is a gain, not a coverage fix. + const { observe, actuator, state } = fakeFleet(); + Object.assign(state.routes, fleet.profiles.standalone.routes); + Object.assign(state.residency, fleet.profiles.standalone.residency); + state.running.delete('qwen3.8-flash-next-atlas@spark-node-01+spark-node-02'); + state.running.add('qwen38-27b-nvfp4@spark-node-03'); + let clock = 0; + const controller = createPoolController({ + observe, + actuator, + now: () => clock, + policy: { mode: 'auto', confirmTicks: 2, cooldownMs: 600000 } + }); + const first = await controller.tick(); + assert.equal(first.action, 'hold'); + assert.match(first.reason, /stable plan \(1\/2\)/); + clock += 60000; + const second = await controller.tick(); + assert.equal(second.action, 'apply'); + assert.equal(second.urgent, false); + await controller.executing; + assert.equal(state.routes['chat-capable'], 'qwen3.8-flash-next-atlas'); +}); + +test('the hourly load budget holds back a non-urgent move but never an urgent one', () => { + const policy = normalizeControllerPolicy({ maxLoadGbPerHour: 400 }); + const reconcile = { + empty: false, + steps: [ + { type: 'start', memoryGbPerNode: 60, nodes: ['a'] }, + { type: 'route', alias: 'x' } + ] + }; + const state = { stableTicks: 5, lastActionAt: null, loads: [{ at: 0, gb: 350 }] }; + const better = { score: { uncovered: 0, external: 0, sloMisses: 0, cost: 50 } }; + const today = { score: { uncovered: 0, external: 0, sloMisses: 0, cost: 100 } }; + const held = decide({ plan: better, stay: today, reconcile, state, policy, now: 1000 }); + assert.equal(held.act, false); + assert.match(held.reason, /hourly load budget/); + const later = decide({ plan: better, stay: today, reconcile, state, policy, now: 3600001 }); + assert.equal(later.act, true, 'the budget is a rolling hour'); + const uncovered = { score: { ...today.score, uncovered: 1 } }; + assert.equal(decide({ plan: better, stay: uncovered, reconcile, state, policy, now: 1000 }).urgent, true); +}); + +test('a failed start is retried elsewhere after backoff instead of repeating', async () => { + const { observe, actuator, state } = fakeFleet({ + cordon: ['spark-node-01', 'spark-node-02'], + failStarts: ['qwen38-27b-nvfp4@spark-node-03'] + }); + let clock = 0; + const controller = createPoolController({ observe, actuator, now: () => clock, policy: { mode: 'auto' } }); + const first = await controller.tick(); + assert.equal(first.action, 'apply'); + const result = await controller.executing; + assert.equal(result.ok, false); + assert.equal( + state.routes['chat-capable'], + 'qwen3.8-flash-next-atlas', + 'routes did not move to a lane that never started' + ); + assert.ok(controller.failures()['qwen38-27b-nvfp4@spark-node-03']); + clock += 60000; + const second = await controller.tick(); + // With the 27B excluded, the only cover left for cognition is the cloud member. + assert.equal(second.plan.routes['chat-capable'].model, 'cloud/openrouter/q38fn'); + assert.equal(second.action, 'apply'); +}); diff --git a/test/pool-planner.test.mjs b/test/pool-planner.test.mjs index 0f1cba2..3e5f8dd 100644 --- a/test/pool-planner.test.mjs +++ b/test/pool-planner.test.mjs @@ -33,6 +33,9 @@ const STANDALONE_RUNNING = [ 'Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03' ]; +// The hand-written profiles were sized against the admission budget alone, so +// profile reproduction turns the memory-guard ceiling off; the guard has its +// own test below. function planFleet({ cordon = [], current, pins = {}, nodes = fleet.nodes, policy = {} } = {}) { return planPool({ nodes, @@ -40,7 +43,7 @@ function planFleet({ cordon = [], current, pins = {}, nodes = fleet.nodes, polic workloads: fleetWorkloads(), pins: { ...fleet.pins, ...pins, cordon }, current, - policy: { ...fleet.policy, ...policy } + policy: { ...fleet.policy, memorySafetyUtilization: null, ...policy } }); } @@ -114,6 +117,22 @@ test('a free pair moves cognition back to SparkQwen as a manual start', () => { assert.deepEqual(plan.diff.residencyChanges, [{ runtime: 'qwen38-27b-nvfp4', from: 'always', to: 'auto' }]); }); +test('the memory guard ceiling shows standalone cannot load video next to a keep-warm 27B', () => { + const plan = planFleet({ + cordon: ['spark-node-01', 'spark-node-02'], + current: currentFromProfile('sparkqwen-pair-rc16', RC16_RUNNING), + policy: { memorySafetyUtilization: 0.9 } + }); + // 128 GB x 0.9 guard ceiling - 4 GB headroom = 111.2 GB; keep-warm 27B, + // embeddings and hear hold 64 GB, and the 48 GB video lane needs 112. + assert.equal(plan.nodes['spark-node-03'].domains[0].budgetGb, 111.2); + assert.deepEqual( + plan.uncovered.map((entry) => entry.workload), + ['video'] + ); + assert.equal(plan.routes['chat-capable'].model, 'qwen38-27b-nvfp4'); +}); + test('a high churn weight keeps the running lane when the gain is small', () => { const plan = planFleet({ current: currentFromProfile('standalone', STANDALONE_RUNNING), @@ -205,20 +224,35 @@ test('memory headroom is conservative', () => { nodes: [spark('s')], variants: [large], workloads: one(), - policy: { maxMemoryUtilization: 0.95, headroomGb: 4 } + policy: { maxMemoryUtilization: 0.95, headroomGb: 4, memorySafetyUtilization: null } }); assert.equal(fits.ok, true); const tight = planPool({ nodes: [spark('s')], variants: [large], workloads: one(), - policy: { maxMemoryUtilization: 0.95, headroomGb: 8 } + policy: { maxMemoryUtilization: 0.95, headroomGb: 8, memorySafetyUtilization: null } }); assert.equal(tight.ok, false); assert.ok(tight.uncovered[0].reasons.some((reason) => reason.includes('does not fit as always'))); - const defaults = planPool({ nodes: [spark('s')], variants: [large], workloads: one() }); - assert.equal(defaults.ok, false, 'default budget is capacity minus reserve minus headroom'); - assert.equal(defaults.nodes.s.domains[0].budgetGb, 116); + const admissionOnly = planPool({ + nodes: [spark('s')], + variants: [large], + workloads: one(), + policy: { memorySafetyUtilization: null } + }); + assert.equal(admissionOnly.ok, false, 'admission budget is capacity minus reserve minus headroom'); + assert.equal(admissionOnly.nodes.s.domains[0].budgetGb, 116); + // By default the runtime memory guard's 90% ceiling caps unified memory too. + const guarded = planPool({ nodes: [spark('s')], variants: [large], workloads: one() }); + assert.equal(guarded.nodes.s.domains[0].budgetGb, 111.2); + const guardedAt95 = planPool({ + nodes: [spark('s')], + variants: [large], + workloads: one(), + policy: { maxMemoryUtilization: 0.95, headroomGb: 4 } + }); + assert.equal(guardedAt95.ok, false, 'the guard aborts a load the 95% admission budget would allow'); }); test('memory is accounted per domain on discrete GPUs', () => { diff --git a/test/pool-reconciler.test.mjs b/test/pool-reconciler.test.mjs new file mode 100644 index 0000000..9d771b4 --- /dev/null +++ b/test/pool-reconciler.test.mjs @@ -0,0 +1,260 @@ +// Pool reconciler tests: step ordering (make before break, swaps, residency) +// and the dependency-aware executor. Pure inputs only. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { planPool } from '../src/pool-planner.mjs'; +import { executeReconcile, planReconcile } from '../src/pool-reconciler.mjs'; + +const fleet = JSON.parse(readFileSync(new URL('./fixtures/pool/spark-fleet.json', import.meta.url), 'utf8')); + +function fleetWorkloads() { + return fleet.workloads.map(({ cognition, ...workload }) => ({ + ...workload, + candidates: cognition ? fleet.cognitionCandidates : workload.candidates + })); +} + +const RC16_RUNNING = [ + 'qwen3.8-flash-next-atlas@spark-node-01+spark-node-02', + 'Qwen/Qwen3-Embedding-4B@spark-node-03', + 'hear@spark-node-03', + 'BennyDaBall/Qwen-Image-2.1-Turbo-NVFP4@spark-node-03', + 'Qwen/Qwen3-TTS-12Hz-1.7B-Base@spark-node-03' +]; + +function placementsFor(keys, variants) { + return keys.map((key) => { + const variant = variants.find((candidate) => candidate.id === key); + return { + key, + models: [variant.gatewayModel], + runtime: variant.runtime ?? null, + nodes: variant.nodes, + memoryGbPerNode: variant.requirements.memoryGbPerNode, + managed: variant.managed !== false, + state: 'ready' + }; + }); +} + +function index(steps) { + return Object.fromEntries(steps.map((step, position) => [step.id, { ...step, position }])); +} + +function dependsOn(steps, id, dependency) { + const byId = index(steps); + const seen = new Set(); + const walk = (current) => { + for (const parent of byId[current]?.after ?? []) { + if (parent === dependency) return true; + if (!seen.has(parent)) { + seen.add(parent); + if (walk(parent)) return true; + } + } + return false; + }; + return walk(id); +} + +test('a pool that already matches the plan needs no steps', () => { + const profile = fleet.profiles['sparkqwen-pair-rc16']; + const current = { routes: profile.routes, residency: profile.residency, running: RC16_RUNNING }; + const plan = planPool({ + nodes: fleet.nodes, + variants: fleet.variants, + workloads: fleetWorkloads(), + pins: fleet.pins, + current, + policy: fleet.policy + }); + const result = planReconcile({ + plan, + current: { ...current, placements: placementsFor(RC16_RUNNING, fleet.variants) } + }); + assert.equal(result.empty, true, JSON.stringify(result.steps)); +}); + +test('taking the pair starts the 27B before moving cognition, and stops the pair only after the moves', () => { + const profile = fleet.profiles['sparkqwen-pair-rc16']; + const current = { routes: profile.routes, residency: profile.residency, running: RC16_RUNNING }; + const plan = planPool({ + nodes: fleet.nodes, + variants: fleet.variants, + workloads: fleetWorkloads(), + pins: { ...fleet.pins, cordon: ['spark-node-01', 'spark-node-02'] }, + current, + policy: fleet.policy + }); + const { steps, interruptions, blocked } = planReconcile({ + plan, + current: { ...current, placements: placementsFor(RC16_RUNNING, fleet.variants) }, + workloads: fleetWorkloads() + }); + assert.deepEqual(interruptions, []); + assert.deepEqual(blocked, []); + const start = steps.find((step) => step.type === 'start'); + assert.equal(start.placement, 'qwen38-27b-nvfp4@spark-node-03'); + const routeSteps = steps.filter((step) => step.type === 'route'); + assert.equal(routeSteps.length, 7); + for (const route of routeSteps) { + assert.equal(route.to, 'qwen38-27b-nvfp4'); + assert.ok(route.after.includes(start.id), `${route.id} waits for the start`); + } + const stop = steps.find((step) => step.type === 'manual-stop'); + assert.equal(stop.placement, 'qwen3.8-flash-next-atlas@spark-node-01+spark-node-02'); + for (const route of routeSteps) assert.ok(dependsOn(steps, stop.id, route.id), `stop waits for ${route.id}`); + const upgrade = steps.find((step) => step.type === 'residency' && step.runtime === 'qwen38-27b-nvfp4'); + assert.deepEqual([upgrade.from, upgrade.to], ['auto', 'always']); + assert.ok(upgrade.after.includes(start.id)); +}); + +// One node with room for one big lane: moving alias `x` from A to B is a swap. +const SWAP_NODE = { + id: 'n1', + computeArchs: ['sm_121'], + memoryDomains: [{ id: 'u', kind: 'unified', capacityGb: 100 }] +}; +const swapVariants = [ + { + id: 'A@n1', + gatewayModel: 'A', + runtime: 'a', + nodes: ['n1'], + kind: 'chat', + requirements: { memoryGbPerNode: 60 }, + performance: { ttftMs: 2000, decodeTps: 20 } + }, + { + id: 'B@n1', + gatewayModel: 'B', + runtime: 'b', + nodes: ['n1'], + kind: 'chat', + requirements: { memoryGbPerNode: 60 }, + performance: { ttftMs: 300, decodeTps: 80 } + }, + { id: 'cloud', gatewayModel: 'cloud', external: true, kind: 'chat', performance: { ttftMs: 700, decodeTps: 100 } } +]; + +function swap({ candidates }) { + const workloads = [{ id: 'x', candidates, residency: 'always', demandPerHour: 50 }]; + const current = { routes: { x: 'A' }, residency: { a: 'always', b: 'auto' }, running: ['A@n1'] }; + const plan = planPool({ + nodes: [SWAP_NODE], + variants: swapVariants, + workloads, + current, + policy: { reserveGb: 0, headroomGb: 0, churnWeight: 0 } + }); + assert.equal(plan.routes.x.model, 'B'); + return planReconcile({ + plan, + current: { ...current, placements: placementsFor(['A@n1'], swapVariants) }, + workloads, + variants: swapVariants + }); +} + +test('a swap moves the alias to an interim member, drains the old lane, then starts the new one', () => { + const { steps, interruptions } = swap({ candidates: ['B', 'A', 'cloud'] }); + assert.deepEqual(interruptions, []); + const byType = (type) => steps.find((step) => step.type === type); + const interim = byType('interim-route'); + assert.deepEqual([interim.alias, interim.to], ['x', 'cloud']); + const stop = byType('stop'); + assert.equal(stop.placement, 'A@n1'); + assert.ok(stop.after.includes(interim.id)); + const downgrade = steps.find((step) => step.type === 'residency' && step.runtime === 'a'); + assert.ok(stop.after.includes(downgrade.id), 'keep-warm is dropped before the stop'); + const start = byType('start'); + assert.ok(start.after.includes(stop.id)); + const route = byType('route'); + assert.deepEqual([route.to, route.after.includes(start.id)], ['B', true]); +}); + +test('a swap with no interim member reports the interruption', () => { + const { steps, interruptions } = swap({ candidates: ['B', 'A'] }); + assert.deepEqual(interruptions, [{ alias: 'x', placement: 'A@n1', until: 'B@n1' }]); + assert.ok(steps.some((step) => step.type === 'stop')); + assert.ok(!steps.some((step) => step.type === 'interim-route')); +}); + +test('a new placement on a node without the weights prefetches first', () => { + const node = { ...SWAP_NODE, id: 'mac', computeArchs: ['apple-metal'], weights: [] }; + const variants = [ + { + id: 'E', + gatewayModel: 'E', + runtime: 'e', + kind: 'embedding', + requirements: { memoryGbPerNode: 8, computeArchs: ['apple-metal'], recipeId: 'mlx-e' }, + performance: { durationMs: 50 } + } + ]; + const workloads = [{ id: 'emb', candidates: ['E'], residency: 'always', demandPerHour: 100 }]; + const plan = planPool({ nodes: [node], variants, workloads, current: {}, policy: { reserveGb: 0 } }); + const { steps } = planReconcile({ + plan, + current: { placements: [], routes: {} }, + workloads, + variants, + nodes: [node] + }); + const prefetch = steps.find((step) => step.type === 'prefetch'); + const start = steps.find((step) => step.type === 'start'); + assert.equal(prefetch.placement, 'E@mac'); + assert.ok(start.after.includes(prefetch.id)); +}); + +test('a failed start skips the route flip and the old lane stop, so the old lane keeps serving', async () => { + const { steps } = swap({ candidates: ['B', 'A', 'cloud'] }); + const calls = []; + const actuator = { + prefetch: async () => {}, + start: async (step) => { + calls.push(step.id); + throw new Error('admission refused'); + }, + stop: async (step) => calls.push(step.id), + route: async (step) => calls.push(step.id), + residency: async (step) => calls.push(step.id), + manual: async () => {} + }; + const result = await executeReconcile(steps, actuator); + assert.equal(result.ok, false); + const route = steps.find((step) => step.type === 'route'); + assert.equal(result.status[route.id], 'skipped'); + assert.match(result.errors[steps.find((step) => step.type === 'start').id], /admission refused/); +}); + +test('make-before-break: a start that fits runs before the old lane stops', async () => { + const profile = fleet.profiles['sparkqwen-pair-rc16']; + const current = { routes: profile.routes, residency: profile.residency, running: RC16_RUNNING }; + const plan = planPool({ + nodes: fleet.nodes, + variants: fleet.variants, + workloads: fleetWorkloads(), + pins: { ...fleet.pins, cordon: ['spark-node-01', 'spark-node-02'] }, + current, + policy: fleet.policy + }); + const { steps } = planReconcile({ + plan, + current: { ...current, placements: placementsFor(RC16_RUNNING, fleet.variants) }, + workloads: fleetWorkloads() + }); + const order = []; + const record = async (step) => order.push(step.id); + const result = await executeReconcile( + steps, + { prefetch: record, start: record, stop: record, route: record, residency: record, manual: record }, + { concurrency: 1 } + ); + assert.equal(result.ok, true, JSON.stringify(result.errors)); + const at = (type) => order.findIndex((id) => id.startsWith(`${type}:`)); + assert.ok(at('start') < at('route')); + assert.ok(at('route') < at('manual-stop')); + assert.equal(order.length, steps.length); +}); diff --git a/test/pool-sim.test.mjs b/test/pool-sim.test.mjs new file mode 100644 index 0000000..80acc90 --- /dev/null +++ b/test/pool-sim.test.mjs @@ -0,0 +1,199 @@ +// Pool simulator tests: the engine's model of today's LLooM behaviour, and the +// fleet scenarios under test/fixtures/pool/scenarios run with and without the +// pool controller. Everything is simulated; nothing touches a gateway. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { checkExpectations, runPoolSimulation } from '../src/pool-sim.mjs'; +import { formatPoolSimulation, loadPoolScenario, runPoolSimulateCommand } from '../src/pool-sim-cli.mjs'; + +const SCENARIOS = fileURLToPath(new URL('./fixtures/pool/scenarios/', import.meta.url)); + +const spark = (id, extra = {}) => ({ + id, + platformId: 'linux-arm64', + computeArchs: ['sm_121'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 128 }], + links: [], + ...extra +}); + +const chat = (id, node, extra = {}) => ({ + id: `${id}@${node}`, + gatewayModel: id, + kind: 'chat', + runtime: id, + nodes: [node], + requirements: { memoryGbPerNode: 40 }, + performance: { ttftMs: 500, decodeTps: 50 }, + sim: { loadSeconds: 60, maxSequences: 4 }, + ...extra +}); + +function tiny(overrides = {}) { + return { + name: 'tiny', + seed: 3, + durationHours: 2, + policy: { reserveGb: 8, headroomGb: 4 }, + nodes: [spark('s1')], + variants: [chat('m', 's1')], + workloads: [{ id: 'chat', candidates: ['m'], residency: 'auto', demand: { perHour: 60 }, slo: { ttftMs: 2000 } }], + initial: { running: [], routes: { chat: 'm' }, residency: { m: 'auto' } }, + ...overrides + }; +} + +test('a cold on-demand lane loads once, and the first requests wait for it', async () => { + const report = await runPoolSimulation(tiny(), { mode: 'off' }); + assert.equal(report.lanes.loads, 1); + assert.ok(report.requests.total > 80); + assert.equal(report.requests.failed, 0); + assert.equal(report.requests.coldWaits, 1); + // The first request waited for the 60 s load; everything else was warm. + assert.ok(report.workloads.chat.ttftP95Ms < 2000); + assert.ok(report.requests.sloAttainment < 1); +}); + +test('the simulation is deterministic for a seed', async () => { + const first = await runPoolSimulation(tiny(), { mode: 'auto' }); + const second = await runPoolSimulation(tiny(), { mode: 'auto' }); + assert.deepEqual(first, second); + const reseeded = await runPoolSimulation(tiny({ seed: 4 }), { mode: 'auto' }); + assert.notEqual(reseeded.requests.total, first.requests.total); +}); + +test('admission evicts an idle on-demand lane but never a keep-warm lane', async () => { + const scenario = tiny({ + nodes: [spark('s1')], + variants: [ + chat('warm', 's1', { requirements: { memoryGbPerNode: 60 } }), + chat('idle', 's1', { requirements: { memoryGbPerNode: 40 } }), + chat('burst', 's1', { requirements: { memoryGbPerNode: 40 } }) + ], + workloads: [ + { id: 'w', candidates: ['warm'], residency: 'always', demand: { perHour: 30 } }, + { id: 'b', candidates: ['burst'], residency: 'auto', demand: { perHour: 6 } } + ], + initial: { + running: ['warm@s1', 'idle@s1'], + routes: { w: 'warm', b: 'burst' }, + residency: { warm: 'always', idle: 'auto', burst: 'auto' } + } + }); + const report = await runPoolSimulation(scenario, { mode: 'off' }); + assert.equal(report.lanes.evictions, 1); + assert.ok(report.final.lanes.includes('warm@s1')); + assert.ok(!report.final.lanes.includes('idle@s1')); + assert.equal(report.workloads.w.failed, 0); +}); + +test('a failed node fails in-flight requests and fails queued ones over to another member', async () => { + const scenario = tiny({ + nodes: [spark('s1'), spark('s2')], + variants: [chat('a', 's1'), chat('b', 's2')], + workloads: [ + { id: 'chat', candidates: ['a', 'b'], residency: 'always', outputTokens: 2000, demand: { perHour: 120 } } + ], + initial: { running: ['a@s1', 'b@s2'], routes: { chat: 'a' }, residency: { a: 'always', b: 'always' } }, + events: [{ atHours: 1, type: 'fail', nodes: ['s1'] }] + }); + const report = await runPoolSimulation(scenario, { mode: 'off' }); + assert.ok(report.requests.failed > 0, 'requests in flight on s1 fail'); + assert.ok(report.requests.failed < 10); + assert.ok(report.workloads.chat.served > report.requests.total * 0.95, 'the rest move to b on s2'); +}); + +test('an unmanaged lane is never cold started; an external member covers instead', async () => { + const scenario = tiny({ + variants: [ + chat('pair', 's1', { managed: false }), + { id: 'cloud', gatewayModel: 'cloud', external: true, kind: 'chat', performance: { ttftMs: 800, decodeTps: 100 } } + ], + workloads: [{ id: 'chat', candidates: ['pair', 'cloud'], residency: 'auto', demand: { perHour: 30 } }], + initial: { running: [], routes: { chat: 'pair' }, residency: {} } + }); + const report = await runPoolSimulation(scenario, { mode: 'off' }); + assert.equal(report.lanes.loads, 0); + assert.equal(report.requests.external, report.requests.total); +}); + +test('the memory guard aborts an underestimated load; with the guard off the node hangs', async () => { + const scenario = tiny({ + variants: [ + chat('m', 's1', { requirements: { memoryGbPerNode: 40 }, sim: { loadSeconds: 60, memoryGbPerNode: 130 } }) + ] + }); + const guarded = await runPoolSimulation(scenario, { mode: 'off' }); + assert.equal(guarded.safety.memoryViolations.length, 0); + assert.ok(guarded.lanes.guardAborts > 0); + const unguarded = await runPoolSimulation({ ...scenario, memoryGuard: { enabled: false } }, { mode: 'off' }); + assert.ok(unguarded.safety.memoryViolations.length > 0); + assert.ok(unguarded.safety.nodeHangs > 0); +}); + +test('expectations check paths with min, max, equals and includes', () => { + const report = { a: { b: 2 }, list: ['x'], routes: { r: 'm' } }; + const checks = checkExpectations(report, { + 'a.b': { min: 1, max: 2 }, + list: { includes: 'x' }, + 'routes.r': { equals: 'm' }, + 'a.missing': { min: 0 } + }); + assert.deepEqual( + checks.map((check) => check.ok), + [true, true, true, false] + ); +}); + +test('scenarios inherit a base and merge arrays by id', async () => { + const scenario = await loadPoolScenario(`${SCENARIOS}mac-joins.json`); + assert.equal(scenario.name, 'mac-joins'); + assert.ok(scenario.nodes.some((node) => node.id === 'mac-studio')); + assert.ok(!scenario.nodes.some((node) => node.id === 'spark-node-01')); + const video = scenario.workloads.find((workload) => workload.id === 'video'); + assert.deepEqual(video.candidates, ['Wan-AI/Wan2.2-TI2V-5B-Diffusers'], 'base workloads are kept'); + const chatCapable = scenario.workloads.find((workload) => workload.id === 'chat-capable'); + assert.equal(chatCapable.candidates[0], 'qwen3.8-flash-next-mlx'); + assert.equal(chatCapable.slo.ttftMs, 3000, 'overlay fields merge into the base workload'); +}); + +// Every scenario's own expectations, in the modes it states them for. +const scenarioFiles = readdirSync(SCENARIOS).filter((file) => file.endsWith('.json') && !file.includes('base')); + +for (const file of scenarioFiles) { + test(`scenario ${file} meets its expectations`, async () => { + const reports = await runPoolSimulateCommand(`${SCENARIOS}${file}`, { modes: ['off', 'advise', 'auto'] }); + for (const [mode, report] of Object.entries(reports)) { + assert.equal(report.safety.memoryViolations.length, 0, `${mode}: no memory violations`); + const failed = report.checks.filter((check) => !check.ok); + assert.deepEqual(failed, [], `${mode}: ${JSON.stringify(failed)}`); + } + // Advise never changes the fleet: it serves exactly what off serves. + assert.deepEqual(reports.advise.requests, reports.off.requests); + assert.deepEqual(reports.advise.final.lanes, reports.off.final.lanes); + assert.ok(formatPoolSimulation(reports).includes(reports.auto.scenario)); + }); +} + +test('pair crash: the controller keeps cognition local and warm where today it cold starts and spills to the cloud', async () => { + const { off, auto } = await runPoolSimulateCommand(`${SCENARIOS}pair-crash.json`, { modes: ['off', 'auto'] }); + const cognition = ['chat-capable', 'chat-fast', 'chat-classifier', 'multimodal', 'enntity-presence', 'omp', 'ds4f']; + const p95 = (report) => Math.max(...cognition.map((id) => report.workloads[id].ttftP95Ms)); + assert.ok(off.requests.external > 0, 'today some cognition spills to the cloud'); + assert.equal(auto.requests.external, 0); + assert.ok(p95(auto) < p95(off) / 5, `cognition p95 ${p95(auto)} ms vs ${p95(off)} ms`); + assert.ok(auto.lanes.loadGb < off.lanes.loadGb, 'fewer reloads'); + assert.equal(auto.final.routes['chat-capable'], 'qwen3.8-flash-next-atlas', 'back on the pair after recovery'); +}); + +test('mac joins: weights are prefetched, cognition moves without dropping a request, and 03 is freed', async () => { + const { auto } = await runPoolSimulateCommand(`${SCENARIOS}mac-joins.json`, { modes: ['auto'] }); + const apply = auto.controller.decisions.find((entry) => entry.action === 'apply'); + const types = apply.steps.map((step) => step.type); + assert.ok(types.indexOf('prefetch') < types.indexOf('start')); + assert.ok(types.lastIndexOf('route') < types.indexOf('stop')); + assert.equal(auto.requests.failed, 0); + assert.ok(!auto.final.lanes.includes('qwen38-27b-nvfp4@spark-node-03')); +}); From a45efb8c965c106eee79d279c121fdcae131f3f6 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 12:23:20 -0700 Subject: [PATCH 3/9] Add pool membership: enrollment tokens, node leases and planner bridge Nodes can join a head with a one-time enrollment token and keep a lease with heartbeats from `serve`, so the pool planner knows who is in the pool. Membership is inert until a token is issued; static cluster.nodes config and `cluster add-node` are unchanged. - src/pool-membership.mjs: pure membership core (tokens, enroll, heartbeat, cordon/drain/leave, ready/suspect/lost/left leases, planner node shape). Tokens and node secrets are stored only as SHA-256 hashes. - src/pool-membership-store.mjs: pool-membership.json next to the config, loaded on first use, written atomically (0600) only on changes. - Head routes: POST /gateway/pool/enroll (token only, rate limited) and /gateway/pool/heartbeat (node secret only) authenticate in their handlers; GET /gateway/pool/members is admin-read; tokens and cordon/uncordon/drain are admin writes. - src/pool-node.mjs: `lloom pool join` and the serve heartbeat loop with backoff; the secret lives in a private file or an env var. - CLI: lloom pool token create | members | cordon | uncordon | drain | join. - Planner: members merge with configured nodes (config wins on endpoint and auth; membership adds liveness, cordon and inventory). Enrolled-only nodes are planned but flagged as needing `cluster add-node` to route. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + README.md | 1 + bin/lloom.mjs | 74 +++++- docs/clusters.md | 2 +- docs/pool-planning.md | 43 +++- src/deployment-fence.mjs | 2 +- src/node-onboarding.mjs | 13 +- src/pool-catalog.mjs | 68 +++++- src/pool-cli.mjs | 126 +++++++++- src/pool-inventory.mjs | 2 + src/pool-membership-store.mjs | 78 ++++++ src/pool-membership.mjs | 374 +++++++++++++++++++++++++++++ src/pool-node.mjs | 202 ++++++++++++++++ src/security.mjs | 9 +- src/server.mjs | 208 +++++++++++++++- test/pool-catalog.test.mjs | 82 +++++++ test/pool-cli.test.mjs | 216 ++++++++++++++++- test/pool-membership-http.test.mjs | 307 +++++++++++++++++++++++ test/pool-membership.test.mjs | 321 +++++++++++++++++++++++++ 19 files changed, 2088 insertions(+), 42 deletions(-) create mode 100644 src/pool-membership-store.mjs create mode 100644 src/pool-membership.mjs create mode 100644 src/pool-node.mjs create mode 100644 test/pool-membership-http.test.mjs create mode 100644 test/pool-membership.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 58459c1..88f2a43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to LLooM will be documented in this file. The format follows ### Added +- Pool membership: nodes join a head with a one-time enrollment token (`lloom pool token create`, `lloom pool join`) and keep a lease with heartbeats from `serve`. The head lists members (`lloom pool members`), cordons, drains and uncordons them, and `lloom pool plan` uses their liveness, cordons and inventory. Membership is inert until a token is issued, keeps only hashed secrets, and does not grant routing: an enrolled-only node still needs `lloom cluster add-node`. See `docs/pool-planning.md`. + - `lloom pool plan`: a dry-run placement plan across every cluster node. It proposes which lanes run where, which residency they get and how each alias routes, and diffs that against what runs now. It covers every required alias first, then minimises demand-weighted latency and the load cost of changes, within per-domain memory headroom, direct-link tensor parallelism, compute architecture and pins. Nodes report a pool inventory through `/gateway/node?inventory=1`. Recipes may declare `requirements.computeArchs`, `requirements.tensorParallel` and `settings.maxSequences`. `/gateway/metrics` adds `requested` buckets, which give per-alias demand. See `docs/pool-planning.md`. - LLooM Hear CPU audio analysis, with structured estimates, optional dashboard images, and opt-in upstream interpretation. File and URL inputs require operator configuration. diff --git a/README.md b/README.md index cd07a57..34942cc 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ lloom route q38fn --suspend-member qwen3.8-flash-next --apply --yes # keep lloom runtime-stop qwen38-flash-next-cluster # stop local TP-2 after suspension lloom route q38fn --resume-member qwen3.8-flash-next --apply --yes # restore preferred-local recovery lloom pool plan --cordon spark-node-02 # dry-run placement across the cluster +lloom pool members # nodes that joined with `lloom pool join` lloom integrate omp --apply --yes lloom add-model mlx-community/Qwen3.6-27B-OptiQ-4bit --keep-warm --default --go # Mac: install, download, start lloom remove-model mlx-community/Qwen3.6-27B-OptiQ-4bit # preview safe cleanup diff --git a/bin/lloom.mjs b/bin/lloom.mjs index 00cc0be..36c2dae 100755 --- a/bin/lloom.mjs +++ b/bin/lloom.mjs @@ -36,7 +36,15 @@ import { selectedRecipeIdFromCommunityPlan } from '../src/community-client.mjs'; import { loadConfig } from '../src/config.mjs'; -import { POOL_USAGE, createPoolPlanReport, formatPoolPlanReport, poolPlanOptions } from '../src/pool-cli.mjs'; +import { + POOL_USAGE, + createPoolPlanReport, + formatPoolMembers, + formatPoolPlanReport, + poolPlanOptions, + poolTokenOptions +} from '../src/pool-cli.mjs'; +import { joinPool } from '../src/pool-node.mjs'; import { mutateConfigSource } from '../src/config-mutation.mjs'; import { runtimeControlTimeoutMs } from '../src/control-timeout.mjs'; import { createDoctorReport } from '../src/doctor.mjs'; @@ -231,6 +239,10 @@ Backends and runtimes: lloom down lloom cluster [status|doctor|discover] [--apply] [--id NAME] [--api-key-env NAME] [--json] lloom pool plan [--cordon NODE[,NODE]] [--pin ALIAS=MODEL] [--profile NAME] [--consider-installs] [--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json] + lloom pool token create [--ttl DURATION] [--uses N] [--label KEY=VALUE] [--json] + lloom pool members [--json] + lloom pool cordon|uncordon|drain [--reason TEXT] + lloom pool join --token-stdin [--node-id ID] [--endpoint URL] [--apply --yes] lloom deployment plan --plan FILE [--json] lloom deployment apply --plan FILE --nodes FILE [--operation-id ID] --apply --yes [--journal FILE] [--json] lloom deployment status --operation-id ID [--journal FILE] [--json] @@ -2502,6 +2514,66 @@ async function main() { }, pool: async ({ args, config }) => { const action = positional(args)[1] ?? 'plan'; + const json = hasFlag(args, '--json'); + if (action === 'token') { + if (positional(args)[2] !== 'create') throw new Error(POOL_USAGE); + const result = await gatewayRequest(config, '/gateway/pool/tokens', { + args, + method: 'POST', + body: poolTokenOptions({ + ttl: argValue(args, '--ttl'), + uses: argValue(args, '--uses'), + labels: argValues(args, '--label') + }), + timeoutMs: 10000 + }); + console.log( + json + ? JSON.stringify(result, null, 2) + : [ + `Enrollment token ${result.id} (shown once; expires ${result.expiresAt}):`, + result.token, + '', + 'On the joining node:', + ` lloom pool join --token-stdin --endpoint --apply --yes` + ].join('\n') + ); + return; + } + if (action === 'members') { + const result = await gatewayRequest(config, '/gateway/pool/members', { args, timeoutMs: 10000 }); + console.log(json ? JSON.stringify(result, null, 2) : formatPoolMembers(result)); + return; + } + if (['cordon', 'uncordon', 'drain'].includes(action)) { + const nodeId = positional(args)[2]; + if (!nodeId) throw new Error(POOL_USAGE); + const reason = argValue(args, '--reason'); + const result = await gatewayRequest(config, `/gateway/pool/nodes/${encodeURIComponent(nodeId)}/${action}`, { + args, + method: 'POST', + body: reason == null ? {} : { reason }, + timeoutMs: 10000 + }); + console.log(JSON.stringify(result, null, 2)); + return; + } + if (action === 'join') { + const headUrl = positional(args)[2]; + if (!headUrl || !hasFlag(args, '--token-stdin')) throw new Error(POOL_USAGE); + const apply = hasFlag(args, '--apply'); + const result = await joinPool(config, { + headUrl, + // The token is read only when it will be used. + token: apply && hasFlag(args, '--yes') ? await readBoundedStdinCredential(process.stdin) : null, + nodeId: argValue(args, '--node-id') ?? undefined, + endpoint: argValue(args, '--endpoint'), + apply, + yes: hasFlag(args, '--yes') + }); + console.log(JSON.stringify(result, null, 2)); + return; + } if (action !== 'plan') throw new Error(POOL_USAGE); const options = poolPlanOptions({ cordon: argValues(args, '--cordon'), diff --git a/docs/clusters.md b/docs/clusters.md index 38f29e6..7238e32 100644 --- a/docs/clusters.md +++ b/docs/clusters.md @@ -256,7 +256,7 @@ before contacting any node. The live dashboard renders node cards between the main LLooM chassis and model cards. Each node card reports architecture, accelerator type, CPU, RAM, GPU compute, GPU memory, temperature, and power when the host can report them. Unsupported fields render as unavailable rather than inventing values; Apple Silicon still reports its profile and shared-memory pressure even when per-GPU utilization is unavailable. Select a node for its complete profile. Replicated models connect to every target node and distributed models connect to every member node. -`GET /gateway/node?inventory=1` and `GET /gateway/cluster?inventory=1` add each node's pool inventory (compute architectures, model-root disk, cached recipe weights, installed images and fabric links). `lloom pool plan` uses it to propose placements across the whole cluster; see [`pool-planning.md`](pool-planning.md). +`GET /gateway/node?inventory=1` and `GET /gateway/cluster?inventory=1` add each node's pool inventory (compute architectures, model-root disk, cached recipe weights, installed images and fabric links). `lloom pool plan` uses it to propose placements across the whole cluster; see [`pool-planning.md`](pool-planning.md). Nodes can also join a head with an enrollment token and keep a lease (`lloom pool join`); membership adds liveness and cordons to planning but does not replace `cluster add-node` for routing. Before sending model traffic, use `lloom cluster doctor --runtime ID` to verify the required nodes and serving endpoint. Whole-topology doctor reports optional offline nodes and missing memory telemetry as warnings; they do not make unrelated runtimes unavailable. Then start a small replicated lane, verify requests alternate under equal load, and only then canary the distributed engine. diff --git a/docs/pool-planning.md b/docs/pool-planning.md index 78ab5c7..989fbaf 100644 --- a/docs/pool-planning.md +++ b/docs/pool-planning.md @@ -73,6 +73,47 @@ Override any of this per alias in config: The currently active routes are not pins; they are what the diff is measured against. To keep a hand-chosen route, pin it. +## Membership + +Nodes can also join a head themselves and keep a lease, so the planner always knows who is in the pool. Membership is optional and inert until used: the head writes nothing and accepts no enrollment until an operator issues a token. Static `cluster.nodes` config, `lloom cluster add-node` and on-demand polling keep working unchanged. + +```bash +# on the head +lloom pool token create --ttl 1h --uses 1 # prints a one-time enrollment token +lloom pool members # state, lease age, inventory, running lanes +lloom pool cordon spark-node-02 --reason "disk swap" +lloom pool drain spark-node-02 --reason rebalance +lloom pool uncordon spark-node-02 + +# on the joining node (token on stdin, never in argv) +lloom pool join http://head-private:8100 --token-stdin --endpoint http://10.10.10.2:8100 --apply --yes +``` + +**Enrollment.** `POST /gateway/pool/enroll` takes the token as a bearer credential and the node's `nodeId`, `endpoint` and optional `labels`. It returns a node secret and the lease length. A token expires (default one hour) and has a use count (default one). Labels set on the token override labels the node sends. Enrolling an existing node id again needs a different token, and rotates the node's secret. It keeps any cordon or drain, so a node cannot re-enroll its way out of one. A node cannot enroll with the head's own node id. Attempts are rate limited per client address. + +`lloom pool join` stores the secret in `pool-node-secret` (mode 0600) next to the node's config and sets: + +```json +"pool": { "head": { "url": "http://head-private:8100", "nodeId": "spark-node-02", "secretFile": "pool-node-secret" } } +``` + +`secretEnv: "NAME"` reads the secret from an environment variable instead. A relative `secretFile` resolves against the config directory, and must be a private regular file. `intervalMs` sets the heartbeat interval (default 10 s). Without `--apply --yes`, `join` only shows what it would do and does not read the token. + +**Leases.** With `pool.head` set, `serve` sends `POST /gateway/pool/heartbeat` with the cached pool inventory, the running runtimes and host telemetry, authenticated with the node secret. It backs off exponentially, up to five minutes, while the head is unreachable. The head's lease is 30 s by default. A member is `ready` while its lease is current and `suspect` once it expires. It is `lost` 90 s after its last heartbeat (`pool.membership.leaseMs` and `lostAfterMs` on the head). `left` means the node left the pool. The head keeps only the inventory fields the planner reads. + +**State.** The head keeps membership in `pool-membership.json` next to its config file (mode 0600, atomic writes). The file is written when a token is issued, a node enrolls, or a node is cordoned, drained or uncordoned. Heartbeats are not written: after a head restart, members show their last heartbeat time until they next check in. The file holds SHA-256 hashes of tokens and node secrets, never the values. Both are 256-bit random strings. + +**Planning.** `lloom pool plan` reads `GET /gateway/pool/members` and merges members with configured nodes: + +- A configured node keeps its endpoint, credentials and resources. +- Membership adds liveness (a `lost` or `left` member is unavailable) and cordon or drain (the node hosts no placements). +- Membership adds inventory when it is more complete than what the head already has. +- An offline plan reads the saved file. It applies cordons, drains and departures there, but not lease ages. + +A node that only enrolled is planned on, but it is not routable. The head has no credential for that node's gateway and no model catalog for it, and enrollment does not create either. So the plan warns that the node needs `lloom cluster add-node --apply --yes` before the head can route there, and marks it `enrolled only`. Enrollment does not grant routing: the endpoint a node reports is not used for traffic, and a configured endpoint always wins. + +**Auth.** Enrollment accepts only an enrollment token, and heartbeats only a node secret. Admin and inference keys are rejected on both routes, and neither route returns or grants an API key. A node secret is not accepted on any other route. `GET /gateway/pool/members` is admin-read. Token creation and cordon, drain and uncordon are admin writes, so remote use needs `security.allowRemoteAdmin` and an admin key. As with cluster keys, put the head on a private network or behind TLS: plain HTTP carries tokens and secrets in the clear. + ## Constraints and objective - **Memory per domain.** Unified memory on Spark and Apple Silicon. Device memory, plus system memory for host-side state, on discrete GPUs. @@ -98,7 +139,7 @@ The search places workloads greedily, pinned and hot ones first, sharing a place The text output lists: -- each node's memory budget and resident set; +- each node's memory budget and resident set, and its membership state; - every placement by tier, with the aliases it serves; - the changes from now: - route changes; diff --git a/src/deployment-fence.mjs b/src/deployment-fence.mjs index cf7eb82..0cb265c 100644 --- a/src/deployment-fence.mjs +++ b/src/deployment-fence.mjs @@ -295,7 +295,7 @@ function fenceRecordShape(record) { return record; } -async function writeAtomicJson(filePath, value) { +export async function writeAtomicJson(filePath, value) { const directory = path.dirname(filePath); await fs.mkdir(directory, { recursive: true, mode: 0o700 }); const temporary = `${filePath}.${process.pid}.${randomUUID()}.tmp`; diff --git a/src/node-onboarding.mjs b/src/node-onboarding.mjs index 5497232..65d6070 100644 --- a/src/node-onboarding.mjs +++ b/src/node-onboarding.mjs @@ -2,6 +2,7 @@ import { promises as fs } from 'node:fs'; import { randomUUID } from 'node:crypto'; import { mutateConfigSource } from './config-mutation.mjs'; import { currentNodeId, federatedNodeConfigFromSnapshot } from './cluster.mjs'; +import { normalizeNodeEndpoint, validNodeId } from './pool-membership.mjs'; export async function readBoundedStdinCredential(stream, { maxBytes = 16384 } = {}) { const chunks = []; @@ -35,18 +36,10 @@ export async function addAuthenticatedNode( fetchFn = fetch } = {} ) { - if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(nodeId ?? '')) throw new Error('Invalid node identity'); + if (!validNodeId(nodeId)) throw new Error('Invalid node identity'); if (nodeId === currentNodeId(config)) throw new Error('Cannot replace the local node with a federation peer'); - let url; - try { - url = new URL(endpoint); - } catch { - throw new Error('Invalid node endpoint'); - } - if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash) - throw new Error('Node endpoint must be an HTTP(S) URL without credentials, query, or fragment'); + const base = normalizeNodeEndpoint(endpoint); if (apply && !yes) throw new Error('Adding this node requires --apply --yes'); - const base = url.toString().replace(/\/+$/, ''); let response; try { response = await fetchFn(base + '/gateway/node', { diff --git a/src/pool-catalog.mjs b/src/pool-catalog.mjs index 291df8d..f30116b 100644 --- a/src/pool-catalog.mjs +++ b/src/pool-catalog.mjs @@ -7,6 +7,7 @@ import { benchmarkRankingEligible, benchmarkScore } from './benchmarks.mjs'; import { clusterNodes, modelTargets, runtimeResourcesByNode } from './cluster.mjs'; import { resolveRouteTarget } from './config-profiles.mjs'; import { inventoryFromProfile } from './pool-inventory.mjs'; +import { memberPlannerNode } from './pool-membership.mjs'; import { residencyPolicy } from './runtime-preferences.mjs'; export const POOL_CATALOG_VERSION = 1; @@ -14,6 +15,7 @@ const LIVE_METRIC_MIN_REQUESTS = 5; const PERIOD_HOURS = { today: 24, '7d': 168, '30d': 720 }; const RESIDENCY_RANK = { auto: 0, preferred: 1, always: 2 }; const ARCH_BY_ACCELERATOR = { gb10: 'sm_121', 'dgx-spark': 'sm_121' }; +const INVENTORY_RANK = { none: 0, config: 0, profile: 1, reported: 2 }; function object(value) { return value && typeof value === 'object' && !Array.isArray(value) ? value : {}; @@ -506,6 +508,64 @@ export function plannerNodesFromStatus(config = {}, nodeStatuses = {}, { env = p .sort((left, right) => left.id.localeCompare(right.id)); } +function preferred(value, fallback) { + return (Array.isArray(value) ? value.length : value != null) ? value : fallback; +} + +// Nodes that joined through pool membership (see pool-membership.mjs). A +// configured cluster node keeps its endpoint, credentials and resources; +// membership adds liveness, cordon/drain and its inventory when that is more +// complete than what the head already knows. The head has no credential or +// model catalog for an enrolled-only node, so it is planned but not routable +// until `lloom cluster add-node`. With `liveness: false` (saved state, no live +// gateway) lease ages are not evidence; only cordons and departures apply. +export function mergeMembershipNodes(nodes = [], members = [], { liveness = true } = {}) { + const byId = new Map(nodes.map((node) => [node.id, node])); + const warnings = []; + for (const member of members) { + const joined = memberPlannerNode(member); + const available = liveness ? joined.available : member.state !== 'left'; + const configured = byId.get(member.id); + if (!configured) { + byId.set(member.id, { ...joined, available, routable: false }); + if (member.state !== 'left') { + warnings.push( + `${member.id} joined the pool by enrollment only; placements there need ` + + `\`lloom cluster add-node ${member.id} ${member.endpoint} --apply --yes\` before the head can route to it` + ); + } + continue; + } + let inventory = {}; + if (INVENTORY_RANK[joined.inventory] > INVENTORY_RANK[configured.inventory]) { + const budgetGb = configured.memoryDomains?.length === 1 ? configured.memoryDomains[0].budgetGb : undefined; + const memoryDomains = preferred(joined.memoryDomains, configured.memoryDomains); + inventory = { + platformId: preferred(joined.platformId, configured.platformId), + computeArchs: preferred(joined.computeArchs, configured.computeArchs), + accelerators: preferred(joined.accelerators, configured.accelerators), + memoryDomains: + budgetGb != null && memoryDomains.length === 1 ? [{ ...memoryDomains[0], budgetGb }] : memoryDomains, + weights: joined.weights, + images: joined.images, + inventory: joined.inventory + }; + } + byId.set( + member.id, + compact({ + ...configured, + ...inventory, + links: unique([...(configured.links ?? []), ...joined.links]).sort(), + available: configured.available !== false && available, + cordoned: configured.cordoned === true || joined.cordoned || null, + membership: member.state + }) + ); + } + return { nodes: [...byId.values()].sort((left, right) => left.id.localeCompare(right.id)), warnings }; +} + export function buildPoolPlanInputs({ config = {}, nodeStatuses = {}, @@ -517,12 +577,18 @@ export function buildPoolPlanInputs({ routes = {}, considerInstalls = false, policy = {}, + members = [], + membershipLiveness = true, env = process.env } = {}) { const catalog = buildPoolCatalog({ config, recipes, benchmarks, metrics, nodeStatuses, env }); const variants = considerInstalls ? catalog.variants : catalog.variants.filter((v) => v.source !== 'recipe'); + const { nodes, warnings } = mergeMembershipNodes(plannerNodesFromStatus(config, nodeStatuses, { env }), members, { + liveness: membershipLiveness + }); return { - nodes: plannerNodesFromStatus(config, nodeStatuses, { env }), + nodes, + warnings, variants, workloads: poolWorkloadsFromConfig(config, { metrics }), pins: poolPinsFromConfig(config, { profile, cordon, routes }), diff --git a/src/pool-cli.mjs b/src/pool-cli.mjs index 854f82e..3c9f5e4 100644 --- a/src/pool-cli.mjs +++ b/src/pool-cli.mjs @@ -1,6 +1,8 @@ // `lloom pool plan`: a read-only pool placement plan for the installed config. -// Live node status, inventory and 7-day metrics come from the running gateway; -// without one, the plan uses the config plus this machine's own inventory. +// Live node status, inventory, membership and 7-day metrics come from the +// running gateway; without one, the plan uses the config, the saved pool +// membership and this machine's own inventory. Also the report helpers for +// `lloom pool token|members|cordon|uncordon|drain|join`. import { loadBenchmarkEvidence } from './benchmarks.mjs'; import { currentNodeId } from './cluster.mjs'; import { readProfile } from './config-profiles.mjs'; @@ -8,12 +10,93 @@ import { profileMachine } from './machine-profile.mjs'; import { modelRootCandidates } from './model-files.mjs'; import { buildPoolPlanInputs } from './pool-catalog.mjs'; import { collectNodeInventory } from './pool-inventory.mjs'; +import { createPoolMembership } from './pool-membership.mjs'; +import { poolMembershipOptions, poolMembershipPath, readPoolMembershipState } from './pool-membership-store.mjs'; import { planPool } from './pool-planner.mjs'; import { loadRecipes } from './recipes.mjs'; -export const POOL_USAGE = +export const POOL_USAGE = [ 'Usage: lloom pool plan [--cordon NODE[,NODE]] [--pin ALIAS=MODEL] [--profile NAME] [--consider-installs] ' + - '[--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json]'; + '[--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json]', + ' lloom pool token create [--ttl DURATION] [--uses N] [--label KEY=VALUE] [--json]', + ' lloom pool members [--json]', + ' lloom pool cordon|uncordon|drain [--reason TEXT]', + ' lloom pool join --token-stdin [--node-id ID] [--endpoint URL] [--apply --yes]' +].join('\n'); + +const DURATION_UNITS = { s: 1000, m: 60000, h: 3600000, d: 86400000 }; + +// `90` is seconds; `30m`, `2h` and `7d` carry a unit. +export function parseDurationMs(value) { + const match = String(value ?? '') + .trim() + .match(/^(\d+(?:\.\d+)?)([smhd]?)$/); + const ms = match ? Number(match[1]) * DURATION_UNITS[match[2] || 's'] : NaN; + if (!Number.isSafeInteger(Math.round(ms)) || ms < 1000) { + throw new Error(`Expected a duration such as 90, 30m, 2h or 7d; got ${value}`); + } + return Math.round(ms); +} + +export function poolTokenOptions({ ttl, uses, labels = [] } = {}) { + const options = {}; + if (ttl != null) options.ttlMs = parseDurationMs(ttl); + if (uses != null) { + if (!/^\d+$/.test(String(uses))) throw new Error('--uses must be a positive integer'); + options.uses = Number(uses); + } + if (labels.length) { + options.labels = Object.fromEntries( + labels.map((label) => { + const index = String(label).indexOf('='); + if (index < 1) throw new Error(`--label expects KEY=VALUE, got ${label}`); + return [label.slice(0, index), label.slice(index + 1)]; + }) + ); + } + return options; +} + +function ago(iso, now) { + const seconds = Math.max(0, Math.round((now() - Date.parse(iso)) / 1000)); + return seconds < 120 + ? `${seconds}s ago` + : seconds < 7200 + ? `${Math.round(seconds / 60)}m ago` + : `${Math.round(seconds / 3600)}h ago`; +} + +export function formatPoolMembers(result, { now = Date.now } = {}) { + const members = result?.members ?? []; + if (!members.length) return 'No pool members. Issue a token with `lloom pool token create`.'; + const rows = members.map((member) => { + const flags = [member.cordoned && 'cordoned', member.draining && 'draining'].filter(Boolean); + const state = [member.state, ...flags].join(', ') + (flags.length && member.reason ? ` (${member.reason})` : ''); + const inventory = member.inventory; + const memoryGb = (inventory?.memoryDomains ?? []).reduce((sum, domain) => sum + (domain.capacityGb ?? 0), 0); + const facts = inventory + ? [...(inventory.computeArchs ?? []), memoryGb ? `${Math.round(memoryGb)} GB` : null].filter(Boolean).join(', ') + : 'no inventory yet'; + return [ + member.id, + state, + member.endpoint, + ago(member.lastSeenAt, now), + facts || '-', + (member.running ?? []).join(', ') || '-' + ]; + }); + const header = ['NODE', 'STATE', 'ENDPOINT', 'LAST SEEN', 'INVENTORY', 'RUNNING']; + const widths = header.map((title, column) => Math.max(title.length, ...rows.map((row) => row[column].length))); + return [header, ...rows] + .map((row) => + row + .map((cell, column) => cell.padEnd(widths[column])) + .join(' ') + .trimEnd() + ) + .join('\n'); +} const METRICS_PERIOD = '7d'; @@ -69,12 +152,21 @@ export function poolPlanOptions({ async function liveState(config, gatewayRequest) { const cluster = await gatewayRequest(config, '/gateway/cluster?inventory=1'); const metrics = await gatewayRequest(config, `/gateway/metrics?period=${METRICS_PERIOD}`).catch(() => null); + // A gateway that predates pool membership has no members route. + const membership = await gatewayRequest(config, '/gateway/pool/members').catch(() => null); return { nodeStatuses: cluster?.cluster?.nodes ?? {}, - metrics: metrics ? { ...metrics, period: METRICS_PERIOD } : null + metrics: metrics ? { ...metrics, period: METRICS_PERIOD } : null, + members: membership?.members ?? [], + membershipLiveness: true }; } +async function savedMembers(config, deps) { + const state = await deps.readPoolMembership(config); + return state ? createPoolMembership({ ...poolMembershipOptions(config), state }).members() : []; +} + async function localState(config, deps) { const profile = await deps.profileMachine(); const inventory = await deps.collectNodeInventory({ @@ -83,7 +175,12 @@ async function localState(config, deps) { recipes: await deps.loadRecipes(), modelRoots: await deps.modelRootCandidates({ config }) }); - return { nodeStatuses: { [currentNodeId(config)]: { reachable: true, profile, inventory } }, metrics: null }; + return { + nodeStatuses: { [currentNodeId(config)]: { reachable: true, profile, inventory } }, + metrics: null, + members: await savedMembers(config, deps), + membershipLiveness: false + }; } export async function createPoolPlanReport(config, options = {}, overrides = {}) { @@ -96,6 +193,7 @@ export async function createPoolPlanReport(config, options = {}, overrides = {}) loadBenchmarkEvidence, modelRootCandidates, readProfile, + readPoolMembership: (target) => readPoolMembershipState(poolMembershipPath(target)), ...overrides }; let state = null; @@ -123,9 +221,12 @@ export async function createPoolPlanReport(config, options = {}, overrides = {}) cordon: options.cordon ?? [], routes: options.routes ?? {}, considerInstalls: options.considerInstalls === true, - policy: options.policy ?? {} + policy: options.policy ?? {}, + members: state.members, + membershipLiveness: state.membershipLiveness }); - const plan = planPool(inputs); + const planned = planPool(inputs); + const plan = { ...planned, warnings: [...inputs.warnings, ...planned.warnings] }; return { ok: plan.ok, dryRun: true, @@ -133,9 +234,11 @@ export async function createPoolPlanReport(config, options = {}, overrides = {}) nodes: inputs.nodes.map((node) => ({ id: node.id, available: node.available, - cordoned: inputs.pins.cordon.includes(node.id), + cordoned: node.cordoned === true || inputs.pins.cordon.includes(node.id), inventory: node.inventory, - computeArchs: node.computeArchs + computeArchs: node.computeArchs, + membership: node.membership ?? null, + routable: node.routable !== false })), counts: { workloads: inputs.workloads.length, variants: inputs.variants.length }, pins: inputs.pins, @@ -173,8 +276,9 @@ export function formatPoolPlanReport(report) { `${domain.id} ${gb(domain.residentGb)} resident of ${gb(domain.budgetGb)} (${gb(domain.freeGb)} free)` ) .join('; '); + const member = node.membership ? ` [member ${node.membership}${node.routable ? '' : ', enrolled only'}]` : ''; lines.push( - ` ${node.id.padEnd(16)} ${summary?.status ?? 'unknown'}${memory ? ` ${memory}` : ''} [inventory: ${node.inventory}]` + ` ${node.id.padEnd(16)} ${summary?.status ?? 'unknown'}${memory ? ` ${memory}` : ''} [inventory: ${node.inventory}]${member}` ); } diff --git a/src/pool-inventory.mjs b/src/pool-inventory.mjs index 0442215..9007e1a 100644 --- a/src/pool-inventory.mjs +++ b/src/pool-inventory.mjs @@ -48,6 +48,8 @@ export function inventoryFromProfile(profile = {}) { return { version: NODE_INVENTORY_VERSION, complete: false, + ...(profile.platformId ? { platformId: profile.platformId } : {}), + accelerators: Array.isArray(profile.accelerators) ? [...profile.accelerators] : [], computeArchs: computeArchsForProfile(profile), memoryDomains: capacityDomains(profile), links: profileLinks(profile) diff --git a/src/pool-membership-store.mjs b/src/pool-membership-store.mjs new file mode 100644 index 0000000..83a54c4 --- /dev/null +++ b/src/pool-membership-store.mjs @@ -0,0 +1,78 @@ +// Head-side persistence for pool membership: `pool-membership.json` next to +// the config file. It is loaded on first use and written only by changes an +// operator or an enrolling node makes (tokens, enrollments, cordons), never by +// heartbeats, so a head that never uses membership never writes the file. +// After a restart, members keep their last heartbeat time and show as lost +// until their next heartbeat. +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { writeAtomicJson } from './deployment-fence.mjs'; +import { createPoolMembership } from './pool-membership.mjs'; + +export const POOL_MEMBERSHIP_FILE = 'pool-membership.json'; + +export function poolMembershipPath(config) { + return config?.sourcePath ? path.join(path.dirname(path.resolve(config.sourcePath)), POOL_MEMBERSHIP_FILE) : null; +} + +export function poolMembershipOptions(config) { + const settings = config?.pool?.membership ?? {}; + return Object.fromEntries( + Object.entries({ leaseMs: settings.leaseMs, lostAfterMs: settings.lostAfterMs }).filter( + ([, value]) => value != null + ) + ); +} + +export async function readPoolMembershipState(filePath) { + if (!filePath) return null; + try { + return JSON.parse(await fs.readFile(filePath, 'utf8')); + } catch (error) { + if (error?.code === 'ENOENT') return null; + throw new Error(`Unreadable pool membership state ${filePath}: ${error?.message ?? error}`, { cause: error }); + } +} + +export function createPoolMembershipStore({ filePath, options = {}, write = writeAtomicJson } = {}) { + let membership = null; + let loading = null; + let queue = Promise.resolve(); + + async function load() { + if (membership) return membership; + loading ??= readPoolMembershipState(filePath) + .then((state) => { + membership = createPoolMembership({ ...options, state }); + return membership; + }) + .finally(() => { + loading = null; + }); + return loading; + } + + return { + load, + // Serialized change plus write. A failed write restores the last persisted + // state, so a token or enrollment never outlives a failed save. + mutate(change) { + const run = queue.then(async () => { + const current = await load(); + const before = current.toJSON(); + const result = change(current); + if (filePath) { + try { + await write(filePath, current.toJSON()); + } catch (error) { + membership = createPoolMembership({ ...options, state: before }); + throw error; + } + } + return result; + }); + queue = run.catch(() => {}); + return run; + } + }; +} diff --git a/src/pool-membership.mjs b/src/pool-membership.mjs new file mode 100644 index 0000000..1238a1b --- /dev/null +++ b/src/pool-membership.mjs @@ -0,0 +1,374 @@ +// Pool membership: nodes join a head with a one-time enrollment token and keep +// a lease with heartbeats, so the pool planner knows who is in the pool. +// Pure: no I/O and an injectable clock. Persistence (`toJSON()` / `state`) +// holds SHA-256 hashes of tokens and node secrets, never the values. Both are +// 256-bit random strings, so an unsalted hash is not guessable. +import { createHash, randomBytes, timingSafeEqual } from 'node:crypto'; + +export const POOL_MEMBERSHIP_VERSION = 1; + +const NODE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const TOKEN_PATTERN = /^lpt_([0-9a-f]{12})_[A-Za-z0-9_-]{43}$/; +const SECRET_PATTERN = /^lps_[A-Za-z0-9_-]{43}$/; +const LABEL_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,63}$/; +const MAX_LABELS = 32; +const MAX_TEXT = 256; +const MAX_LIST = 4096; +const MAX_TOKEN_TTL_MS = 30 * 24 * 3600 * 1000; +const MAX_TOKEN_USES = 1000; +const MAX_TELEMETRY_CHARS = 65536; +// Compared when a node id is unknown, so a miss costs the same as a mismatch. +const DUMMY_HASH = sha256('lloom-pool-membership'); + +function membershipError(statusCode, code, message) { + return Object.assign(new Error(message), { statusCode, code }); +} + +const invalid = (message) => membershipError(400, 'invalid_request', message); + +function sha256(value) { + return createHash('sha256').update(String(value)).digest('hex'); +} + +function hashMatches(value, expectedHash) { + const actual = Buffer.from(sha256(value), 'hex'); + const expected = Buffer.from(typeof expectedHash === 'string' ? expectedHash : DUMMY_HASH, 'hex'); + return expected.length === actual.length && timingSafeEqual(actual, expected); +} + +export function validNodeId(value) { + return typeof value === 'string' && NODE_ID_PATTERN.test(value); +} + +// Shared with `lloom cluster add-node`: an HTTP(S) URL with no credentials, +// query or fragment, returned without a trailing slash. +export function normalizeNodeEndpoint(endpoint) { + let url; + try { + url = new URL(endpoint); + } catch { + throw invalid('Invalid node endpoint'); + } + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash) + throw invalid('Node endpoint must be an HTTP(S) URL without credentials, query, or fragment'); + return url.toString().replace(/\/+$/, ''); +} + +function text(value, name) { + if (value == null || value === '') return null; + if (typeof value !== 'string' || value.length > MAX_TEXT) + throw invalid(`${name} must be text up to ${MAX_TEXT} characters`); + return value; +} + +function normalizeLabels(labels) { + if (labels == null) return {}; + const entries = labels && typeof labels === 'object' && !Array.isArray(labels) ? Object.entries(labels) : null; + if ( + !entries || + entries.length > MAX_LABELS || + entries.some(([key, value]) => !LABEL_KEY_PATTERN.test(key) || typeof value !== 'string' || value.length > MAX_TEXT) + ) { + throw invalid(`labels must map up to ${MAX_LABELS} names to text values`); + } + return Object.fromEntries(entries.sort(([a], [b]) => a.localeCompare(b))); +} + +const isObject = (value) => value && typeof value === 'object' && !Array.isArray(value); +const finite = (value) => (typeof value === 'number' && Number.isFinite(value) ? value : null); +const strings = (value) => + (Array.isArray(value) ? value : []) + .filter((item) => typeof item === 'string' && item && item.length <= MAX_TEXT) + .slice(0, MAX_LIST); +const objects = (value, map) => + (Array.isArray(value) ? value : []) + .slice(0, MAX_LIST) + .map((item) => (isObject(item) || typeof item === 'string' ? map(item) : null)) + .filter(Boolean); +const compact = (value) => Object.fromEntries(Object.entries(value).filter(([, entry]) => entry != null)); +const pickText = (value) => (typeof value === 'string' && value.length <= MAX_TEXT ? value : null); + +// A node reports its own inventory, so keep only the fields the planner reads, +// with the types it expects (see collectNodeInventory in pool-inventory.mjs). +function sanitizeInventory(inventory) { + if (!isObject(inventory)) return null; + const has = (key) => Object.hasOwn(inventory, key); + const backends = isObject(inventory.backends) ? inventory.backends : null; + return compact({ + version: finite(inventory.version), + complete: inventory.complete === true, + collectedAt: pickText(inventory.collectedAt), + platformId: pickText(inventory.platformId), + computeArchs: has('computeArchs') ? strings(inventory.computeArchs) : null, + accelerators: has('accelerators') ? strings(inventory.accelerators) : null, + memoryDomains: has('memoryDomains') + ? objects(inventory.memoryDomains, (domain) => + pickText(domain.id) && finite(domain.capacityGb) != null + ? compact({ id: domain.id, kind: pickText(domain.kind), capacityGb: domain.capacityGb }) + : null + ) + : null, + links: has('links') + ? objects(inventory.links, (link) => { + const peerNodeId = typeof link === 'string' ? link : link.peerNodeId; + return validNodeId(peerNodeId) + ? compact({ + peerNodeId, + localInterface: pickText(link.localInterface), + peerAddress: pickText(link.peerAddress) + }) + : null; + }) + : null, + disks: has('disks') + ? objects(inventory.disks, (disk) => + pickText(disk.root) + ? compact({ root: disk.root, freeGb: finite(disk.freeGb), totalGb: finite(disk.totalGb) }) + : null + ) + : null, + weights: has('weights') + ? objects(inventory.weights, (entry) => + pickText(entry.recipeId) + ? compact({ recipeId: entry.recipeId, root: pickText(entry.root), complete: entry.complete === true }) + : null + ) + : null, + backends: backends + ? { + commands: Object.fromEntries( + Object.entries(isObject(backends.commands) ? backends.commands : {}) + .filter(([name, value]) => value === true && name.length <= MAX_TEXT) + .slice(0, MAX_LIST) + ), + dockerImages: strings(backends.dockerImages) + } + : null + }); +} + +function sanitizeTelemetry(telemetry) { + if (!isObject(telemetry)) return null; + return JSON.stringify(telemetry).length <= MAX_TELEMETRY_CHARS ? telemetry : null; +} + +function positiveInteger(value, name, max) { + if (!Number.isSafeInteger(value) || value < 1 || value > max) + throw invalid(`${name} must be an integer from 1 to ${max}`); + return value; +} + +const iso = (ms) => new Date(ms).toISOString(); + +// Planner node shape (see plannerNodesFromStatus in pool-catalog.mjs). Lost and +// left nodes host nothing; cordoned and draining nodes take no placements. +export function memberPlannerNode(member) { + const inventory = member.inventory ?? null; + return compact({ + id: member.id, + available: !['lost', 'left'].includes(member.state), + cordoned: member.cordoned === true || member.draining === true, + membership: member.state, + platformId: inventory?.platformId ?? null, + computeArchs: inventory?.computeArchs ?? [], + accelerators: inventory?.accelerators ?? [], + memoryDomains: inventory?.memoryDomains ?? [], + links: [...new Set((inventory?.links ?? []).map((link) => link.peerNodeId))].sort(), + weights: (inventory?.weights ?? []).filter((entry) => entry.complete).map((entry) => entry.recipeId), + images: inventory?.backends?.dockerImages ?? [], + inventory: inventory ? (inventory.complete ? 'reported' : 'profile') : 'none' + }); +} + +function restoreState(state) { + if (state == null) return { tokens: new Map(), nodes: new Map() }; + if (state.version !== POOL_MEMBERSHIP_VERSION) { + throw new Error(`Unsupported pool membership state version ${state.version}`); + } + const tokens = new Map( + (Array.isArray(state.tokens) ? state.tokens : []) + .filter((token) => /^[0-9a-f]{12}$/.test(token?.id ?? '') && /^[0-9a-f]{64}$/.test(token.hash ?? '')) + .map((token) => [token.id, { ...token, labels: normalizeLabels(token.labels) }]) + ); + const nodes = new Map( + (Array.isArray(state.nodes) ? state.nodes : []) + .filter((node) => validNodeId(node?.id)) + .map((node) => [ + node.id, + { + ...node, + labels: normalizeLabels(node.labels), + inventory: sanitizeInventory(node.inventory), + running: strings(node.running), + telemetry: null + } + ]) + ); + return { tokens, nodes }; +} + +export function createPoolMembership({ leaseMs = 30000, lostAfterMs = 3 * leaseMs, now = Date.now, state } = {}) { + if (!Number.isFinite(leaseMs) || leaseMs <= 0) throw new Error('leaseMs must be a positive number'); + if (!Number.isFinite(lostAfterMs) || lostAfterMs < leaseMs) throw new Error('lostAfterMs must be at least leaseMs'); + const { tokens, nodes } = restoreState(state); + + function liveToken(token, at) { + return token && token.used < token.uses && Date.parse(token.expiresAt) > at; + } + + function memberState(node, at) { + if (node.left) return 'left'; + const lastSeen = Date.parse(node.lastSeenAt); + if (at < lastSeen + leaseMs) return 'ready'; + return at < lastSeen + lostAfterMs ? 'suspect' : 'lost'; + } + + function view(node, at = now()) { + return { + id: node.id, + endpoint: node.endpoint, + labels: { ...node.labels }, + state: memberState(node, at), + cordoned: node.cordoned, + draining: node.draining, + reason: node.reason, + enrolledAt: node.enrolledAt, + lastSeenAt: node.lastSeenAt, + leaseExpiresAt: iso(Date.parse(node.lastSeenAt) + leaseMs), + inventory: node.inventory ? structuredClone(node.inventory) : null, + running: [...node.running], + telemetry: node.telemetry ? structuredClone(node.telemetry) : null + }; + } + + function requireNode(nodeId) { + const node = typeof nodeId === 'string' ? nodes.get(nodeId) : null; + if (!node) throw membershipError(404, 'unknown_pool_node', `unknown pool node ${nodeId}`); + return node; + } + + function newSecret() { + return 'lps_' + randomBytes(32).toString('base64url'); + } + + return { + leaseMs, + lostAfterMs, + + issueEnrollmentToken({ ttlMs = 3600000, uses = 1, labels = {} } = {}) { + positiveInteger(ttlMs, 'ttlMs', MAX_TOKEN_TTL_MS); + positiveInteger(uses, 'uses', MAX_TOKEN_USES); + const tokenLabels = normalizeLabels(labels); + const at = now(); + for (const [id, token] of tokens) if (!liveToken(token, at)) tokens.delete(id); + let id; + do id = randomBytes(6).toString('hex'); + while (tokens.has(id)); + const token = `lpt_${id}_${randomBytes(32).toString('base64url')}`; + const expiresAt = iso(at + ttlMs); + tokens.set(id, { id, hash: sha256(token), createdAt: iso(at), expiresAt, uses, used: 0, labels: tokenLabels }); + return { token, id, expiresAt }; + }, + + enroll({ token, nodeId, endpoint, inventory, labels } = {}) { + const at = now(); + const match = typeof token === 'string' ? token.match(TOKEN_PATTERN) : null; + const record = match ? tokens.get(match[1]) : null; + // One generic failure: an attacker learns nothing about which check failed. + const tokenOk = hashMatches(token ?? '', record?.hash) && liveToken(record, at); + const existing = validNodeId(nodeId) ? nodes.get(nodeId) : null; + if (!tokenOk || existing?.tokenId === record.id) { + throw membershipError(401, 'invalid_enrollment_token', 'enrollment token is invalid, expired, or used up'); + } + if (!validNodeId(nodeId)) throw invalid('Invalid node identity'); + const normalizedEndpoint = normalizeNodeEndpoint(endpoint); + const nodeLabels = { ...normalizeLabels(labels), ...record.labels }; + const nodeSecret = newSecret(); + record.used += 1; + nodes.set(nodeId, { + id: nodeId, + endpoint: normalizedEndpoint, + labels: normalizeLabels(nodeLabels), + secretHash: sha256(nodeSecret), + tokenId: record.id, + enrolledAt: iso(at), + lastSeenAt: iso(at), + cordoned: existing?.cordoned === true, + draining: existing?.draining === true, + reason: existing?.reason ?? null, + left: false, + inventory: sanitizeInventory(inventory) ?? existing?.inventory ?? null, + running: existing?.running ?? [], + telemetry: null + }); + return { nodeId, nodeSecret, leaseMs }; + }, + + heartbeat({ nodeId, nodeSecret, inventory, running, telemetry } = {}) { + const node = typeof nodeId === 'string' ? nodes.get(nodeId) : null; + const secretOk = + typeof nodeSecret === 'string' && SECRET_PATTERN.test(nodeSecret) && hashMatches(nodeSecret, node?.secretHash); + if (!secretOk || !node || node.left) { + throw membershipError(401, 'invalid_node_credential', 'pool node credential is invalid'); + } + const at = now(); + node.lastSeenAt = iso(at); + const nextInventory = sanitizeInventory(inventory); + if (nextInventory) node.inventory = nextInventory; + if (running !== undefined) node.running = strings(running); + if (telemetry !== undefined) node.telemetry = sanitizeTelemetry(telemetry); + return { ok: true, leaseExpiresAt: iso(at + leaseMs), cordoned: node.cordoned, draining: node.draining }; + }, + + cordon(nodeId, { reason } = {}) { + const node = requireNode(nodeId); + const nextReason = text(reason, 'reason'); + node.cordoned = true; + node.reason = nextReason ?? node.reason; + return view(node); + }, + + uncordon(nodeId) { + const node = requireNode(nodeId); + Object.assign(node, { cordoned: false, draining: false, reason: null }); + return view(node); + }, + + drain(nodeId, { reason } = {}) { + const node = requireNode(nodeId); + const nextReason = text(reason, 'reason'); + node.draining = true; + node.reason = nextReason ?? node.reason; + return view(node); + }, + + leave(nodeId) { + const node = requireNode(nodeId); + Object.assign(node, { left: true, secretHash: null, running: [], telemetry: null }); + return view(node); + }, + + members() { + const at = now(); + return [...nodes.values()].map((node) => view(node, at)).sort((a, b) => a.id.localeCompare(b.id)); + }, + + plannerNodes() { + return this.members().map(memberPlannerNode); + }, + + toJSON() { + const at = now(); + return { + version: POOL_MEMBERSHIP_VERSION, + tokens: [...tokens.values()] + .filter((token) => liveToken(token, at)) + .map((token) => ({ ...token })) + .sort((a, b) => a.id.localeCompare(b.id)), + nodes: [...nodes.values()] + .map(({ telemetry: _telemetry, ...node }) => ({ ...node })) + .sort((a, b) => a.id.localeCompare(b.id)) + }; + } + }; +} diff --git a/src/pool-node.mjs b/src/pool-node.mjs new file mode 100644 index 0000000..1f3709a --- /dev/null +++ b/src/pool-node.mjs @@ -0,0 +1,202 @@ +// Node side of pool membership: `lloom pool join` enrolls this node with a +// head and stores the node secret in a private file; `serve` then keeps the +// lease with a heartbeat loop when `pool.head` is configured. +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { currentNodeId } from './cluster.mjs'; +import { mutateConfigSource } from './config-mutation.mjs'; +import { normalizeNodeEndpoint, validNodeId } from './pool-membership.mjs'; + +export const POOL_NODE_SECRET_FILE = 'pool-node-secret'; +export const DEFAULT_HEARTBEAT_INTERVAL_MS = 10000; +const MAX_BACKOFF_MS = 300000; +const REQUEST_TIMEOUT_MS = 10000; +const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; + +function headBaseUrl(url) { + try { + return normalizeNodeEndpoint(url); + } catch { + throw new Error('Pool head URL must be an HTTP(S) URL without credentials, query, or fragment'); + } +} + +const configDir = (configPath) => path.dirname(path.resolve(String(configPath))); + +// `pool.head.secretEnv` names an environment variable; `pool.head.secretFile` +// is a private regular file, relative to the config directory. +export async function resolvePoolNodeSecret(head = {}, { configPath, env = process.env } = {}) { + if (head.secretEnv) { + if (!ENV_NAME.test(head.secretEnv)) throw new Error('pool.head.secretEnv must be an environment variable name'); + const value = env[head.secretEnv]; + if (!value) throw new Error(`pool.head.secretEnv ${head.secretEnv} is empty or unset`); + return value.trim(); + } + if (head.secretFile) { + const file = path.resolve(configDir(configPath), String(head.secretFile)); + const stat = await fs.lstat(file); + if (!stat.isFile()) throw new Error('pool.head.secretFile must be a regular file'); + if (stat.mode & 0o077) throw new Error('pool.head.secretFile must have private (0600) permissions'); + return (await fs.readFile(file, 'utf8')).trim(); + } + throw new Error('pool.head needs secretEnv or secretFile'); +} + +async function writePrivateFile(file, content) { + const temporary = `${file}.tmp-${process.pid}-${randomUUID()}`; + try { + await fs.writeFile(temporary, content, { mode: 0o600, flag: 'wx' }); + await fs.rename(temporary, file); + } catch (error) { + await fs.rm(temporary, { force: true }); + throw error; + } +} + +async function errorMessage(response) { + try { + const message = (await response.json())?.error?.message; + return typeof message === 'string' ? message.slice(0, 300) : ''; + } catch { + return ''; + } +} + +export async function joinPool( + config, + { headUrl, token, nodeId = currentNodeId(config), endpoint, labels, apply = false, yes = false, fetchFn = fetch } = {} +) { + const head = headBaseUrl(headUrl); + if (!validNodeId(nodeId)) throw new Error('Invalid node identity; pass --node-id'); + const advertised = endpoint ?? config.cluster?.nodes?.[nodeId]?.endpoint; + if (!advertised) throw new Error("Pass --endpoint with this node's gateway URL as the head should reach it"); + const nodeEndpoint = normalizeNodeEndpoint(advertised); + if (!config.sourcePath) throw new Error('Joining a pool requires an installed config file'); + const secretPath = path.join(configDir(config.sourcePath), POOL_NODE_SECRET_FILE); + const summary = { + ok: true, + applied: false, + head, + nodeId, + endpoint: nodeEndpoint, + configPath: config.sourcePath, + secretFile: secretPath + }; + if (!apply) + return { ...summary, next: 'Re-run with --apply --yes and the enrollment token on stdin (--token-stdin).' }; + if (!yes) throw new Error('Joining a pool requires --apply --yes'); + if (!token) throw new Error('Pass the enrollment token on stdin with --token-stdin'); + let response; + try { + response = await fetchFn(`${head}/gateway/pool/enroll`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ nodeId, endpoint: nodeEndpoint, ...(labels ? { labels } : {}) }), + redirect: 'error', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }); + } catch { + throw new Error(`Pool head ${head} is unreachable`); + } + if (!response.ok) { + const detail = await errorMessage(response); + throw new Error(`Pool head rejected the enrollment (${response.status})${detail ? `: ${detail}` : ''}`); + } + const result = await response.json().catch(() => null); + if (typeof result?.nodeSecret !== 'string' || !/^lps_[A-Za-z0-9_-]{43}$/.test(result.nodeSecret)) { + throw new Error('Pool head returned an invalid enrollment response'); + } + await writePrivateFile(secretPath, `${result.nodeSecret}\n`); + const { changed } = await mutateConfigSource(config, (raw) => { + raw.pool ??= {}; + raw.pool.head = { url: head, nodeId, secretFile: POOL_NODE_SECRET_FILE }; + }); + return { ...summary, applied: true, changed, leaseMs: result.leaseMs ?? null }; +} + +// One heartbeat per interval, with exponential backoff (and +/-10% jitter) +// while the head is unreachable or rejects the node. The head config is read +// on every tick, so a config reload adds, changes or removes it. +export function createPoolHeartbeatLoop({ + getHead, + configPath, + collect, + env = process.env, + fetchImpl = fetch, + logger = console, + schedule = setTimeout, + cancel = clearTimeout, + random = Math.random +} = {}) { + let timer = null; + let inFlight = null; + let started = false; + let stopped = false; + let failures = 0; + let last = null; + + async function beat(head) { + const secret = await resolvePoolNodeSecret(head, { configPath, env }); + const body = { nodeId: head.nodeId, ...(await collect()) }; + const response = await fetchImpl(`${headBaseUrl(head.url)}/gateway/pool/heartbeat`, { + method: 'POST', + headers: { authorization: `Bearer ${secret}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + redirect: 'error', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }); + if (!response.ok) { + const detail = await errorMessage(response); + throw new Error(`pool head answered ${response.status}${detail ? `: ${detail}` : ''}`); + } + return response.json(); + } + + async function tick() { + const head = getHead(); + const intervalMs = Number(head?.intervalMs) > 0 ? Number(head.intervalMs) : DEFAULT_HEARTBEAT_INTERVAL_MS; + if (head?.url) { + inFlight = beat(head); + try { + last = { at: new Date().toISOString(), ...(await inFlight) }; + if (failures) logger.info?.(`pool heartbeat to ${head.url} recovered`); + failures = 0; + } catch (error) { + failures += 1; + // Log the first failure and then every tenth, not every retry. + if (failures === 1 || failures % 10 === 0) { + logger.warn?.(`pool heartbeat to ${head.url} failed (${failures}x): ${error?.message ?? error}`); + } + } finally { + inFlight = null; + } + } + if (stopped) return; + const backoff = Math.min(MAX_BACKOFF_MS, intervalMs * 2 ** failures); + timer = schedule(() => tick().catch(() => {}), Math.round(backoff * (0.9 + random() * 0.2))); + timer?.unref?.(); + } + + return { + tick, + // Idempotent: a config reload may call it again while the loop runs. + start() { + if (started) return; + started = true; + stopped = false; + timer = schedule(() => tick().catch(() => {}), 0); + timer?.unref?.(); + }, + async stop() { + started = false; + stopped = true; + if (timer) cancel(timer); + timer = null; + await inFlight?.catch(() => {}); + }, + status() { + return { failures, last }; + } + }; +} diff --git a/src/security.mjs b/src/security.mjs index 4bd0344..a98df1c 100644 --- a/src/security.mjs +++ b/src/security.mjs @@ -60,15 +60,20 @@ export function adminKeysConfigured(config) { return keySet(config.security?.adminApiKeys).size > 0; } +const POOL_NODE_ROUTES = new Set(['/gateway/pool/enroll', '/gateway/pool/heartbeat']); + /** * @param {string} method * @param {string} pathname - * @returns {'public' | 'inference' | 'admin-read' | 'admin-write'} + * @returns {'public' | 'inference' | 'admin-read' | 'admin-write' | 'pool-node'} */ export function classifyRoute(method, pathname) { const verb = String(method ?? 'GET').toUpperCase(); const path = String(pathname ?? '/'); if (verb === 'OPTIONS') return 'public'; + // Pool enrollment and heartbeats authenticate in their handlers with an + // enrollment token or a node secret. API keys neither grant nor replace them. + if (verb === 'POST' && POOL_NODE_ROUTES.has(path)) return 'pool-node'; if ( verb === 'GET' && (path === '/health' || path === '/' || path === '/gateway/dashboard' || path === '/gateway/security') @@ -102,7 +107,7 @@ export function authorizeRequest(req, config, { method, pathname } = {}) { const validInferenceKey = hasValidApiKey(req, config, { admin: false }); const validAdminKey = hasValidApiKey(req, config, { admin: true }); - if (routeKind === 'public') { + if (routeKind === 'public' || routeKind === 'pool-node') { return { ok: true, routeKind }; } diff --git a/src/server.mjs b/src/server.mjs index 5f03ade..88a7f60 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -60,6 +60,8 @@ import { applyRecipePack, createRecipePackPlan } from './recipe-pack.mjs'; import { buildRecipeIndexReport } from './recipe-index.mjs'; import { loadRecipes } from './recipes.mjs'; import { collectNodeInventory, createNodeInventoryCache } from './pool-inventory.mjs'; +import { createPoolMembershipStore, poolMembershipOptions, poolMembershipPath } from './pool-membership-store.mjs'; +import { createPoolHeartbeatLoop } from './pool-node.mjs'; import { modelRootCandidates } from './model-files.mjs'; import { createRegistry, UnknownModelError } from './registry.mjs'; import { routeProfileStatus, writeRouteMemberSuspension, writeRouteProfile } from './route-control.mjs'; @@ -68,6 +70,7 @@ import { createModelMaintenanceController } from './model-maintenance-control.mj import { acquireRateLimitSlot, awaitRateBudget, + createGcra, createRateLimitRegistry, RATE_LIMIT_QUEUE_CAPACITY, RateQueueFullError @@ -133,6 +136,7 @@ import { assertBindAllowed, authorizeRequest, corsHeaders, + extractApiKey, hasValidApiKey, securityPublicStatus } from './security.mjs'; @@ -946,6 +950,52 @@ async function readJson(req) { return JSON.parse(raw); } +const POOL_BODY_LIMIT_BYTES = 1024 * 1024; +const POOL_BODY_DRAIN_BYTES = 16 * 1024 * 1024; +// Per client: ten enrollment attempts at once, then one every six seconds. +const POOL_ENROLL_RATE = { rateMs: 6000, burst: 9 }; +const POOL_ENROLL_CLIENTS_MAX = 4096; + +function poolRequestError(statusCode, code, message) { + return Object.assign(new Error(message), { statusCode, code }); +} + +// Pool node routes accept bodies from not-yet-authenticated callers: bound +// them and report malformed input as a client error. An oversized body is +// drained (up to a hard cap) so the client reads the 413 instead of a reset. +function readPoolBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + let total = 0; + const tooLarge = () => + poolRequestError(413, 'payload_too_large', `pool request body exceeds ${POOL_BODY_LIMIT_BYTES} bytes`); + req.on('data', (chunk) => { + total += chunk.length; + if (total <= POOL_BODY_LIMIT_BYTES) chunks.push(chunk); + else if (total > POOL_BODY_DRAIN_BYTES) req.destroy(); + }); + req.once('end', () => + total > POOL_BODY_LIMIT_BYTES ? reject(tooLarge()) : resolve(Buffer.concat(chunks).toString('utf8')) + ); + req.once('close', () => reject(tooLarge())); + req.once('error', reject); + }); +} + +async function readPoolJson(req) { + const raw = await readPoolBody(req); + let value; + try { + value = raw.trim() ? JSON.parse(raw) : {}; + } catch { + value = null; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw poolRequestError(400, 'invalid_request', 'pool request body must be a JSON object'); + } + return value; +} + function upstreamUrl(backend, path) { // An absolute URL is already resolved against the backend; joining it to the // baseUrl would corrupt it. Callers use this for routes a backend serves @@ -2330,7 +2380,9 @@ export function createLloomServer( upstreamDispatcher = null, // Tests may point identity verification at a disposable atomic release; // production leaves this unset and uses the installed package root. - deploymentPackageRoot = null + deploymentPackageRoot = null, + // Tests may supply a fixed pool inventory instead of probing this machine. + nodeInventory: injectedNodeInventory = null } = {} ) { assertStandbyConfig(config); @@ -2352,16 +2404,39 @@ export function createLloomServer( }); // Pool inventory is collected on request only and cached; it scans model // roots and lists container images, which is too slow for the status path. - const nodeInventory = createNodeInventoryCache({ - collect: async () => - collectNodeInventory({ - nodeId: clusterCoordinator.nodeId, - profile: (await machineProfile) ?? {}, - recipes: await loadRecipes().catch(() => []), - modelRoots: await modelRootCandidates({ config: clusterCoordinator.config }) - }) - }); + const nodeInventory = + injectedNodeInventory ?? + createNodeInventoryCache({ + collect: async () => + collectNodeInventory({ + nodeId: clusterCoordinator.nodeId, + profile: (await machineProfile) ?? {}, + recipes: await loadRecipes().catch(() => []), + modelRoots: await modelRootCandidates({ config: clusterCoordinator.config }) + }) + }); clusterCoordinator.inventory ??= () => nodeInventory.get(); + // Pool membership is inert until used: the head loads its state on the first + // pool request and writes it only when a token is issued or a node enrolls. + const poolMembership = createPoolMembershipStore({ + filePath: poolMembershipPath(config), + options: poolMembershipOptions(config) + }); + const poolEnrollLimiters = new Map(); + // A node with `pool.head` keeps its lease from `serve`. + const poolHeartbeat = createPoolHeartbeatLoop({ + getHead: () => (isStandby(config) ? null : config.pool?.head), + configPath: config.sourcePath, + logger, + collect: async () => ({ + inventory: await nodeInventory.get().catch(() => null), + running: Object.entries((await runtimeManager.status({ localOnly: true }))?.runtimes ?? {}) + .filter(([, runtime]) => ['running', 'external'].includes(runtime?.status)) + .map(([id]) => id) + .sort(), + telemetry: await hostTelemetry.snapshot().catch(() => null) + }) + }); runtimeManager ??= new RuntimeManager(config, { logger, clusterCoordinator }); if (!runtimeManager.clusterCoordinator) { runtimeManager.clusterCoordinator = clusterCoordinator; @@ -2549,6 +2624,111 @@ export function createLloomServer( })); } + function startPoolHeartbeat() { + if (config.pool?.head && !isStandby(config)) poolHeartbeat.start(); + } + + function poolEnrollRetryMs(req) { + const client = req.socket?.remoteAddress ?? 'unknown'; + if (!poolEnrollLimiters.has(client) && poolEnrollLimiters.size >= POOL_ENROLL_CLIENTS_MAX) + poolEnrollLimiters.clear(); + if (!poolEnrollLimiters.has(client)) poolEnrollLimiters.set(client, createGcra(POOL_ENROLL_RATE)); + const admission = poolEnrollLimiters.get(client).tryAcquire(Date.now()); + return admission.allowed ? 0 : admission.retryAfterMs; + } + + async function handlePoolMembership(req, res, url) { + const action = url.pathname.match(/^\/gateway\/pool\/nodes\/([^/]+)\/(cordon|uncordon|drain)$/); + try { + if (req.method === 'POST' && url.pathname === '/gateway/pool/enroll') { + const retryAfterMs = poolEnrollRetryMs(req); + if (retryAfterMs) { + sendJson( + res, + 429, + errorBody('too many enrollment attempts; retry later', { + type: 'rate_limit_error', + code: 'pool_enroll_rate_limited' + }), + { 'retry-after': String(Math.max(1, Math.ceil(retryAfterMs / 1000))) }, + config + ); + return; + } + const body = await readPoolJson(req); + if (body.nodeId === clusterCoordinator.nodeId) { + throw poolRequestError(409, 'pool_node_conflict', "a node cannot enroll with the head's own node id"); + } + const enrolled = await poolMembership.mutate((membership) => + membership.enroll({ + token: extractApiKey(req), + nodeId: body.nodeId, + endpoint: body.endpoint, + inventory: body.inventory, + labels: body.labels + }) + ); + logger.info?.(`pool node ${enrolled.nodeId} enrolled`); + sendJson(res, 200, { ok: true, ...enrolled }, {}, config); + return; + } + if (req.method === 'POST' && url.pathname === '/gateway/pool/heartbeat') { + const body = await readPoolJson(req); + const membership = await poolMembership.load(); + const lease = membership.heartbeat({ + nodeId: body.nodeId, + nodeSecret: extractApiKey(req), + inventory: body.inventory, + running: body.running, + telemetry: body.telemetry + }); + sendJson(res, 200, lease, {}, config); + return; + } + if (req.method === 'GET' && url.pathname === '/gateway/pool/members') { + const membership = await poolMembership.load(); + sendJson( + res, + 200, + { ok: true, leaseMs: membership.leaseMs, lostAfterMs: membership.lostAfterMs, members: membership.members() }, + {}, + config + ); + return; + } + if (req.method === 'POST' && url.pathname === '/gateway/pool/tokens') { + const body = await readPoolJson(req); + const issued = await poolMembership.mutate((membership) => + membership.issueEnrollmentToken({ ttlMs: body.ttlMs, uses: body.uses, labels: body.labels }) + ); + sendJson(res, 200, { ok: true, ...issued }, {}, config); + return; + } + if (req.method === 'POST' && action) { + const body = await readPoolJson(req); + let nodeId; + try { + nodeId = decodeURIComponent(action[1]); + } catch { + throw poolRequestError(400, 'invalid_request', 'invalid pool node id'); + } + const member = await poolMembership.mutate((membership) => + membership[action[2]](nodeId, { reason: body.reason }) + ); + logger.info?.(`pool node ${nodeId}: ${action[2]}`); + sendJson(res, 200, { ok: true, member }, {}, config); + return; + } + sendJson(res, 404, errorBody(`unknown route: ${url.pathname}`, { code: 'not_found' }), {}, config); + } catch (error) { + const status = errorStatusCode(error); + // Expected rejections (bad token, unknown node) are answered, not logged. + if (!status || status >= 500) throw error; + const type = status === 401 ? 'authentication_error' : status === 403 ? 'permission_error' : undefined; + sendJson(res, status, errorBody(error.message, { type, code: error.code ?? 'invalid_request' }), {}, config); + } + } + function reloadConfig() { if (!configPath) return; if (runtimeManager.shuttingDown) return Promise.resolve({ skipped: 'gateway-closing' }); @@ -2604,6 +2784,7 @@ export function createLloomServer( // admission queues synchronously, so policy must already match. requestScheduler.refresh(config.server?.requestScheduling?.mode, { notify: false }); rateLimitRegistry.sync(rateLimitSettingsFor(config)); + if (server.listening) startPoolHeartbeat(); logger.info?.(`reloaded LLooM config; changed runtimes: ${result.changed.join(', ') || 'none'}`); } catch (error) { registry = previousRegistry; @@ -5124,6 +5305,11 @@ export function createLloomServer( return; } + if (url.pathname.startsWith('/gateway/pool/')) { + await handlePoolMembership(req, res, url); + return; + } + if (req.method === 'GET' && url.pathname === '/gateway/cluster/readiness') { const runtimeId = firstQueryParam(url.searchParams, ['runtime', 'runtime_id', 'runtime-id']); if (!runtimeId) { @@ -5968,6 +6154,7 @@ export function createLloomServer( }) .catch((error) => logger.error?.(error)); if (configPath && !isStandby(config)) watchFile(configPath, { interval: 500 }, reloadConfig); + startPoolHeartbeat(); resolve(server); }); }); @@ -5975,6 +6162,7 @@ export function createLloomServer( async close({ stopRuntimes = true, httpGraceMs = 5000 } = {}) { runtimeManager.shuttingDown = true; if (configPath && !isStandby(config)) unwatchFile(configPath, reloadConfig); + await poolHeartbeat.stop(); const closingError = Object.assign(new Error('gateway is closing'), { statusCode: 503, code: 'REQUEST_SCHEDULER_CLOSED' diff --git a/test/pool-catalog.test.mjs b/test/pool-catalog.test.mjs index 907d8ba..653a3e3 100644 --- a/test/pool-catalog.test.mjs +++ b/test/pool-catalog.test.mjs @@ -7,6 +7,7 @@ import { buildPoolCatalog, buildPoolPlanInputs, currentPoolState, + mergeMembershipNodes, plannerNodesFromStatus, poolPinsFromConfig, poolWorkloadsFromConfig, @@ -332,3 +333,84 @@ test('planner inputs from config produce a coherent plan', () => { ['manual-stop'] ); }); + +// Shape of GET /gateway/pool/members entries. +function poolMember(id, overrides = {}) { + return { + id, + endpoint: `http://${id}:8100`, + labels: {}, + state: 'ready', + cordoned: false, + draining: false, + reason: null, + enrolledAt: '2026-10-11T12:00:00.000Z', + lastSeenAt: '2026-10-11T12:00:00.000Z', + leaseExpiresAt: '2026-10-11T12:00:30.000Z', + inventory: { + complete: true, + platformId: 'linux-arm64', + accelerators: ['cuda', 'gb10'], + computeArchs: ['sm_121'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 128 }], + links: [], + weights: [{ recipeId: 'spark-27b', complete: true }] + }, + running: [], + ...overrides + }; +} + +test('enrolled members add liveness, cordon and inventory; configured nodes keep identity', () => { + const config = fleetConfig(); + delete config.pool.cordon; + const statuses = { ...fleetStatuses(), 'spark-node-02': { reachable: true, profile: null } }; + const members = [ + poolMember('spark-node-01', { state: 'lost' }), + poolMember('spark-node-02', { + draining: true, + reason: 'disk swap', + inventory: { ...poolMember('x').inventory, links: [{ peerNodeId: 'spark-node-01' }] } + }), + poolMember('spark-node-04'), + poolMember('spark-node-05', { state: 'left' }) + ]; + const inputs = buildPoolPlanInputs({ config, nodeStatuses: statuses, metrics, members }); + const nodes = Object.fromEntries(inputs.nodes.map((node) => [node.id, node])); + assert.equal(nodes['spark-node-01'].available, false, 'a lost lease makes a configured node unavailable'); + assert.equal(nodes['spark-node-01'].membership, 'lost'); + assert.equal(nodes['spark-node-01'].maxMemoryUtilization, 0.9, 'configured resources are kept'); + assert.equal(nodes['spark-node-02'].available, true); + assert.equal(nodes['spark-node-02'].cordoned, true, 'draining comes from membership'); + assert.equal(nodes['spark-node-02'].inventory, 'reported', 'the member inventory beats config-only facts'); + assert.deepEqual(nodes['spark-node-02'].weights, ['spark-27b']); + assert.deepEqual(nodes['spark-node-02'].links, ['spark-node-01']); + assert.equal(nodes['spark-node-03'].membership, undefined, 'configured-only nodes are unchanged'); + assert.equal(nodes['spark-node-04'].routable, false); + assert.equal(nodes['spark-node-04'].available, true); + assert.equal(nodes['spark-node-05'].available, false); + assert.equal(inputs.warnings.length, 1); + assert.match( + inputs.warnings[0], + /spark-node-04 .*lloom cluster add-node spark-node-04 http:\/\/spark-node-04:8100 --apply --yes/ + ); + + const plan = planPool(inputs); + assert.equal(plan.routes['chat-capable'].model, 'qwen38-27b-nvfp4', 'the pair is lost and draining'); + assert.equal(plan.nodes['spark-node-02'].status, 'cordoned'); +}); + +test('without live state only cordons and departures apply from saved membership', () => { + const configured = plannerNodesFromStatus(fleetConfig(), {}); + const { nodes, warnings } = mergeMembershipNodes( + configured, + [poolMember('spark-node-01', { state: 'lost', cordoned: true }), poolMember('spark-node-09', { state: 'lost' })], + { liveness: false } + ); + const byId = Object.fromEntries(nodes.map((node) => [node.id, node])); + assert.equal(byId['spark-node-01'].available, true, 'a stale lease is not evidence offline'); + assert.equal(byId['spark-node-01'].cordoned, true); + assert.equal(byId['spark-node-09'].available, true); + assert.equal(warnings.length, 1); + assert.deepEqual(mergeMembershipNodes(configured, []).nodes, configured, 'no members, no change'); +}); diff --git a/test/pool-cli.test.mjs b/test/pool-cli.test.mjs index a67f6bc..b69af58 100644 --- a/test/pool-cli.test.mjs +++ b/test/pool-cli.test.mjs @@ -2,12 +2,22 @@ // the spawned CLI checks only argument handling and the offline JSON shape. import { test, after } from 'node:test'; import assert from 'node:assert/strict'; -import { execFile } from 'node:child_process'; +import { execFile, spawn } from 'node:child_process'; import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { createPoolPlanReport, formatPoolPlanReport, poolPlanOptions } from '../src/pool-cli.mjs'; +import { + createPoolPlanReport, + formatPoolMembers, + formatPoolPlanReport, + parseDurationMs, + poolPlanOptions, + poolTokenOptions +} from '../src/pool-cli.mjs'; +import { loadConfig } from '../src/config.mjs'; +import { createPoolMembership } from '../src/pool-membership.mjs'; +import { createLloomServer } from '../src/server.mjs'; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const cli = path.join(repoRoot, 'bin', 'lloom.mjs'); @@ -48,7 +58,8 @@ const deps = (gatewayRequest) => ({ profileMachine: async () => spark().profile, collectNodeInventory: async ({ nodeId }) => ({ version: 1, complete: true, nodeId, computeArchs: ['sm_121'] }), modelRootCandidates: async () => [], - readProfile: async (_config, name) => ({ name, routes: { chat: 'small' }, residency: {} }) + readProfile: async (_config, name) => ({ name, routes: { chat: 'small' }, residency: {} }), + readPoolMembership: async () => null }); test('pool plan options parse cordons, pins and policy flags', () => { @@ -87,7 +98,7 @@ test('a gateway report plans from live status and metrics', async () => { return { period: '7d', models: [], routes: [], requested: [{ id: 'chat', requests: 336 }] }; }) ); - assert.deepEqual(requested, ['/gateway/cluster?inventory=1', '/gateway/metrics?period=7d']); + assert.deepEqual(requested, ['/gateway/cluster?inventory=1', '/gateway/metrics?period=7d', '/gateway/pool/members']); assert.equal(report.dryRun, true); assert.deepEqual(report.source, { mode: 'gateway', @@ -170,3 +181,200 @@ test('the CLI prints an offline JSON plan and rejects unknown actions', async () assert.notEqual(bad.code, 0); assert.match(bad.stderr, /Usage: lloom pool plan/); }); + +function member(id, overrides = {}) { + return { + id, + endpoint: `http://${id}:8100`, + labels: {}, + state: 'ready', + cordoned: false, + draining: false, + reason: null, + enrolledAt: '2026-10-11T12:00:00.000Z', + lastSeenAt: '2026-10-11T12:00:00.000Z', + leaseExpiresAt: '2026-10-11T12:00:30.000Z', + inventory: null, + running: [], + ...overrides + }; +} + +test('a plan reflects membership drains and flags enrolled-only nodes', async () => { + const report = await createPoolPlanReport( + config(), + poolPlanOptions({}), + deps(async (_config, pathname) => { + if (pathname.startsWith('/gateway/cluster')) return { cluster: { nodes: { head: spark(), pair: spark() } } }; + if (pathname === '/gateway/pool/members') { + return { members: [member('pair', { draining: true, reason: 'swap' }), member('extra')] }; + } + return null; + }) + ); + const nodes = Object.fromEntries(report.nodes.map((node) => [node.id, node])); + assert.equal(nodes.pair.cordoned, true); + assert.equal(nodes.pair.membership, 'ready'); + assert.equal(nodes.pair.routable, true); + assert.equal(nodes.extra.routable, false); + assert.equal(report.plan.routes.chat.model, 'small', 'the drained pair takes no placement'); + assert.match(report.plan.warnings[0], /extra joined the pool by enrollment only/); + const text = formatPoolPlanReport(report); + assert.match(text, /pair +cordoned .*\[member ready\]/); + assert.match(text, /extra +.*\[member ready, enrolled only\]/); +}); + +test('an offline plan applies cordons from the saved membership file', async () => { + const membership = createPoolMembership(); + const { token } = membership.issueEnrollmentToken({}); + membership.enroll({ token, nodeId: 'pair', endpoint: 'http://pair:8100' }); + membership.cordon('pair', { reason: 'maintenance' }); + const state = membership.toJSON(); + const report = await createPoolPlanReport(config(), poolPlanOptions({ offline: true }), { + ...deps(async () => { + throw new Error('offline plans do not call the gateway'); + }), + readPoolMembership: async () => state + }); + assert.equal(report.nodes.find((node) => node.id === 'pair').cordoned, true); +}); + +test('token options parse durations, uses and labels', () => { + assert.equal(parseDurationMs('90'), 90000); + assert.equal(parseDurationMs('30m'), 1800000); + assert.equal(parseDurationMs('2h'), 7200000); + assert.equal(parseDurationMs('7d'), 604800000); + assert.throws(() => parseDurationMs('soon'), /duration/); + assert.throws(() => parseDurationMs('0'), /duration/); + assert.deepEqual(poolTokenOptions({ ttl: '10m', uses: '3', labels: ['role=gpu', 'zone=a'] }), { + ttlMs: 600000, + uses: 3, + labels: { role: 'gpu', zone: 'a' } + }); + assert.deepEqual(poolTokenOptions({}), {}); + assert.throws(() => poolTokenOptions({ uses: 'many' }), /--uses/); + assert.throws(() => poolTokenOptions({ labels: ['novalue'] }), /KEY=VALUE/); +}); + +test('members print state, cordons and inventory at a glance', () => { + const text = formatPoolMembers( + { + leaseMs: 30000, + members: [ + member('spark-node-01', { + inventory: { + complete: true, + computeArchs: ['sm_121'], + memoryDomains: [{ id: 'unified:0', capacityGb: 128 }] + }, + running: ['q27'] + }), + member('spark-node-02', { state: 'lost', draining: true, reason: 'disk swap' }) + ] + }, + { now: () => Date.parse('2026-10-11T12:01:00.000Z') } + ); + assert.match(text, /spark-node-01 +ready +http:\/\/spark-node-01:8100 +60s ago +sm_121, 128 GB +q27/); + assert.match(text, /spark-node-02 +lost, draining \(disk swap\)/); + assert.equal(formatPoolMembers({ members: [] }), 'No pool members. Issue a token with `lloom pool token create`.'); +}); + +function runCliWithInput(args, env, input) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [cli, ...args], { env }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => (stdout += chunk)); + child.stderr.on('data', (chunk) => (stderr += chunk)); + child.on('close', (code) => resolve({ code, stdout, stderr })); + child.stdin.end(input ?? ''); + }); +} + +test('the CLI issues tokens, joins a node, lists members and cordons', async (t) => { + const admin = 'fixture-pool-admin'; + const headDir = await fs.mkdtemp(path.join(os.tmpdir(), 'lloom-pool-head-')); + const nodeDir = await fs.mkdtemp(path.join(os.tmpdir(), 'lloom-pool-node-')); + directories.push(headDir, nodeDir); + const headConfigPath = path.join(headDir, 'config.json'); + const headRaw = { + server: { host: '127.0.0.1', port: 0 }, + security: { adminApiKeys: [admin] }, + cluster: { nodeId: 'head' }, + backends: {}, + models: [] + }; + await fs.writeFile(headConfigPath, JSON.stringify(headRaw), { mode: 0o600 }); + const app = createLloomServer(await loadConfig(headConfigPath, { env: {} }), { + logger: { info() {}, warn() {}, error() {} } + }); + await app.listen(); + t.after(() => app.close({ stopRuntimes: false, httpGraceMs: 0 })); + const port = app.server.address().port; + // The CLI reaches the head through its configured port. + await fs.writeFile(headConfigPath, JSON.stringify({ ...headRaw, server: { host: '127.0.0.1', port } }), { + mode: 0o600 + }); + const nodeConfigPath = path.join(nodeDir, 'config.json'); + await fs.writeFile(nodeConfigPath, JSON.stringify({ cluster: { nodeId: 'worker' }, backends: {}, models: [] }), { + mode: 0o600 + }); + const env = { ...process.env, HOME: headDir, LLOOM_ADMIN_API_KEY: admin, LLOOM_CONFIG: '' }; + delete env.LLOOM_NODE_ID; + const head = ['--config', headConfigPath]; + const node = ['--config', nodeConfigPath]; + + const created = await runCli(['pool', 'token', 'create', '--ttl', '10m', '--json', ...head], env); + assert.equal(created.code, 0, created.stderr); + const { token } = JSON.parse(created.stdout); + assert.match(token, /^lpt_/); + + const headUrl = `http://127.0.0.1:${port}`; + const preview = await runCliWithInput( + ['pool', 'join', headUrl, '--token-stdin', '--endpoint', 'http://10.0.0.2:8100', ...node], + env, + token + ); + assert.equal(preview.code, 0, preview.stderr); + assert.equal(JSON.parse(preview.stdout).applied, false); + const joined = await runCliWithInput( + ['pool', 'join', headUrl, '--token-stdin', '--endpoint', 'http://10.0.0.2:8100', '--apply', '--yes', ...node], + env, + token + '\n' + ); + assert.equal(joined.code, 0, joined.stderr); + const result = JSON.parse(joined.stdout); + assert.equal(result.applied, true); + assert.equal(result.nodeId, 'worker'); + assert.ok(!joined.stdout.includes('lps_'), 'the node secret is never printed'); + const secretFile = path.join(nodeDir, 'pool-node-secret'); + assert.equal((await fs.stat(secretFile)).mode & 0o777, 0o600); + assert.match(await fs.readFile(secretFile, 'utf8'), /^lps_/); + const nodeConfig = JSON.parse(await fs.readFile(nodeConfigPath, 'utf8')); + assert.deepEqual(nodeConfig.pool.head, { url: headUrl, nodeId: 'worker', secretFile: 'pool-node-secret' }); + + const reused = await runCliWithInput( + ['pool', 'join', headUrl, '--token-stdin', '--endpoint', 'http://10.0.0.2:8100', '--apply', '--yes', ...node], + env, + token + ); + assert.notEqual(reused.code, 0); + assert.match(reused.stderr, /rejected the enrollment \(401\)/); + assert.ok(!reused.stderr.includes(token)); + + const cordoned = await runCli(['pool', 'drain', 'worker', '--reason', 'rebalance', ...head], env); + assert.equal(cordoned.code, 0, cordoned.stderr); + assert.equal(JSON.parse(cordoned.stdout).member.draining, true); + + const members = await runCli(['pool', 'members', '--json', ...head], env); + assert.equal(members.code, 0, members.stderr); + const [listed] = JSON.parse(members.stdout).members; + assert.equal(listed.id, 'worker'); + assert.equal(listed.draining, true); + const table = await runCli(['pool', 'members', ...head], env); + assert.match(table.stdout, /worker +ready, draining \(rebalance\) +http:\/\/10\.0\.0\.2:8100/); + + const usage = await runCli(['pool', 'cordon', ...head], env); + assert.notEqual(usage.code, 0); + assert.match(usage.stderr, /lloom pool cordon\|uncordon\|drain /); +}); diff --git a/test/pool-membership-http.test.mjs b/test/pool-membership-http.test.mjs new file mode 100644 index 0000000..e025666 --- /dev/null +++ b/test/pool-membership-http.test.mjs @@ -0,0 +1,307 @@ +// Pool membership over HTTP: an in-process head on a temp config. Covers the +// auth boundary (enrollment token, node secret, admin keys), persistence and +// the node-side heartbeat loop. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { loadConfig } from '../src/config.mjs'; +import { createPoolHeartbeatLoop, resolvePoolNodeSecret } from '../src/pool-node.mjs'; +import { authorizeRequest, classifyRoute } from '../src/security.mjs'; +import { createLloomServer } from '../src/server.mjs'; + +const ADMIN = 'fixture-admin-key'; +const INFERENCE = 'fixture-inference-key'; +const logger = { info() {}, warn() {}, error() {} }; + +async function tempDir(t) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'lloom-pool-membership-')); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + return dir; +} + +async function startHead(t, { dir, pool, serverOptions = {} } = {}) { + dir ??= await tempDir(t); + const configPath = path.join(dir, 'config.json'); + const raw = { + server: { host: '127.0.0.1', port: 0 }, + security: { adminApiKeys: [ADMIN], apiKeys: [INFERENCE] }, + cluster: { nodeId: 'head' }, + ...(pool ? { pool } : {}), + backends: {}, + models: [] + }; + await fs.writeFile(configPath, JSON.stringify(raw, null, 2), { mode: 0o600 }); + const config = await loadConfig(configPath, { env: {} }); + const app = createLloomServer(config, { logger, ...serverOptions }); + await app.listen(); + t.after(() => app.close({ stopRuntimes: false, httpGraceMs: 0 })); + const base = `http://127.0.0.1:${app.server.address().port}`; + const call = async (pathname, { key, body, method = body === undefined ? 'GET' : 'POST' } = {}) => { + const response = await fetch(base + pathname, { + method, + headers: { + ...(key ? { authorization: `Bearer ${key}` } : {}), + ...(body === undefined ? {} : { 'content-type': 'application/json' }) + }, + body: body === undefined ? undefined : JSON.stringify(body) + }); + return { status: response.status, headers: response.headers, json: await response.json() }; + }; + return { dir, base, call, membershipPath: path.join(dir, 'pool-membership.json') }; +} + +const exists = (file) => + fs.stat(file).then( + () => true, + () => false + ); + +test('pool node routes authenticate in their handlers, not with API keys', () => { + assert.equal(classifyRoute('POST', '/gateway/pool/enroll'), 'pool-node'); + assert.equal(classifyRoute('POST', '/gateway/pool/heartbeat'), 'pool-node'); + assert.equal(classifyRoute('GET', '/gateway/pool/enroll'), 'admin-read'); + assert.equal(classifyRoute('GET', '/gateway/pool/members'), 'admin-read'); + assert.equal(classifyRoute('POST', '/gateway/pool/tokens'), 'admin-write'); + assert.equal(classifyRoute('POST', '/gateway/pool/nodes/a/cordon'), 'admin-write'); + assert.equal(classifyRoute('POST', '/gateway/pool/enroll/x'), 'admin-write'); + // A non-loopback head with no remote admin still accepts enrollment and + // heartbeats; the handler checks the token or secret. + const remote = { server: { host: '10.0.0.1' }, security: { apiKeys: [INFERENCE] } }; + const req = (pathname, origin) => ({ + headers: { host: '10.0.0.1:8100', ...(origin ? { origin } : {}) }, + method: 'POST', + url: pathname + }); + assert.equal( + authorizeRequest(req('/gateway/pool/enroll'), remote, { method: 'POST', pathname: '/gateway/pool/enroll' }).ok, + true + ); + assert.equal( + authorizeRequest(req('/gateway/pool/tokens'), remote, { method: 'POST', pathname: '/gateway/pool/tokens' }).ok, + false + ); + assert.equal( + authorizeRequest(req('/gateway/pool/enroll', 'https://evil.example'), remote, { + method: 'POST', + pathname: '/gateway/pool/enroll' + }).code, + 'admin_origin_denied', + 'browsers on another origin cannot reach the enrollment route' + ); +}); + +test('membership stays inert and writes nothing until a token is issued', async (t) => { + const head = await startHead(t); + const members = await head.call('/gateway/pool/members', { key: ADMIN }); + assert.equal(members.status, 200); + assert.deepEqual(members.json.members, []); + assert.equal((await head.call('/gateway/pool/heartbeat', { body: { nodeId: 'x' }, key: 'lps_x' })).status, 401); + assert.equal((await head.call('/gateway/pool/enroll', { body: { nodeId: 'x' } })).status, 401); + assert.equal((await head.call('/gateway/pool/nodes/x/cordon', { key: ADMIN, body: {} })).status, 404); + assert.equal(await exists(head.membershipPath), false); +}); + +test('enroll, heartbeat, cordon and restart over HTTP', async (t) => { + const head = await startHead(t); + for (const key of [undefined, INFERENCE, 'lps_' + 'A'.repeat(43)]) { + assert.equal((await head.call('/gateway/pool/tokens', { key, body: {} })).status, 401, `token create as ${key}`); + } + const issued = await head.call('/gateway/pool/tokens', { key: ADMIN, body: { ttlMs: 600000, uses: 2 } }); + assert.equal(issued.status, 200); + assert.match(issued.json.token, /^lpt_/); + const stat = await fs.stat(head.membershipPath); + assert.equal(stat.mode & 0o777, 0o600); + assert.ok(!(await fs.readFile(head.membershipPath, 'utf8')).includes(issued.json.token.split('_').at(-1))); + + // An admin key is not an enrollment token, and enrollment never grants one. + const asAdmin = await head.call('/gateway/pool/enroll', { + key: ADMIN, + body: { nodeId: 'spark-node-01', endpoint: 'http://10.0.0.1:8100' } + }); + assert.equal(asAdmin.status, 401); + assert.equal(asAdmin.json.error.code, 'invalid_enrollment_token'); + const self = await head.call('/gateway/pool/enroll', { + key: issued.json.token, + body: { nodeId: 'head', endpoint: 'http://10.0.0.1:8100' } + }); + assert.equal(self.status, 409, 'a node cannot enroll as the head'); + + const enrolled = await head.call('/gateway/pool/enroll', { + key: issued.json.token, + body: { nodeId: 'spark-node-01', endpoint: 'http://10.0.0.1:8100', labels: { rack: 'a' } } + }); + assert.equal(enrolled.status, 200); + const { nodeSecret } = enrolled.json; + assert.match(nodeSecret, /^lps_/); + assert.deepEqual(Object.keys(enrolled.json).sort(), ['leaseMs', 'nodeId', 'nodeSecret', 'ok']); + assert.ok(!(await fs.readFile(head.membershipPath, 'utf8')).includes(nodeSecret.slice(4))); + + // The node secret only works for heartbeats. + for (const [pathname, body] of [ + ['/gateway/pool/members', undefined], + ['/gateway/pool/tokens', {}], + ['/gateway/pool/nodes/spark-node-01/uncordon', {}], + ['/gateway/status', undefined] + ]) { + assert.equal((await head.call(pathname, { key: nodeSecret, body })).status, 401, pathname); + } + assert.equal( + (await head.call('/gateway/pool/heartbeat', { key: ADMIN, body: { nodeId: 'spark-node-01' } })).status, + 401, + 'an admin key is not a node secret' + ); + const heartbeat = { + nodeId: 'spark-node-01', + running: ['small'], + inventory: { complete: true, computeArchs: ['sm_121'] } + }; + let beat = await head.call('/gateway/pool/heartbeat', { key: nodeSecret, body: heartbeat }); + assert.equal(beat.status, 200); + assert.equal(beat.json.cordoned, false); + + const cordoned = await head.call('/gateway/pool/nodes/spark-node-01/drain', { + key: ADMIN, + body: { reason: 'swap disks' } + }); + assert.equal(cordoned.status, 200); + assert.equal(cordoned.json.member.draining, true); + beat = await head.call('/gateway/pool/heartbeat', { key: nodeSecret, body: heartbeat }); + assert.equal(beat.json.draining, true); + + const listed = await head.call('/gateway/pool/members', { key: ADMIN }); + assert.equal(listed.json.leaseMs, 30000); + const [member] = listed.json.members; + assert.equal(member.id, 'spark-node-01'); + assert.equal(member.state, 'ready'); + assert.equal(member.reason, 'swap disks'); + assert.deepEqual(member.running, ['small']); + assert.ok(!JSON.stringify(listed.json).includes(nodeSecret)); + + const bad = await head.call('/gateway/pool/nodes/spark-node-01/cordon', { key: ADMIN, body: { reason: 7 } }); + assert.equal(bad.status, 400); + + // A restarted head keeps members, cordons and secrets. + const restarted = await startHead(t, { dir: await tempDir(t) }); + await fs.copyFile(head.membershipPath, restarted.membershipPath); + beat = await restarted.call('/gateway/pool/heartbeat', { key: nodeSecret, body: heartbeat }); + assert.equal(beat.status, 200); + assert.equal(beat.json.draining, true); +}); + +test('enrollment attempts are rate limited per client', async (t) => { + const head = await startHead(t); + const statuses = []; + for (let index = 0; index < 14; index += 1) { + const response = await head.call('/gateway/pool/enroll', { + key: 'lpt_000000000000_' + 'A'.repeat(43), + body: { nodeId: 'x', endpoint: 'http://x:8100' } + }); + statuses.push(response.status); + if (response.status === 429) { + assert.ok(Number(response.headers.get('retry-after')) > 0); + assert.equal(response.json.error.code, 'pool_enroll_rate_limited'); + } + } + assert.ok(statuses.slice(0, 10).every((status) => status === 401)); + assert.equal(statuses.at(-1), 429); +}); + +test('oversized and malformed pool bodies are rejected', async (t) => { + const head = await startHead(t); + const huge = await fetch(head.base + '/gateway/pool/heartbeat', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ nodeId: 'x', inventory: { padding: 'x'.repeat(2 * 1024 * 1024) } }) + }); + assert.equal(huge.status, 413); + const malformed = await fetch(head.base + '/gateway/pool/heartbeat', { method: 'POST', body: '{' }); + assert.equal(malformed.status, 400); +}); + +test('a node with pool.head heartbeats to the head from serve', async (t) => { + const head = await startHead(t); + const issued = await head.call('/gateway/pool/tokens', { key: ADMIN, body: {} }); + const enrolled = await head.call('/gateway/pool/enroll', { + key: issued.json.token, + body: { nodeId: 'worker', endpoint: 'http://10.0.0.2:8100' } + }); + const nodeDir = await tempDir(t); + await fs.writeFile(path.join(nodeDir, 'pool-node-secret'), enrolled.json.nodeSecret + '\n', { mode: 0o600 }); + await startHead(t, { + dir: nodeDir, + pool: { head: { url: head.base, nodeId: 'worker', secretFile: 'pool-node-secret', intervalMs: 50 } }, + serverOptions: { + nodeInventory: { get: async () => ({ nodeId: 'worker', complete: true, computeArchs: ['sm_121'] }) } + } + }); + const deadline = Date.now() + 20000; + let member; + while (Date.now() < deadline) { + [member] = (await head.call('/gateway/pool/members', { key: ADMIN })).json.members; + if (member.inventory) break; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + assert.equal(member.state, 'ready'); + assert.equal(member.inventory.nodeId, undefined, 'only planner fields are kept'); + assert.deepEqual(member.inventory.computeArchs, ['sm_121']); + assert.deepEqual(member.running, []); +}); + +test('node secrets come from an env var or a private file', async (t) => { + const dir = await tempDir(t); + const configPath = path.join(dir, 'config.json'); + const secret = 'lps_' + 'B'.repeat(43); + assert.equal( + await resolvePoolNodeSecret({ secretEnv: 'POOL_SECRET' }, { configPath, env: { POOL_SECRET: secret } }), + secret + ); + await assert.rejects(resolvePoolNodeSecret({ secretEnv: 'POOL_SECRET' }, { configPath, env: {} }), /POOL_SECRET/); + const file = path.join(dir, 'secret'); + await fs.writeFile(file, secret + '\n', { mode: 0o600 }); + assert.equal(await resolvePoolNodeSecret({ secretFile: 'secret' }, { configPath }), secret); + await fs.chmod(file, 0o644); + await assert.rejects(resolvePoolNodeSecret({ secretFile: 'secret' }, { configPath }), /private/); + await fs.chmod(file, 0o600); + await fs.symlink(file, path.join(dir, 'link')); + await assert.rejects(resolvePoolNodeSecret({ secretFile: 'link' }, { configPath }), /regular file/); + await assert.rejects(resolvePoolNodeSecret({}, { configPath }), /secretEnv or secretFile/); +}); + +test('the heartbeat loop backs off on failure and resets on success', async () => { + const delays = []; + let failures = 2; + const sent = []; + const loop = createPoolHeartbeatLoop({ + getHead: () => ({ url: 'http://head.test:8100', nodeId: 'worker', secretEnv: 'S', intervalMs: 1000 }), + configPath: '/nonexistent/config.json', + env: { S: 'lps_' + 'C'.repeat(43) }, + collect: async () => ({ inventory: { complete: true }, running: ['a'], telemetry: null }), + fetchImpl: async (url, options) => { + sent.push({ url, options }); + if (failures-- > 0) return new Response('{}', { status: 503 }); + return Response.json({ ok: true, cordoned: false, draining: false }); + }, + logger, + schedule: (fn, ms) => { + delays.push(ms); + return { unref() {}, fn }; + }, + cancel: () => {}, + random: () => 0.5 + }); + for (let index = 0; index < 3; index += 1) await loop.tick(); + assert.deepEqual(delays, [2000, 4000, 1000]); + assert.equal(sent[0].url, 'http://head.test:8100/gateway/pool/heartbeat'); + assert.equal(sent[0].options.headers.authorization, 'Bearer lps_' + 'C'.repeat(43)); + assert.equal(sent[0].options.redirect, 'error'); + assert.deepEqual(JSON.parse(sent[0].options.body), { + nodeId: 'worker', + inventory: { complete: true }, + running: ['a'], + telemetry: null + }); + assert.equal(loop.status().failures, 0); + await loop.stop(); +}); diff --git a/test/pool-membership.test.mjs b/test/pool-membership.test.mjs new file mode 100644 index 0000000..0793956 --- /dev/null +++ b/test/pool-membership.test.mjs @@ -0,0 +1,321 @@ +// Pool membership: enrollment tokens, node leases and planner node shapes. +// Time is injected, so every lease transition is deterministic. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createPoolMembership, memberPlannerNode } from '../src/pool-membership.mjs'; + +function clock(start = Date.parse('2026-10-11T12:00:00Z')) { + let at = start; + const now = () => at; + now.advance = (ms) => { + at += ms; + }; + return now; +} + +const inventory = { + version: 1, + complete: true, + platformId: 'linux-arm64', + accelerators: ['cuda', 'gb10'], + computeArchs: ['sm_121'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 128 }], + links: [{ peerNodeId: 'spark-node-02', localInterface: 'enp1s0f0np0' }], + weights: [ + { recipeId: 'spark-27b', root: '/models', complete: true }, + { recipeId: 'partial', root: '/models', complete: false } + ], + backends: { commands: { docker: true }, dockerImages: ['vllm/vllm-openai@sha256:abc'] } +}; + +function enrolled(options = {}) { + const now = clock(); + const membership = createPoolMembership({ leaseMs: 30000, now, ...options }); + const { token } = membership.issueEnrollmentToken({}); + const joined = membership.enroll({ + token, + nodeId: 'spark-node-01', + endpoint: 'http://10.0.0.1:8100', + inventory, + labels: { rack: 'a' } + }); + return { now, membership, ...joined }; +} + +const member = (membership, id = 'spark-node-01') => membership.members().find((entry) => entry.id === id); + +test('a token enrolls a node with a fresh lease and a secret only the node sees', () => { + const { membership, nodeId, nodeSecret, leaseMs } = enrolled(); + assert.equal(nodeId, 'spark-node-01'); + assert.equal(leaseMs, 30000); + assert.match(nodeSecret, /^lps_[A-Za-z0-9_-]{43}$/); + const entry = member(membership); + assert.equal(entry.state, 'ready'); + assert.equal(entry.endpoint, 'http://10.0.0.1:8100'); + assert.deepEqual(entry.labels, { rack: 'a' }); + assert.equal(entry.cordoned, false); + assert.equal(entry.draining, false); + assert.equal(entry.enrolledAt, '2026-10-11T12:00:00.000Z'); + assert.equal(entry.lastSeenAt, '2026-10-11T12:00:00.000Z'); + assert.equal(entry.leaseExpiresAt, '2026-10-11T12:00:30.000Z'); + assert.deepEqual(entry.inventory.computeArchs, ['sm_121']); + assert.ok(!JSON.stringify(membership.members()).includes(nodeSecret)); +}); + +test('the lease runs ready, suspect, lost; a heartbeat brings the node back', () => { + const { now, membership, nodeSecret } = enrolled(); + now.advance(29999); + assert.equal(member(membership).state, 'ready'); + now.advance(1); + assert.equal(member(membership).state, 'suspect', 'lease expired, lostAfterMs not reached'); + now.advance(59999); + assert.equal(member(membership).state, 'suspect'); + now.advance(1); + assert.equal(member(membership).state, 'lost', 'lost 3 x leaseMs after the last heartbeat'); + + const beat = membership.heartbeat({ + nodeId: 'spark-node-01', + nodeSecret, + running: ['qwen38-27b-nvfp4'], + telemetry: { memory: { availableBytes: 1 } } + }); + assert.deepEqual(beat, { + ok: true, + leaseExpiresAt: new Date(now() + 30000).toISOString(), + cordoned: false, + draining: false + }); + const entry = member(membership); + assert.equal(entry.state, 'ready'); + assert.deepEqual(entry.running, ['qwen38-27b-nvfp4']); + assert.deepEqual(entry.inventory.computeArchs, ['sm_121'], 'a heartbeat without inventory keeps the last one'); +}); + +test('lostAfterMs is configurable but never shorter than the lease', () => { + const { now, membership } = enrolled({ lostAfterMs: 45000 }); + now.advance(44999); + assert.equal(member(membership).state, 'suspect'); + now.advance(1); + assert.equal(member(membership).state, 'lost'); + assert.throws(() => createPoolMembership({ leaseMs: 30000, lostAfterMs: 1000 }), /lostAfterMs/); + assert.throws(() => createPoolMembership({ leaseMs: 0 }), /leaseMs/); +}); + +test('a bad secret, an unknown node and a malformed secret are all rejected the same way', () => { + const { membership, nodeSecret } = enrolled(); + const rejected = (call) => + assert.throws(call, (error) => error.statusCode === 401 && error.code === 'invalid_node_credential'); + rejected(() => membership.heartbeat({ nodeId: 'spark-node-01', nodeSecret: nodeSecret.slice(0, -1) + 'x' })); + rejected(() => membership.heartbeat({ nodeId: 'spark-node-01', nodeSecret: 'short' })); + rejected(() => membership.heartbeat({ nodeId: 'spark-node-01' })); + rejected(() => membership.heartbeat({ nodeId: 'other', nodeSecret })); +}); + +test('tokens expire and run out of uses', () => { + const now = clock(); + const membership = createPoolMembership({ now }); + const rejected = (call) => + assert.throws(call, (error) => error.statusCode === 401 && error.code === 'invalid_enrollment_token'); + const single = membership.issueEnrollmentToken({ ttlMs: 60000 }); + assert.match(single.token, /^lpt_[0-9a-f]{12}_[A-Za-z0-9_-]{43}$/); + assert.equal(single.expiresAt, '2026-10-11T12:01:00.000Z'); + membership.enroll({ token: single.token, nodeId: 'a', endpoint: 'http://a:8100' }); + rejected(() => membership.enroll({ token: single.token, nodeId: 'b', endpoint: 'http://b:8100' })); + + const twice = membership.issueEnrollmentToken({ uses: 2, ttlMs: 60000 }); + membership.enroll({ token: twice.token, nodeId: 'b', endpoint: 'http://b:8100' }); + membership.enroll({ token: twice.token, nodeId: 'c', endpoint: 'http://c:8100' }); + rejected(() => membership.enroll({ token: twice.token, nodeId: 'd', endpoint: 'http://d:8100' })); + + const expiring = membership.issueEnrollmentToken({ ttlMs: 60000 }); + now.advance(60000); + rejected(() => membership.enroll({ token: expiring.token, nodeId: 'e', endpoint: 'http://e:8100' })); + rejected(() => + membership.enroll({ token: 'lpt_000000000000_' + 'A'.repeat(43), nodeId: 'e', endpoint: 'http://e:8100' }) + ); + rejected(() => membership.enroll({ token: 'not-a-token', nodeId: 'e', endpoint: 'http://e:8100' })); + rejected(() => membership.enroll({ nodeId: 'e', endpoint: 'http://e:8100' })); + assert.deepEqual( + membership.members().map((entry) => entry.id), + ['a', 'b', 'c'] + ); +}); + +test('token options and enrollment input are validated before a use is consumed', () => { + const membership = createPoolMembership({ now: clock() }); + assert.throws(() => membership.issueEnrollmentToken({ ttlMs: 0 }), /ttlMs/); + assert.throws(() => membership.issueEnrollmentToken({ uses: 1.5 }), /uses/); + assert.throws(() => membership.issueEnrollmentToken({ labels: { bad: 1 } }), /labels/); + const { token } = membership.issueEnrollmentToken({}); + const invalid = (input, pattern) => + assert.throws( + () => membership.enroll({ token, endpoint: 'http://a:8100', nodeId: 'a', ...input }), + (error) => error.statusCode === 400 && pattern.test(error.message) + ); + invalid({ nodeId: '../etc' }, /node identity/); + invalid({ nodeId: '' }, /node identity/); + invalid({ endpoint: 'ftp://a' }, /HTTP\(S\)/); + invalid({ endpoint: 'http://user:pass@a:8100' }, /HTTP\(S\)/); + invalid({ endpoint: 'http://a:8100/?x=1' }, /HTTP\(S\)/); + invalid({ endpoint: 'not a url' }, /endpoint/); + invalid({ labels: { k: { nested: true } } }, /labels/); + // The token is still unused after every rejected attempt. + membership.enroll({ token, nodeId: 'a', endpoint: 'http://a:8100/' }); + assert.equal(member(membership, 'a').endpoint, 'http://a:8100'); +}); + +test('token labels win over node-supplied labels', () => { + const membership = createPoolMembership({ now: clock() }); + const { token } = membership.issueEnrollmentToken({ labels: { role: 'gpu' } }); + membership.enroll({ token, nodeId: 'a', endpoint: 'http://a:8100', labels: { role: 'head', zone: 'x' } }); + assert.deepEqual(member(membership, 'a').labels, { role: 'gpu', zone: 'x' }); +}); + +test('re-enrolling needs a new token and rotates the secret', () => { + const { membership, nodeSecret } = enrolled(); + const reuse = membership.issueEnrollmentToken({ uses: 3 }); + const first = membership.enroll({ token: reuse.token, nodeId: 'spark-node-02', endpoint: 'http://b:8100' }); + assert.throws( + () => membership.enroll({ token: reuse.token, nodeId: 'spark-node-02', endpoint: 'http://b:8100' }), + (error) => error.statusCode === 401 && error.code === 'invalid_enrollment_token', + 'the token that enrolled a node cannot re-enroll it' + ); + membership.cordon('spark-node-01', { reason: 'maintenance' }); + const { token } = membership.issueEnrollmentToken({}); + const rotated = membership.enroll({ token, nodeId: 'spark-node-01', endpoint: 'http://10.0.0.9:8100' }); + assert.notEqual(rotated.nodeSecret, nodeSecret); + assert.throws(() => membership.heartbeat({ nodeId: 'spark-node-01', nodeSecret }), /credential/); + assert.equal(membership.heartbeat({ nodeId: 'spark-node-01', nodeSecret: rotated.nodeSecret }).cordoned, true); + const entry = member(membership); + assert.equal(entry.endpoint, 'http://10.0.0.9:8100'); + assert.equal(entry.cordoned, true, 'a node cannot re-enroll its way out of a cordon'); + assert.equal(entry.reason, 'maintenance'); + assert.equal( + membership.heartbeat({ nodeId: 'spark-node-02', nodeSecret: first.nodeSecret }).ok, + true, + 'other nodes keep their secrets' + ); +}); + +test('cordon, drain, uncordon and leave', () => { + const { membership, nodeSecret } = enrolled(); + assert.equal(membership.cordon('spark-node-01', { reason: 'disk swap' }).cordoned, true); + let beat = membership.heartbeat({ nodeId: 'spark-node-01', nodeSecret }); + assert.equal(beat.cordoned, true); + assert.equal(beat.draining, false); + assert.equal(member(membership).reason, 'disk swap'); + + const drained = membership.drain('spark-node-01', { reason: 'rebalance' }); + assert.equal(drained.draining, true); + assert.equal(drained.reason, 'rebalance'); + beat = membership.heartbeat({ nodeId: 'spark-node-01', nodeSecret }); + assert.equal(beat.draining, true); + + const cleared = membership.uncordon('spark-node-01'); + assert.equal(cleared.cordoned, false); + assert.equal(cleared.draining, false); + assert.equal(cleared.reason, null); + + const left = membership.leave('spark-node-01'); + assert.equal(left.state, 'left'); + assert.throws(() => membership.heartbeat({ nodeId: 'spark-node-01', nodeSecret }), /credential/); + for (const action of ['cordon', 'uncordon', 'drain', 'leave']) { + assert.throws( + () => membership[action]('missing'), + (error) => error.statusCode === 404 + ); + } + assert.throws(() => membership.cordon('spark-node-01', { reason: 'x'.repeat(1000) }), /reason/); +}); + +test('planner nodes carry liveness, cordon and inventory in the planner shape', () => { + const { now, membership } = enrolled(); + let [node] = membership.plannerNodes(); + assert.deepEqual(node, { + id: 'spark-node-01', + available: true, + cordoned: false, + membership: 'ready', + platformId: 'linux-arm64', + computeArchs: ['sm_121'], + accelerators: ['cuda', 'gb10'], + memoryDomains: [{ id: 'unified:0', kind: 'unified', capacityGb: 128 }], + links: ['spark-node-02'], + weights: ['spark-27b'], + images: ['vllm/vllm-openai@sha256:abc'], + inventory: 'reported' + }); + now.advance(30000); + assert.equal(membership.plannerNodes()[0].available, true, 'a suspect node is still planned on'); + now.advance(60000); + assert.equal(membership.plannerNodes()[0].available, false, 'lost'); + membership.drain('spark-node-01', {}); + [node] = membership.plannerNodes(); + assert.equal(node.cordoned, true, 'draining nodes take no new placements'); + membership.leave('spark-node-01'); + assert.equal(membership.plannerNodes()[0].available, false); + assert.equal(memberPlannerNode({ id: 'bare', state: 'ready' }).inventory, 'none'); +}); + +test('untrusted inventory is reduced to the fields the planner reads', () => { + const membership = createPoolMembership({ now: clock() }); + const { token } = membership.issueEnrollmentToken({}); + membership.enroll({ + token, + nodeId: 'a', + endpoint: 'http://a:8100', + inventory: { + complete: 'yes', + computeArchs: ['sm_121', 7, { x: 1 }], + memoryDomains: [{ id: 'm', kind: 'unified', capacityGb: 'lots' }, { id: 'n', capacityGb: 64 }, 'bad'], + links: ['b', { peerNodeId: 'c' }, { nope: true }], + weights: 'all', + secret: 'do not keep', + backends: { dockerImages: [1, 'img'], commands: { docker: 'true', uv: true } } + } + }); + const { inventory: stored } = member(membership, 'a'); + assert.deepEqual(stored, { + complete: false, + computeArchs: ['sm_121'], + memoryDomains: [{ id: 'n', capacityGb: 64 }], + links: [{ peerNodeId: 'b' }, { peerNodeId: 'c' }], + weights: [], + backends: { commands: { uv: true }, dockerImages: ['img'] } + }); + assert.throws( + () => membership.heartbeat({ nodeId: 'a', nodeSecret: 'x', running: 'nope' }), + /credential/, + 'authentication comes before input validation' + ); +}); + +test('persisted state has hashes but no secrets or tokens, and restores', () => { + const now = clock(); + const membership = createPoolMembership({ now }); + const pending = membership.issueEnrollmentToken({ uses: 2, labels: { role: 'gpu' } }); + const { nodeSecret } = membership.enroll({ token: pending.token, nodeId: 'a', endpoint: 'http://a:8100', inventory }); + membership.cordon('a', { reason: 'test' }); + membership.heartbeat({ nodeId: 'a', nodeSecret, telemetry: { load: 1 } }); + const state = JSON.parse(JSON.stringify(membership.toJSON())); + const text = JSON.stringify(state); + assert.ok(!text.includes(nodeSecret)); + assert.ok(!text.includes(pending.token)); + assert.ok(!text.includes(nodeSecret.slice(4)), 'not even the secret body'); + assert.ok(!text.includes(pending.token.split('_').at(-1))); + assert.ok(!text.includes('"load"'), 'telemetry is live state only'); + assert.equal(state.version, 1); + + const restored = createPoolMembership({ now, state }); + assert.deepEqual( + restored.members().map(({ telemetry: _telemetry, ...entry }) => entry), + membership.members().map(({ telemetry: _telemetry, ...entry }) => entry) + ); + assert.equal(restored.heartbeat({ nodeId: 'a', nodeSecret }).cordoned, true, 'the restored secret hash verifies'); + restored.enroll({ token: pending.token, nodeId: 'b', endpoint: 'http://b:8100' }); + assert.throws(() => restored.enroll({ token: pending.token, nodeId: 'c', endpoint: 'http://c:8100' }), /token/); + + now.advance(3600000); + assert.deepEqual(restored.toJSON().tokens, [], 'spent and expired tokens are pruned'); + assert.throws(() => createPoolMembership({ state: { version: 99 } }), /version/); +}); From 280ed3cb53d14300a5d2490e12267be878cdbf43 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 12:28:44 -0700 Subject: [PATCH 4/9] Keep admission below the memory guard; learn from guard aborts The fleet simulator showed admission could approve a load the runtime memory guard was certain to abort: admission kept only reserveMemoryGb (8 GB) free, while the guard trips at 90% host use (12.8 GB free on a 128 GB Spark). After an abort the runtime is latched until a manual start, so the lane stayed down. - runtime-policy: the host check keeps max(reserve, guard margin) free and cluster node budgets are capped at the guard ceiling. - runtime-memory-safety: memorySafetyCeiling() shared by the guard, admission and pool planning. - runtime-manager: a guard abort records usedGb and consumedGb (the load's lower-bound footprint) and status reports memorySafetyLatched. - pool-catalog: raises a variant's memory to the abort lower bound and excludes a latched lane until it is started by hand. - reconciler: per-step timeouts so a stuck step cannot wedge the controller; simulator found and now covers a drain that never settled. - simulator: models the latch and fixed admission (legacyAdmission keeps the old behaviour for comparison) and logs controller hold reasons. Co-Authored-By: Claude Opus 5.5 --- src/pool-catalog.mjs | 28 ++++++++- src/pool-controller.mjs | 12 +++- src/pool-planner.mjs | 2 +- src/pool-reconciler.mjs | 35 +++++++++-- src/pool-sim.mjs | 60 ++++++++++++++++--- src/runtime-manager.mjs | 29 ++++++++- src/runtime-memory-safety.mjs | 16 +++-- src/runtime-policy.mjs | 31 +++++++--- .../pool/scenarios/memory-estimate-low.json | 29 +++++++-- test/pool-catalog.test.mjs | 33 ++++++++++ test/pool-reconciler.test.mjs | 27 +++++++++ test/pool-sim.test.mjs | 9 +++ test/runtime-memory-safety.test.mjs | 8 +++ test/runtime-policy.test.mjs | 34 +++++++++++ 14 files changed, 319 insertions(+), 34 deletions(-) diff --git a/src/pool-catalog.mjs b/src/pool-catalog.mjs index bcaf598..953b7ea 100644 --- a/src/pool-catalog.mjs +++ b/src/pool-catalog.mjs @@ -172,6 +172,23 @@ function measuredMemoryGb(nodeStatuses, nodes, runtimeIds) { return largest ? round(largest) : null; } +// A memory-safety abort is evidence the estimate is too low: the load had +// taken `consumedGb` when the guard stopped it. The runtime stays latched (no +// automatic restarts) until someone starts it by hand. +function memorySafetyEvidence(nodeStatuses, nodes, runtimeIds) { + let abortGb = 0; + const latched = []; + for (const nodeId of nodes) { + const runtimes = object(nodeStatuses?.[nodeId]?.runtimeManager?.runtimes); + for (const runtimeId of runtimeIds) { + const status = object(runtimes[runtimeId]); + abortGb = Math.max(abortGb, num(status.memorySafetyAbort?.consumedGb) ?? 0); + if (status.memorySafetyLatched === true && !latched.includes(nodeId)) latched.push(nodeId); + } + } + return { abortGb: abortGb ? round(abortGb) : null, latched }; +} + function memberRuntimeIds(runtime, runtimeId) { const members = runtime?.placement?.mode === 'distributed' ? (runtime.placement.members ?? []) : []; return members.length ? members.map((member) => member.runtime).filter(Boolean) : [runtimeId]; @@ -210,7 +227,9 @@ function configVariants(config, { nodeStatuses, env }) { const nodes = Object.keys(resources).sort(); if (!nodes.length) continue; const configuredGb = Math.max(0, ...nodes.map((nodeId) => resources[nodeId].memoryGb)); - const measuredGb = measuredMemoryGb(nodeStatuses, nodes, memberRuntimeIds(runtime, model.runtime)); + const runtimeIds = memberRuntimeIds(runtime, model.runtime); + const measuredGb = measuredMemoryGb(nodeStatuses, nodes, runtimeIds); + const safety = memorySafetyEvidence(nodeStatuses, nodes, runtimeIds); variants.push({ ...base, id: `${model.id}@${nodes.join('+')}`, @@ -219,13 +238,16 @@ function configVariants(config, { nodeStatuses, env }) { nodes, requirements: compact({ nodeCount: nodes.length, - memoryGbPerNode: Math.max(configuredGb, measuredGb ?? 0) || null, + memoryGbPerNode: Math.max(configuredGb, measuredGb ?? 0, safety.abortGb ?? 0) || null, contextWindow: num(model.contextWindow ?? runtime.contextWindow), maxSequences: num(runtime.maxActiveRequests), recipeId: runtime.recipe?.id ?? null, images: runtime.bootstrap?.image ? [runtime.bootstrap.image] : null }), - ...(measuredGb ? { measured: { memoryGbPerNode: measuredGb, source: 'live-rss' } } : {}) + ...(measuredGb ? { measured: { memoryGbPerNode: measuredGb, source: 'live-rss' } } : {}), + ...(safety.latched.length + ? { excludeNodes: safety.latched, excludeReason: 'memory safety abort; start it by hand to retry' } + : {}) }); continue; } diff --git a/src/pool-controller.mjs b/src/pool-controller.mjs index 83e0151..ea2472a 100644 --- a/src/pool-controller.mjs +++ b/src/pool-controller.mjs @@ -125,13 +125,22 @@ export function decide({ plan, stay, reconcile, state, policy, now }) { } /** + * timer(ms): optional promise-returning delay for step timeouts (a simulated + * clock passes its own). * observe(): Promise<{ nodes, variants, workloads, pins, policy, current }> * current = { routes, residency, running: [placement keys], * placements: [{ key, models, runtime, nodes, memoryGbPerNode, managed, state }], * missingWeights: { placementKey: true } } * actuator: see executeReconcile(). */ -export function createPoolController({ observe, actuator, policy = {}, now = Date.now, onEvent = () => {} } = {}) { +export function createPoolController({ + observe, + actuator, + policy = {}, + now = Date.now, + timer, + onEvent = () => {} +} = {}) { const controllerPolicy = normalizeControllerPolicy(policy); const state = { signature: null, @@ -190,6 +199,7 @@ export function createPoolController({ observe, actuator, policy = {}, now = Dat state.loads = state.loads.filter((load) => at - load.at < 3600000); state.executing = executeReconcile(reconcile.steps, actuator, { concurrency: controllerPolicy.concurrency, + ...(timer ? { timer } : {}), onStep: ({ step, status, error }) => { onEvent({ at: now(), action: 'step', step: step.id, status, ...(error ? { error } : {}) }); if (status === 'failed' && (step.type === 'start' || step.type === 'manual-start')) { diff --git a/src/pool-planner.mjs b/src/pool-planner.mjs index c653972..14da8d2 100644 --- a/src/pool-planner.mjs +++ b/src/pool-planner.mjs @@ -144,7 +144,7 @@ function variantMemoryGb(variant) { function nodeIneligibility(variant, node) { if (node.cordoned) return 'cordoned'; if (node.available === false) return 'unavailable'; - if (list(variant.excludeNodes).includes(node.id)) return 'excluded after a failed start'; + if (list(variant.excludeNodes).includes(node.id)) return variant.excludeReason ?? 'excluded after a failed start'; const requirements = variant.requirements ?? {}; const platforms = list(requirements.platforms); if (platforms.length && !platforms.includes(node.platformId)) return `needs platform ${platforms.join('|')}`; diff --git a/src/pool-reconciler.mjs b/src/pool-reconciler.mjs index 147b145..bd0e467 100644 --- a/src/pool-reconciler.mjs +++ b/src/pool-reconciler.mjs @@ -328,9 +328,32 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ * actuator: { prefetch(step), start(step), stop(step), route(step), residency(step), manual(step) } * Every method returns a promise that settles when the step is done (a start * when the lane is ready, a stop when it has drained). Manual steps resolve - * when the actuator observes the operator's change or rejects on timeout. + * when the actuator observes the operator's change. A step that does not + * settle within its timeout fails, so one stuck step cannot hold the + * controller forever; `timer(ms)` is injectable for simulated clocks. */ -export async function executeReconcile(steps, actuator, { concurrency = 4, onStep = () => {} } = {}) { +export const DEFAULT_STEP_TIMEOUTS_MS = Object.freeze({ + prefetch: 4 * 60 * 60 * 1000, + start: 45 * 60 * 1000, + stop: 20 * 60 * 1000, + route: 60 * 1000, + 'interim-route': 60 * 1000, + residency: 60 * 1000, + 'manual-start': 2 * 60 * 60 * 1000, + 'manual-stop': 2 * 60 * 60 * 1000 +}); + +const defaultTimer = (ms) => + new Promise((resolve) => { + const handle = setTimeout(resolve, ms); + handle.unref?.(); + }); + +export async function executeReconcile( + steps, + actuator, + { concurrency = 4, onStep = () => {}, timeoutsMs = {}, timer = defaultTimer } = {} +) { const status = new Map(steps.map((step) => [step.id, 'pending'])); const errors = {}; const byId = new Map(steps.map((step) => [step.id, step])); @@ -368,8 +391,12 @@ export async function executeReconcile(steps, actuator, { concurrency = 4, onSte status.set(step.id, 'running'); running.add(step.id); onStep({ step, status: 'running' }); - Promise.resolve() - .then(() => actuator[method(step.type)](step)) + const limitMs = timeoutsMs[step.type] ?? DEFAULT_STEP_TIMEOUTS_MS[step.type]; + const timedOut = timer(limitMs).then(() => { + throw new Error(`${step.type} timed out after ${Math.round(limitMs / 1000)} s`); + }); + timedOut.catch(() => {}); + Promise.race([Promise.resolve().then(() => actuator[method(step.type)](step)), timedOut]) .then( () => { status.set(step.id, 'done'); diff --git a/src/pool-sim.mjs b/src/pool-sim.mjs index f3b1f86..ee21199 100644 --- a/src/pool-sim.mjs +++ b/src/pool-sim.mjs @@ -21,6 +21,7 @@ import { createPoolController } from './pool-controller.mjs'; import { memoryBudgetGb } from './resource-fit.mjs'; +import { memorySafetyCeiling } from './runtime-memory-safety.mjs'; export const POOL_SIM_VERSION = 1; @@ -155,7 +156,11 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c const demandWindowMs = (num(scenario.demandWindowHours) ?? 24) * HOUR; const operator = scenario.operator ?? {}; const memoryGuard = scenario.memoryGuard ?? {}; - const guardUtilization = num(memoryGuard.maxMemoryUtilization) ?? 0.9; + const guardUtilization = + memorySafetyCeiling({ + maxMemoryUtilization: num(policy.maxMemoryUtilization) ?? undefined, + memorySafety: { maxMemoryUtilization: num(memoryGuard.maxMemoryUtilization) ?? undefined } + }) ?? 1; const osGb = num(scenario.osMemoryGb) ?? 4; let now = 0; @@ -177,7 +182,8 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c stepFailures: 0, interruptions: 0, firstProposal: null, - decisions: [] + decisions: [], + holds: [] }, timeline: [] }; @@ -214,6 +220,7 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c const served = new Map(); const arrivals = new Map(workloads.map((workload) => [workload.id, []])); const flexiblePlacements = new Map(); + const latched = new Set(); const truth = (variant) => variant.sim ?? {}; const memoryTruth = (variant) => @@ -234,6 +241,15 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c maxMemoryUtilization: num(nodes.get(id)?.spec.maxMemoryUtilization) ?? num(policy.maxMemoryUtilization), reserveMemoryGb: num(policy.reserveGb) ?? 8 }); + // Admission's host check keeps this much free after a load. Legacy admission + // kept only the reserve, below the guard's margin, so it could admit loads + // the guard was certain to abort. + const hostReserve = (id) => { + const reserve = num(policy.reserveGb) ?? 8; + if (scenario.legacyAdmission === true || memoryGuard.enabled === false) return reserve; + return Math.max(reserve, capacity(id) * (1 - guardUtilization)); + }; + const admissionLimit = (id) => Math.min(budget(id), capacity(id) - osGb - hostReserve(id)); const lanesOn = (id) => [...lanes.values()].filter((lane) => lane.nodes.includes(id)); const usedTruth = (id) => lanesOn(id).reduce((sum, lane) => sum + lane.memoryGb, 0); const laneModels = (variant, laneNodes) => @@ -314,8 +330,9 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c } function removeLane(lane, reason) { - if (!lanes.has(lane.key) || lanes.get(lane.key) !== lane) return; - lanes.delete(lane.key); + if (lane.removed) return; + lane.removed = true; + if (lanes.get(lane.key) === lane) lanes.delete(lane.key); report.lanes.stops += reason === 'evicted' || reason === 'drained' || reason === 'stopped' ? 1 : 0; timeline('lane-stop', { lane: lane.key, reason }); for (const resolve of lane.onRemoved.splice(0)) resolve(); @@ -343,7 +360,7 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c for (const id of laneNodes) { let used = usedTruth(id) - evictions.filter((lane) => lane.nodes.includes(id)).reduce((s, l) => s + l.memoryGb, 0); - if (used + need <= budget(id)) continue; + if (used + need <= admissionLimit(id)) continue; if (!evict) return null; const idle = lanesOn(id) .filter( @@ -352,11 +369,11 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c .filter((lane) => RESIDENCY_RANK[tierOf(lane)] <= protect) .sort((a, b) => RESIDENCY_RANK[tierOf(a)] - RESIDENCY_RANK[tierOf(b)] || a.lastUsedAt - b.lastUsedAt); for (const lane of idle) { - if (used + need <= budget(id)) break; + if (used + need <= admissionLimit(id)) break; evictions.push(lane); used -= lane.memoryGb; } - if (used + need > budget(id)) return null; + if (used + need > admissionLimit(id)) return null; } return evictions; } @@ -364,8 +381,14 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c function startLane(variant, laneNodes, { reason, protect = 1, fetch = true, background = false } = {}) { const key = placementKey(variant, laneNodes); const existing = lanes.get(key); - if (existing && existing.state !== 'draining') return existing; + // A lane being retired is not restarted under it; requests go elsewhere. + if (existing) return existing.state === 'draining' ? null : existing; if (!laneNodes.every(nodeUsable)) return null; + // After a memory-safety abort LLooM refuses automatic restarts until a + // manual start (runtime-manager memorySafetyFailures). + const manual = reason === 'operator'; + if (latched.has(variant.id) && !manual) return null; + latched.delete(variant.id); const evictions = admit(variant, laneNodes, { protect }); if (!evictions) { // The residency reconciler retries every 30 s; only count refusals someone waits on. @@ -402,10 +425,15 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c const guarded = variant.managed !== false && memoryGuard.enabled !== false; const overGuard = laneNodes.find((id) => usedTruth(id) + osGb > capacity(id) * guardUtilization); if (guarded && overGuard) { + // The load had taken this much when host use crossed the ceiling: a + // lower bound on its footprint, which the catalog learns. + const consumedGb = Math.max(0, capacity(overGuard) * guardUtilization - (usedTruth(overGuard) - memoryGb + osGb)); queue.push(now + seconds * 500, () => { if (lanes.get(key) !== lane || lane.state !== 'loading') return; report.lanes.guardAborts += 1; report.lanes.startFailures += 1; + latched.add(variant.id); + learnedMemory.set(variant.id, Math.max(learnedMemory.get(variant.id) ?? 0, round(consumedGb, 1))); timeline('memory-guard-abort', { lane: key, node: overGuard, usedGb: round(usedTruth(overGuard) + osGb, 1) }); killLane(lane, 'memory safety abort'); }); @@ -608,6 +636,13 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c const mean = samples.reduce((sum, value) => sum + value, 0) / samples.length; next = { ...next, performance: { ...next.performance, ttftMs: Math.round(mean), source: 'observed' } }; } + if (latched.has(variant.id)) { + next = { + ...next, + excludeNodes: knownPlacement(variant) ?? [], + excludeReason: 'memory safety abort; start it by hand to retry' + }; + } const { sim: _sim, ...plannerVariant } = next; return plannerVariant; }); @@ -759,12 +794,21 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c observe: async () => observe(), actuator, now: () => now, + timer: (ms) => new Promise((resolve) => queue.push(now + ms, resolve)), policy: { ...(scenario.controller ?? {}), ...controllerOverrides, mode: mode === 'advise' ? 'advise' : 'auto' }, onEvent: (entry) => { + // Each change in why the controller is holding, so a run explains itself. + if ((entry.action === 'hold' && entry.reason !== 'no changes') || entry.action === 'busy') { + const reason = (entry.reason ?? 'still applying the last change').replace(/\(\d+\/\d+\)|[\d.]+%/g, '#'); + const holds = report.controller.holds; + if (holds.at(-1)?.reason !== reason && holds.length < 200) { + holds.push({ atMinutes: round(entry.at / MINUTE, 2), reason }); + } + } if (entry.action === 'apply' || entry.action === 'propose') { if (entry.action === 'propose') report.controller.proposals += 1; else report.controller.applies += 1; diff --git a/src/runtime-manager.mjs b/src/runtime-manager.mjs index 0fe540b..c73a893 100644 --- a/src/runtime-manager.mjs +++ b/src/runtime-manager.mjs @@ -18,6 +18,7 @@ import { } from './cluster.mjs'; import { cleanupPortListener, terminateProcessTree } from './process-control.mjs'; import { memorySafetyPolicy, createMemorySafetyGuard, RuntimeMemorySafetyError } from './runtime-memory-safety.mjs'; +import { readHostMemory } from './host-memory.mjs'; import { createRuntimeMemoryUsageSampler } from './runtime-memory-usage.mjs'; import { @@ -48,6 +49,21 @@ function nowIso() { return new Date().toISOString(); } +// What a memory-safety abort tells pool planning: host memory in use when the +// guard tripped, and how much of it this load had taken (a lower bound on its +// footprint, since the load was cut short). +function memorySafetyAbortRecord(baseline, snapshot) { + const gib = (bytes) => Math.round((bytes / 1024 ** 3) * 10) / 10; + const total = numberOrNull(snapshot?.totalBytes); + const available = numberOrNull(snapshot?.availableBytes); + const before = numberOrNull(baseline?.availableBytes); + return { + at: nowIso(), + usedGb: total != null && available != null ? gib(total - available) : null, + consumedGb: before != null && available != null ? gib(Math.max(0, before - available)) : null + }; +} + const MTPLX_SESSION_CACHE_FLAGS = new Set([ '--ssd-session-cache', '--ssd-session-cache-dir', @@ -1412,6 +1428,8 @@ export class RuntimeManager { stoppedAt: state.stoppedAt, lastWarmup: state.lastWarmup, lastError: state.lastError, + memorySafetyLatched: this.memorySafetyFailures.has(runtimeId), + memorySafetyAbort: state.memorySafetyAbort ?? null, watchdog: { ...runtimeWatchdogConfig(runtime), ...state.watchdog @@ -2339,12 +2357,21 @@ export class RuntimeManager { }); } })()); + // The first sample is the host before this load; at an abort, the + // difference is a lower bound on what the load really needs. + const sampleHost = this.memorySampler ?? (() => readHostMemory({ strict: true })); + let baseline = null; const guard = createMemorySafetyGuard({ policy, runtimeId, - sample: this.memorySampler, + sample: async () => { + const snapshot = await sampleHost(); + baseline ??= snapshot; + return snapshot; + }, onAbort: (error) => { this.memorySafetyFailures.set(runtimeId, error); + this.stateFor(runtimeId).memorySafetyAbort = memorySafetyAbortRecord(baseline, error.snapshot); this.record({ runtimeId, event: 'memory-safety-abort', message: error.message, snapshot: error.snapshot }); this.logger.error?.(error.message); void cleanup().catch(() => {}); diff --git a/src/runtime-memory-safety.mjs b/src/runtime-memory-safety.mjs index 20afda4..000a617 100644 --- a/src/runtime-memory-safety.mjs +++ b/src/runtime-memory-safety.mjs @@ -30,14 +30,22 @@ export class MemorySafetyConfigError extends Error { } } +// The host utilization at which the guard aborts a load, or null when the +// guard is off. Admission and pool planning keep below it. +export function memorySafetyCeiling(runtimePolicy = {}) { + const input = runtimePolicy?.memorySafety ?? {}; + if (input.mode === 'yolo') return null; + return Math.min(input.maxMemoryUtilization ?? 0.9, runtimePolicy?.maxMemoryUtilization ?? 0.9); +} + export function memorySafetyPolicy(config, totalMemoryGb = os.totalmem() / GiB) { const input = config.runtimePolicy?.memorySafety ?? {}; const configuredReserve = config.runtimePolicy?.reserveMemoryGb; const minAvailableMemoryGb = input.minAvailableMemoryGb ?? Math.min(configuredReserve ?? 8, totalMemoryGb * 0.15); - const maxMemoryUtilization = Math.min( - input.maxMemoryUtilization ?? 0.9, - config.runtimePolicy?.maxMemoryUtilization ?? 0.9 - ); + const maxMemoryUtilization = memorySafetyCeiling({ + ...config.runtimePolicy, + memorySafety: { ...input, mode: undefined } + }); const pollIntervalMs = input.pollIntervalMs ?? 250; const mode = input.mode ?? 'enforce'; if ( diff --git a/src/runtime-policy.mjs b/src/runtime-policy.mjs index 20aafb2..d02b560 100644 --- a/src/runtime-policy.mjs +++ b/src/runtime-policy.mjs @@ -2,6 +2,7 @@ import { assertMaintenanceStartAllowed } from './model-maintenance.mjs'; import os from 'node:os'; import { readHostMemory } from './host-memory.mjs'; import { memoryBudgetGb as hostMemoryBudgetGb } from './resource-fit.mjs'; +import { memorySafetyCeiling } from './runtime-memory-safety.mjs'; import { RuntimeManager } from './runtime-manager.mjs'; import { clusterNodes, currentNodeId, runtimeAuthority, runtimeResourcesByNode } from './cluster.mjs'; @@ -159,6 +160,15 @@ async function liveMemoryProfile(profile = {}) { }; } +// Free memory the host check keeps after a load: the configured reserve, or +// the margin below the memory safety guard's ceiling if that is larger, so +// admission never allows a load the guard is certain to abort. +function hostReserveGb(runtimePolicy, totalMemoryGb, reserveMemoryGb) { + const ceiling = memorySafetyCeiling(runtimePolicy); + const guardMarginGb = ceiling == null ? 0 : Math.round(totalMemoryGb * (1 - ceiling) * 10) / 10; + return Math.max(reserveMemoryGb, guardMarginGb); +} + function policyConfig(config, profile = {}) { const policy = config.runtimePolicy ?? {}; const totalMemoryGb = numberOrNull(profile.totalMemoryGb) ?? Math.round(os.totalmem() / 1024 / 1024 / 1024); @@ -175,6 +185,7 @@ function policyConfig(config, profile = {}) { autoEvict: policy.autoEvict === true, totalMemoryGb, reserveMemoryGb, + hostReserveGb: hostReserveGb(policy, totalMemoryGb, reserveMemoryGb), memoryBudgetGb, maxMemoryUtilization, protectActiveRequests: policy.protectActiveRequests !== false @@ -331,13 +342,17 @@ function clusterRuntimePolicyPlan( const maxMemoryUtilization = numberOrNull( configuredNodes[nodeId].resources?.maxMemoryUtilization ?? policyTemplate.maxMemoryUtilization ); - const memoryBudgetGb = hostMemoryBudgetGb({ - totalMemoryGb, - memoryBudgetGb: - numberOrNull(configuredNodes[nodeId].resources?.memoryBudgetGb) ?? numberOrNull(policyTemplate.memoryBudgetGb), - maxMemoryUtilization, - reserveMemoryGb - }); + const memoryBudgetGb = Math.min( + hostMemoryBudgetGb({ + totalMemoryGb, + memoryBudgetGb: + numberOrNull(configuredNodes[nodeId].resources?.memoryBudgetGb) ?? + numberOrNull(policyTemplate.memoryBudgetGb), + maxMemoryUtilization, + reserveMemoryGb + }), + totalMemoryGb - hostReserveGb({ ...policyTemplate, maxMemoryUtilization }, totalMemoryGb, reserveMemoryGb) + ); const loadedMemoryGb = rows .filter((row) => row.loaded) .reduce((sum, row) => sum + numberOrNull(row.resourcesByNode?.[nodeId]?.memoryGb), 0); @@ -533,7 +548,7 @@ export async function createRuntimePolicyPlan( .filter((row) => row.loaded && protectedReasons(row, policy).length === 0) .reduce((sum, row) => sum + row.memoryGb, 0); const hostShortfallGb = predictive - ? Math.max(0, policy.reserveMemoryGb - memoryProfile.availableMemoryGb + requestedAddsMemory - evictionFreesGb) + ? Math.max(0, policy.hostReserveGb - memoryProfile.availableMemoryGb + requestedAddsMemory - evictionFreesGb) : 0; overBudgetGb = Math.max(overBudgetGb, hostShortfallGb); diff --git a/test/fixtures/pool/scenarios/memory-estimate-low.json b/test/fixtures/pool/scenarios/memory-estimate-low.json index eed11ae..af60878 100644 --- a/test/fixtures/pool/scenarios/memory-estimate-low.json +++ b/test/fixtures/pool/scenarios/memory-estimate-low.json @@ -1,13 +1,34 @@ { "base": "spark-fleet-base.json", "name": "memory-estimate-low", - "description": "The video lane's configured memory estimate (48 GB) is well below what it really uses (74 GB). Video requests arrive while spark-node-03 also holds its resident set. The memory guard must abort the load before the node runs out of unified memory; nothing may hang.", + "description": "The video lane's configured memory estimate (48 GB) is well below what it really uses (74 GB). Video requests arrive while spark-node-03 also holds its resident set. The memory guard must abort the load before the node runs out of unified memory, nothing may hang, and the lane must not be retried automatically: LLooM latches it until a manual start, and the pool planner learns the aborted load's lower-bound footprint and plans video as unavailable.", "durationHours": 6, "variants": [ - { "id": "Wan-AI/Wan2.2-TI2V-5B-Diffusers@spark-node-03", "sim": { "memoryGbPerNode": 74, "loadSeconds": 180 } } + { + "id": "Wan-AI/Wan2.2-TI2V-5B-Diffusers@spark-node-03", + "sim": { + "memoryGbPerNode": 74, + "loadSeconds": 180 + } + } + ], + "workloads": [ + { + "id": "video", + "demand": { + "perHour": 2, + "diurnal": null + } + } ], - "workloads": [{ "id": "video", "demand": { "perHour": 2, "diurnal": null } }], "expect": { - "all": { "safety.memoryViolations.length": { "max": 0 } } + "all": { + "safety.memoryViolations.length": { + "max": 0 + }, + "lanes.guardAborts": { + "max": 1 + } + } } } diff --git a/test/pool-catalog.test.mjs b/test/pool-catalog.test.mjs index 907d8ba..d10348a 100644 --- a/test/pool-catalog.test.mjs +++ b/test/pool-catalog.test.mjs @@ -236,6 +236,39 @@ test('the catalog groups installed lanes, recipes and evidence by model', () => ); }); +test('a memory-safety abort raises the footprint to its lower bound and latches the lane out of the plan', () => { + const statuses = fleetStatuses(); + statuses['spark-node-03'].runtimeManager.runtimes['qwen38-27b-nvfp4'] = { + status: 'failed', + memorySafetyLatched: true, + memorySafetyAbort: { at: '2026-10-11T00:00:00.000Z', usedGb: 115, consumedGb: 61 } + }; + const catalog = buildPoolCatalog({ + config: fleetConfig(), + recipes: [], + benchmarks: [], + metrics, + nodeStatuses: statuses + }); + const q27 = catalog.variants.find((variant) => variant.id === 'qwen38-27b-nvfp4@spark-node-03'); + assert.equal(q27.requirements.memoryGbPerNode, 61, 'the aborted load needed at least 61 GB'); + assert.deepEqual(q27.excludeNodes, ['spark-node-03']); + assert.match(q27.excludeReason, /memory safety abort/); + + // Once someone retries it by hand the latch clears, but the evidence stays. + statuses['spark-node-03'].runtimeManager.runtimes['qwen38-27b-nvfp4'].memorySafetyLatched = false; + const retried = buildPoolCatalog({ + config: fleetConfig(), + recipes: [], + benchmarks: [], + metrics, + nodeStatuses: statuses + }); + const after = retried.variants.find((variant) => variant.id === 'qwen38-27b-nvfp4@spark-node-03'); + assert.equal(after.requirements.memoryGbPerNode, 61); + assert.equal(after.excludeNodes, undefined); +}); + test('workloads come from alias routes, pool overrides and observed demand', () => { const workloads = Object.fromEntries( poolWorkloadsFromConfig(fleetConfig(), { metrics }).map((workload) => [workload.id, workload]) diff --git a/test/pool-reconciler.test.mjs b/test/pool-reconciler.test.mjs index 9d771b4..78d5bb1 100644 --- a/test/pool-reconciler.test.mjs +++ b/test/pool-reconciler.test.mjs @@ -258,3 +258,30 @@ test('make-before-break: a start that fits runs before the old lane stops', asyn assert.ok(at('route') < at('manual-stop')); assert.equal(order.length, steps.length); }); + +test('a step that never settles times out, fails, and skips its dependents', async () => { + const { steps } = swap({ candidates: ['B', 'A', 'cloud'] }); + const pending = []; + const timer = (ms) => new Promise((resolve) => pending.push({ ms, resolve })); + const done = async () => {}; + const actuator = { + prefetch: done, + start: () => new Promise(() => {}), + stop: done, + route: done, + residency: done, + manual: done + }; + const running = executeReconcile(steps, actuator, { timer, timeoutsMs: { start: 5000 } }); + // Fire timers until the run settles; only the hung start's timer matters. + let result; + running.then((value) => (result = value)); + for (let round = 0; round < 50 && !result; round += 1) { + await new Promise((resolve) => setImmediate(resolve)); + for (const entry of pending.splice(0)) if (entry.ms === 5000) entry.resolve(); + } + const start = steps.find((step) => step.type === 'start'); + assert.equal(result.status[start.id], 'failed'); + assert.match(result.errors[start.id], /start timed out after 5 s/); + assert.equal(result.status[steps.find((step) => step.type === 'route').id], 'skipped'); +}); diff --git a/test/pool-sim.test.mjs b/test/pool-sim.test.mjs index 80acc90..122fa58 100644 --- a/test/pool-sim.test.mjs +++ b/test/pool-sim.test.mjs @@ -197,3 +197,12 @@ test('mac joins: weights are prefetched, cognition moves without dropping a requ assert.equal(auto.requests.failed, 0); assert.ok(!auto.final.lanes.includes('qwen38-27b-nvfp4@spark-node-03')); }); + +test('admission that keeps the memory guard margin free ends the abort-and-latch cycle', async () => { + const scenario = await loadPoolScenario(`${SCENARIOS}pair-crash.json`); + const legacy = await runPoolSimulation({ ...scenario, legacyAdmission: true }, { mode: 'off' }); + const fixed = await runPoolSimulation(scenario, { mode: 'off' }); + assert.ok(legacy.lanes.guardAborts > 0, 'legacy admission lets in loads the guard then aborts'); + assert.equal(fixed.lanes.guardAborts, 0); + assert.ok(fixed.requests.failed < legacy.requests.failed); +}); diff --git a/test/runtime-memory-safety.test.mjs b/test/runtime-memory-safety.test.mjs index 38baa61..70401be 100644 --- a/test/runtime-memory-safety.test.mjs +++ b/test/runtime-memory-safety.test.mjs @@ -324,6 +324,14 @@ default: process.exit(4); ); await assert.rejects(manager.start('demo'), (e) => e.code === 'runtime_memory_safety_abort'); assert.equal(await fs.readFile(killed, 'utf8'), 'killed'); + // The abort is recorded with how much host memory the load had taken, a + // lower bound on its real footprint that pool planning uses, and the + // runtime stays latched until a manual start. + const status = (await manager.status()).runtimes.demo; + assert.equal(status.memorySafetyLatched, true); + assert.equal(status.memorySafetyAbort.consumedGb, 57); + assert.equal(status.memorySafetyAbort.usedGb, 93); + assert.ok(Date.parse(status.memorySafetyAbort.at)); }); test('supervisor preflight rejection cannot race an immediate health success', async (t) => { diff --git a/test/runtime-policy.test.mjs b/test/runtime-policy.test.mjs index 21c259f..6a49859 100644 --- a/test/runtime-policy.test.mjs +++ b/test/runtime-policy.test.mjs @@ -634,4 +634,38 @@ const applied = await applyRuntimePolicyPlan(unpinnedPolicyConfig, fakePolicyMan assert.equal(applied.dryRun, false); assert.deepEqual(policyOperations, ['stop:warm', 'start:big:unit-admit']); +// Admission must never allow a load the memory safety guard is certain to +// abort: the guard trips at min(memorySafety, runtimePolicy) utilization (0.9 +// by default), so the host check keeps that margin free, not just the reserve. +const guardMarginConfig = { + runtimePolicy: { enabled: true, autoEvict: false, maxMemoryUtilization: 0.95 }, + runtimes: { video: { enabled: true, memoryGb: 20 } } +}; +const guardMargin = await createRuntimePolicyPlan(guardMarginConfig, { + requestedRuntimeId: 'video', + status: { runtimes: { video: { healthy: false, status: 'idle' } } }, + profile: { totalMemoryGb: 128, availableMemoryGb: 30 } +}); +assert.equal(guardMargin.policy.hostReserveGb, 12.8); +assert.equal(guardMargin.admission.allowed, false, '10 GB left is under the 12.8 GB the guard needs free'); +const guardMarginFits = await createRuntimePolicyPlan(guardMarginConfig, { + requestedRuntimeId: 'video', + status: { runtimes: { video: { healthy: false, status: 'idle' } } }, + profile: { totalMemoryGb: 128, availableMemoryGb: 34 } +}); +assert.equal(guardMarginFits.admission.allowed, true); +const guardLooser = await createRuntimePolicyPlan( + { + ...guardMarginConfig, + runtimePolicy: { ...guardMarginConfig.runtimePolicy, memorySafety: { maxMemoryUtilization: 0.97 } } + }, + { + requestedRuntimeId: 'video', + status: { runtimes: { video: { healthy: false, status: 'idle' } } }, + profile: { totalMemoryGb: 128, availableMemoryGb: 30 } + } +); +assert.equal(guardLooser.policy.hostReserveGb, 8, 'a 0.95 ceiling leaves 6.4 GB, so the 8 GB reserve governs'); +assert.equal(guardLooser.admission.allowed, true); + console.log('runtime-policy tests passed'); From 3edd86dcc1d856852564c1a5993a789dcba45ac0 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 12:32:34 -0700 Subject: [PATCH 5/9] Pool head: membership leases in the simulator; controller preemption - The simulator can run nodes through pool membership (enroll, heartbeat, lease loss) so node loss is detected the way a head detects it; new lease-loss scenario (crash plus a harmless 60 s partition). - The scenario found the controller blocked for 30 minutes on a manual restart of a lane whose node had just died. Stability now tracks the moves, not just the plan; an urgent plan for a different target cancels a change in progress (executeReconcile takes an AbortSignal); the same target never preempts itself. - Reconciler re-issues an idempotent start for lanes still loading so route flips wait for them. - Register membership tests in check:js and test:unit. Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- src/pool-controller.mjs | 30 +++++-- src/pool-reconciler.mjs | 83 ++++++++++++-------- src/pool-sim.mjs | 65 +++++++++++++-- test/fixtures/pool/scenarios/lease-loss.json | 59 ++++++++++++++ test/pool-controller.test.mjs | 31 ++++++++ test/pool-reconciler.test.mjs | 22 ++++++ 7 files changed, 246 insertions(+), 46 deletions(-) create mode 100644 test/fixtures/pool/scenarios/lease-loss.json diff --git a/package.json b/package.json index e98d9b3..5b25891 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "check": "npm run check:cluster && npm run check:js && npm run check:entity-stagger && npm run check:python && npm run check:shell && npm run check:spark-deploy && npm run check:workers", "check:cluster": "node --check src/cluster.mjs && node --check test/cluster.test.mjs && node --check test/cluster-availability.test.mjs && node --check test/cluster-readiness.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs", "check:entity-stagger": "node --check scripts/entity-stagger-benchmark.mjs && node --check test/entity-stagger-benchmark.test.mjs", - "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/request-scheduler.mjs && node --check src/rate-limit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/deployment-fence.mjs && node --check test/deployment-fence.test.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/standby.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check test/spark-audio-recipes.test.mjs && node --check src/runtime-residency.mjs && node --check test/ltx-pipelines.test.mjs && node --check test/ltx-pipelines-recipe.test.mjs && node --check src/video-contracts.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs && node --check src/pool-planner.mjs && node --check src/pool-catalog.mjs && node --check src/pool-inventory.mjs && node --check src/pool-cli.mjs && node --check test/pool-planner.test.mjs && node --check test/pool-catalog.test.mjs && node --check test/pool-inventory.test.mjs && node --check test/pool-cli.test.mjs && node --check src/pool-reconciler.mjs && node --check src/pool-controller.mjs && node --check src/pool-sim.mjs && node --check src/pool-sim-cli.mjs && node --check test/pool-reconciler.test.mjs && node --check test/pool-controller.test.mjs && node --check test/pool-sim.test.mjs && node --check src/runtime-policy-config.mjs && node --check test/runtime-policy-config.test.mjs && node --check src/audio-providers.mjs && node --check src/protocol/openrouter-provider.mjs && node --check test/audio-providers.test.mjs && node --check test/openrouter-provider.test.mjs && node --check backends/atlas-sparkglm/verify-pins.mjs && node --check test/atlas-sparkglm-recipe.test.mjs && node --check src/head-preparation.mjs && node --check src/head-transfer.mjs && node --check src/service-control.mjs && node --check src/gateway-relay.mjs && node --check src/client-retarget.mjs && node --check src/head-promotion.mjs && node --check src/head-promotion-transfer.mjs && node --check src/gateway-client.mjs && node --check test/cli-diagnostics.test.mjs && node --check src/service-install.mjs && node --check test/service-install.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check src/node-release-agent.mjs && node --check src/node-gateway-adapter.mjs && node --check src/node-agent-cli.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs && node --check test/node-release-agent.test.mjs && node --check test/node-gateway-adapter.test.mjs && node --check test/node-gateway-adapter-e2e.test.mjs && node --check test/deployment-bundle.test.mjs && node --check test/node-agent-cli.test.mjs && node --check test/ssh-deployment-transport.test.mjs && node --check scripts/build-deployment-bundle.mjs", + "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/request-scheduler.mjs && node --check src/rate-limit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/deployment-fence.mjs && node --check test/deployment-fence.test.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/standby.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check test/spark-audio-recipes.test.mjs && node --check src/runtime-residency.mjs && node --check test/ltx-pipelines.test.mjs && node --check test/ltx-pipelines-recipe.test.mjs && node --check src/video-contracts.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs && node --check src/pool-planner.mjs && node --check src/pool-catalog.mjs && node --check src/pool-inventory.mjs && node --check src/pool-cli.mjs && node --check test/pool-planner.test.mjs && node --check test/pool-catalog.test.mjs && node --check test/pool-inventory.test.mjs && node --check test/pool-cli.test.mjs && node --check src/pool-reconciler.mjs && node --check src/pool-controller.mjs && node --check src/pool-sim.mjs && node --check src/pool-sim-cli.mjs && node --check test/pool-reconciler.test.mjs && node --check test/pool-controller.test.mjs && node --check test/pool-sim.test.mjs test/pool-membership.test.mjs test/pool-membership-http.test.mjs && node --check src/pool-membership.mjs && node --check src/pool-membership-store.mjs && node --check src/pool-node.mjs && node --check test/pool-membership.test.mjs && node --check test/pool-membership-http.test.mjs && node --check src/runtime-policy-config.mjs && node --check test/runtime-policy-config.test.mjs && node --check src/audio-providers.mjs && node --check src/protocol/openrouter-provider.mjs && node --check test/audio-providers.test.mjs && node --check test/openrouter-provider.test.mjs && node --check backends/atlas-sparkglm/verify-pins.mjs && node --check test/atlas-sparkglm-recipe.test.mjs && node --check src/head-preparation.mjs && node --check src/head-transfer.mjs && node --check src/service-control.mjs && node --check src/gateway-relay.mjs && node --check src/client-retarget.mjs && node --check src/head-promotion.mjs && node --check src/head-promotion-transfer.mjs && node --check src/gateway-client.mjs && node --check test/cli-diagnostics.test.mjs && node --check src/service-install.mjs && node --check test/service-install.test.mjs && node --check src/cluster-deployment.mjs && node --check src/cluster-deployment-cli.mjs && node --check src/ssh-deployment-transport.mjs && node --check src/node-release-agent.mjs && node --check src/node-gateway-adapter.mjs && node --check src/node-agent-cli.mjs && node --check test/cluster-deployment.test.mjs && node --check test/cluster-deployment-cli.test.mjs && node --check test/cluster-deployment-e2e.test.mjs && node --check test/node-release-agent.test.mjs && node --check test/node-gateway-adapter.test.mjs && node --check test/node-gateway-adapter-e2e.test.mjs && node --check test/deployment-bundle.test.mjs && node --check test/node-agent-cli.test.mjs && node --check test/ssh-deployment-transport.test.mjs && node --check scripts/build-deployment-bundle.mjs", "check:python": "python3 -m py_compile backends/dspark-vllm/apply-patch-pack.py backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/vision_exp/*.py backends/qwen38-vllm/apply-ple-fp8-patch.py backends/qwen38-sglang/*.py backends/glm53-exl3/*.py backends/mlx-audio/lloom_audio_server.py backends/hear/lloom_hear_server.py backends/chatterbox/lloom_chatterbox_server.py patches/apply_mtplx_longctx_fix.py backends/comfyui-media/bridge/*.py backends/comfyui-media/graphs/*.py backends/comfyui-media/build/*.py backends/comfyui-media/install.py backends/qwen-image-diffusers/*.py backends/ace-step-diffusers/*.py backends/ace-step-diffusers/tests/*.py backends/hear/test/test_hear_security.py backends/spark-audio/install.py backends/spark-audio/lloom_audio_cuda_server.py backends/spark-audio/test_lloom_audio_cuda_server.py backends/ltx-pipelines/*.py src/darwin-memory-usage.py", "check:shell": "bash -n backends/dspark-vllm/entrypoint.sh backends/dspark-vllm/runtime/build-hardened-image.sh backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/apply-runtime.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.sh backends/qwen38-vllm/entrypoint.sh backends/qwen38-sglang/entrypoint.sh backends/glm53-exl3/entrypoint.sh backends/mlx-audio/install.sh backends/hear/install.sh backends/chatterbox/install.sh clients/examples/lloom-claude clients/examples/lloom-codex clients/examples/lloom-hermes clients/examples/lloom-opencode clients/examples/lloom-zero scripts/remote-install-spark.sh scripts/remote-stage-spark-worker.sh backends/atlas-sparkglm/install.sh backends/atlas-sparkglm/convert-overlay.sh", "check:spark-deploy": "node --check test/spark-deploy.test.mjs && node test/spark-deploy.test.mjs", diff --git a/src/pool-controller.mjs b/src/pool-controller.mjs index ea2472a..61e0df0 100644 --- a/src/pool-controller.mjs +++ b/src/pool-controller.mjs @@ -161,13 +161,9 @@ export function createPoolController({ async function tick() { const at = now(); if (controllerPolicy.mode === 'off') return record({ at, action: 'off' }); - if (state.executing) return record({ at, action: 'busy' }); const observed = await observe(); const inputs = { ...observed, variants: excludeFailures(observed.variants ?? [], state.failures, at) }; const plan = planPool(inputs); - const signature = planSignature(plan); - state.stableTicks = signature === state.signature ? state.stableTicks + 1 : 1; - state.signature = signature; const stay = stayPlan(inputs); const reconcile = planReconcile({ plan, @@ -176,7 +172,28 @@ export function createPoolController({ variants: inputs.variants, nodes: inputs.nodes }); + // Stable means the same moves, not just the same target: a lane that + // just vanished changes the moves even when the plan does not. + const signature = `${planSignature(plan)}|${reconcile.steps.map((step) => step.id).join(',')}`; + state.stableTicks = signature === state.signature ? state.stableTicks + 1 : 1; + state.signature = signature; const decision = decide({ plan, stay, reconcile, state, policy: controllerPolicy, now: at }); + if (state.executing) { + // Only a plan that restores coverage may cancel work in progress (for + // example a manual start that is waiting on a node which just died). + // ...and only for a different target, not the same one with fewer steps left. + const preempt = + decision.act && + decision.urgent && + controllerPolicy.mode === 'auto' && + planSignature(plan) !== state.executingTarget; + if (!preempt) { + return record({ at, action: 'busy', reason: 'still applying the last change' }); + } + state.abort?.abort(); + await state.executing; + record({ at: now(), action: 'preempted', reason: decision.reason }); + } const entry = { at, action: decision.act ? (controllerPolicy.mode === 'auto' ? 'apply' : 'propose') : 'hold', @@ -197,8 +214,11 @@ export function createPoolController({ state.loads.push({ at, gb: (num(step.memoryGbPerNode) ?? 0) * (step.nodes?.length ?? 1) }); } state.loads = state.loads.filter((load) => at - load.at < 3600000); + state.abort = new AbortController(); + state.executingTarget = planSignature(plan); state.executing = executeReconcile(reconcile.steps, actuator, { concurrency: controllerPolicy.concurrency, + signal: state.abort.signal, ...(timer ? { timer } : {}), onStep: ({ step, status, error }) => { onEvent({ at: now(), action: 'step', step: step.id, status, ...(error ? { error } : {}) }); @@ -213,7 +233,7 @@ export function createPoolController({ } }).then((result) => { state.executing = null; - record({ at: now(), action: 'applied', ok: result.ok, errors: result.errors }); + record({ at: now(), action: 'applied', ok: result.ok, cancelled: result.cancelled, errors: result.errors }); return result; }); return entry; diff --git a/src/pool-reconciler.mjs b/src/pool-reconciler.mjs index bd0e467..9e68230 100644 --- a/src/pool-reconciler.mjs +++ b/src/pool-reconciler.mjs @@ -119,8 +119,11 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ .sort((a, b) => a.runtime.localeCompare(b.runtime)) .map((change) => add({ type: 'residency', ...change })); + // A lane still loading gets a start too: starts are idempotent and resolve + // when the lane is ready, which is what its route flips must wait for. + const loading = (placement) => running.get(placement.placement)?.state === 'starting'; const pendingStarts = planned - .filter((placement) => placement.residency !== 'auto' && !running.has(placement.placement)) + .filter((placement) => (placement.residency !== 'auto' && !running.has(placement.placement)) || loading(placement)) .sort( (a, b) => RESIDENCY_RANK[b.residency] - RESIDENCY_RANK[a.residency] || a.placement.localeCompare(b.placement) ); @@ -219,7 +222,8 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ // Start lanes that fit next to everything still running (make before break). for (const placement of [...pendingStarts]) { - if (startStep.has(placement.placement) || !fits(placement)) continue; + const counted = loading(placement); + if (startStep.has(placement.placement) || (!counted && !fits(placement))) continue; const id = add( { type: placement.managed === false ? 'manual-start' : 'start', @@ -232,8 +236,11 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ startAfter(placement) ); startStep.set(placement.placement, id); - for (const node of placement.nodes) - usage.set(node, (usage.get(node) ?? 0) + (num(placement.memoryGbPerNode) ?? 0)); + if (!counted) { + for (const node of placement.nodes) { + usage.set(node, (usage.get(node) ?? 0) + (num(placement.memoryGbPerNode) ?? 0)); + } + } for (const model of placement.models) readyStep.set(model, id); running.set(placement.placement, { key: placement.placement, ...placement, state: 'starting' }); pendingStarts.splice(pendingStarts.indexOf(placement), 1); @@ -352,11 +359,10 @@ const defaultTimer = (ms) => export async function executeReconcile( steps, actuator, - { concurrency = 4, onStep = () => {}, timeoutsMs = {}, timer = defaultTimer } = {} + { concurrency = 4, onStep = () => {}, timeoutsMs = {}, timer = defaultTimer, signal } = {} ) { const status = new Map(steps.map((step) => [step.id, 'pending'])); const errors = {}; - const byId = new Map(steps.map((step) => [step.id, step])); const running = new Set(); const method = (type) => ({ @@ -369,21 +375,44 @@ export async function executeReconcile( 'manual-start': 'manual', 'manual-stop': 'manual' })[type]; + // Cancelling (a newer, urgent plan) stops new steps at once and abandons + // running ones; what they already did stays and the next plan sees it. + const cancelled = signal + ? new Promise((_, reject) => { + const abort = () => reject(new Error('cancelled')); + if (signal.aborted) abort(); + else signal.addEventListener('abort', abort, { once: true }); + }) + : null; + cancelled?.catch(() => {}); return new Promise((resolve) => { - const finish = () => + let settled = false; + const finish = () => { + if (settled) return; + settled = true; resolve({ ok: [...status.values()].every((value) => value === 'done'), + cancelled: signal?.aborted === true, status: Object.fromEntries(status), errors }); + }; + const settle = (step, value, error) => { + status.set(step.id, value); + if (error) errors[step.id] = error; + onStep({ step, status: value, ...(error ? { error } : {}) }); + }; const pump = () => { for (const step of steps) { if (status.get(step.id) !== 'pending') continue; + if (signal?.aborted) { + settle(step, 'cancelled'); + continue; + } const deps = step.after.map((id) => status.get(id)); - if (deps.some((value) => value === 'failed' || value === 'skipped')) { - status.set(step.id, 'skipped'); - onStep({ step, status: 'skipped' }); + if (deps.some((value) => value === 'failed' || value === 'skipped' || value === 'cancelled')) { + settle(step, 'skipped'); continue; } if (deps.some((value) => value !== 'done' && value !== undefined)) continue; @@ -396,38 +425,24 @@ export async function executeReconcile( throw new Error(`${step.type} timed out after ${Math.round(limitMs / 1000)} s`); }); timedOut.catch(() => {}); - Promise.race([Promise.resolve().then(() => actuator[method(step.type)](step)), timedOut]) + Promise.race( + [Promise.resolve().then(() => actuator[method(step.type)](step)), timedOut, cancelled].filter(Boolean) + ) .then( - () => { - status.set(step.id, 'done'); - onStep({ step, status: 'done' }); - }, - (error) => { - status.set(step.id, 'failed'); - errors[step.id] = String(error?.message ?? error); - onStep({ step, status: 'failed', error: errors[step.id] }); - } + () => settle(step, 'done'), + (error) => + signal?.aborted ? settle(step, 'cancelled') : settle(step, 'failed', String(error?.message ?? error)) ) .finally(() => { running.delete(step.id); pump(); }); } - // Skips can cascade without any step running; settle once nothing is left. - if (!running.size) { - const stillPending = [...status.entries()].some(([id, value]) => value === 'pending' && byId.get(id)); - if (stillPending) { - const progressed = steps.some( - (step) => - status.get(step.id) === 'pending' && - step.after.every((id) => ['done', 'failed', 'skipped', undefined].includes(status.get(id))) - ); - if (progressed) return pump(); - for (const [id, value] of status) if (value === 'pending') status.set(id, 'skipped'); - } - finish(); - } + // Steps are ordered after their dependencies, so one pass settles every + // skip; with nothing running, nothing is left to wait for. + if (!running.size) finish(); }; + signal?.addEventListener('abort', () => pump(), { once: true }); pump(); }); } diff --git a/src/pool-sim.mjs b/src/pool-sim.mjs index ee21199..26a299d 100644 --- a/src/pool-sim.mjs +++ b/src/pool-sim.mjs @@ -20,6 +20,7 @@ // `hangMinutes`. Plans should never cause one. import { createPoolController } from './pool-controller.mjs'; +import { createPoolMembership } from './pool-membership.mjs'; import { memoryBudgetGb } from './resource-fit.mjs'; import { memorySafetyCeiling } from './runtime-memory-safety.mjs'; @@ -222,6 +223,38 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c const flexiblePlacements = new Map(); const latched = new Set(); + // With `membership`, nodes enroll with the head and heartbeat; the planner + // learns that a node is gone only when its lease is lost, as a real head + // would. Without it, the planner sees node failures immediately. + const membershipConfig = scenario.membership ?? null; + const SIM_EPOCH = Date.parse('2026-01-01T00:00:00.000Z'); + const membership = membershipConfig + ? createPoolMembership({ + leaseMs: (num(membershipConfig.leaseSeconds) ?? 30) * 1000, + lostAfterMs: (num(membershipConfig.lostAfterSeconds) ?? 90) * 1000, + now: () => SIM_EPOCH + now + }) + : null; + const heartbeatMs = (num(membershipConfig?.heartbeatSeconds) ?? 10) * 1000; + const nodeSecrets = new Map(); + function enroll(id) { + if (!membership) return; + const { token } = membership.issueEnrollmentToken(); + const { nodeSecret } = membership.enroll({ token, nodeId: id, endpoint: `http://${id}.pool.test:8100` }); + nodeSecrets.set(id, nodeSecret); + const beat = () => { + const node = nodes.get(id); + if (!node.joined || nodeSecrets.get(id) !== nodeSecret) return; + // A crashed, hung or partitioned node stops heartbeating but stays enrolled. + if (node.available && node.hungUntil <= now && !(node.partitionedUntil > now)) { + membership.heartbeat({ nodeId: id, nodeSecret, running: lanesOn(id).map((lane) => lane.key) }); + } + queue.push(now + heartbeatMs, beat); + }; + queue.push(now + heartbeatMs, beat); + } + const memberById = () => new Map((membership?.members() ?? []).map((member) => [member.id, member])); + const truth = (variant) => variant.sim ?? {}; const memoryTruth = (variant) => num(truth(variant).memoryGbPerNode) ?? num(variant.requirements?.memoryGbPerNode) ?? 0; @@ -667,12 +700,15 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c } function observe() { + const members = memberById(); const plannerNodes = [...nodes.values()] - .filter((node) => node.joined) + .filter((node) => + membership ? members.has(node.spec.id) && members.get(node.spec.id).state !== 'left' : node.joined + ) .map((node) => ({ ...node.spec, - available: node.available && node.hungUntil <= now, - cordoned: node.cordoned, + available: membership ? members.get(node.spec.id).state !== 'lost' : node.available && node.hungUntil <= now, + cordoned: membership ? members.get(node.spec.id).cordoned || members.get(node.spec.id).draining : node.cordoned, weights: [...node.weights], images: [...node.images] })); @@ -865,10 +901,19 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c }); switch (event.type) { case 'cordon': - for (const id of ids) nodes.get(id).cordoned = true; + for (const id of ids) { + nodes.get(id).cordoned = true; + membership?.cordon(id, { reason: event.note }); + } break; case 'uncordon': - for (const id of ids) nodes.get(id).cordoned = false; + for (const id of ids) { + nodes.get(id).cordoned = false; + membership?.uncordon(id); + } + break; + case 'partition': + for (const id of ids) nodes.get(id).partitionedUntil = now + (num(event.minutes) ?? 1) * MINUTE; break; case 'fail': for (const id of ids) { @@ -880,11 +925,18 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c for (const id of ids) nodes.get(id).available = true; break; case 'join': - for (const id of ids) nodes.get(id).joined = true; + for (const id of ids) { + nodes.get(id).joined = true; + enroll(id); + } break; case 'leave': for (const id of ids) { nodes.get(id).joined = false; + if (membership) { + membership.leave(id); + nodeSecrets.delete(id); + } for (const lane of lanesOn(id)) killLane(lane, `node ${id} left`); } break; @@ -920,6 +972,7 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c queue.push(event.atHours * HOUR, () => applyEvent(event)); } for (const node of nodes.values()) { + if (node.joined) enroll(node.spec.id); if (node.spec.joinAtHours != null) queue.push(node.spec.joinAtHours * HOUR, () => applyEvent({ type: 'join', nodes: [node.spec.id] })); } diff --git a/test/fixtures/pool/scenarios/lease-loss.json b/test/fixtures/pool/scenarios/lease-loss.json new file mode 100644 index 0000000..515b6ea --- /dev/null +++ b/test/fixtures/pool/scenarios/lease-loss.json @@ -0,0 +1,59 @@ +{ + "base": "pair-crash.json", + "name": "lease-loss", + "description": "The pair crash again, but the head only learns about it from membership leases: nodes heartbeat every 10 s, a lease lasts 30 s and a node is lost 90 s after its last heartbeat. Before the crash, spark-node-02 drops off the network for 60 s and comes back; that is a suspect lease, not a lost node, and must not move anything.", + "membership": { + "heartbeatSeconds": 10, + "leaseSeconds": 30, + "lostAfterSeconds": 90 + }, + "events": [ + { + "atHours": 1, + "type": "partition", + "nodes": ["spark-node-02"], + "minutes": 1 + }, + { + "atHours": 3, + "type": "fail", + "nodes": ["spark-node-01"] + }, + { + "atHours": 7, + "type": "recover", + "nodes": ["spark-node-01"] + }, + { + "atHours": 7.5, + "type": "start-lane", + "modes": ["off"], + "placement": "qwen3.8-flash-next-atlas@spark-node-01+spark-node-02" + } + ], + "expect": { + "auto": { + "safety.memoryViolations.length": { + "max": 0 + }, + "controller.decisions.0.atMinutes": { + "min": 181.5, + "max": 183 + }, + "controller.decisions.0.urgent": { + "equals": true + }, + "final.routes.chat-capable": { + "equals": "qwen3.8-flash-next-atlas" + }, + "controller.stepFailures": { + "max": 0 + } + }, + "off": { + "safety.memoryViolations.length": { + "max": 0 + } + } + } +} diff --git a/test/pool-controller.test.mjs b/test/pool-controller.test.mjs index 71bdab5..cb4e971 100644 --- a/test/pool-controller.test.mjs +++ b/test/pool-controller.test.mjs @@ -195,3 +195,34 @@ test('a failed start is retried elsewhere after backoff instead of repeating', a assert.equal(second.plan.routes['chat-capable'].model, 'cloud/openrouter/q38fn'); assert.equal(second.action, 'apply'); }); + +test('an urgent new target cancels a change stuck in progress; the same target never preempts itself', async () => { + const { observe, actuator, state } = fakeFleet({ cordon: ['spark-node-01', 'spark-node-02'] }); + const hung = []; + actuator.start = (step) => + new Promise(() => { + hung.push(step.id); + }); + const events = []; + let clock = 0; + const controller = createPoolController({ + observe, + actuator, + now: () => clock, + onEvent: (entry) => events.push(entry.action), + policy: { mode: 'auto' } + }); + assert.equal((await controller.tick()).action, 'apply'); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(hung, ['start:qwen38-27b-nvfp4@spark-node-03']); + clock += 60000; + assert.equal((await controller.tick()).action, 'busy', 'the same target waits for the start'); + // Now spark-node-03 goes too: only the cloud member is left for cognition. + state.cordon.add('spark-node-03'); + clock += 60000; + const entry = await controller.tick(); + assert.equal(entry.action, 'apply'); + assert.ok(events.includes('preempted')); + await controller.executing; + assert.equal(state.routes['chat-capable'], 'cloud/openrouter/q38fn'); +}); diff --git a/test/pool-reconciler.test.mjs b/test/pool-reconciler.test.mjs index 78d5bb1..f2c739e 100644 --- a/test/pool-reconciler.test.mjs +++ b/test/pool-reconciler.test.mjs @@ -285,3 +285,25 @@ test('a step that never settles times out, fails, and skips its dependents', asy assert.match(result.errors[start.id], /start timed out after 5 s/); assert.equal(result.status[steps.find((step) => step.type === 'route').id], 'skipped'); }); + +test('routes to a lane that is still loading wait on an idempotent start', () => { + const profile = fleet.profiles['sparkqwen-pair-rc16']; + const running = [...RC16_RUNNING, 'qwen38-27b-nvfp4@spark-node-03']; + const current = { routes: profile.routes, residency: profile.residency, running }; + const plan = planPool({ + nodes: fleet.nodes, + variants: fleet.variants, + workloads: fleetWorkloads(), + pins: { ...fleet.pins, cordon: ['spark-node-01', 'spark-node-02'] }, + current, + policy: { ...fleet.policy, memorySafetyUtilization: null } + }); + const placements = placementsFor(running, fleet.variants).map((placement) => + placement.key === 'qwen38-27b-nvfp4@spark-node-03' ? { ...placement, state: 'starting' } : placement + ); + const { steps, blocked } = planReconcile({ plan, current: { ...current, placements }, workloads: fleetWorkloads() }); + assert.deepEqual(blocked, []); + const start = steps.find((step) => step.type === 'start'); + assert.equal(start.placement, 'qwen38-27b-nvfp4@spark-node-03'); + for (const route of steps.filter((step) => step.type === 'route')) assert.ok(route.after.includes(start.id)); +}); From e53f7a1d45530718e5e1143c4768c86338d53ee0 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 12:34:56 -0700 Subject: [PATCH 6/9] Pool head: lloom pool simulate, docs, and a flaky token assertion - lloom pool simulate [--mode off|advise|auto|compare]: runs a scenario through the planner, reconciler and controller; exits non-zero when expectations fail. - docs/pool-planning.md: reconciling, the controller, simulation and the scenario results; runtime-policy.md: the guard margin in admission. - Membership tests sliced the token body with split('_'), but base64url bodies can contain '_', so a short fragment sometimes matched the state JSON; slice the fixed prefix instead. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++ bin/lloom.mjs | 22 +++++++ docs/pool-planning.md | 93 +++++++++++++++++++++++++++++- docs/runtime-policy.md | 11 ++++ src/pool-cli.mjs | 3 +- test/pool-membership-http.test.mjs | 4 +- test/pool-membership.test.mjs | 3 +- 7 files changed, 134 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88f2a43..8ea50aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to LLooM will be documented in this file. The format follows ### Added +- Pool reconciler and controller, tested in simulation. `lloom pool simulate ` runs a fleet scenario through the real planner, reconciler and controller in a deterministic discrete-event model, and compares today's behaviour with the controller's. The reconciler orders make-before-break steps (prefetch, start, route flip, drain and stop), with swaps through interim members, per-step timeouts, and dependents skipped when a step fails. The controller has `off`, `advise` and `auto` modes, urgency, stability, cooldowns, a load budget, failure backoff, and preemption by an urgent new target. Neither is wired to a live gateway yet. See `docs/pool-planning.md`. + - Pool membership: nodes join a head with a one-time enrollment token (`lloom pool token create`, `lloom pool join`) and keep a lease with heartbeats from `serve`. The head lists members (`lloom pool members`), cordons, drains and uncordons them, and `lloom pool plan` uses their liveness, cordons and inventory. Membership is inert until a token is issued, keeps only hashed secrets, and does not grant routing: an enrolled-only node still needs `lloom cluster add-node`. See `docs/pool-planning.md`. - `lloom pool plan`: a dry-run placement plan across every cluster node. It proposes which lanes run where, which residency they get and how each alias routes, and diffs that against what runs now. It covers every required alias first, then minimises demand-weighted latency and the load cost of changes, within per-domain memory headroom, direct-link tensor parallelism, compute architecture and pins. Nodes report a pool inventory through `/gateway/node?inventory=1`. Recipes may declare `requirements.computeArchs`, `requirements.tensorParallel` and `settings.maxSequences`. `/gateway/metrics` adds `requested` buckets, which give per-alias demand. See `docs/pool-planning.md`. @@ -35,6 +37,8 @@ All notable changes to LLooM will be documented in this file. The format follows ### Changed +- Admission keeps the memory safety guard's margin free. The host check now reserves the larger of `reserveMemoryGb` and the guard's margin (`total × (1 − ceiling)`, 12.8 GB on a 128 GB host at the default 0.9), and cluster node budgets are capped at the ceiling. Before, admission could approve a load the guard was certain to abort and then latch until a manual start. A guard abort now records `memorySafetyAbort` (`usedGb`, and `consumedGb`, a lower bound on the load's footprint) and `memorySafetyLatched` in runtime status. Pool planning caps host budgets at the same ceiling and learns from aborts. + - LLooM now requires Node.js 22.19 or newer, the minimum for its undici 8 HTTP client. It already failed to start on Node 20 after that upgrade; the engine range and CI matrix now say so. - The action view frames only models serving traffic and their live requests. Active cards cluster near the loom; idle models and machine racks no longer pull the camera outward. Manual zoom holds independently of the automatic fit until Reset view restores it. diff --git a/bin/lloom.mjs b/bin/lloom.mjs index 36c2dae..33dce54 100755 --- a/bin/lloom.mjs +++ b/bin/lloom.mjs @@ -45,6 +45,12 @@ import { poolTokenOptions } from '../src/pool-cli.mjs'; import { joinPool } from '../src/pool-node.mjs'; +import { + POOL_SIMULATE_USAGE, + formatPoolSimulation, + poolSimulateOptions, + runPoolSimulateCommand +} from '../src/pool-sim-cli.mjs'; import { mutateConfigSource } from '../src/config-mutation.mjs'; import { runtimeControlTimeoutMs } from '../src/control-timeout.mjs'; import { createDoctorReport } from '../src/doctor.mjs'; @@ -243,6 +249,7 @@ Backends and runtimes: lloom pool members [--json] lloom pool cordon|uncordon|drain [--reason TEXT] lloom pool join --token-stdin [--node-id ID] [--endpoint URL] [--apply --yes] + lloom pool simulate [--mode off|advise|auto|compare] [--hours N] [--seed N] [--json] lloom deployment plan --plan FILE [--json] lloom deployment apply --plan FILE --nodes FILE [--operation-id ID] --apply --yes [--journal FILE] [--json] lloom deployment status --operation-id ID [--journal FILE] [--json] @@ -2574,6 +2581,21 @@ async function main() { console.log(JSON.stringify(result, null, 2)); return; } + if (action === 'simulate') { + const scenario = positional(args)[2]; + if (!scenario) throw new Error(POOL_SIMULATE_USAGE); + const reports = await runPoolSimulateCommand( + scenario, + poolSimulateOptions({ + mode: argValue(args, '--mode') ?? undefined, + hours: argValue(args, '--hours'), + seed: argValue(args, '--seed') + }) + ); + console.log(json ? JSON.stringify(reports, null, 2) : formatPoolSimulation(reports)); + if (Object.values(reports).some((report) => !report.ok)) process.exitCode = 1; + return; + } if (action !== 'plan') throw new Error(POOL_USAGE); const options = poolPlanOptions({ cordon: argValues(args, '--cordon'), diff --git a/docs/pool-planning.md b/docs/pool-planning.md index 989fbaf..cbabcbc 100644 --- a/docs/pool-planning.md +++ b/docs/pool-planning.md @@ -1,6 +1,8 @@ # Pool planning -`lloom pool plan` treats every node in the cluster as one pool and proposes which model lanes run where, how each alias routes, and what would change from today. It is a dry run: it never starts, stops or reroutes anything, and there is no apply step yet. Act on a plan with `lloom fleet`, `lloom route`, `lloom keep-warm` and `lloom suspend`/`resume`. +`lloom pool plan` treats every node in the cluster as one pool and proposes which model lanes run where, how each alias routes, and what would change from today. It is a dry run: it never starts, stops or reroutes anything. Act on a plan with `lloom fleet`, `lloom route`, `lloom keep-warm` and `lloom suspend`/`resume`. + +The reconciler and pool controller that would apply plans are built and tested in a fleet simulator (`lloom pool simulate`, below), but they are not wired to a live gateway yet. ```bash lloom pool plan # live status + 7-day demand from the gateway @@ -33,10 +35,15 @@ A gateway that predates inventory still reports its machine profile, and the pla - **Unmanaged lanes:** declared with a `pool` block on the model, for example a hand-run TP2 server on the pair: ```json - { "id": "qwen3.8-flash-next-atlas", "backend": "atlas", "pool": { "nodes": ["spark-node-01", "spark-node-02"], "memoryGbPerNode": 110 } } + { + "id": "qwen3.8-flash-next-atlas", + "backend": "atlas", + "pool": { "nodes": ["spark-node-01", "spark-node-02"], "memoryGbPerNode": 110 } + } ``` LLooM cannot start or stop these, so the plan lists them as manual actions. + - **External models:** cloud and other models with no node. With `--consider-installs`, bundled recipes add variants that may go on any eligible node set. @@ -117,9 +124,10 @@ A node that only enrolled is planned on, but it is not routable. The head has no ## Constraints and objective - **Memory per domain.** Unified memory on Spark and Apple Silicon. Device memory, plus system memory for host-side state, on discrete GPUs. - - The budget follows admission: `memoryBudgetGb`, else capacity × `maxMemoryUtilization`, else capacity minus `reserveMemoryGb`. Pool planning then subtracts `headroomGb` (default 4 GB), because exhausting unified memory hangs a Spark. + - The budget follows admission: `memoryBudgetGb`, else capacity × `maxMemoryUtilization`, else capacity minus `reserveMemoryGb`. Host memory (unified and system) is also capped at the runtime memory guard's ceiling: `min(runtimePolicy.memorySafety.maxMemoryUtilization, runtimePolicy.maxMemoryUtilization)`, 0.9 by default. The guard aborts any load past that, so a plan that only fits under a 0.95 admission budget would be aborted mid-load. Pool planning then subtracts `headroomGb` (default 4 GB), because exhausting unified memory hangs a Spark. On a 128 GB Spark the default budget is 128 × 0.9 − 4 = 111.2 GB. - `always` and `preferred` lanes must fit together. Every on-demand lane must fit next to the `always` set. - A `preferred` lane that does not fit becomes on demand, with a warning. + - A runtime the memory guard aborted counts at least the memory the load had taken when it was stopped (`memorySafetyAbort.consumedGb` in runtime status). While it is latched (no automatic restarts until someone starts it by hand), the planner plans around it. - **Tensor parallel only across directly linked nodes.** Every pair in a multi-node set must share a reported fabric link. - **Compute architecture, platform and accelerators** must match the node. - **Cordoned and unavailable nodes** host nothing. @@ -152,3 +160,82 @@ The text output lists: `--json` returns the full report. It includes the plan's profile-shaped `routes` and `residency`, `placements`, per-node memory, the `diff`, and the `score`. On a three-Spark fleet, the planner's fixture tests (`test/fixtures/pool/spark-fleet.json`) reproduce the hand-written fleet profiles. With every node present, it gives the `sparkqwen-pair-rc16` routes and residency. With the 01/02 pair cordoned or offline, it gives `standalone`: cognition moves to the 27B on spark-node-03, which becomes `always`, and the SparkQwen lane gets a manual stop. + +With the memory guard ceiling applied, the same fixture shows a conflict the hand-written `standalone` profile hides: a keep-warm 27B, embeddings and hear hold 64 GB on spark-node-03, so the 48 GB video lane (112 GB) no longer fits under the 111.2 GB budget, and video is reported uncovered. + +## Reconciling a plan + +`src/pool-reconciler.mjs` turns a plan and the observed state into ordered steps that never drop a route: + +1. **Prefetch** weights for a new placement whose node lacks them. +2. **Downgrade residency** for lanes leaving keep-warm, so LLooM does not restart them while they drain. +3. **Start** new lanes that fit next to everything still running (make before break). +4. **Flip routes** once their target is ready. +5. **Drain and stop** lanes no route uses any more. +6. **Upgrade residency** for lanes that just started. + +When a new lane only fits after an old one is gone, the move is a swap. The old lane's aliases go to an interim member that is already serving, such as another running lane or an external fallback. Then the old lane drains and stops, the new lane starts, and the aliases flip to it. If there is no interim member, the plan reports an interruption instead of hiding it. Unmanaged lanes (for example a hand-run TP2 server) become `manual-start` and `manual-stop` steps for an operator. + +Steps run with dependencies. When a step fails, everything that depends on it is skipped. A lane that fails to start therefore never strands its aliases: their route flips are skipped, and so is the stop of the old lane, which keeps serving. Every step has a timeout (45 minutes for a start, 20 for a drain, 2 hours for a manual step), so one stuck step cannot hold the controller forever. + +## Pool controller + +`src/pool-controller.mjs` plans, decides and reconciles in a loop: + +- **off** never plans. +- **advise** plans and records proposals that a person applies. +- **auto** applies proposals itself. + +Moving a model costs minutes of load time, so the controller only acts on a plan worth moving for: + +- **Urgent changes act at once.** A change is urgent when the plan covers a required workload that keeping today's routes would leave uncovered: a node went away, a lane died, or memory was freed. An urgent plan for a _different_ target can cancel a change still in progress. For example, it cancels a manual restart that is waiting on a node which has just died. The same target never preempts itself. +- **Other changes are held back.** The steps must be the same for `confirmTicks` (2) ticks in a row. They must also either cut the latency cost by at least `minGain` (10%) or improve SLOs or local coverage. After that come a `cooldownMs` (15 min) cooldown and a `maxLoadGbPerHour` (400 GB) budget for new loads. +- **Residency-only changes follow noisy demand,** so they get a slower signal: `maintenanceConfirmTicks` (10) and `maintenanceCooldownMs` (1 hour). +- **Failed starts are skipped for a while.** A lane that fails to start is excluded from planning on that node for `failureBackoffMs` (30 min). + +## Simulation + +`lloom pool simulate` runs a fleet scenario through the real planner, reconciler and controller. It uses a deterministic discrete-event model of the fleet and reads or changes nothing on the gateway or any node. `--mode compare` (the default) runs the scenario twice: + +- `off`: what LLooM does today without a pool controller; +- `auto`: with the controller. + +The command exits non-zero when a scenario's expectations fail. + +```bash +lloom pool simulate test/fixtures/pool/scenarios/pair-crash.json +lloom pool simulate my-fleet.json --mode auto --hours 48 --seed 3 --json +``` + +The simulator owns the truth: real footprints, load times, request service times and failures. The planner sees only what LLooM would see: configured estimates, footprints measured once a lane has run, observed latency and demand, and node liveness. In membership scenarios, liveness comes from leases. The simulator also models what LLooM already does: + +- requests go to the alias's routed member, else the first ready candidate, else a cold start that admission allows, else an external member; +- admission evicts idle on-demand lanes first, then idle preferred lanes, and never evicts keep-warm lanes; +- keep-warm and preferred lanes are restored; +- the memory guard aborts a load past its ceiling and latches the runtime. + +A lane whose real footprint overflows physical memory is a memory violation: the node hangs for `hangMinutes`. Plans must never cause one. + +**Scenarios** are JSON files with these fields: + +- `nodes`, `variants` and `workloads`, in planner shapes. Variants add `sim` truth such as `memoryGbPerNode`, `loadSeconds`, `maxSequences`, `failsOn` and `startFailRate`. Workloads add `demand` (`perHour`, `diurnal`, `bursts`, `changes`). +- `initial` running lanes, routes and residency. +- `events`: `cordon`, `uncordon`, `fail`, `recover`, `join`, `leave`, `partition`, `start-lane`, `stop-lane` and `apply-profile`. An event may be limited to some modes, for example an operator's manual fleet switch that happens only without a controller. +- `operator`: how long manual steps take. +- `membership`: heartbeat, lease and lost-after seconds. +- `expect`: checks per mode, as `{ "path.in.report": { "min" | "max" | "equals" | "includes": … } }`. + +A scenario can extend a `base` file. Objects merge, arrays of objects merge by `id` (`"remove": true` drops one), and anything else replaces the base value. The scenarios in `test/fixtures/pool/scenarios/` start from the three-Spark fleet: + +| Scenario | What happens | Today (`off`) | With the controller (`auto`) | +| --------------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------- | +| `pair-taken` | Operator takes the 01/02 pair for 6 hours and gives it back | Operator switches fleet profiles by hand | Cordon and uncordon only; the same moves, one apply each | +| `pair-crash` | spark-node-01 crashes for 4 hours, nobody watching | 60 cold waits, 12 requests to the cloud, 1414 GB reloaded, cognition p95 up to 106 s | 26 cold waits, none to the cloud, 1008 GB reloaded, cognition p95 0.9 s, back on the pair after recovery | +| `lease-loss` | The same crash, detected only through leases, after a 60 s partition | as above | Acts 2 min after the crash (lease lost at 90 s); the partition moves nothing | +| `mac-joins` | A Mac Studio with no weights joins a one-Spark pool | Cognition stays on the 27B | Prefetches weights, moves cognition to the Mac, releases the 27B, drops no requests | +| `memory-estimate-low` | The video lane needs 74 GB but is configured as 48 GB | One guard abort, then the lane is latched | One guard abort; the planner learns the lower bound and plans video as unavailable | +| `demand-flap` | Image and music demand swing hard every 30 minutes | n/a | At most one residency change an hour and no extra loads | + +Numbers are from seeded runs of these fixtures. They show behaviour, and are not benchmarks. + +**Limits of the model.** Lane capacity is modelled (sequences, decode slowdown with concurrency, queues), but the planner does not use it yet, so an overloaded lane queues rather than gaining a replica. GPU interference between co-located lanes is not modelled. Demand is Poisson with diurnal shape and bursts. diff --git a/docs/runtime-policy.md b/docs/runtime-policy.md index 92196a5..5ea9441 100644 --- a/docs/runtime-policy.md +++ b/docs/runtime-policy.md @@ -50,6 +50,17 @@ numeric options. Unrelated settings, routes and credentials are preserved. `runtime-plan` remains read-only. Without numeric options, `runtime-policy` retains its historical read-only runtime-plan behavior. +Admission never approves a load the memory guard is certain to abort. Its host +check keeps whichever is larger free after the load: the reserve, or the margin +below the guard's ceiling. The ceiling is +`min(memorySafety.maxMemoryUtilization, maxMemoryUtilization)`, which defaults +to 0.9, so on a 128 GB host the margin is 12.8 GB, more than the default 8 GB +reserve. For the same reason, cluster node budgets are capped at the ceiling. +When the guard does abort a load, the runtime stays latched until a manual +start. Its status then reports `memorySafetyLatched` and `memorySafetyAbort`: +`usedGb`, and `consumedGb`, the memory the load had taken when it was stopped. +Pool planning treats that memory as a lower bound on the runtime's footprint. + ## Idle restoration The preferred-restore pass starts after all owned, enabled keep-warm pins are diff --git a/src/pool-cli.mjs b/src/pool-cli.mjs index 3c9f5e4..c060495 100644 --- a/src/pool-cli.mjs +++ b/src/pool-cli.mjs @@ -21,7 +21,8 @@ export const POOL_USAGE = [ ' lloom pool token create [--ttl DURATION] [--uses N] [--label KEY=VALUE] [--json]', ' lloom pool members [--json]', ' lloom pool cordon|uncordon|drain [--reason TEXT]', - ' lloom pool join --token-stdin [--node-id ID] [--endpoint URL] [--apply --yes]' + ' lloom pool join --token-stdin [--node-id ID] [--endpoint URL] [--apply --yes]', + ' lloom pool simulate [--mode off|advise|auto|compare] [--hours N] [--seed N] [--json]' ].join('\n'); const DURATION_UNITS = { s: 1000, m: 60000, h: 3600000, d: 86400000 }; diff --git a/test/pool-membership-http.test.mjs b/test/pool-membership-http.test.mjs index e025666..d1f06a4 100644 --- a/test/pool-membership-http.test.mjs +++ b/test/pool-membership-http.test.mjs @@ -113,7 +113,9 @@ test('enroll, heartbeat, cordon and restart over HTTP', async (t) => { assert.match(issued.json.token, /^lpt_/); const stat = await fs.stat(head.membershipPath); assert.equal(stat.mode & 0o777, 0o600); - assert.ok(!(await fs.readFile(head.membershipPath, 'utf8')).includes(issued.json.token.split('_').at(-1))); + assert.ok( + !(await fs.readFile(head.membershipPath, 'utf8')).includes(issued.json.token.slice('lpt_000000000000_'.length)) + ); // An admin key is not an enrollment token, and enrollment never grants one. const asAdmin = await head.call('/gateway/pool/enroll', { diff --git a/test/pool-membership.test.mjs b/test/pool-membership.test.mjs index 0793956..55af386 100644 --- a/test/pool-membership.test.mjs +++ b/test/pool-membership.test.mjs @@ -302,7 +302,8 @@ test('persisted state has hashes but no secrets or tokens, and restores', () => assert.ok(!text.includes(nodeSecret)); assert.ok(!text.includes(pending.token)); assert.ok(!text.includes(nodeSecret.slice(4)), 'not even the secret body'); - assert.ok(!text.includes(pending.token.split('_').at(-1))); + // The body follows the fixed `lpt_<12 hex>_` prefix; base64url may itself contain `_`. + assert.ok(!text.includes(pending.token.slice('lpt_000000000000_'.length))); assert.ok(!text.includes('"load"'), 'telemetry is live state only'); assert.equal(state.version, 1); From c90e2c52e0caa68abc4bf12bca455917a56fc615 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 12:41:05 -0700 Subject: [PATCH 7/9] Pool membership: close node-id takeover and other review findings Security review findings: - Medium: any enrollment token could re-enroll any node id, replacing a live node's secret, endpoint and reported capacity (and flipping a configured peer unavailable in plans). Tokens can now be bound to a node id (lloom pool token create --node-id); claiming a current member or a configured cluster.nodes id needs a token bound to it (409 otherwise). lloom pool leave / POST /gateway/pool/nodes/:id/leave frees an id. - Heartbeats are refused before the body is read unless the bearer has the node-secret shape, and are rate limited per client. - Rate limiters evict least recently used clients instead of clearing the whole map when full. - Node-reported text with control characters is dropped (no escape sequences in lloom pool members output). - join reports the saved secret path and the pool.head to set when the config update fails after enrolling, and warns about plain HTTP heads off loopback and tailnets. Co-Authored-By: Claude Opus 5.5 --- bin/lloom.mjs | 9 ++-- docs/pool-planning.md | 10 ++++- src/pool-cli.mjs | 7 +-- src/pool-membership.mjs | 49 ++++++++++++++++----- src/pool-node.mjs | 38 +++++++++++++--- src/server.mjs | 71 ++++++++++++++++++++++-------- test/pool-cli.test.mjs | 12 ++++- test/pool-membership-http.test.mjs | 66 +++++++++++++++++++++++++-- test/pool-membership.test.mjs | 65 +++++++++++++++++++++++++-- 9 files changed, 274 insertions(+), 53 deletions(-) diff --git a/bin/lloom.mjs b/bin/lloom.mjs index 33dce54..7ffbf91 100755 --- a/bin/lloom.mjs +++ b/bin/lloom.mjs @@ -245,9 +245,9 @@ Backends and runtimes: lloom down lloom cluster [status|doctor|discover] [--apply] [--id NAME] [--api-key-env NAME] [--json] lloom pool plan [--cordon NODE[,NODE]] [--pin ALIAS=MODEL] [--profile NAME] [--consider-installs] [--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json] - lloom pool token create [--ttl DURATION] [--uses N] [--label KEY=VALUE] [--json] + lloom pool token create [--node-id ID] [--ttl DURATION] [--uses N] [--label KEY=VALUE] [--json] lloom pool members [--json] - lloom pool cordon|uncordon|drain [--reason TEXT] + lloom pool cordon|uncordon|drain|leave [--reason TEXT] lloom pool join --token-stdin [--node-id ID] [--endpoint URL] [--apply --yes] lloom pool simulate [--mode off|advise|auto|compare] [--hours N] [--seed N] [--json] lloom deployment plan --plan FILE [--json] @@ -2530,7 +2530,8 @@ async function main() { body: poolTokenOptions({ ttl: argValue(args, '--ttl'), uses: argValue(args, '--uses'), - labels: argValues(args, '--label') + labels: argValues(args, '--label'), + nodeId: argValue(args, '--node-id') ?? undefined }), timeoutMs: 10000 }); @@ -2552,7 +2553,7 @@ async function main() { console.log(json ? JSON.stringify(result, null, 2) : formatPoolMembers(result)); return; } - if (['cordon', 'uncordon', 'drain'].includes(action)) { + if (['cordon', 'uncordon', 'drain', 'leave'].includes(action)) { const nodeId = positional(args)[2]; if (!nodeId) throw new Error(POOL_USAGE); const reason = argValue(args, '--reason'); diff --git a/docs/pool-planning.md b/docs/pool-planning.md index cbabcbc..7509ecc 100644 --- a/docs/pool-planning.md +++ b/docs/pool-planning.md @@ -91,12 +91,20 @@ lloom pool members # state, lease age, inventory, lloom pool cordon spark-node-02 --reason "disk swap" lloom pool drain spark-node-02 --reason rebalance lloom pool uncordon spark-node-02 +lloom pool leave spark-node-02 # decommission; the id may enroll again +lloom pool token create --node-id spark-node-02 # re-enroll a node that is still a member # on the joining node (token on stdin, never in argv) lloom pool join http://head-private:8100 --token-stdin --endpoint http://10.10.10.2:8100 --apply --yes ``` -**Enrollment.** `POST /gateway/pool/enroll` takes the token as a bearer credential and the node's `nodeId`, `endpoint` and optional `labels`. It returns a node secret and the lease length. A token expires (default one hour) and has a use count (default one). Labels set on the token override labels the node sends. Enrolling an existing node id again needs a different token, and rotates the node's secret. It keeps any cordon or drain, so a node cannot re-enroll its way out of one. A node cannot enroll with the head's own node id. Attempts are rate limited per client address. +**Enrollment.** `POST /gateway/pool/enroll` takes the token as a bearer credential and the node's `nodeId`, `endpoint` and optional `labels`. It returns a node secret and the lease length. A token expires (default one hour) and has a use count (default one). Labels set on the token override labels the node sends. A token issued with `--node-id` enrolls only that node. + +A token can claim an id that is a current member, or a node configured in `cluster.nodes`, only if it was issued for that id. Otherwise the head answers 409. This means no other node's token can take over a node's liveness, endpoint or reported capacity. + +Re-enrolling rotates the node's secret and keeps any cordon or drain, so a node cannot re-enroll its way out of one. After `lloom pool leave`, the id is free again. A node cannot enroll with the head's own node id. + +Enrollment attempts are rate limited per client address. Heartbeats are rate limited per address too, and are refused before the body is read unless the bearer has the node-secret format. Node-reported text that contains control characters is dropped. `join` warns when the head URL is plain HTTP and the host is neither loopback nor a tailnet address. `lloom pool join` stores the secret in `pool-node-secret` (mode 0600) next to the node's config and sets: diff --git a/src/pool-cli.mjs b/src/pool-cli.mjs index c060495..5f1a1c3 100644 --- a/src/pool-cli.mjs +++ b/src/pool-cli.mjs @@ -18,9 +18,9 @@ import { loadRecipes } from './recipes.mjs'; export const POOL_USAGE = [ 'Usage: lloom pool plan [--cordon NODE[,NODE]] [--pin ALIAS=MODEL] [--profile NAME] [--consider-installs] ' + '[--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json]', - ' lloom pool token create [--ttl DURATION] [--uses N] [--label KEY=VALUE] [--json]', + ' lloom pool token create [--node-id ID] [--ttl DURATION] [--uses N] [--label KEY=VALUE] [--json]', ' lloom pool members [--json]', - ' lloom pool cordon|uncordon|drain [--reason TEXT]', + ' lloom pool cordon|uncordon|drain|leave [--reason TEXT]', ' lloom pool join --token-stdin [--node-id ID] [--endpoint URL] [--apply --yes]', ' lloom pool simulate [--mode off|advise|auto|compare] [--hours N] [--seed N] [--json]' ].join('\n'); @@ -39,8 +39,9 @@ export function parseDurationMs(value) { return Math.round(ms); } -export function poolTokenOptions({ ttl, uses, labels = [] } = {}) { +export function poolTokenOptions({ ttl, uses, labels = [], nodeId } = {}) { const options = {}; + if (nodeId != null) options.nodeId = String(nodeId); if (ttl != null) options.ttlMs = parseDurationMs(ttl); if (uses != null) { if (!/^\d+$/.test(String(uses))) throw new Error('--uses must be a positive integer'); diff --git a/src/pool-membership.mjs b/src/pool-membership.mjs index 1238a1b..920e38c 100644 --- a/src/pool-membership.mjs +++ b/src/pool-membership.mjs @@ -76,17 +76,18 @@ function normalizeLabels(labels) { const isObject = (value) => value && typeof value === 'object' && !Array.isArray(value); const finite = (value) => (typeof value === 'number' && Number.isFinite(value) ? value : null); -const strings = (value) => - (Array.isArray(value) ? value : []) - .filter((item) => typeof item === 'string' && item && item.length <= MAX_TEXT) - .slice(0, MAX_LIST); +// Node-reported text reaches admin terminals; control characters (escape +// sequences) are dropped rather than printed. +const CONTROL = /[\u0000-\u001f\u007f-\u009f]/; +const cleanText = (item) => typeof item === 'string' && item && item.length <= MAX_TEXT && !CONTROL.test(item); +const strings = (value) => (Array.isArray(value) ? value : []).filter(cleanText).slice(0, MAX_LIST); const objects = (value, map) => (Array.isArray(value) ? value : []) .slice(0, MAX_LIST) .map((item) => (isObject(item) || typeof item === 'string' ? map(item) : null)) .filter(Boolean); const compact = (value) => Object.fromEntries(Object.entries(value).filter(([, entry]) => entry != null)); -const pickText = (value) => (typeof value === 'string' && value.length <= MAX_TEXT ? value : null); +const pickText = (value) => (cleanText(value) ? value : null); // A node reports its own inventory, so keep only the fields the planner reads, // with the types it expects (see collectNodeInventory in pool-inventory.mjs). @@ -255,9 +256,12 @@ export function createPoolMembership({ leaseMs = 30000, lostAfterMs = 3 * leaseM leaseMs, lostAfterMs, - issueEnrollmentToken({ ttlMs = 3600000, uses = 1, labels = {} } = {}) { + // A token bound to `nodeId` enrolls only that node, and is the only way to + // re-enroll a node that is enrolled (or configured on the head). + issueEnrollmentToken({ ttlMs = 3600000, uses = 1, labels = {}, nodeId = null } = {}) { positiveInteger(ttlMs, 'ttlMs', MAX_TOKEN_TTL_MS); positiveInteger(uses, 'uses', MAX_TOKEN_USES); + if (nodeId != null && !validNodeId(nodeId)) throw invalid('Invalid node identity'); const tokenLabels = normalizeLabels(labels); const at = now(); for (const [id, token] of tokens) if (!liveToken(token, at)) tokens.delete(id); @@ -266,21 +270,42 @@ export function createPoolMembership({ leaseMs = 30000, lostAfterMs = 3 * leaseM while (tokens.has(id)); const token = `lpt_${id}_${randomBytes(32).toString('base64url')}`; const expiresAt = iso(at + ttlMs); - tokens.set(id, { id, hash: sha256(token), createdAt: iso(at), expiresAt, uses, used: 0, labels: tokenLabels }); - return { token, id, expiresAt }; + tokens.set(id, { + id, + hash: sha256(token), + createdAt: iso(at), + expiresAt, + uses, + used: 0, + labels: tokenLabels, + ...(nodeId ? { nodeId } : {}) + }); + return { token, id, expiresAt, ...(nodeId ? { nodeId } : {}) }; }, - enroll({ token, nodeId, endpoint, inventory, labels } = {}) { + enroll({ token, nodeId, endpoint, inventory, labels, protectedNodeIds = [] } = {}) { const at = now(); const match = typeof token === 'string' ? token.match(TOKEN_PATTERN) : null; const record = match ? tokens.get(match[1]) : null; // One generic failure: an attacker learns nothing about which check failed. - const tokenOk = hashMatches(token ?? '', record?.hash) && liveToken(record, at); - const existing = validNodeId(nodeId) ? nodes.get(nodeId) : null; - if (!tokenOk || existing?.tokenId === record.id) { + const tokenOk = + hashMatches(token ?? '', record?.hash) && liveToken(record, at) && (!record.nodeId || record.nodeId === nodeId); + if (!tokenOk) { throw membershipError(401, 'invalid_enrollment_token', 'enrollment token is invalid, expired, or used up'); } if (!validNodeId(nodeId)) throw invalid('Invalid node identity'); + const existing = nodes.get(nodeId) ?? null; + // Claiming a live or configured identity would hand its liveness, + // endpoint and reported capacity to whoever holds any token. + const claimed = + (existing && !existing.left) || (Array.isArray(protectedNodeIds) && protectedNodeIds.includes(nodeId)); + if (claimed && record.nodeId !== nodeId) { + throw membershipError( + 409, + 'pool_node_conflict', + `node ${nodeId} is already in the pool; re-enroll it with a token issued for it (lloom pool token create --node-id ${nodeId})` + ); + } const normalizedEndpoint = normalizeNodeEndpoint(endpoint); const nodeLabels = { ...normalizeLabels(labels), ...record.labels }; const nodeSecret = newSecret(); diff --git a/src/pool-node.mjs b/src/pool-node.mjs index 1f3709a..dee6d75 100644 --- a/src/pool-node.mjs +++ b/src/pool-node.mjs @@ -22,6 +22,19 @@ function headBaseUrl(url) { } } +// The enrollment token and the node secret are bearer credentials. Plain HTTP +// is fine on loopback or a tailnet (WireGuard encrypts it), not across a LAN. +export function cleartextWarning(head) { + const url = new URL(head); + if (url.protocol !== 'http:') return null; + const host = url.hostname.replace(/^\[|\]$/g, ''); + const loopback = host === 'localhost' || host === '::1' || /^127\./.test(host); + const tailnet = /\.ts\.net$/i.test(host) || /^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./.test(host); + return loopback || tailnet + ? null + : `pool head ${head} is plain HTTP: the enrollment token and node secret cross the network unencrypted; use HTTPS or a tailnet address`; +} + const configDir = (configPath) => path.dirname(path.resolve(String(configPath))); // `pool.head.secretEnv` names an environment variable; `pool.head.secretFile` @@ -108,11 +121,26 @@ export async function joinPool( throw new Error('Pool head returned an invalid enrollment response'); } await writePrivateFile(secretPath, `${result.nodeSecret}\n`); - const { changed } = await mutateConfigSource(config, (raw) => { - raw.pool ??= {}; - raw.pool.head = { url: head, nodeId, secretFile: POOL_NODE_SECRET_FILE }; - }); - return { ...summary, applied: true, changed, leaseMs: result.leaseMs ?? null }; + let changed; + try { + ({ changed } = await mutateConfigSource(config, (raw) => { + raw.pool ??= {}; + raw.pool.head = { url: head, nodeId, secretFile: POOL_NODE_SECRET_FILE }; + })); + } catch (error) { + // The token is spent and the secret saved; say how to finish by hand. + throw new Error( + `Enrolled as ${nodeId}, and the node secret is saved in ${secretPath}, but the config update failed ` + + `(${error.message}). Set pool.head to ${JSON.stringify({ url: head, nodeId, secretFile: POOL_NODE_SECRET_FILE })} by hand.` + ); + } + return { + ...summary, + applied: true, + changed, + leaseMs: result.leaseMs ?? null, + ...(cleartextWarning(head) ? { warning: cleartextWarning(head) } : {}) + }; } // One heartbeat per interval, with exponential backoff (and +/-10% jitter) diff --git a/src/server.mjs b/src/server.mjs index 88a7f60..f5d8c45 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -954,7 +954,10 @@ const POOL_BODY_LIMIT_BYTES = 1024 * 1024; const POOL_BODY_DRAIN_BYTES = 16 * 1024 * 1024; // Per client: ten enrollment attempts at once, then one every six seconds. const POOL_ENROLL_RATE = { rateMs: 6000, burst: 9 }; -const POOL_ENROLL_CLIENTS_MAX = 4096; +// Heartbeats arrive every ~10 s per node; several nodes may share an address. +const POOL_HEARTBEAT_RATE = { rateMs: 500, burst: 59 }; +const POOL_RATE_CLIENTS_MAX = 4096; +const POOL_NODE_SECRET_SHAPE = /^lps_[A-Za-z0-9_-]{43}$/; function poolRequestError(statusCode, code, message) { return Object.assign(new Error(message), { statusCode, code }); @@ -2423,6 +2426,7 @@ export function createLloomServer( options: poolMembershipOptions(config) }); const poolEnrollLimiters = new Map(); + const poolHeartbeatLimiters = new Map(); // A node with `pool.head` keeps its lease from `serve`. const poolHeartbeat = createPoolHeartbeatLoop({ getHead: () => (isStandby(config) ? null : config.pool?.head), @@ -2628,30 +2632,42 @@ export function createLloomServer( if (config.pool?.head && !isStandby(config)) poolHeartbeat.start(); } - function poolEnrollRetryMs(req) { + // Per-client limiters, least recently used first: a flood of new source + // addresses evicts only the oldest entries, never every client's budget. + function poolRetryMs(limiters, rate, req) { const client = req.socket?.remoteAddress ?? 'unknown'; - if (!poolEnrollLimiters.has(client) && poolEnrollLimiters.size >= POOL_ENROLL_CLIENTS_MAX) - poolEnrollLimiters.clear(); - if (!poolEnrollLimiters.has(client)) poolEnrollLimiters.set(client, createGcra(POOL_ENROLL_RATE)); - const admission = poolEnrollLimiters.get(client).tryAcquire(Date.now()); + let limiter = limiters.get(client); + if (limiter) limiters.delete(client); + else { + if (limiters.size >= POOL_RATE_CLIENTS_MAX) limiters.delete(limiters.keys().next().value); + limiter = createGcra(rate); + } + limiters.set(client, limiter); + const admission = limiter.tryAcquire(Date.now()); return admission.allowed ? 0 : admission.retryAfterMs; } + function sendPoolRateLimited(res, retryAfterMs, message, code) { + sendJson( + res, + 429, + errorBody(message, { type: 'rate_limit_error', code }), + { 'retry-after': String(Math.max(1, Math.ceil(retryAfterMs / 1000))) }, + config + ); + } + async function handlePoolMembership(req, res, url) { - const action = url.pathname.match(/^\/gateway\/pool\/nodes\/([^/]+)\/(cordon|uncordon|drain)$/); + const action = url.pathname.match(/^\/gateway\/pool\/nodes\/([^/]+)\/(cordon|uncordon|drain|leave)$/); try { if (req.method === 'POST' && url.pathname === '/gateway/pool/enroll') { - const retryAfterMs = poolEnrollRetryMs(req); + const retryAfterMs = poolRetryMs(poolEnrollLimiters, POOL_ENROLL_RATE, req); if (retryAfterMs) { - sendJson( + sendPoolRateLimited( res, - 429, - errorBody('too many enrollment attempts; retry later', { - type: 'rate_limit_error', - code: 'pool_enroll_rate_limited' - }), - { 'retry-after': String(Math.max(1, Math.ceil(retryAfterMs / 1000))) }, - config + retryAfterMs, + 'too many enrollment attempts; retry later', + 'pool_enroll_rate_limited' ); return; } @@ -2665,7 +2681,9 @@ export function createLloomServer( nodeId: body.nodeId, endpoint: body.endpoint, inventory: body.inventory, - labels: body.labels + labels: body.labels, + // Configured peers keep their identity unless the token names them. + protectedNodeIds: Object.keys(config.cluster?.nodes ?? {}) }) ); logger.info?.(`pool node ${enrolled.nodeId} enrolled`); @@ -2673,11 +2691,21 @@ export function createLloomServer( return; } if (req.method === 'POST' && url.pathname === '/gateway/pool/heartbeat') { + // Refuse anything that cannot be a node secret before buffering a body. + const nodeSecret = extractApiKey(req); + if (!POOL_NODE_SECRET_SHAPE.test(nodeSecret ?? '')) { + throw poolRequestError(401, 'invalid_node_credential', 'pool node credential is invalid'); + } + const retryAfterMs = poolRetryMs(poolHeartbeatLimiters, POOL_HEARTBEAT_RATE, req); + if (retryAfterMs) { + sendPoolRateLimited(res, retryAfterMs, 'too many heartbeats; retry later', 'pool_heartbeat_rate_limited'); + return; + } const body = await readPoolJson(req); const membership = await poolMembership.load(); const lease = membership.heartbeat({ nodeId: body.nodeId, - nodeSecret: extractApiKey(req), + nodeSecret, inventory: body.inventory, running: body.running, telemetry: body.telemetry @@ -2699,7 +2727,12 @@ export function createLloomServer( if (req.method === 'POST' && url.pathname === '/gateway/pool/tokens') { const body = await readPoolJson(req); const issued = await poolMembership.mutate((membership) => - membership.issueEnrollmentToken({ ttlMs: body.ttlMs, uses: body.uses, labels: body.labels }) + membership.issueEnrollmentToken({ + ttlMs: body.ttlMs, + uses: body.uses, + labels: body.labels, + nodeId: body.nodeId ?? null + }) ); sendJson(res, 200, { ok: true, ...issued }, {}, config); return; diff --git a/test/pool-cli.test.mjs b/test/pool-cli.test.mjs index b69af58..12d66dc 100644 --- a/test/pool-cli.test.mjs +++ b/test/pool-cli.test.mjs @@ -376,5 +376,15 @@ test('the CLI issues tokens, joins a node, lists members and cordons', async (t) const usage = await runCli(['pool', 'cordon', ...head], env); assert.notEqual(usage.code, 0); - assert.match(usage.stderr, /lloom pool cordon\|uncordon\|drain /); + assert.match(usage.stderr, /lloom pool cordon\|uncordon\|drain\|leave /); +}); + +test('token options bind a node id; joins warn about cleartext credentials off loopback and tailnets', async () => { + assert.deepEqual(poolTokenOptions({ nodeId: 'spark-node-01' }), { nodeId: 'spark-node-01' }); + const { cleartextWarning } = await import('../src/pool-node.mjs'); + assert.equal(cleartextWarning('http://127.0.0.1:8100'), null); + assert.equal(cleartextWarning('http://100.119.155.43:8100'), null, 'tailnet CGNAT range'); + assert.equal(cleartextWarning('http://head.example.ts.net:8100'), null); + assert.equal(cleartextWarning('https://head.example.com'), null); + assert.match(cleartextWarning('http://192.168.1.20:8100'), /plain HTTP/); }); diff --git a/test/pool-membership-http.test.mjs b/test/pool-membership-http.test.mjs index d1f06a4..1d899f6 100644 --- a/test/pool-membership-http.test.mjs +++ b/test/pool-membership-http.test.mjs @@ -21,13 +21,13 @@ async function tempDir(t) { return dir; } -async function startHead(t, { dir, pool, serverOptions = {} } = {}) { +async function startHead(t, { dir, pool, clusterNodes, serverOptions = {} } = {}) { dir ??= await tempDir(t); const configPath = path.join(dir, 'config.json'); const raw = { server: { host: '127.0.0.1', port: 0 }, security: { adminApiKeys: [ADMIN], apiKeys: [INFERENCE] }, - cluster: { nodeId: 'head' }, + cluster: { nodeId: 'head', ...(clusterNodes ? { nodes: clusterNodes } : {}) }, ...(pool ? { pool } : {}), backends: {}, models: [] @@ -210,18 +210,76 @@ test('enrollment attempts are rate limited per client', async (t) => { assert.equal(statuses.at(-1), 429); }); +const SECRET_SHAPED = 'lps_' + 'A'.repeat(43); + test('oversized and malformed pool bodies are rejected', async (t) => { const head = await startHead(t); const huge = await fetch(head.base + '/gateway/pool/heartbeat', { method: 'POST', - headers: { 'content-type': 'application/json' }, + headers: { 'content-type': 'application/json', authorization: `Bearer ${SECRET_SHAPED}` }, body: JSON.stringify({ nodeId: 'x', inventory: { padding: 'x'.repeat(2 * 1024 * 1024) } }) }); assert.equal(huge.status, 413); - const malformed = await fetch(head.base + '/gateway/pool/heartbeat', { method: 'POST', body: '{' }); + const malformed = await fetch(head.base + '/gateway/pool/heartbeat', { + method: 'POST', + headers: { authorization: `Bearer ${SECRET_SHAPED}` }, + body: '{' + }); assert.equal(malformed.status, 400); }); +test('heartbeats without a secret-shaped bearer are refused before the body is read, and are rate limited', async (t) => { + const head = await startHead(t); + const unauthenticated = await fetch(head.base + '/gateway/pool/heartbeat', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ nodeId: 'x', inventory: { padding: 'x'.repeat(2 * 1024 * 1024) } }) + }); + assert.equal(unauthenticated.status, 401, 'no 413: the body was never buffered'); + const statuses = []; + for (let index = 0; index < 70; index += 1) { + statuses.push((await head.call('/gateway/pool/heartbeat', { key: SECRET_SHAPED, body: { nodeId: 'x' } })).status); + } + assert.ok(statuses.slice(0, 30).every((status) => status === 401)); + assert.equal(statuses.at(-1), 429); +}); + +test('configured node ids and enrolled nodes cannot be claimed by an unbound token; leave frees an id', async (t) => { + const head = await startHead(t, { pool: {}, clusterNodes: { head: {}, peer: { endpoint: 'http://peer:8100' } } }); + const configured = await head.call('/gateway/pool/tokens', { key: ADMIN, body: { uses: 2 } }); + const claimPeer = await head.call('/gateway/pool/enroll', { + key: configured.json.token, + body: { nodeId: 'peer', endpoint: 'http://x:8100' } + }); + assert.equal(claimPeer.status, 409, 'a configured cluster peer needs a token bound to it'); + const first = await head.call('/gateway/pool/enroll', { + key: configured.json.token, + body: { nodeId: 'n1', endpoint: 'http://n1:8100' } + }); + assert.equal(first.status, 200); + const other = await head.call('/gateway/pool/tokens', { key: ADMIN, body: {} }); + const takeover = await head.call('/gateway/pool/enroll', { + key: other.json.token, + body: { nodeId: 'n1', endpoint: 'http://evil:1' } + }); + assert.equal(takeover.status, 409); + assert.equal(takeover.json.error.code, 'pool_node_conflict'); + const bound = await head.call('/gateway/pool/tokens', { key: ADMIN, body: { nodeId: 'n1' } }); + assert.equal(bound.json.nodeId, 'n1'); + const rotated = await head.call('/gateway/pool/enroll', { + key: bound.json.token, + body: { nodeId: 'n1', endpoint: 'http://n1:8100' } + }); + assert.equal(rotated.status, 200); + const left = await head.call('/gateway/pool/nodes/n1/leave', { key: ADMIN, body: {} }); + assert.equal(left.json.member.state, 'left'); + const rejoin = await head.call('/gateway/pool/enroll', { + key: other.json.token, + body: { nodeId: 'n1', endpoint: 'http://n1:8100' } + }); + assert.equal(rejoin.status, 200, 'a node that left may join again with any token'); +}); + test('a node with pool.head heartbeats to the head from serve', async (t) => { const head = await startHead(t); const issued = await head.call('/gateway/pool/tokens', { key: ADMIN, body: {} }); diff --git a/test/pool-membership.test.mjs b/test/pool-membership.test.mjs index 55af386..9465bf1 100644 --- a/test/pool-membership.test.mjs +++ b/test/pool-membership.test.mjs @@ -171,17 +171,30 @@ test('token labels win over node-supplied labels', () => { assert.deepEqual(member(membership, 'a').labels, { role: 'gpu', zone: 'x' }); }); -test('re-enrolling needs a new token and rotates the secret', () => { +test('re-enrolling a live node needs a token issued for that node, and rotates the secret', () => { const { membership, nodeSecret } = enrolled(); const reuse = membership.issueEnrollmentToken({ uses: 3 }); const first = membership.enroll({ token: reuse.token, nodeId: 'spark-node-02', endpoint: 'http://b:8100' }); assert.throws( () => membership.enroll({ token: reuse.token, nodeId: 'spark-node-02', endpoint: 'http://b:8100' }), - (error) => error.statusCode === 401 && error.code === 'invalid_enrollment_token', - 'the token that enrolled a node cannot re-enroll it' + (error) => error.statusCode === 409 && error.code === 'pool_node_conflict', + 'an enrolled node cannot be claimed again by an unbound token' + ); + // Another node's (unbound) token cannot take over spark-node-01. + const other = membership.issueEnrollmentToken({ labels: { role: 'mac' } }); + assert.throws( + () => membership.enroll({ token: other.token, nodeId: 'spark-node-01', endpoint: 'http://evil:1' }), + (error) => error.statusCode === 409 && error.code === 'pool_node_conflict' + ); + assert.equal(membership.heartbeat({ nodeId: 'spark-node-01', nodeSecret }).ok, true, 'the real node is untouched'); + // A token bound to another id cannot enroll this one. + const boundElsewhere = membership.issueEnrollmentToken({ nodeId: 'spark-node-09' }); + assert.throws( + () => membership.enroll({ token: boundElsewhere.token, nodeId: 'spark-node-03', endpoint: 'http://c:8100' }), + (error) => error.statusCode === 401 && error.code === 'invalid_enrollment_token' ); membership.cordon('spark-node-01', { reason: 'maintenance' }); - const { token } = membership.issueEnrollmentToken({}); + const { token } = membership.issueEnrollmentToken({ nodeId: 'spark-node-01' }); const rotated = membership.enroll({ token, nodeId: 'spark-node-01', endpoint: 'http://10.0.0.9:8100' }); assert.notEqual(rotated.nodeSecret, nodeSecret); assert.throws(() => membership.heartbeat({ nodeId: 'spark-node-01', nodeSecret }), /credential/); @@ -197,6 +210,50 @@ test('re-enrolling needs a new token and rotates the secret', () => { ); }); +test('protected (configured) node ids need a token bound to them; a node that left may re-enroll', () => { + const membership = createPoolMembership({ now: clock() }); + const unbound = membership.issueEnrollmentToken({ uses: 2 }); + assert.throws( + () => + membership.enroll({ + token: unbound.token, + nodeId: 'spark-node-03', + endpoint: 'http://x:8100', + protectedNodeIds: ['spark-node-03'] + }), + (error) => error.statusCode === 409 && error.code === 'pool_node_conflict' + ); + const bound = membership.issueEnrollmentToken({ nodeId: 'spark-node-03' }); + assert.equal( + membership.enroll({ + token: bound.token, + nodeId: 'spark-node-03', + endpoint: 'http://x:8100', + protectedNodeIds: ['spark-node-03'] + }).nodeId, + 'spark-node-03' + ); + membership.enroll({ token: unbound.token, nodeId: 'mac', endpoint: 'http://mac:8100' }); + membership.leave('mac'); + const again = membership.issueEnrollmentToken({}); + assert.equal(membership.enroll({ token: again.token, nodeId: 'mac', endpoint: 'http://mac:8100' }).nodeId, 'mac'); +}); + +test('node-reported text with control characters is dropped', () => { + const membership = createPoolMembership({ now: clock() }); + const { token } = membership.issueEnrollmentToken({}); + const { nodeSecret } = membership.enroll({ token, nodeId: 'n', endpoint: 'http://n:8100' }); + membership.heartbeat({ + nodeId: 'n', + nodeSecret, + running: ['ok-lane', 'bad\u001b[2Jlane'], + inventory: { computeArchs: ['sm_121', '\u001b]0;pwned\u0007'] } + }); + const entry = membership.members()[0]; + assert.deepEqual(entry.running, ['ok-lane']); + assert.deepEqual(entry.inventory.computeArchs, ['sm_121']); +}); + test('cordon, drain, uncordon and leave', () => { const { membership, nodeSecret } = enrolled(); assert.equal(membership.cordon('spark-node-01', { reason: 'disk swap' }).cordoned, true); From 10c88f8380ffb8af5ebc65c9aa0471a0ed6df5e0 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 12:41:19 -0700 Subject: [PATCH 8/9] Pool join: keep the config error as the cause Co-Authored-By: Claude Opus 5.5 --- src/pool-node.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pool-node.mjs b/src/pool-node.mjs index dee6d75..87c590d 100644 --- a/src/pool-node.mjs +++ b/src/pool-node.mjs @@ -131,7 +131,8 @@ export async function joinPool( // The token is spent and the secret saved; say how to finish by hand. throw new Error( `Enrolled as ${nodeId}, and the node secret is saved in ${secretPath}, but the config update failed ` + - `(${error.message}). Set pool.head to ${JSON.stringify({ url: head, nodeId, secretFile: POOL_NODE_SECRET_FILE })} by hand.` + `(${error.message}). Set pool.head to ${JSON.stringify({ url: head, nodeId, secretFile: POOL_NODE_SECRET_FILE })} by hand.`, + { cause: error } ); } return { From 9487f53e8c153078447909d3154f43490ca08dd9 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Sun, 11 Oct 2026 12:47:46 -0700 Subject: [PATCH 9/9] Pool head: fix correctness review findings Bugs (each with a test): - A model that moved between nodes left its old lane running: an unplanned lane is now retired once every planned lane serving its models is up. - Lanes already stopping were treated as freed memory; they now hold it and get an idempotent stop that starts on that node wait for. - The planner could plan past the guard when only memorySafety was set; the catalog now passes memorySafetyCeiling() itself. - A cluster node with unknown capacity got a negative admission budget, blocking no-request plans; the clamp is floored at zero, and the guard margin rounds up. - Memory-safety abort evidence never cleared; a healthy start clears it and a measured footprint supersedes it. - An alias moved to an interim member during a swap was never moved back; the flip back is queued and ordered after the interim flip. Hardening: ticks never overlap; throwing observers cannot wedge the executor or controller; concurrency is at least 1; a lane already loading is not charged to the load budget again; one move larger than the budget may go in an empty hour; only route changes preempt; vacating a cordoned or draining node is urgent. The simulator reports draining lanes as stopping and the controller's last plan, and memory-estimate-low now asserts the abort, the latch and video planned as uncovered. Co-Authored-By: Claude Opus 5.5 --- docs/pool-planning.md | 12 ++- docs/runtime-policy.md | 3 +- src/pool-catalog.mjs | 11 ++- src/pool-controller.mjs | 72 ++++++++++++++---- src/pool-reconciler.mjs | 59 ++++++++++++--- src/pool-sim.mjs | 30 ++++---- src/runtime-manager.mjs | 2 + src/runtime-policy.mjs | 8 +- .../pool/scenarios/memory-estimate-low.json | 13 ++++ test/pool-catalog.test.mjs | 11 +++ test/pool-controller.test.mjs | 57 ++++++++++++++ test/pool-reconciler.test.mjs | 74 +++++++++++++++++++ test/pool-sim.test.mjs | 27 +++++++ test/runtime-memory-safety.test.mjs | 8 ++ test/runtime-policy.test.mjs | 29 ++++++++ 15 files changed, 363 insertions(+), 53 deletions(-) diff --git a/docs/pool-planning.md b/docs/pool-planning.md index 7509ecc..c233ff2 100644 --- a/docs/pool-planning.md +++ b/docs/pool-planning.md @@ -179,7 +179,7 @@ With the memory guard ceiling applied, the same fixture shows a conflict the han 2. **Downgrade residency** for lanes leaving keep-warm, so LLooM does not restart them while they drain. 3. **Start** new lanes that fit next to everything still running (make before break). 4. **Flip routes** once their target is ready. -5. **Drain and stop** lanes no route uses any more. +5. **Drain and stop** lanes no route uses any more. This includes a lane whose model moved to another node: it stops once the new lane is up. A lane already draining from an earlier change counts as holding its memory until the drain ends. 6. **Upgrade residency** for lanes that just started. When a new lane only fits after an old one is gone, the move is a swap. The old lane's aliases go to an interim member that is already serving, such as another running lane or an external fallback. Then the old lane drains and stops, the new lane starts, and the aliases flip to it. If there is no interim member, the plan reports an interruption instead of hiding it. Unmanaged lanes (for example a hand-run TP2 server) become `manual-start` and `manual-stop` steps for an operator. @@ -196,10 +196,16 @@ Steps run with dependencies. When a step fails, everything that depends on it is Moving a model costs minutes of load time, so the controller only acts on a plan worth moving for: -- **Urgent changes act at once.** A change is urgent when the plan covers a required workload that keeping today's routes would leave uncovered: a node went away, a lane died, or memory was freed. An urgent plan for a _different_ target can cancel a change still in progress. For example, it cancels a manual restart that is waiting on a node which has just died. The same target never preempts itself. -- **Other changes are held back.** The steps must be the same for `confirmTicks` (2) ticks in a row. They must also either cut the latency cost by at least `minGain` (10%) or improve SLOs or local coverage. After that come a `cooldownMs` (15 min) cooldown and a `maxLoadGbPerHour` (400 GB) budget for new loads. +- **Urgent changes act at once.** A change is urgent in two cases: + - The plan covers a required workload that keeping today's routes would leave uncovered: a node went away, a lane died, or memory was freed. + - The plan vacates a cordoned or draining node. + + An urgent plan with _different routes_ can cancel a change still in progress. For example, it cancels a manual restart that is waiting on a node which has just died. The same routes never preempt themselves, and residency changes never preempt. + +- **Other changes are held back.** The steps must be the same for `confirmTicks` (2) ticks in a row. They must also either cut the latency cost by at least `minGain` (10%) or improve SLOs or local coverage. After that come a `cooldownMs` (15 min) cooldown and a `maxLoadGbPerHour` (400 GB) budget for new loads. A lane already loading does not count against the budget again. A single change larger than the whole budget may still go when nothing else loaded in the past hour. - **Residency-only changes follow noisy demand,** so they get a slower signal: `maintenanceConfirmTicks` (10) and `maintenanceCooldownMs` (1 hour). - **Failed starts are skipped for a while.** A lane that fails to start is excluded from planning on that node for `failureBackoffMs` (30 min). +- **One tick at a time.** Ticks never overlap, and an exception thrown by an observer cannot wedge the controller. ## Simulation diff --git a/docs/runtime-policy.md b/docs/runtime-policy.md index 5ea9441..c1b092b 100644 --- a/docs/runtime-policy.md +++ b/docs/runtime-policy.md @@ -55,7 +55,8 @@ check keeps whichever is larger free after the load: the reserve, or the margin below the guard's ceiling. The ceiling is `min(memorySafety.maxMemoryUtilization, maxMemoryUtilization)`, which defaults to 0.9, so on a 128 GB host the margin is 12.8 GB, more than the default 8 GB -reserve. For the same reason, cluster node budgets are capped at the ceiling. +reserve. For the same reason, cluster node budgets are capped at the ceiling, +including an explicit per-node `memoryBudgetGb` set above it. When the guard does abort a load, the runtime stays latched until a manual start. Its status then reports `memorySafetyLatched` and `memorySafetyAbort`: `usedGb`, and `consumedGb`, the memory the load had taken when it was stopped. diff --git a/src/pool-catalog.mjs b/src/pool-catalog.mjs index 5764aec..eec260c 100644 --- a/src/pool-catalog.mjs +++ b/src/pool-catalog.mjs @@ -8,6 +8,7 @@ import { clusterNodes, modelTargets, runtimeResourcesByNode } from './cluster.mj import { resolveRouteTarget } from './config-profiles.mjs'; import { inventoryFromProfile } from './pool-inventory.mjs'; import { memberPlannerNode } from './pool-membership.mjs'; +import { memorySafetyCeiling } from './runtime-memory-safety.mjs'; import { residencyPolicy } from './runtime-preferences.mjs'; export const POOL_CATALOG_VERSION = 1; @@ -240,7 +241,8 @@ function configVariants(config, { nodeStatuses, env }) { nodes, requirements: compact({ nodeCount: nodes.length, - memoryGbPerNode: Math.max(configuredGb, measuredGb ?? 0, safety.abortGb ?? 0) || null, + // A measured footprint supersedes an abort's lower bound. + memoryGbPerNode: Math.max(configuredGb, measuredGb ?? safety.abortGb ?? 0) || null, contextWindow: num(model.contextWindow ?? runtime.contextWindow), maxSequences: num(runtime.maxActiveRequests), recipeId: runtime.recipe?.id ?? null, @@ -618,11 +620,8 @@ export function buildPoolPlanInputs({ policy: compact({ maxMemoryUtilization: num(config.runtimePolicy?.maxMemoryUtilization), reserveGb: num(config.runtimePolicy?.reserveMemoryGb), - // "yolo" turns the load guard off, so it no longer caps the budget. - memorySafetyUtilization: - config.runtimePolicy?.memorySafety?.mode === 'yolo' - ? 1 - : num(config.runtimePolicy?.memorySafety?.maxMemoryUtilization), + // The guard's own ceiling (1 when "yolo" turns it off). + memorySafetyUtilization: memorySafetyCeiling(config.runtimePolicy ?? {}) ?? 1, ...object(config.pool?.policy), ...policy }), diff --git a/src/pool-controller.mjs b/src/pool-controller.mjs index 61e0df0..30cd8a7 100644 --- a/src/pool-controller.mjs +++ b/src/pool-controller.mjs @@ -44,15 +44,20 @@ export function normalizeControllerPolicy(policy = {}) { if (key === 'mode') merged.mode = POOL_CONTROLLER_MODES.includes(value) ? value : DEFAULT_CONTROLLER_POLICY.mode; else merged[key] = num(value) ?? merged[key]; } + merged.concurrency = Math.max(1, Math.floor(merged.concurrency)); return merged; } +function routeSignature(plan) { + return Object.entries(plan.routes ?? {}) + .map(([alias, route]) => `${alias}=${route.model}`) + .sort(); +} + // What the plan changes, independent of scores: two plans with the same // signature would make the same moves. export function planSignature(plan) { - const routes = Object.entries(plan.routes ?? {}) - .map(([alias, route]) => `${alias}=${route.model}`) - .sort(); + const routes = routeSignature(plan); const placements = (plan.placements ?? []) .filter((placement) => !placement.external && placement.residency !== 'auto') .map((placement) => `${placement.placement}:${placement.residency}`) @@ -82,12 +87,25 @@ function stayPlan(inputs) { return planPool({ ...inputs, pins: { ...inputs.pins, routes: { ...viable, ...(inputs.pins?.routes ?? {}) } } }); } -export function decide({ plan, stay, reconcile, state, policy, now }) { +// Memory that new loads bring in; a lane already loading (`resume`) is not new. +function newLoadGb(steps) { + return steps + .filter((step) => step.type === 'start' && !step.resume) + .reduce((sum, step) => sum + (num(step.memoryGbPerNode) ?? 0) * (step.nodes?.length ?? 1), 0); +} + +export function decide({ plan, stay, reconcile, state, policy, now, cordoned = new Set() }) { if (reconcile.empty) return { act: false, reason: 'no changes' }; const restoresCoverage = plan.score.uncovered < stay.score.uncovered || (stay.pinViolations?.length ?? 0) > (plan.pinViolations?.length ?? 0); if (restoresCoverage) return { act: true, urgent: true, reason: 'restores coverage' }; + // A cordon or drain is an operator asking for the node back: vacate it now. + const vacates = reconcile.steps.some( + (step) => + (step.type === 'stop' || step.type === 'manual-stop') && (step.nodes ?? []).some((node) => cordoned.has(node)) + ); + if (vacates) return { act: true, urgent: true, reason: 'vacates a cordoned node' }; if (plan.score.uncovered > stay.score.uncovered) return { act: false, reason: 'plan covers less than today' }; const quality = plan.score.sloMisses < stay.score.sloMisses || plan.score.external < stay.score.external; const gain = stay.score.cost > 0 ? (stay.score.cost - plan.score.cost) / stay.score.cost : 0; @@ -106,11 +124,11 @@ export function decide({ plan, stay, reconcile, state, policy, now }) { if (state.lastActionAt != null && now - state.lastActionAt < cooldownMs) { return { act: false, reason: 'cooldown after the last change' }; } - const loadGb = reconcile.steps - .filter((step) => step.type === 'start') - .reduce((sum, step) => sum + (num(step.memoryGbPerNode) ?? 0) * (step.nodes?.length ?? 1), 0); + const loadGb = newLoadGb(reconcile.steps); const recentGb = state.loads.filter((entry) => now - entry.at < 3600000).reduce((sum, entry) => sum + entry.gb, 0); - if (recentGb + loadGb > policy.maxLoadGbPerHour) { + // One change larger than the whole budget may still go when nothing else + // loaded within the hour; otherwise it could never happen. + if (recentGb > 0 && recentGb + loadGb > policy.maxLoadGbPerHour) { return { act: false, reason: `hourly load budget (${Math.round(recentGb + loadGb)} GB > ${policy.maxLoadGbPerHour} GB)` @@ -151,14 +169,33 @@ export function createPoolController({ executing: null, history: [] }; + const emit = (entry) => { + try { + onEvent(entry); + } catch { + // A throwing observer must not wedge the controller. + } + }; const record = (entry) => { state.history.push(entry); if (state.history.length > controllerPolicy.historyLimit) state.history.shift(); - onEvent(entry); + emit(entry); return entry; }; + // One tick at a time: a tick awaits observation and, when preempting, the + // cancelled change, so overlapping timer ticks would otherwise race. async function tick() { + if (state.ticking) return record({ at: now(), action: 'busy', reason: 'a tick is already running' }); + state.ticking = true; + try { + return await tickOnce(); + } finally { + state.ticking = false; + } + } + + async function tickOnce() { const at = now(); if (controllerPolicy.mode === 'off') return record({ at, action: 'off' }); const observed = await observe(); @@ -177,7 +214,9 @@ export function createPoolController({ const signature = `${planSignature(plan)}|${reconcile.steps.map((step) => step.id).join(',')}`; state.stableTicks = signature === state.signature ? state.stableTicks + 1 : 1; state.signature = signature; - const decision = decide({ plan, stay, reconcile, state, policy: controllerPolicy, now: at }); + const cordoned = new Set((inputs.nodes ?? []).filter((node) => node.cordoned).map((node) => node.id)); + for (const node of inputs.pins?.cordon ?? []) cordoned.add(node); + const decision = decide({ plan, stay, reconcile, state, policy: controllerPolicy, now: at, cordoned }); if (state.executing) { // Only a plan that restores coverage may cancel work in progress (for // example a manual start that is waiting on a node which just died). @@ -186,7 +225,7 @@ export function createPoolController({ decision.act && decision.urgent && controllerPolicy.mode === 'auto' && - planSignature(plan) !== state.executingTarget; + routeSignature(plan).join() !== state.executingTarget; if (!preempt) { return record({ at, action: 'busy', reason: 'still applying the last change' }); } @@ -210,18 +249,19 @@ export function createPoolController({ record(entry); state.lastActionAt = at; state.stableTicks = 0; - for (const step of reconcile.steps.filter((candidate) => candidate.type === 'start')) { - state.loads.push({ at, gb: (num(step.memoryGbPerNode) ?? 0) * (step.nodes?.length ?? 1) }); - } + const loadGb = newLoadGb(reconcile.steps); + if (loadGb) state.loads.push({ at, gb: loadGb }); state.loads = state.loads.filter((load) => at - load.at < 3600000); state.abort = new AbortController(); - state.executingTarget = planSignature(plan); + // Only a change of routes counts as a new target for preemption, so + // residency that follows demand cannot keep cancelling an urgent move. + state.executingTarget = routeSignature(plan).join(); state.executing = executeReconcile(reconcile.steps, actuator, { concurrency: controllerPolicy.concurrency, signal: state.abort.signal, ...(timer ? { timer } : {}), onStep: ({ step, status, error }) => { - onEvent({ at: now(), action: 'step', step: step.id, status, ...(error ? { error } : {}) }); + emit({ at: now(), action: 'step', step: step.id, status, ...(error ? { error } : {}) }); if (status === 'failed' && (step.type === 'start' || step.type === 'manual-start')) { const placement = plan.placements.find((entry) => entry.placement === step.placement); state.failures.set(step.placement, { diff --git a/src/pool-reconciler.mjs b/src/pool-reconciler.mjs index 9e68230..2c5c1ed 100644 --- a/src/pool-reconciler.mjs +++ b/src/pool-reconciler.mjs @@ -70,10 +70,10 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ return unique; }; + // Lanes already stopping (a drain from an earlier, preempted change) still + // hold their memory; they get an idempotent stop that later starts wait on. const running = new Map( - list(current.placements) - .filter((placement) => placement.state !== 'stopping') - .map((placement) => [placement.key, { ...placement, models: list(placement.models) }]) + list(current.placements).map((placement) => [placement.key, { ...placement, models: list(placement.models) }]) ); const planned = list(plan.placements).filter((placement) => !placement.external); const plannedKeys = new Set(planned.map((placement) => placement.placement)); @@ -186,7 +186,8 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ const routeTo = (alias, to, type = 'route') => { const target = plannedByModel.get(to); - const after = [readyStep.get(to), ...downgradeSteps]; + // Flips of one alias stay in order (an interim member, then back). + const after = [readyStep.get(to), routeStep.get(alias), ...downgradeSteps]; const id = add( { type, alias, from: routes[alias] ?? null, to, ...(target ? { placement: target.placement } : {}) }, after @@ -196,6 +197,20 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ return id; }; + // Another planned lane that serves `model`: { after: stepId|null } once it + // is ready or has a start step, undefined while it is neither. + const replacementFor = (model, excludeKey) => { + for (const placement of planned) { + if (placement.placement === excludeKey || !placement.models.includes(model)) continue; + if (startStep.has(placement.placement)) return { after: startStep.get(placement.placement) }; + const lane = running.get(placement.placement); + if (lane && (lane.state === 'ready' || lane.state == null)) return { after: null }; + } + return undefined; + }; + + for (const placement of [...running.values()].filter((entry) => entry.state === 'stopping')) release(placement); + for (let guard = 0; guard < 1000; guard += 1) { let progress = false; @@ -209,14 +224,18 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ return false; }); - // Retire lanes that are not planned and no longer routed to. + // Retire lanes that are not planned once nothing needs them: no alias is + // routed to their models, or every such model is (about to be) served by + // another planned lane, as when a model moves between nodes. for (const placement of [...running.values()].sort((a, b) => a.key.localeCompare(b.key))) { - if (plannedKeys.has(placement.key) || aliasesOn(placement).length) continue; + if (plannedKeys.has(placement.key)) continue; + const replacements = aliasesOn(placement).map((alias) => replacementFor(routes[alias], placement.key)); + if (replacements.some((replacement) => replacement === undefined)) continue; const after = Object.entries(routes) .filter(([alias]) => routeStep.has(alias)) .filter(([alias]) => placement.models.includes(current.routes?.[alias])) .map(([alias]) => routeStep.get(alias)); - release(placement, after); + release(placement, [...after, ...replacements.map((replacement) => replacement.after)]); progress = true; } @@ -231,7 +250,8 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ nodes: placement.nodes, runtime: placement.runtime, residency: placement.residency, - memoryGbPerNode: placement.memoryGbPerNode + memoryGbPerNode: placement.memoryGbPerNode, + ...(counted ? { resume: true } : {}) }, startAfter(placement) ); @@ -307,8 +327,14 @@ export function planReconcile({ plan, current = {}, workloads = [], variants = [ placement.models.includes(model) )) ); - if (interim) interimAfter.push(routeTo(alias, interim, 'interim-route')); - else interruptions.push({ alias, placement: victim.key, until: blockedStart.placement }); + if (interim) { + interimAfter.push(routeTo(alias, interim, 'interim-route')); + // The interim member is temporary: queue the flip back to the plan. + const target = plan.routes?.[alias]?.model; + if (target && target !== interim && !pendingRoutes.some((change) => change.alias === alias)) { + pendingRoutes.push({ alias, from: interim, to: target }); + } + } else interruptions.push({ alias, placement: victim.key, until: blockedStart.placement }); } release(victim, interimAfter); } @@ -364,6 +390,7 @@ export async function executeReconcile( const status = new Map(steps.map((step) => [step.id, 'pending'])); const errors = {}; const running = new Set(); + concurrency = Math.max(1, Math.floor(Number(concurrency) || 1)); const method = (type) => ({ prefetch: 'prefetch', @@ -398,10 +425,18 @@ export async function executeReconcile( errors }); }; + // A throwing observer must not wedge the run. + const notify = (event) => { + try { + onStep(event); + } catch { + // ignored + } + }; const settle = (step, value, error) => { status.set(step.id, value); if (error) errors[step.id] = error; - onStep({ step, status: value, ...(error ? { error } : {}) }); + notify({ step, status: value, ...(error ? { error } : {}) }); }; const pump = () => { for (const step of steps) { @@ -419,7 +454,7 @@ export async function executeReconcile( if (running.size >= concurrency) break; status.set(step.id, 'running'); running.add(step.id); - onStep({ step, status: 'running' }); + notify({ step, status: 'running' }); const limitMs = timeoutsMs[step.type] ?? DEFAULT_STEP_TIMEOUTS_MS[step.type]; const timedOut = timer(limitMs).then(() => { throw new Error(`${step.type} timed out after ${Math.round(limitMs / 1000)} s`); diff --git a/src/pool-sim.mjs b/src/pool-sim.mjs index 26a299d..d1859c9 100644 --- a/src/pool-sim.mjs +++ b/src/pool-sim.mjs @@ -222,6 +222,7 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c const arrivals = new Map(workloads.map((workload) => [workload.id, []])); const flexiblePlacements = new Map(); const latched = new Set(); + let lastPlan = null; // With `membership`, nodes enroll with the head and heartbeat; the planner // learns that a node is gone only when its lease is lost, as a real head @@ -712,17 +713,15 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c weights: [...node.weights], images: [...node.images] })); - const placements = [...lanes.values()] - .filter((lane) => lane.state !== 'draining') - .map((lane) => ({ - key: lane.key, - models: [...lane.models], - runtime: lane.variant.runtime ?? null, - nodes: lane.nodes, - memoryGbPerNode: lane.memoryGb, - managed: lane.variant.managed !== false, - state: lane.state === 'loading' ? 'starting' : 'ready' - })); + const placements = [...lanes.values()].map((lane) => ({ + key: lane.key, + models: [...lane.models], + runtime: lane.variant.runtime ?? null, + nodes: lane.nodes, + memoryGbPerNode: lane.memoryGb, + managed: lane.variant.managed !== false, + state: lane.state === 'loading' ? 'starting' : lane.state === 'draining' ? 'stopping' : 'ready' + })); return { nodes: plannerNodes, variants: observedVariants(), @@ -732,7 +731,7 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c current: { routes: { ...routes }, residency: { ...residency }, - running: placements.map((placement) => placement.key), + running: placements.filter((placement) => placement.state !== 'stopping').map((placement) => placement.key), placements } }; @@ -837,6 +836,7 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c mode: mode === 'advise' ? 'advise' : 'auto' }, onEvent: (entry) => { + if (entry.plan) lastPlan = entry.plan; // Each change in why the controller is holding, so a run explains itself. if ((entry.action === 'hold' && entry.reason !== 'no changes') || entry.action === 'busy') { const reason = (entry.reason ?? 'still applying the last change').replace(/\(\d+\/\d+\)|[\d.]+%/g, '#'); @@ -1010,7 +1010,11 @@ export async function runPoolSimulation(scenario, { mode = 'auto', controller: c report.final = { routes: { ...routes }, residency: { ...residency }, - lanes: [...lanes.values()].map((lane) => lane.key).sort() + lanes: [...lanes.values()].map((lane) => lane.key).sort(), + // The controller's last plan: what it would cover now. + plan: lastPlan + ? { ok: lastPlan.ok, uncovered: lastPlan.uncovered.map((entry) => entry.workload), score: lastPlan.score } + : null }; report.checks = checkExpectations(report, scenario.expect?.[mode] ?? scenario.expect?.all ?? {}); report.ok = report.checks.every((check) => check.ok); diff --git a/src/runtime-manager.mjs b/src/runtime-manager.mjs index c73a893..a033a56 100644 --- a/src/runtime-manager.mjs +++ b/src/runtime-manager.mjs @@ -2394,6 +2394,8 @@ export class RuntimeManager { await guard.check(); combinedSignal.throwIfAborted(); if (owned.child?.connected) owned.child.send({ type: 'memory-safety-complete' }, () => {}); + // A completed load is measured from here on; old abort evidence no longer applies. + if (result?.healthy) this.stateFor(runtimeId).memorySafetyAbort = null; return result; } catch (cause) { const error = guard.signal.aborted ? guard.signal.reason : cause; diff --git a/src/runtime-policy.mjs b/src/runtime-policy.mjs index d02b560..4c9fb9c 100644 --- a/src/runtime-policy.mjs +++ b/src/runtime-policy.mjs @@ -165,7 +165,8 @@ async function liveMemoryProfile(profile = {}) { // admission never allows a load the guard is certain to abort. function hostReserveGb(runtimePolicy, totalMemoryGb, reserveMemoryGb) { const ceiling = memorySafetyCeiling(runtimePolicy); - const guardMarginGb = ceiling == null ? 0 : Math.round(totalMemoryGb * (1 - ceiling) * 10) / 10; + // Rounded up to 0.1 GB so the margin never ends up below the guard's. + const guardMarginGb = ceiling == null ? 0 : Math.ceil(totalMemoryGb * (1 - ceiling) * 10 - 1e-9) / 10; return Math.max(reserveMemoryGb, guardMarginGb); } @@ -351,7 +352,10 @@ function clusterRuntimePolicyPlan( maxMemoryUtilization, reserveMemoryGb }), - totalMemoryGb - hostReserveGb({ ...policyTemplate, maxMemoryUtilization }, totalMemoryGb, reserveMemoryGb) + Math.max( + 0, + totalMemoryGb - hostReserveGb({ ...policyTemplate, maxMemoryUtilization }, totalMemoryGb, reserveMemoryGb) + ) ); const loadedMemoryGb = rows .filter((row) => row.loaded) diff --git a/test/fixtures/pool/scenarios/memory-estimate-low.json b/test/fixtures/pool/scenarios/memory-estimate-low.json index af60878..ae4bd83 100644 --- a/test/fixtures/pool/scenarios/memory-estimate-low.json +++ b/test/fixtures/pool/scenarios/memory-estimate-low.json @@ -27,8 +27,21 @@ "max": 0 }, "lanes.guardAborts": { + "min": 1, "max": 1 } + }, + "auto": { + "safety.memoryViolations.length": { + "max": 0 + }, + "lanes.guardAborts": { + "min": 1, + "max": 1 + }, + "final.plan.uncovered": { + "includes": "video" + } } } } diff --git a/test/pool-catalog.test.mjs b/test/pool-catalog.test.mjs index bfedb43..162bc60 100644 --- a/test/pool-catalog.test.mjs +++ b/test/pool-catalog.test.mjs @@ -352,7 +352,18 @@ test('planner inputs from config produce a coherent plan', () => { recipes: [tp2Recipe] }); assert.equal(inputs.policy.maxMemoryUtilization, 0.95); + assert.equal(inputs.policy.memorySafetyUtilization, 0.9, 'the guard ceiling, not the admission utilization'); assert.ok(!inputs.variants.some((variant) => variant.source === 'recipe'), 'installs are opt-in'); + const looser = buildPoolPlanInputs({ + config: { ...fleetConfig(), runtimePolicy: { memorySafety: { maxMemoryUtilization: 0.95 } } }, + nodeStatuses: fleetStatuses() + }); + assert.equal(looser.policy.memorySafetyUtilization, 0.9, 'runtimePolicy.maxMemoryUtilization defaults to 0.9'); + const yolo = buildPoolPlanInputs({ + config: { ...fleetConfig(), runtimePolicy: { memorySafety: { mode: 'yolo' } } }, + nodeStatuses: fleetStatuses() + }); + assert.equal(yolo.policy.memorySafetyUtilization, 1); const plan = planPool(inputs); // spark-node-02 is cordoned and unreachable, so the pair lane cannot serve. assert.equal(plan.routes['chat-capable'].model, 'qwen38-27b-nvfp4'); diff --git a/test/pool-controller.test.mjs b/test/pool-controller.test.mjs index cb4e971..827ce5e 100644 --- a/test/pool-controller.test.mjs +++ b/test/pool-controller.test.mjs @@ -226,3 +226,60 @@ test('an urgent new target cancels a change stuck in progress; the same target n await controller.executing; assert.equal(state.routes['chat-capable'], 'cloud/openrouter/q38fn'); }); + +test('ticks do not overlap, throwing observers do not wedge it, and one large move fits an empty hour', async () => { + const { observe, actuator, state } = fakeFleet({ cordon: ['spark-node-01', 'spark-node-02'] }); + let release; + const gate = new Promise((resolve) => (release = resolve)); + const controller = createPoolController({ + observe: async () => { + await gate; + return observe(); + }, + actuator, + onEvent: () => { + throw new Error('observer bug'); + }, + policy: { mode: 'auto', concurrency: 0 } + }); + assert.equal(controller.policy.concurrency, 1); + const first = controller.tick(); + const second = await controller.tick(); + assert.equal(second.action, 'busy'); + release(); + assert.equal((await first).action, 'apply'); + const result = await controller.executing; + assert.equal(result.ok, true); + assert.equal(state.routes['chat-capable'], 'qwen38-27b-nvfp4'); + + const policy = normalizeControllerPolicy({ maxLoadGbPerHour: 10 }); + const big = { empty: false, steps: [{ type: 'start', memoryGbPerNode: 110, nodes: ['a', 'b'] }, { type: 'route' }] }; + const score = { uncovered: 0, external: 0, sloMisses: 0 }; + const quiet = { stableTicks: 5, lastActionAt: null, loads: [] }; + const decision = decide({ + plan: { score: { ...score, cost: 10 } }, + stay: { score: { ...score, cost: 100 } }, + reconcile: big, + state: quiet, + policy, + now: 0 + }); + assert.equal(decision.act, true, 'a 220 GB move is not impossible under a 10 GB budget'); + const resumed = { + empty: false, + steps: [{ type: 'start', resume: true, memoryGbPerNode: 110, nodes: ['a'] }, { type: 'route' }] + }; + const busyHour = { ...quiet, loads: [{ at: 0, gb: 5 }] }; + assert.equal( + decide({ + plan: { score: { ...score, cost: 10 } }, + stay: { score: { ...score, cost: 100 } }, + reconcile: resumed, + state: busyHour, + policy, + now: 1 + }).act, + true, + 'a lane already loading is not counted again' + ); +}); diff --git a/test/pool-reconciler.test.mjs b/test/pool-reconciler.test.mjs index f2c739e..8c3f608 100644 --- a/test/pool-reconciler.test.mjs +++ b/test/pool-reconciler.test.mjs @@ -307,3 +307,77 @@ test('routes to a lane that is still loading wait on an idempotent start', () => assert.equal(start.placement, 'qwen38-27b-nvfp4@spark-node-03'); for (const route of steps.filter((step) => step.type === 'route')) assert.ok(route.after.includes(start.id)); }); + +const twoNodes = { + a: { domains: [{ kind: 'unified', budgetGb: 100 }] }, + b: { domains: [{ kind: 'unified', budgetGb: 100 }] } +}; + +test('a model that moves between nodes retires its old lane once the new one is up', () => { + const { steps } = planReconcile({ + plan: { + nodes: twoNodes, + placements: [ + { placement: 'v@b', variant: 'v', models: ['M'], nodes: ['b'], residency: 'always', memoryGbPerNode: 60 } + ], + routes: { chat: { model: 'M' } } + }, + current: { + placements: [{ key: 'v@a', models: ['M'], nodes: ['a'], memoryGbPerNode: 60, state: 'ready' }], + routes: { chat: 'M' } + } + }); + const start = steps.find((step) => step.type === 'start'); + const stop = steps.find((step) => step.type === 'stop'); + assert.equal(start.placement, 'v@b'); + assert.equal(stop?.placement, 'v@a', 'the old lane is released'); + assert.ok(stop.after.includes(start.id), 'only after the new lane is ready'); +}); + +test('a lane already stopping still holds its memory until its drain finishes', () => { + const { steps } = planReconcile({ + plan: { + nodes: twoNodes, + placements: [ + { placement: 'n', variant: 'n', models: ['N'], nodes: ['a'], residency: 'always', memoryGbPerNode: 90 } + ], + routes: {} + }, + current: { + placements: [{ key: 'old', models: ['X'], nodes: ['a'], memoryGbPerNode: 90, state: 'stopping' }], + routes: {} + } + }); + const wait = steps.find((step) => step.type === 'stop' && step.placement === 'old'); + const start = steps.find((step) => step.type === 'start'); + assert.ok(wait, 'an idempotent stop waits for the drain in progress'); + assert.ok(start.after.includes(wait.id)); +}); + +test('an alias moved to an interim member during a swap goes back to its planned target', () => { + const { steps } = planReconcile({ + plan: { + nodes: twoNodes, + placements: [ + { placement: 'big', variant: 'big', models: ['B'], nodes: ['a'], residency: 'always', memoryGbPerNode: 80 }, + { placement: 'small', variant: 'small', models: ['S'], nodes: ['a'], residency: 'auto', memoryGbPerNode: 40 }, + { placement: 'other', variant: 'other', models: ['O'], nodes: ['b'], residency: 'always', memoryGbPerNode: 40 } + ], + routes: { chat: { model: 'S' }, code: { model: 'B' }, o: { model: 'O' } } + }, + current: { + placements: [ + { key: 'small', models: ['S'], nodes: ['a'], memoryGbPerNode: 40, state: 'ready' }, + { key: 'other', models: ['O'], nodes: ['b'], memoryGbPerNode: 40, state: 'ready' } + ], + routes: { chat: 'S', code: 'B', o: 'O' } + }, + workloads: [{ id: 'chat', candidates: ['S', 'O'] }] + }); + const routes = {}; + for (const step of steps) if (step.type === 'route' || step.type === 'interim-route') routes[step.alias] = step.to; + assert.equal(routes.chat, 'S', 'the last word on chat is its planned route'); + const interim = steps.find((step) => step.type === 'interim-route'); + const back = steps.filter((step) => step.type === 'route' && step.alias === 'chat').at(-1); + assert.ok(back.after.includes(interim.id), 'the flip back runs after the interim flip'); +}); diff --git a/test/pool-sim.test.mjs b/test/pool-sim.test.mjs index 122fa58..6709389 100644 --- a/test/pool-sim.test.mjs +++ b/test/pool-sim.test.mjs @@ -206,3 +206,30 @@ test('admission that keeps the memory guard margin free ends the abort-and-latch assert.equal(fixed.lanes.guardAborts, 0); assert.ok(fixed.requests.failed < legacy.requests.failed); }); + +test('cordoning a node moves its lanes off promptly and retires the old lane', async () => { + const scenario = tiny({ + durationHours: 3, + nodes: [spark('s1'), spark('s2')], + variants: [ + { + id: 'recipe:x:m', + gatewayModel: 'm', + kind: 'chat', + runtime: 'm', + requirements: { memoryGbPerNode: 40 }, + performance: { ttftMs: 500, decodeTps: 50 }, + sim: { loadSeconds: 60 } + } + ], + workloads: [{ id: 'chat', candidates: ['m'], residency: 'always', demand: { perHour: 60 } }], + initial: { running: [{ variant: 'recipe:x:m', nodes: ['s1'] }], routes: { chat: 'm' }, residency: { m: 'always' } }, + events: [{ atHours: 1, type: 'cordon', nodes: ['s1'] }] + }); + const report = await runPoolSimulation(scenario, { mode: 'auto' }); + assert.deepEqual(report.final.lanes, ['recipe:x:m@s2']); + const apply = report.controller.decisions.find((entry) => entry.action === 'apply'); + assert.ok(apply.atMinutes <= 61, `acted at ${apply.atMinutes} min`); + assert.equal(apply.reason, 'vacates a cordoned node'); + assert.equal(report.requests.failed, 0); +}); diff --git a/test/runtime-memory-safety.test.mjs b/test/runtime-memory-safety.test.mjs index 70401be..54fca5f 100644 --- a/test/runtime-memory-safety.test.mjs +++ b/test/runtime-memory-safety.test.mjs @@ -192,6 +192,14 @@ test('explicit YOLO is the only guard bypass', async (t) => { assert(alive((await f.ids()).backend)); }); +test('a successful start clears earlier memory-safety abort evidence', async (t) => { + const f = await fixture(t, { yolo: true }); + f.manager.stateFor('demo').memorySafetyAbort = { at: '2026-10-11T00:00:00.000Z', usedGb: 90, consumedGb: 40 }; + const result = await f.manager.start('demo'); + assert.equal(result.healthy, true); + assert.equal((await f.manager.status()).runtimes.demo.memorySafetyAbort, null); +}); + test('a stale sample cannot abort a completed operation', async () => { let release; let aborts = 0; diff --git a/test/runtime-policy.test.mjs b/test/runtime-policy.test.mjs index 6a49859..7edc617 100644 --- a/test/runtime-policy.test.mjs +++ b/test/runtime-policy.test.mjs @@ -668,4 +668,33 @@ const guardLooser = await createRuntimePolicyPlan( assert.equal(guardLooser.policy.hostReserveGb, 8, 'a 0.95 ceiling leaves 6.4 GB, so the 8 GB reserve governs'); assert.equal(guardLooser.admission.allowed, true); +// A cluster node with unknown capacity has a zero budget, not a negative one +// that would block planning on the nodes that are known. +const unknownCapacity = await createRuntimePolicyPlan( + { + cluster: { nodeId: 'a', leaderNode: 'a', nodes: { a: { resources: { memoryGb: 128 } }, b: {} } }, + runtimePolicy: { autoEvict: true }, + runtimes: { r: { enabled: true, node: 'a', memoryGb: 16 } } + }, + { + status: { + runtimes: {}, + cluster: { + nodes: { + a: { + local: true, + reachable: true, + telemetry: { memory: { totalBytes: 128 * 1024 ** 3, availableBytes: 100 * 1024 ** 3 } } + }, + b: { reachable: true } + } + } + }, + profile: { totalMemoryGb: 128, availableMemoryGb: 100 } + } +); +assert.equal(unknownCapacity.admission.nodes.b.memoryBudgetGb, 0); +assert.equal(unknownCapacity.admission.overBudgetGb, 0); +assert.equal(unknownCapacity.admission.allowed, true); + console.log('runtime-policy tests passed');