diff --git a/apps/aflow-executor-ai/src/handlers/ai/aiClient.ts b/apps/aflow-executor-ai/src/handlers/ai/aiClient.ts index cb36d9d1..914c1f42 100644 --- a/apps/aflow-executor-ai/src/handlers/ai/aiClient.ts +++ b/apps/aflow-executor-ai/src/handlers/ai/aiClient.ts @@ -7,6 +7,7 @@ import { } from '@aflow/ai-client'; import type { ExecutorContext } from '@aflow/executor-runtime'; import { localRuntimeId, type AflowError } from '@aflow/schemas'; +import { readLocalMachine, type HostMachine, type Redis } from '@aflow/redis'; import { CredentialResolver, credentialMissingMessage, @@ -132,6 +133,25 @@ let credentialResolver: CredentialResolver | null = null; /** * Initialize the credential resolver. Call once at executor startup. */ +// The machine a local runtime runs on, from the paired machine's inventory. +// Read at most this often: it changes when the machine does. +const LOCAL_MACHINE_TTL_MS = 60_000; +let machineSource: Redis | null = null; +let machineReading: { value: HostMachine | undefined; expiresAt: number } | null = null; + +/** Where the machine a local runtime runs on is read from (Plan 323 D8). */ +export function initLocalMachineSource(redis: Redis): void { + machineSource = redis; +} + +async function localMachine(): Promise { + if (!machineSource) return undefined; + if (machineReading && machineReading.expiresAt > Date.now()) return machineReading.value; + const value = await readLocalMachine(machineSource).catch(() => undefined); + machineReading = { value, expiresAt: Date.now() + LOCAL_MACHINE_TTL_MS }; + return value; +} + export function initCredentialResolver(loader: CredentialLoader): void { credentialResolver = new CredentialResolver({ loader }); } @@ -301,15 +321,21 @@ export async function getAIClientForContext( // 4. Check client cache (keyed by credentialId + updatedAt for rotation safety) // When a credential is rotated in-place, updatedAt changes → cache miss → new client - const cacheKey = `${resolved.credentialId}:${resolved.updatedAt}`; + + // 5. Build single-provider AIClient + // The machine is part of the key: a GPU figure that arrives after the first + // client was built changes the window that client would size. + const machine = aiProvider === 'ollama' ? await localMachine() : undefined; + const cacheKey = `${resolved.credentialId}:${resolved.updatedAt}:${machine?.gpuWorkingSetBytes ?? '-'}`; const cached = CLIENT_CACHE.get(cacheKey); if (cached && cached.expiresAt > Date.now()) { return cached.client; } - // 5. Build single-provider AIClient const client = createAIClient({ - providers: { [aiProvider]: providerConfigFromCredential(credentialProviderId, resolved) }, + providers: { + [aiProvider]: providerConfigFromCredential(credentialProviderId, resolved, { machine }), + }, defaultProvider: aiProvider, }); diff --git a/apps/aflow-executor-ai/src/index.ts b/apps/aflow-executor-ai/src/index.ts index 72ff750a..d13f6223 100644 --- a/apps/aflow-executor-ai/src/index.ts +++ b/apps/aflow-executor-ai/src/index.ts @@ -40,6 +40,7 @@ import { createServiceLogger } from '@aflow/executor-runtime'; import { AiHandler } from './handlers/ai/index.js'; import { initCredentialResolver, + initLocalMachineSource, getCredentialResolver, invalidateAllCredentialCaches, } from './handlers/ai/aiClient.js'; @@ -140,6 +141,7 @@ async function main(): Promise { if (shouldRunStepExecutor) { const db = database.db; + initLocalMachineSource(redis); initCredentialResolver(async (tenantId, providerId) => { const tenantContext = createTenantContext(tenantId as TenantId); try { diff --git a/apps/aflow-executor-host/src/__tests__/harnessBoundary.test.ts b/apps/aflow-executor-host/src/__tests__/harnessBoundary.test.ts index 57357039..9df17238 100644 --- a/apps/aflow-executor-host/src/__tests__/harnessBoundary.test.ts +++ b/apps/aflow-executor-host/src/__tests__/harnessBoundary.test.ts @@ -198,7 +198,9 @@ describe('one spawn path', () => { // pair, which Node has no call for; it runs nothing else. keepAwake.ts // starts the `caffeinate` or `systemd-inhibit` that holds the machine awake // while a step runs, a system call by another name that reads and runs no - // workload. + // workload. machine.ts asks Metal (through `swift`) or `nvidia-smi` how much + // GPU memory a model runtime on this machine has, once, and runs nothing it + // was handed. // Named rather than matched, so a new unconfined spawn is argued for here. expect(importers).toEqual([ 'browser/harnessBrowser.ts', @@ -206,6 +208,7 @@ describe('one spawn path', () => { 'folderPicker.ts', 'harnessDiscovery.ts', 'keepAwake.ts', + 'machine.ts', 'orphans.ts', 'runtimes.ts', 'sandboxedRun.ts', diff --git a/apps/aflow-executor-host/src/__tests__/machine.test.ts b/apps/aflow-executor-host/src/__tests__/machine.test.ts new file mode 100644 index 00000000..6588a73c --- /dev/null +++ b/apps/aflow-executor-host/src/__tests__/machine.test.ts @@ -0,0 +1,88 @@ +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import { createMachineReader } from '../machine.js'; + +const GIB = 1024 ** 3; + +async function hostDir(): Promise { + return mkdtemp(join(tmpdir(), 'aflow-machine-')); +} + +describe('createMachineReader', () => { + it('reads Metal’s working set on Apple silicon once and keeps it for the next start', async () => { + const dir = await hostDir(); + const run = vi.fn(async () => '19069665280\n'); + const options = { + hostDir: dir, + platform: 'darwin' as const, + arch: 'arm64', + totalMemoryBytes: 24 * GIB, + osRelease: '25.3.0', + run, + }; + + const first = createMachineReader(options); + expect(first.current()).toEqual({ totalMemoryBytes: 24 * GIB, unifiedMemory: true }); + await first.start(); + expect(first.current().gpuWorkingSetBytes).toBe(19_069_665_280); + + const second = createMachineReader(options); + await second.start(); + expect(second.current().gpuWorkingSetBytes).toBe(19_069_665_280); + expect(run).toHaveBeenCalledTimes(1); + }); + + it('reads it again once the machine or its OS has changed', async () => { + const dir = await hostDir(); + await writeFile( + join(dir, 'machine.json'), + JSON.stringify({ totalMemoryBytes: 24 * GIB, osRelease: '25.2.0', gpuWorkingSetBytes: 1 }), + ); + const run = vi.fn(async () => '19069665280'); + const reader = createMachineReader({ + hostDir: dir, + platform: 'darwin', + arch: 'arm64', + totalMemoryBytes: 24 * GIB, + osRelease: '25.3.0', + run, + }); + await reader.start(); + expect(run).toHaveBeenCalledTimes(1); + expect(JSON.parse(await readFile(join(dir, 'machine.json'), 'utf8'))).toMatchObject({ + osRelease: '25.3.0', + }); + }); + + it('takes the largest NVIDIA device, and leaves the figure out where no probe answers', async () => { + const nvidia = createMachineReader({ + hostDir: await hostDir(), + platform: 'linux', + arch: 'x64', + totalMemoryBytes: 64 * GIB, + osRelease: '6.8', + run: async () => '24576\n49152\n', + }); + await nvidia.start(); + expect(nvidia.current()).toEqual({ + totalMemoryBytes: 64 * GIB, + unifiedMemory: false, + gpuWorkingSetBytes: 49152 * 1024 ** 2, + }); + + const none = createMachineReader({ + hostDir: await hostDir(), + platform: 'linux', + arch: 'x64', + totalMemoryBytes: 16 * GIB, + osRelease: '6.8', + run: async () => { + throw new Error('nvidia-smi: command not found'); + }, + }); + await none.start(); + expect(none.current().gpuWorkingSetBytes).toBeUndefined(); + }); +}); diff --git a/apps/aflow-executor-host/src/index.ts b/apps/aflow-executor-host/src/index.ts index 7ee4f90b..812123b6 100644 --- a/apps/aflow-executor-host/src/index.ts +++ b/apps/aflow-executor-host/src/index.ts @@ -81,6 +81,7 @@ import { loadPairedEnv } from './pairedEnv.js'; import { followPolicy, watchPolicy } from './policyWatch.js'; import { killAllProcesses, killProcessesForBinding, reapWithdrawn } from './sandboxedRun.js'; import { observeRuntimes } from './runtimes.js'; +import { createMachineReader } from './machine.js'; import { publishingFolders } from './pushApproval.js'; import { startUnderSignals } from './startUnderSignals.js'; import { createStartupTimings } from './startupTimings.js'; @@ -288,6 +289,10 @@ async function main(): Promise { let lastFolders: HostInventoryFolders = []; let lastMaxConcurrentHarnessRuns = HOST_HARNESS_CONCURRENCY_DEFAULT; let lastBrowsers: HostInventoryBrowsers = []; + const machine = createMachineReader({ hostDir: dirname(policyPath) }); + // Read in the background: the probe compiles a Swift program, and the + // inventory goes out without the GPU figure until it is in. + void machine.start(); // Published with a lifetime rather than stored: an inventory that outlives the // executor describes a machine nobody is listening on, and inviting a run @@ -340,6 +345,7 @@ async function main(): Promise { const inventory: HostInventory = { hostname, observedAt: new Date().toISOString(), + machine: machine.current(), runtimes, harnesses, maxConcurrentHarnessRuns, diff --git a/apps/aflow-executor-host/src/machine.ts b/apps/aflow-executor-host/src/machine.ts new file mode 100644 index 00000000..56796973 --- /dev/null +++ b/apps/aflow-executor-host/src/machine.ts @@ -0,0 +1,111 @@ +/** + * What this machine offers a model runtime on it (Plan 323 D8), for the + * inventory. + * + * The figure that decides fit is the GPU working set, and nothing cheap + * reports it: on Apple silicon it is Metal's `recommendedMaxWorkingSetSize`, + * which no `sysctl` carries and Ollama logs but does not serve. Reading it + * through Metal compiles a one-line Swift program, which takes seconds to + * minutes, so it is read once in the background and cached beside the policy + * file, keyed by the machine's memory and OS release so an upgrade reads it + * again. Until it is read — or where it cannot be — it is absent, and fit + * says it does not know rather than guessing. + */ +import { execFile } from 'node:child_process'; +import { readFile, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import { join } from 'node:path'; +import type { HostMachine } from '@aflow/redis'; + +const CACHE_FILE = 'machine.json'; +/** A first `swift` run compiles its toolchain's caches; minutes, not seconds. */ +const PROBE_TIMEOUT_MS = 300_000; +const METAL_PROBE = + 'import Metal; print(MTLCreateSystemDefaultDevice()?.recommendedMaxWorkingSetSize ?? 0)'; + +interface CachedReading { + totalMemoryBytes: number; + osRelease: string; + gpuWorkingSetBytes: number; +} + +export type RunCommand = (file: string, args: string[], timeoutMs: number) => Promise; + +const runCommand: RunCommand = (file, args, timeoutMs) => + new Promise((resolve, reject) => { + execFile(file, args, { timeout: timeoutMs, encoding: 'utf8' }, (error, stdout) => { + if (error) reject(new Error(`${file} failed: ${error.message}`, { cause: error })); + else resolve(stdout); + }); + }); + +function positive(value: string): number | undefined { + const parsed = Number(value.trim()); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : undefined; +} + +async function readGpuWorkingSet(platform: string, run: RunCommand): Promise { + if (platform === 'darwin') { + return positive(await run('swift', ['-e', METAL_PROBE], PROBE_TIMEOUT_MS)); + } + // A discrete GPU's memory is the device's own; the largest one is what a + // model that does not split across devices can use. + const out = await run( + 'nvidia-smi', + ['--query-gpu=memory.total', '--format=csv,noheader,nounits'], + PROBE_TIMEOUT_MS, + ); + const mib = out + .split('\n') + .map(positive) + .filter((v): v is number => v !== undefined); + return mib.length > 0 ? Math.max(...mib) * 1024 ** 2 : undefined; +} + +export interface MachineReader { + /** The machine as read so far; the GPU figure arrives once the probe has run. */ + current(): HostMachine; + /** Reads the GPU figure, from the cache when it still describes this machine. */ + start(): Promise; +} + +export function createMachineReader(options: { + hostDir: string; + platform?: NodeJS.Platform; + arch?: string; + totalMemoryBytes?: number; + osRelease?: string; + run?: RunCommand; +}): MachineReader { + const platform = options.platform ?? process.platform; + const totalMemoryBytes = options.totalMemoryBytes ?? os.totalmem(); + const osRelease = options.osRelease ?? os.release(); + const run = options.run ?? runCommand; + const cachePath = join(options.hostDir, CACHE_FILE); + const machine: HostMachine = { + totalMemoryBytes, + unifiedMemory: platform === 'darwin' && (options.arch ?? process.arch) === 'arm64', + }; + + return { + current: () => ({ ...machine }), + async start() { + const cached = await readFile(cachePath, 'utf8') + .then((text) => JSON.parse(text) as CachedReading) + .catch(() => undefined); + if ( + cached?.totalMemoryBytes === totalMemoryBytes && + cached.osRelease === osRelease && + cached.gpuWorkingSetBytes > 0 + ) { + machine.gpuWorkingSetBytes = cached.gpuWorkingSetBytes; + return; + } + const gpu = await readGpuWorkingSet(platform, run).catch(() => undefined); + if (gpu === undefined) return; + machine.gpuWorkingSetBytes = gpu; + const reading: CachedReading = { totalMemoryBytes, osRelease, gpuWorkingSetBytes: gpu }; + await writeFile(cachePath, `${JSON.stringify(reading)}\n`).catch(() => undefined); + }, + }; +} diff --git a/apps/aflow-orchestrator/src/services/cybernetic/__tests__/postRunHooks.test.ts b/apps/aflow-orchestrator/src/services/cybernetic/__tests__/postRunHooks.test.ts index 0292e997..e9818d0c 100644 --- a/apps/aflow-orchestrator/src/services/cybernetic/__tests__/postRunHooks.test.ts +++ b/apps/aflow-orchestrator/src/services/cybernetic/__tests__/postRunHooks.test.ts @@ -66,6 +66,7 @@ vi.mock('@aflow/database', () => ({ const mockAppendEntityEvent = vi.fn(); vi.mock('@aflow/redis', () => ({ runtimeAdmission: () => async () => ({ release: async () => undefined }), + readLocalMachine: async () => undefined, appendEntityEvent: (...args: unknown[]) => mockAppendEntityEvent(...args), })); diff --git a/apps/aflow-orchestrator/src/services/cybernetic/postRunHooks.ts b/apps/aflow-orchestrator/src/services/cybernetic/postRunHooks.ts index b65f00e2..fa38827c 100644 --- a/apps/aflow-orchestrator/src/services/cybernetic/postRunHooks.ts +++ b/apps/aflow-orchestrator/src/services/cybernetic/postRunHooks.ts @@ -41,7 +41,7 @@ import { createTenantContext, withTenantSchema, spaces } from '@aflow/database'; import { createByokAiClientFactory } from '@aflow/credential-resolver'; import { eq } from 'drizzle-orm'; import { EntityDirectivesSchema } from '@aflow/schemas'; -import { appendEntityEvent, runtimeAdmission } from '@aflow/redis'; +import { appendEntityEvent, readLocalMachine, runtimeAdmission } from '@aflow/redis'; import type { PayloadStore } from '@aflow/payload-store'; import { randomUUID } from 'node:crypto'; @@ -149,7 +149,10 @@ export async function fireCyberneticPostRunHooksStandalone( // Step 1: Run eval — unconditional when a suite exists; the outcome is // recorded on the run as a typed evaluation envelope either way. - const byokFactory = createByokAiClientFactory(db, runtimeAdmission(redis, { wait: true })); + const byokFactory = createByokAiClientFactory(db, { + admission: runtimeAdmission(redis, { wait: true }), + machine: () => readLocalMachine(redis), + }); let evalOutcome: EvalRunOutcome; try { evalOutcome = await runEvaluation({ diff --git a/apps/aflow-orchestrator/src/services/sessionMetadataTask.test.ts b/apps/aflow-orchestrator/src/services/sessionMetadataTask.test.ts index a274c16b..915f22ad 100644 --- a/apps/aflow-orchestrator/src/services/sessionMetadataTask.test.ts +++ b/apps/aflow-orchestrator/src/services/sessionMetadataTask.test.ts @@ -17,6 +17,7 @@ const mocks = vi.hoisted(() => ({ vi.mock('@aflow/redis', () => ({ runtimeAdmission: () => async () => ({ release: async () => undefined }), + readLocalMachine: async () => undefined, claimSessionMetadataCandidates: async () => [ { tenantId: '00000000-0000-0000-0000-000000000001', diff --git a/apps/aflow-orchestrator/src/services/sessionMetadataTask.ts b/apps/aflow-orchestrator/src/services/sessionMetadataTask.ts index 37cbe1f7..7c3b424a 100644 --- a/apps/aflow-orchestrator/src/services/sessionMetadataTask.ts +++ b/apps/aflow-orchestrator/src/services/sessionMetadataTask.ts @@ -20,6 +20,7 @@ import { settleSessionMetadata, sessionMetadataLeaseHeld, appendEntityEvent, + readLocalMachine, runtimeAdmission, type SessionMetadataCandidate, } from '@aflow/redis'; @@ -100,10 +101,10 @@ export function createSessionMetadataTask(deps: SessionMetadataTaskDeps): Backgr // A title is not worth queueing behind an agent turn on this computer's // runtime; a busy one fails this conversation's cycle, and it is due again. - byokFactory ??= createByokAiClientFactory( - deps.db, - runtimeAdmission(deps.redis, { wait: false }), - ); + byokFactory ??= createByokAiClientFactory(deps.db, { + admission: runtimeAdmission(deps.redis, { wait: false }), + machine: () => readLocalMachine(deps.redis), + }); let processed = 0; let failed = 0; diff --git a/docs/plans/323-models-that-run-on-this-computer.md b/docs/plans/323-models-that-run-on-this-computer.md index 3b70230f..f6f3dcf2 100644 --- a/docs/plans/323-models-that-run-on-this-computer.md +++ b/docs/plans/323-models-that-run-on-this-computer.md @@ -1,6 +1,6 @@ # Plan 323 — Models that run on this computer -**Status:** 🔨 P0 met on one machine (§9): every gate answered on an Apple M3 with 24 GB and Ollama 0.40.5, and the decisions they served revised. P1 built and unit-tested, and its adapter run against the live runtime; its exit, a Helmsman conversation through the web UI on the dev stack, has not been run. P0 and P1 together are the first complete slice: one model on one machine. +**Status:** 🔨 P0 met on one machine (§9). P1 merged, and its main exit met on the dev stack: Helmsman on Gemma 4 26B held a tool-using conversation through the web UI on the 24 GB machine, slowly, because the model spilled past the GPU working set (§9 G6). P3a built and tested: each model's window is sized from the machine it runs on, never below what the platform's agents need, and readiness says how it fits. P0, P1 and P3a together are the first complete slice. **Supersedes for the local edition:** Tier 0 of archive plan 98 (open-weight inference). Plan 98 put a platform-managed `selfhosted` provider behind Ollama's OpenAI-compatible `/v1` endpoint. This plan uses Ollama's native API instead (D3) and makes the runtime the operator's own (D2). Plan 98's hosted tiers (Cloud Run GPU, GKE/vLLM) are untouched; the OpenAI-compatible adapter in P6 is the piece they would share. @@ -63,18 +63,14 @@ Using one should look like using any other model. You choose it in the same pick - **Callers outside a step** take the same lease through the client: `createByokAiClientFactory(db, admission)` wraps a client for an Ollama model so every request holds a lease while it runs. Session titles pass `wait: false` and decline when the runtime is busy, since the conversation is due again on the next cycle; post-run judging waits. The eval-batch engine, the UI and API executors' generation and the memory embedder are not yet admitted, and are P5's coverage list. - **The idle clock:** every AI operation on a local model runs under an idle window of `AI_LOCAL_MODEL_IDLE_TIMEOUT_MS` (15 min) and the agent turn's ceiling, in place of the flat 120 s or the 300 s stream idle window. Prefill is a silent stretch on every local request, and at 250 tokens a second a 75K-token prompt outlasts the cloud window (§9 G5). _Rejected:_ a per-connection first-token timeout. The timeout is chosen before any credential is read, so a connection field would be a setting nothing reads. -**D8 — Fit is an estimate until the model is loaded, and both are labelled as what they are.** +**D8 — The window is sized from the machine, and fit is an estimate until the model is loaded.** -- **The estimate:** computed from: - - the catalog footprint, `local: { weightsBytes, kvBytesPerToken, measuredUnder: { contextTokens, parallel, kvCacheType, runtimeVersion } }`, written by the probe. `/api/ps` cannot measure `kvBytesPerToken` on the MLX runner: it reports the same size and a 4,096-token context whatever `num_ctx` asks for, because that runner grows its cache with the prompt (§9 G5). The probe measures it from the runner process's resident memory before and after a long prompt; - - the host inventory, which gains `machine: { totalMemoryBytes, availableMemoryBytes, gpuMemoryLimitBytes, chip }`. On macOS the GPU limit is Metal's `recommendedMaxWorkingSetSize`, about 74% of RAM on the measured machine (19.07 GB of 24), which no `sysctl` reports. The host executor reads it through Metal. On Linux it is VRAM per device; - - the models already resident (`/api/ps`); - - the connection's `max_parallel`. Parallel requests multiply the KV cache, which grows with context × parallel. -- **The headroom:** the estimate keeps headroom for the stack, the operating system and a browser, and reports the conditions it assumes. -- **The four answers:** _fits_, _tight_, _does not fit_, or _unknown_. The answer is _unknown_ when the inputs are missing: an unpaired machine, a KV cache type the footprint was not measured with, or a build that differs from the one measured (D9). -- **Verified state:** once the model is loaded, `/api/ps` reports its actual size and its GPU/CPU split, and the host executor reads the runner process's memory as it grows. That is shown as verified and replaces the estimate. Free memory and swap are part of it: on the measured machine a 16 GB model left about 80 MB free and 6.6 GB in swap with the stack's databases running, which is _tight_ however the model's own numbers read. -- **A trial load is an action:** on an unpaired machine, the operator starts a trial load explicitly ("load to check"). Reading diagnostics never loads a model. -- **What over-sizing does:** a model that does not fit may load partly on CPU and run slowly, or fail to load, or press the whole machine into swap. Fit informs and never blocks a choice the operator makes. +- **What a model costs** (§9 G6): its weights, plus a constant cost per context token for the KV cache — 16.18 GB and about 118 KB a token for Gemma 4 26B at the `q8_0` cache type. On the MLX runner the cache grows with the prompts the model has actually read, keeps more than one conversation's, and is freed only when the model unloads, so a loaded model's cost climbs to its whole window and stays there. Fit is therefore computed for the full window × `max_parallel`, never for the prompt in hand. +- **What the machine allows:** on Apple silicon, Metal's working set (`recommendedMaxWorkingSetSize`, 19.07 GB of 24 on the measured machine), less the runtime's own reserve. Past it the model does not fail: it spills, and prefill fell from about 240 to 130 tokens a second as swap grew. On a machine with discrete GPUs it is VRAM per device. The host executor reads the figure once through Metal, caches it, and publishes it in its inventory as `machine: { totalMemoryBytes, gpuWorkingSetBytes }`; no `sysctl` reports it and Ollama's API does not expose it. +- **The default window is the largest that fits, and never less than the agents need.** A connection that sets no `context_tokens` gets, per model, the largest window in 4,096-token steps whose footprint stays inside the GPU working set, capped by the model's catalog window, and never below 32K: Helmsman's first prompt is about 18K tokens of instructions and tool definitions before any conversation, so a smaller window refuses it outright, and slow is better than refused. A 64 GB machine gets Gemma 4 26B's full 256K; the 24 GB Mac fits only 16K, so it gets 32K and is told the model will spill. Without a footprint or a machine figure the window is 32K. An operator who sets `context_tokens` gets exactly that, fitting or not: fit informs, it never overrides a configured choice. A smaller Helmsman prompt for local models — fewer tools on the surface — is what would let a 24 GB machine run at full speed, and is outside this plan's phases until measured. +- **The answers:** _fits_, _tight_ (inside the GPU working set, but the model plus the memory the operating system, the stack and a browser take — about 8 GB measured — exceeds the machine's memory, so it will swap), _does not fit_ (even the smallest useful window spills), or _unknown_ (no footprint, no machine figure, or a build the footprint was not measured on). +- **Verified state:** once loaded, `/api/ps` reports the model's actual size as its cache grows; that replaces the estimate. A loaded size above the GPU working set is a diagnostic (D15) with two remedies: a smaller window, or unloading the model to free the conversations it is holding. +- **A trial load is an action:** reading diagnostics never loads a model. **D9 — Local models are curated catalog entries, measured by a local conformance probe.** @@ -118,7 +114,8 @@ Which local models are _recommended_ for which role follows from the usefulness - **No specialized agent.** The work is reading one record and proposing one of its remedies, and Helmsman with these operations does both. A separate system agent would be a second surface whose guidance could drift from the UI's. - **Runner holds none of these operations.** They are Helmsman's alone, like the eval plane. A skill's run does not reconfigure the machine it runs on. -- **One GPU, several roles.** Helmsman, clerk, judge, titles and embeddings all compete for one runtime, and each model switch costs a load. The default for a local space is therefore a preset that assigns one chat model to every role (plus the embedding model). Mixing models per role is the operator's choice, and the models page shows its cost: total resident memory, and the load time per switch. On the measured machine a switch from a 9B to a 26B model took 49 s, and the MLX runner's first prompt after a load took 25–56 s whatever its length (§9 G5), so a switch in the middle of a run is a pause the operator can see. Only the model a role is about to use is warmed. `keep_alive` comes from the connection, default 30 min. +- **One GPU, several roles.** Helmsman, clerk, judge, titles and embeddings all compete for one runtime, and each model switch costs a load. The default for a local space is therefore a preset that assigns one chat model to every role (plus the embedding model). Mixing models per role is the operator's choice, and the models page shows its cost: total resident memory, and the load time per switch. On the measured machine a switch from a 9B to a 26B model took 49 s, and the MLX runner's first prompt after a load took 25–56 s whatever its length (§9 G5), so a switch in the middle of a run is a pause the operator can see. `keep_alive` comes from the connection, default 30 min. +- **Warming is the operator's, not a side effect.** Beside each role on a model on this computer, the operator sees whether it is loaded and how large, and has **Load now** (at the window its requests will use, so the first request does not reload it) and **Unload**. _Rejected:_ loading on assignment. Picking a model is not a request to take 16 GB of the machine at that moment, and on a small machine an unasked load is the starvation it would be meant to prevent. - **Lifecycle actions are durable.** - **Pull:** one pull per runtime and tag, observed and not duplicated. A second request joins the first; the runtime already joins two pulls of one tag, reporting the same progress to both (§9 G3). - **Disk space:** a pull is refused before it starts when the host inventory's free disk cannot hold the download. @@ -164,7 +161,7 @@ Which local models are _recommended_ for which role follows from the usefulness - **The adapter:** an `openai-compatible` adapter on chat completions, built from the Fireworks adapter (the OpenAI adapter's Responses API is not what local servers serve). It reaches vLLM, SGLang, llama-server and LM Studio, and through vLLM a GPU box serving many requests at once. - **Pools:** several connections to the same catalog model form a pool, chosen by least in-flight against each one's `max_parallel`. -- **A runtime on the network:** it needs an operator-admitted origin and a token, because an Ollama port has no authentication of its own. +- **A runtime on the network:** it needs an operator-admitted origin (`CREDENTIAL_EXTRA_BASE_URL_ORIGINS`) and a token, because an Ollama port has no authentication of its own. Its connection states that machine's GPU memory (`gpu_memory_gb`), so each model's window is sized for the machine it runs on; this machine's inventory is never applied to a runtime elsewhere, and without the figure its windows are the 32K default. Admission is keyed by its host, so a runtime on the network is one capacity however many connections name it. **D15 — One diagnostics record says what is wrong and what fixes it, and every surface reads it.** @@ -308,7 +305,7 @@ _Exit:_ - three concurrent sessions on it run one at a time and finish with no timeout; - a prompt near the window survives its prefill. -_Built and tested:_ everything in the list above, with unit tests for each proof below, and the adapter run against Ollama 0.40.5 on the 24 GB machine: plain text, two parallel native tool calls, streaming with thinking, JSON through the tool fallback after the MLX runner's 501, and a refusal before sending of a prompt the llama.cpp runner would have cut. `yarn models:verify-local` reports the `gemma4:26b` entry as matching its build. _Not yet run:_ the exit itself, which needs the dev stack started from this branch. +_Built and tested:_ everything in the list above, with unit tests for each proof below, and the adapter run against Ollama 0.40.5 on the 24 GB machine: plain text, two parallel native tool calls, streaming with thinking, JSON through the tool fallback after the MLX runner's 501, and a refusal before sending of a prompt the llama.cpp runner would have cut. `yarn models:verify-local` reports the `gemma4:26b` entry as matching its build. _Run on the dev stack_ (§9, P1 on the dev stack): Helmsman on Gemma 4 26B held a tool-using conversation through the web UI. _Not yet run:_ three concurrent sessions, and a prompt near the window. _Proof:_ @@ -324,6 +321,24 @@ Each lineup entry (§4) is qualified on its own: both probe tiers, the build ide _Exit for each entry:_ its usefulness results, repeated runs included, are in the entry, and its recommended roles follow from them. +### P3a — The window fits the machine (D8) + +**Build:** + +- the footprint in the catalog entry, `local.footprint: { weightsBytes, kvBytesPerToken }`, measured by `models:verify-local` from two prompt sizes and checked for drift — the per-token cost scales with the runtime's KV cache type, which its API does not report, so a changed cache type shows as drift rather than being recorded; +- `machine: { totalMemoryBytes, gpuWorkingSetBytes }` in the host inventory, the GPU figure read once through Metal and cached; +- `localModelFit` in `@aflow/ai-client`: the answer, the largest fitting window, and the bytes at the window in use; +- the default window: `context_tokens` left unset means the largest fitting window, per model, with 32K when the inputs are missing; +- the fit and the window in use on the connection's readiness, with what the runtime holds now: the model's loaded size beside the GPU working set, so a model spilling because it holds every conversation's cache says so; +- **Load now** and **Unload** beside the role, through operator-only routes refused outside the local edition; +- `gpu_memory_gb` on the connection for a runtime on another machine. + +_Exit:_ on the 24 GB machine, a connection with no window set runs Gemma 4 26B at 32K and reports that it spills, naming the 16K it fits; the same connection on a machine with 64 GB or more reports _fits_ at the model's full window. + +_Proof:_ a footprint and machine for which the full window does not fit yield a smaller window; a configured window is never replaced; a missing machine figure yields _unknown_ and 32K. + +_Built and tested:_ all of the above. The probe measures Gemma 4 26B at 16.15 GB and 120,469 bytes a token, inside its entry's tolerances; the host's Metal probe reads 19.07 GB on the 24 GB machine and caches it; the fit there is 16K, below the agents' 32K, so the automatic window is 32K and readiness names the spill. Load and unload were run against Ollama on the 24 GB machine: the model loaded at a 32K window and unloading freed it at once. _Not yet run:_ the dev stack with this branch, where the host executor publishes the machine and readiness shows the note and the actions. + ### P3 — The operator's surface (D8, D10, D13, D15) **Build:** @@ -488,6 +503,26 @@ The built-in retrieval set scored 1.0 recall in every configuration, so it separ Metal's working-set limit is 19.07 GB. With `gemma4:26b` loaded and the stack's Postgres and Redis running, free memory was about 80 MB and swap held 6.6 GB. At 253 tokens a second, a prompt of about 75,000 tokens takes longer to read than the 300 s stream idle window allows (D7). +### G6 — Footprint and caching under real prompts + +Measured after P1 with prompts long enough to fill the cache, `gemma4:26b` at a 32K window, the KV cache at `q8_0`: + +| Prompt tokens | Size reported by `/api/ps` | Request time | +| ----------------------- | -------------------------- | ------------ | +| 96 | 16.18 GB | 78 s, cold | +| 2,906 | 16.66 GB | 12 s | +| 12,106 | 17.77 GB | 50 s | +| 25,306 | 19.15 GB | 190 s | +| then a different 14,834 | 19.97 GB | 298 s | + +The cost per token is steady at about 118 KB. The size does not fall back for a shorter prompt, and grows again for a different one: the runner keeps more than one conversation's cache until the model unloads. The P1 live session reached 22.7 GB at a 32K window. Past Metal's 19.07 GB working set prefill fell from about 240 to 130 tokens a second and swap grew by 2.5 GB. The runner process's own resident memory stays near 40 MB; the model's memory is what `/api/ps` reports. + +The prefix cache keeps checkpoints at 8,192-token steps and at the end of each prompt. An identical prompt is read back whole (`cached=13140` of 13,144, 1.9 s), and so is a prompt counted with one token and then sent (4.3 s + 4.7 s). A prompt that extends the previous one, as an agent's next turn does, diverges just before the previous end checkpoint, where the request's closing turn tokens were, and falls back to the 8,192 checkpoint: 4,958 of 13,150 tokens read again, 92 s under memory pressure. Every agent turn on the MLX runner past 8K tokens pays for that. The llama.cpp runner read an extended prompt from its cache in P0; whether a GGUF build of the same model is the faster one for agent loops on Apple silicon is a P2 question. + +### P1 on the dev stack + +Helmsman on `gemma-local` at the 32K default, on the 24 GB machine with the dev stack and a browser running: the first turn, model load included, took 6 m 42 s; the turns after it 15–56 s each at 17.7–19.9K prompt tokens. Helmsman answered from `catalog.tool.search` and `catalog.tool.list` correctly, and recovered on its own from one rejected call (`stepTypes: ['platform']`, not a step type). The model held 22.7 GB, past the working set, which accounts for the pace. + ### Call sites outside role resolution Every production model call was inventoried. The ones a `local_only` space (D12) would miss today: diff --git a/packages/ai-client/src/catalogModels.ts b/packages/ai-client/src/catalogModels.ts index 4c790094..fc0082b6 100644 --- a/packages/ai-client/src/catalogModels.ts +++ b/packages/ai-client/src/catalogModels.ts @@ -91,8 +91,10 @@ export const builtInModels: ModelDefinition[] = [ runner: 'mlx', quantization: 'nvfp4', runtimeVersion: '0.40.5', - probeVersion: 1, + probeVersion: 2, measuredAt: '2026-10-16', + // Measured at OLLAMA_KV_CACHE_TYPE=q8_0 (§9 G6). + footprint: { weightsBytes: 16_180_670_828, kvBytesPerToken: 120_000 }, }, }, diff --git a/packages/ai-client/src/client.ts b/packages/ai-client/src/client.ts index 33c506d9..b0f5e8b2 100644 --- a/packages/ai-client/src/client.ts +++ b/packages/ai-client/src/client.ts @@ -1,6 +1,8 @@ /** * AIClient - Unified interface for AI operations. */ +import { OLLAMA_DEFAULT_CONTEXT_TOKENS } from '@aflow/schemas'; +import { localModelFit } from './localModelFit.js'; import type { GenerateTextRequest, GenerateTextResponse, @@ -290,9 +292,9 @@ export function createAIClient(config: AIClientConfig): AIClient { maxOutputTokens: model.maxOutputTokens, fromCatalog: true, vision: model.capabilities.vision, - contextWindow: effectiveContextWindow( - model.contextWindow, - config.providers[explicitProvider ?? model.provider]?.contextTokens, + contextWindow: contextWindowForModel( + model, + config.providers[explicitProvider ?? model.provider], ), }; } @@ -756,9 +758,26 @@ export function createAIClient(config: AIClientConfig): AIClient { }; } -function effectiveContextWindow( - catalogWindow: number, - connectionWindow: number | undefined, +/** + * The window a request for this model gets. A window the connection sets is + * the operator's choice and is used as set, within the model's own. A model on + * the operator's machine with no window set gets the largest that fits the + * machine (Plan 323 D8), or a conservative default when fit cannot be known. + */ +export function contextWindowForModel( + model: ModelDefinition, + connection: ProviderConfig | undefined, ): number { - return connectionWindow === undefined ? catalogWindow : Math.min(catalogWindow, connectionWindow); + if (connection?.contextTokens !== undefined) + return Math.min(model.contextWindow, connection.contextTokens); + if (!model.local) return model.contextWindow; + const fit = localModelFit({ + footprint: model.local.footprint, + machine: connection?.machine, + catalogWindow: model.contextWindow, + parallel: connection?.parallel ?? 1, + }); + return fit.status === 'unknown' + ? Math.min(model.contextWindow, OLLAMA_DEFAULT_CONTEXT_TOKENS) + : fit.window; } diff --git a/packages/ai-client/src/index.ts b/packages/ai-client/src/index.ts index 98929d20..3a25f071 100644 --- a/packages/ai-client/src/index.ts +++ b/packages/ai-client/src/index.ts @@ -56,6 +56,7 @@ export type { ModelPricing, ModelDefinition, LocalModelBuild, + LocalModelFootprint, ModelTraits, UsageRecord, AIErrorCode, @@ -88,7 +89,7 @@ export { // Client export type { AIClient, AIClientConfig } from './client.js'; -export { createAIClient } from './client.js'; +export { createAIClient, contextWindowForModel } from './client.js'; // Adapter interface export type { AIProviderAdapter } from './adapter.js'; @@ -182,3 +183,19 @@ export { resolveDecisionAnswers } from './decision.js'; // Plan 323: the answer floor prompt budgeting and a local runtime's sizing share export { MIN_ANSWER_TOKENS } from './providers/ollamaPromptSize.js'; +export { + localModelFit, + footprintBytes, + WINDOW_STEP_TOKENS, + MIN_USEFUL_WINDOW_TOKENS, + RUNTIME_GPU_RESERVE_BYTES, + SYSTEM_RESERVE_BYTES, + type LocalMachine, + type LocalModelFit, + type LocalModelFitStatus, +} from './localModelFit.js'; +export { + localRuntimeClient, + type LocalRuntimeClient, + type LoadedLocalModel, +} from './providers/ollamaRuntime.js'; diff --git a/packages/ai-client/src/localModelFit.test.ts b/packages/ai-client/src/localModelFit.test.ts new file mode 100644 index 00000000..d4b1a01a --- /dev/null +++ b/packages/ai-client/src/localModelFit.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from 'vitest'; +import { createAIClient } from './client.js'; +import { localModelFit } from './localModelFit.js'; + +const GB = 1e9; +const GEMMA = { weightsBytes: 16_180_670_828, kvBytesPerToken: 120_000 }; +const M3_24GB = { + totalMemoryBytes: 25_769_803_776, + gpuWorkingSetBytes: 19_069_665_280, + unifiedMemory: true, +}; +const MAC_64GB = { + totalMemoryBytes: 68_719_476_736, + gpuWorkingSetBytes: 51_539_607_552, + unifiedMemory: true, +}; + +describe('localModelFit', () => { + it('never sizes below the window the platform’s agents need, and says the model will spill', () => { + // The 24 GB machine fits 16K, and Helmsman's first prompt alone is ~18K. + const fit = localModelFit({ + footprint: GEMMA, + machine: M3_24GB, + catalogWindow: 262_144, + parallel: 1, + }); + expect(fit).toMatchObject({ + window: 32_768, + status: 'does_not_fit', + largestFittingWindow: 16_384, + }); + }); + + it('calls a window inside the GPU working set but crowding the machine’s memory tight', () => { + const fit = localModelFit({ + footprint: GEMMA, + machine: M3_24GB, + catalogWindow: 262_144, + parallel: 1, + window: 16_384, + }); + expect(fit).toMatchObject({ status: 'tight', window: 16_384 }); + expect(fit.bytesAtWindow).toBeLessThan(M3_24GB.gpuWorkingSetBytes); + }); + + it('gives a large machine the model’s whole window', () => { + const fit = localModelFit({ + footprint: GEMMA, + machine: MAC_64GB, + catalogWindow: 262_144, + parallel: 1, + }); + expect(fit).toMatchObject({ window: 262_144, status: 'fits' }); + }); + + it('sizes for every request served at once, since each holds a window', () => { + const one = localModelFit({ + footprint: GEMMA, + machine: MAC_64GB, + catalogWindow: 262_144, + parallel: 1, + }); + const four = localModelFit({ + footprint: GEMMA, + machine: MAC_64GB, + catalogWindow: 262_144, + parallel: 4, + }); + expect(four.window).toBeLessThan(one.window); + }); + + it('reports a configured window that spills as not fitting, and keeps it', () => { + const fit = localModelFit({ + footprint: GEMMA, + machine: M3_24GB, + catalogWindow: 262_144, + parallel: 1, + window: 32_768, + }); + expect(fit).toMatchObject({ + window: 32_768, + status: 'does_not_fit', + largestFittingWindow: 16_384, + }); + }); + + it('reports a model whose weights alone fill the GPU as not fitting at any window', () => { + const fit = localModelFit({ + footprint: { weightsBytes: 65 * GB, kvBytesPerToken: 120_000 }, + machine: M3_24GB, + catalogWindow: 131_072, + parallel: 1, + }); + expect(fit.status).toBe('does_not_fit'); + }); + + it('does not call a discrete GPU tight for the memory the operating system uses', () => { + const rtx = { + totalMemoryBytes: 32 * GB, + gpuWorkingSetBytes: 24 * 1024 ** 3, + unifiedMemory: false, + }; + expect( + localModelFit({ footprint: GEMMA, machine: rtx, catalogWindow: 262_144, parallel: 1 }).status, + ).toBe('fits'); + }); + + it('says it does not know when the machine has not been read', () => { + expect( + localModelFit({ footprint: GEMMA, machine: undefined, catalogWindow: 262_144, parallel: 1 }) + .status, + ).toBe('unknown'); + }); +}); + +describe('the window a request is sent with', () => { + it('is the largest that fits when the connection sets none', () => { + const client = createAIClient({ + providers: { ollama: { machine: MAC_64GB } }, + defaultProvider: 'ollama', + }); + expect(client.contextWindowFor('gemma-local')).toBe(262_144); + }); + + it('is the agents’ floor on a machine that fits less', () => { + const client = createAIClient({ + providers: { ollama: { machine: M3_24GB } }, + defaultProvider: 'ollama', + }); + expect(client.contextWindowFor('gemma-local')).toBe(32_768); + }); + + it('is the operator’s when the connection sets one, fitting or not', () => { + const client = createAIClient({ + providers: { ollama: { machine: M3_24GB, contextTokens: 65_536 } }, + defaultProvider: 'ollama', + }); + expect(client.contextWindowFor('gemma-local')).toBe(65_536); + }); + + it('is a conservative 32K when the machine is unknown', () => { + const client = createAIClient({ providers: { ollama: {} }, defaultProvider: 'ollama' }); + expect(client.contextWindowFor('gemma-local')).toBe(32_768); + }); +}); diff --git a/packages/ai-client/src/localModelFit.ts b/packages/ai-client/src/localModelFit.ts new file mode 100644 index 00000000..1d87f395 --- /dev/null +++ b/packages/ai-client/src/localModelFit.ts @@ -0,0 +1,99 @@ +/** + * Whether a model on the operator's machine fits it, and the largest window + * that does (Plan 323 D8). + * + * A model costs its weights plus a constant per context token, and on the MLX + * runner a loaded model's cache climbs to its whole window and stays there + * until it unloads (§9 G6) — so fit is computed for the full window, never for + * the prompt in hand. The ceiling is the GPU working set: past it the model + * does not fail, it spills, and every request slows with the machine. + */ +import { OLLAMA_DEFAULT_CONTEXT_TOKENS } from '@aflow/schemas'; +import type { LocalModelFootprint } from './types.js'; + +/** What the machine the runtime runs on offers a model. */ +export interface LocalMachine { + totalMemoryBytes: number; + /** Metal's working set on Apple silicon; VRAM on a discrete GPU. Absent when it could not be read. */ + gpuWorkingSetBytes?: number | undefined; + /** The GPU shares the machine's memory (Apple silicon), so a model competes with everything else for it. */ + unifiedMemory: boolean; +} + +/** Windows are offered in these steps, so a figure an operator reads is a familiar one. */ +export const WINDOW_STEP_TOKENS = 4096; +/** + * The smallest window the platform's own agents run in. Helmsman's first + * prompt, before a word of conversation, is about 18K tokens of instructions + * and tool definitions (§9, P1 on the dev stack), so a smaller window refuses + * it outright. A machine that fits less gets this window and is told the model + * will spill: slow is better than refused. + */ +export const MIN_USEFUL_WINDOW_TOKENS = OLLAMA_DEFAULT_CONTEXT_TOKENS; +/** Held back by the runtime for itself (Ollama's `minimum` in its GPU memory report). */ +export const RUNTIME_GPU_RESERVE_BYTES = 512 * 1024 ** 2; +/** + * What the operating system, the stack and a browser take beside the model on + * unified memory: on the 24 GB machine, with Postgres, Redis, the services and + * a browser running, a 16 GB model left 80 MB free and 6.6 GB in swap (§9 G5). + */ +export const SYSTEM_RESERVE_BYTES = 8 * 1024 ** 3; + +export type LocalModelFitStatus = 'fits' | 'tight' | 'does_not_fit' | 'unknown'; + +export interface LocalModelFit { + status: LocalModelFitStatus; + /** The largest window, in steps, that stays inside the GPU working set; absent when unknown. */ + largestFittingWindow?: number; + /** The model's cost at `window`, for the operator to read; absent when unknown. */ + bytesAtWindow?: number; + /** The window the answer is about. */ + window: number; +} + +export function footprintBytes( + footprint: LocalModelFootprint, + window: number, + parallel: number, +): number { + return footprint.weightsBytes + footprint.kvBytesPerToken * window * parallel; +} + +/** + * The fit of a model at `window` (or, when none is given, at the largest + * window that fits), capped by the model's own window. + */ +export function localModelFit(input: { + footprint: LocalModelFootprint | undefined; + machine: LocalMachine | undefined; + catalogWindow: number; + parallel: number; + window?: number | undefined; +}): LocalModelFit { + const { footprint, machine, catalogWindow, parallel } = input; + if (!footprint || !machine?.gpuWorkingSetBytes) { + return { status: 'unknown', window: input.window ?? catalogWindow }; + } + const budget = machine.gpuWorkingSetBytes - RUNTIME_GPU_RESERVE_BYTES; + const tokens = Math.floor( + (budget - footprint.weightsBytes) / (footprint.kvBytesPerToken * parallel), + ); + const largest = Math.min( + catalogWindow, + Math.floor(tokens / WINDOW_STEP_TOKENS) * WINDOW_STEP_TOKENS, + ); + const window = input.window ?? Math.max(largest, MIN_USEFUL_WINDOW_TOKENS); + const bytes = footprintBytes(footprint, window, parallel); + const status: LocalModelFitStatus = + bytes > budget + ? 'does_not_fit' + : machine.unifiedMemory && bytes > machine.totalMemoryBytes - SYSTEM_RESERVE_BYTES + ? 'tight' + : 'fits'; + return { + status, + window, + bytesAtWindow: bytes, + ...(largest > 0 ? { largestFittingWindow: largest } : {}), + }; +} diff --git a/packages/ai-client/src/providers/ollamaRuntime.test.ts b/packages/ai-client/src/providers/ollamaRuntime.test.ts new file mode 100644 index 00000000..d6647f27 --- /dev/null +++ b/packages/ai-client/src/providers/ollamaRuntime.test.ts @@ -0,0 +1,60 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { localRuntimeClient } from './ollamaRuntime.js'; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('localRuntimeClient', () => { + it('reports what the runtime holds, with each model’s size as loaded', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response( + JSON.stringify({ + models: [ + { + name: 'gemma4:26b', + size: 22_677_887_340, + context_length: 32_768, + expires_at: 'x', + }, + ], + }), + ), + ), + ); + expect(await localRuntimeClient({}).loaded()).toEqual([ + { tag: 'gemma4:26b', sizeBytes: 22_677_887_340, contextLength: 32_768, expiresAt: 'x' }, + ]); + }); + + it('answers null, not an empty list, when the runtime does not answer', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new TypeError('fetch failed'); + }), + ); + expect(await localRuntimeClient({}).loaded()).toBeNull(); + }); + + it('loads at the window requests will use, and unloads by asking to keep nothing', async () => { + const bodies: unknown[] = []; + vi.stubGlobal( + 'fetch', + vi.fn(async (_url: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body))); + return new Response('{}'); + }), + ); + const client = localRuntimeClient({ keepAlive: '30m' }); + await client.warm('gemma4:26b', 32_768); + await client.unload('gemma4:26b'); + expect(bodies).toEqual([ + { model: 'gemma4:26b', keep_alive: '30m', options: { num_ctx: 32_768 } }, + { model: 'gemma4:26b', keep_alive: 0 }, + ]); + }); +}); diff --git a/packages/ai-client/src/providers/ollamaRuntime.ts b/packages/ai-client/src/providers/ollamaRuntime.ts new file mode 100644 index 00000000..635ea18f --- /dev/null +++ b/packages/ai-client/src/providers/ollamaRuntime.ts @@ -0,0 +1,85 @@ +/** + * The operator's actions on a local runtime's models, and what it has loaded + * (Plan 323 D10). Separate from the adapter because these are not model calls: + * nothing here generates, and nothing here is reachable by an agent. + */ +import { OLLAMA_DEFAULT_BASE_URL } from '@aflow/schemas'; +import type { ProviderConfig } from '../types.js'; + +/** Readiness asks this on every read; a runtime slower than this to say what it holds is reported unreachable. */ +const STATUS_TIMEOUT_MS = 2_000; +/** Loading weights from disk takes tens of seconds to a few minutes (§9 G5). */ +const LOAD_TIMEOUT_MS = 600_000; + +export interface LoadedLocalModel { + tag: string; + sizeBytes: number; + contextLength: number | null; + expiresAt: string | null; +} + +export interface LocalRuntimeClient { + /** What the runtime holds now, or null when it does not answer. */ + loaded(): Promise; + /** Load a model at the window its requests will use, so the first one does not wait for it. */ + warm(tag: string, window: number): Promise; + /** Unload a model, freeing its weights and every conversation's cache it holds. */ + unload(tag: string): Promise; +} + +export function localRuntimeClient( + config: Pick, +): LocalRuntimeClient { + const base = (config.baseUrl ?? OLLAMA_DEFAULT_BASE_URL).replace(/\/+$/, ''); + const headers: Record = { + 'content-type': 'application/json', + ...(config.apiKey ? { authorization: `Bearer ${config.apiKey}` } : {}), + }; + const generate = async (body: Record, timeoutMs: number): Promise => { + const res = await fetch(`${base}/api/generate`, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) throw new Error(`Ollama ${res.status}: ${await res.text()}`); + await res.text(); + }; + return { + async loaded() { + try { + const res = await fetch(`${base}/api/ps`, { + headers, + signal: AbortSignal.timeout(STATUS_TIMEOUT_MS), + }); + if (!res.ok) return null; + const body = (await res.json()) as { + models?: Array<{ + name?: string; + size?: number; + context_length?: number; + expires_at?: string; + }>; + }; + return (body.models ?? []).map((m) => ({ + tag: m.name ?? '', + sizeBytes: m.size ?? 0, + contextLength: m.context_length ?? null, + expiresAt: m.expires_at ?? null, + })); + } catch { + return null; + } + }, + warm: (tag, window) => + generate( + { + model: tag, + ...(config.keepAlive ? { keep_alive: config.keepAlive } : {}), + options: { num_ctx: window }, + }, + LOAD_TIMEOUT_MS, + ), + unload: (tag) => generate({ model: tag, keep_alive: 0 }, STATUS_TIMEOUT_MS * 5), + }; +} diff --git a/packages/ai-client/src/types.ts b/packages/ai-client/src/types.ts index 34047ae8..fcddf16d 100644 --- a/packages/ai-client/src/types.ts +++ b/packages/ai-client/src/types.ts @@ -2,6 +2,7 @@ * Core types for the AI client layer. */ import { z } from 'zod'; +import type { LocalMachine } from './localModelFit.js'; import { StepImageSchema } from '@aflow/schemas'; import type { TenantId, @@ -790,6 +791,13 @@ export interface ProviderConfig { contextTokens?: number | undefined; /** How long a local runtime keeps a model loaded after a request, in its own notation. */ keepAlive?: string | undefined; + /** + * The machine a local runtime runs on, when it is known. A connection that + * sets no `contextTokens` gets, per model, the largest window that fits it. + */ + machine?: LocalMachine | undefined; + /** How many requests at once a local runtime serves; each holds a window's cache. */ + parallel?: number | undefined; } // ============================================================================ @@ -968,6 +976,18 @@ export interface ModelDefinition { local?: LocalModelBuild; } +/** What a local model costs in memory, as `yarn models:verify-local` measured it. */ +export interface LocalModelFootprint { + /** The loaded size before any context. */ + weightsBytes: number; + /** + * The KV cache's cost per context token. It scales with the runtime's cache + * type (`OLLAMA_KV_CACHE_TYPE`), which the API does not report, so the probe + * measures it rather than reading it, and a change shows as drift. + */ + kvBytesPerToken: number; +} + /** The build of a local model that `yarn models:verify-local` measured. */ export interface LocalModelBuild { /** The tag the runtime serves it under. */ @@ -980,6 +1000,7 @@ export interface LocalModelBuild { runtimeVersion: string; probeVersion: number; measuredAt: string; + footprint?: LocalModelFootprint; } // ============================================================================ diff --git a/packages/credential-resolver/src/byokAiClient.ts b/packages/credential-resolver/src/byokAiClient.ts index a483a54c..1788a9ef 100644 --- a/packages/credential-resolver/src/byokAiClient.ts +++ b/packages/credential-resolver/src/byokAiClient.ts @@ -10,6 +10,7 @@ import { inferProviderForModelRef, type AIClient, type GenerateTextResponse, + type LocalMachine, } from '@aflow/ai-client'; import { localRuntimeId } from '@aflow/schemas'; import { CredentialResolver } from './resolver.js'; @@ -134,12 +135,20 @@ function admitted( }; } +/** What a factory needs to reach a runtime on the operator's machine as the AI executor does. */ +export interface LocalRuntimeOptions { + /** A lease on the runtime for each request (Plan 323 D7). */ + admission?: RuntimeAdmission | undefined; + /** The machine it runs on, which sizes each model's window (D8). */ + machine?: (() => Promise) | undefined; +} + export class ByokAiClientFactory { private cache = new Map(); constructor( private resolver: CredentialResolver, - private admission?: RuntimeAdmission, + private local: LocalRuntimeOptions = {}, ) {} /** Whether `model`'s provider resolves a credential in this context. */ @@ -175,22 +184,25 @@ export class ByokAiClientFactory { throw new ByokCredentialError(credentialMissingMessage(providerId), providerId); } - const cacheKey = `${resolved.credentialId}:${resolved.updatedAt}`; + const machine = + providerId === 'ollama' ? await this.local.machine?.().catch(() => undefined) : undefined; + const cacheKey = `${resolved.credentialId}:${resolved.updatedAt}:${machine?.gpuWorkingSetBytes ?? '-'}`; const hit = this.cache.get(cacheKey); if (hit && hit.expiresAt > Date.now()) return { client: hit.client, providerId }; - const providerConfig = providerConfigFromCredential(providerId, resolved); + const providerConfig = providerConfigFromCredential(providerId, resolved, { machine }); const plain = createAIClient({ providers: { [providerId]: providerConfig }, defaultProvider: providerId as 'openai', }); + const admission = this.local.admission; const client = - providerId === 'ollama' && this.admission + providerId === 'ollama' && admission ? admitted( plain, localRuntimeId(providerConfig.baseUrl), localRuntimeSettings(providerId, resolved).maxParallel, - this.admission, + admission, ) : plain; this.cache.set(cacheKey, { client, expiresAt: Date.now() + CLIENT_CACHE_TTL_MS }); @@ -200,10 +212,10 @@ export class ByokAiClientFactory { export function createByokAiClientFactory( db: PostgresJsDatabase, - admission?: RuntimeAdmission, + local: LocalRuntimeOptions = {}, ): ByokAiClientFactory { return new ByokAiClientFactory( new CredentialResolver({ loader: createProviderCredentialDbLoader(db) }), - admission, + local, ); } diff --git a/packages/credential-resolver/src/index.ts b/packages/credential-resolver/src/index.ts index 5f9b7255..61f6f693 100644 --- a/packages/credential-resolver/src/index.ts +++ b/packages/credential-resolver/src/index.ts @@ -16,9 +16,11 @@ export { LocalRuntimeBusyError, type ByokClientContext, type RuntimeAdmission, + type LocalRuntimeOptions, } from './byokAiClient.js'; export { providerConfigFromCredential, missingRequiredSecrets, localRuntimeSettings, + localMachineFor, } from './providerConfig.js'; diff --git a/packages/credential-resolver/src/providerConfig.test.ts b/packages/credential-resolver/src/providerConfig.test.ts index 630902d0..4558ae57 100644 --- a/packages/credential-resolver/src/providerConfig.test.ts +++ b/packages/credential-resolver/src/providerConfig.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { + localMachineFor, localRuntimeSettings, missingRequiredSecrets, providerConfigFromCredential, @@ -20,11 +21,12 @@ const resolved = ( describe('providerConfigFromCredential', () => { it('reaches the runtime the registry describes when the connection sets nothing', () => { + // No window: the client sizes one from the machine, per model. expect(providerConfigFromCredential('ollama', resolved({}))).toEqual({ apiKey: '', baseUrl: 'http://127.0.0.1:11434', - contextTokens: 32_768, keepAlive: '30m', + parallel: 1, }); }); @@ -70,3 +72,35 @@ describe('localRuntimeSettings', () => { expect(localRuntimeSettings('ollama', resolved({}, { max_parallel: '2' })).maxParallel).toBe(2); }); }); + +describe('localMachineFor', () => { + const THIS_MAC = { totalMemoryBytes: 24e9, gpuWorkingSetBytes: 19e9, unifiedMemory: true }; + + it('describes a runtime on this machine by this machine', () => { + expect(localMachineFor({ config: {}, inventoryMachine: THIS_MAC })).toEqual(THIS_MAC); + expect( + localMachineFor({ + config: { base_url: 'http://host.docker.internal:11434' }, + inventoryMachine: THIS_MAC, + }), + ).toEqual(THIS_MAC); + }); + + it('never describes a runtime on another machine by this one', () => { + expect( + localMachineFor({ + config: { base_url: 'http://gpu-box.lan:11434' }, + inventoryMachine: THIS_MAC, + }), + ).toBeUndefined(); + }); + + it('takes the GPU memory the connection states, over what this machine reads', () => { + expect( + localMachineFor({ + config: { base_url: 'http://gpu-box.lan:11434', gpu_memory_gb: '96' }, + inventoryMachine: THIS_MAC, + }), + ).toEqual({ totalMemoryBytes: 96e9, gpuWorkingSetBytes: 96e9, unifiedMemory: false }); + }); +}); diff --git a/packages/credential-resolver/src/providerConfig.ts b/packages/credential-resolver/src/providerConfig.ts index d90f07e0..cc3d47dc 100644 --- a/packages/credential-resolver/src/providerConfig.ts +++ b/packages/credential-resolver/src/providerConfig.ts @@ -3,8 +3,13 @@ * shared by every process that builds an AI client from a credential, so the * executor and the API server cannot disagree about where a request goes. */ -import type { ProviderConfig } from '@aflow/ai-client'; -import { getProviderDefinition, getRequiredSecretFieldIds } from '@aflow/schemas'; +import type { LocalMachine, ProviderConfig } from '@aflow/ai-client'; +import { + getProviderDefinition, + getRequiredSecretFieldIds, + localRuntimeId, + THIS_MACHINE_RUNTIME_HOST, +} from '@aflow/schemas'; import type { ResolvedProvider } from './types.js'; /** @@ -28,9 +33,31 @@ function positiveInteger(value: string | undefined): number | undefined { return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; } +/** + * The machine a local runtime's models run on (Plan 323 D8). A GPU memory + * figure on the connection is the operator's word for a runtime on another + * machine, and wins. Otherwise only a runtime on this machine is described by + * this machine's inventory; one elsewhere is unknown, and its windows fall + * back to the conservative default rather than being sized for the wrong box. + */ +export function localMachineFor(input: { + config: Readonly> | null | undefined; + inventoryMachine: LocalMachine | undefined; +}): LocalMachine | undefined { + const configuredGb = Number(input.config?.['gpu_memory_gb']); + if (Number.isFinite(configuredGb) && configuredGb > 0) { + const bytes = Math.round(configuredGb * 1e9); + return { totalMemoryBytes: bytes, gpuWorkingSetBytes: bytes, unifiedMemory: false }; + } + return localRuntimeId(input.config?.['base_url']).startsWith(`${THIS_MACHINE_RUNTIME_HOST}:`) + ? input.inventoryMachine + : undefined; +} + export function providerConfigFromCredential( providerId: string, resolved: ResolvedProvider, + local: { machine?: LocalMachine | undefined } = {}, ): ProviderConfig { // Always set, empty when the credential carries none: an SDK handed no key // reads the platform's own from the environment, and a space's request must @@ -46,6 +73,10 @@ export function providerConfigFromCredential( if (contextTokens !== undefined) config.contextTokens = contextTokens; const keepAlive = configValue(providerId, resolved, 'keep_alive'); if (keepAlive) config.keepAlive = keepAlive; + const parallel = positiveInteger(configValue(providerId, resolved, 'max_parallel')); + if (parallel !== undefined) config.parallel = parallel; + const machine = localMachineFor({ config: resolved.config, inventoryMachine: local.machine }); + if (machine) config.machine = machine; return config; } diff --git a/packages/redis/src/hostInventory.ts b/packages/redis/src/hostInventory.ts index 0536592c..5383d119 100644 --- a/packages/redis/src/hostInventory.ts +++ b/packages/redis/src/hostInventory.ts @@ -161,9 +161,22 @@ export type HostInventoryLocalPort = z.infer; + export const HostInventorySchema = z.object({ hostname: z.string(), observedAt: z.string(), + machine: HostMachineSchema.optional(), runtimes: z.array(z.object({ name: z.string(), version: z.string() })), harnesses: z.array(z.object({ id: z.string(), label: z.string().optional() })), /** @@ -300,3 +313,17 @@ export async function readLiveHostInventories( } return inventories; } + +/** + * The machine a model runtime on the appliance's own host runs on, as the + * paired machine publishes it. The local edition has one host; a second + * publisher would be a second machine, and only the first answers. + */ +export async function readLocalMachine( + redis: HostInventoryReader, +): Promise { + for (const inventory of await readLiveHostInventories(redis)) { + if (inventory.machine) return inventory.machine; + } + return undefined; +} diff --git a/packages/schemas/src/credentials/registry.ts b/packages/schemas/src/credentials/registry.ts index 5d49169c..f79c6b10 100644 --- a/packages/schemas/src/credentials/registry.ts +++ b/packages/schemas/src/credentials/registry.ts @@ -5,6 +5,8 @@ export const OLLAMA_DEFAULT_BASE_URL = 'http://127.0.0.1:11434'; export const OLLAMA_DEFAULT_CONTEXT_TOKENS = 32_768; const THIS_MACHINE_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]', 'host.docker.internal']); +/** The host every name for this machine's loopback folds into. */ +export const THIS_MACHINE_RUNTIME_HOST = 'this-machine'; /** * The runtime a base URL reaches, as one id however it is spelled (Plan 323 @@ -15,7 +17,7 @@ const THIS_MACHINE_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]', 'host.doc export function localRuntimeId(baseUrl: string | undefined): string { const url = new URL(baseUrl || OLLAMA_DEFAULT_BASE_URL); const port = url.port || (url.protocol === 'https:' ? '443' : '80'); - const host = THIS_MACHINE_HOSTS.has(url.hostname) ? 'this-machine' : url.hostname; + const host = THIS_MACHINE_HOSTS.has(url.hostname) ? THIS_MACHINE_RUNTIME_HOST : url.hostname; return `${host}:${port}`; } @@ -181,9 +183,9 @@ export const PROVIDER_REGISTRY: readonly ProviderDefinition[] = [ label: 'Context window (tokens)', type: 'number', required: false, - defaultValue: String(OLLAMA_DEFAULT_CONTEXT_TOKENS), + placeholder: 'Automatic', helpText: - 'The largest prompt and answer a request may use. A larger window holds longer conversations and costs memory.', + 'Leave empty for the largest window this computer fits, per model. A larger window holds longer conversations and costs memory; a window set here is used as set.', }, { fieldId: 'max_parallel', @@ -194,6 +196,15 @@ export const PROVIDER_REGISTRY: readonly ProviderDefinition[] = [ helpText: 'How many requests this computer serves at the same time. Match OLLAMA_NUM_PARALLEL; each one needs its own context memory.', }, + { + fieldId: 'gpu_memory_gb', + label: 'GPU memory (GB)', + type: 'number', + required: false, + placeholder: 'Read from this computer', + helpText: + 'For Ollama on another machine: the GPU memory it has, so each model gets the largest window that fits there. Leave empty for Ollama on this computer, which is read directly.', + }, { fieldId: 'keep_alive', label: 'Keep loaded for', diff --git a/packages/server-runtime/src/compose/coreSurfaces.ts b/packages/server-runtime/src/compose/coreSurfaces.ts index 0c8e99a3..15342839 100644 --- a/packages/server-runtime/src/compose/coreSurfaces.ts +++ b/packages/server-runtime/src/compose/coreSurfaces.ts @@ -27,6 +27,7 @@ import { spacePolicyRoutes } from '../routes/spacePolicyRoutes.js'; import { spaceLifecycleRoutes } from '../routes/spaceLifecycleRoutes.js'; import { activeMemoryRoutes } from '../routes/activeMemory.js'; import { spaceLlmReadinessRoutes } from '../routes/spaceLlmReadiness.js'; +import { localModelRoutes } from '../routes/localModels.js'; import { spaceConnectionsRoutes } from '../routes/spaceConnections.js'; import { cascadesRoutes } from '../routes/cascades.js'; import { workflowsRoutes } from '../routes/workflows/index.js'; @@ -152,6 +153,10 @@ function v1Surfaces({ actionCenterAggregator }: V1SurfaceDeps): ServerSurface[] name: 'space-llm-readiness', register: (s) => s.register(spaceLlmReadinessRoutes, { prefix: '/spaces' }), }, + { + name: 'local-models', + register: (s) => s.register(localModelRoutes, { prefix: '/spaces' }), + }, { name: 'space-connections', register: (s) => s.register(spaceConnectionsRoutes, { prefix: '/spaces' }), diff --git a/packages/server-runtime/src/lib/localRuntime.ts b/packages/server-runtime/src/lib/localRuntime.ts new file mode 100644 index 00000000..abb8af82 --- /dev/null +++ b/packages/server-runtime/src/lib/localRuntime.ts @@ -0,0 +1,42 @@ +/** + * The space's runtime on the operator's machine, resolved as the AI executor + * resolves it — same credential chain, same defaults, same machine — so what + * readiness reports and what the operator's actions reach is the runtime the + * space's models actually run on (Plan 323). + */ +import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'; +import { localRuntimeClient, type LocalRuntimeClient, type ProviderConfig } from '@aflow/ai-client'; +import { + CredentialResolver, + createProviderCredentialDbLoader, + providerConfigFromCredential, +} from '@aflow/credential-resolver'; +import { readLocalMachine, type Redis } from '@aflow/redis'; + +export interface ResolvedLocalRuntime { + config: ProviderConfig; + client: LocalRuntimeClient; +} + +export async function resolveSpaceLocalRuntime(input: { + db: PostgresJsDatabase; + redis: Redis | undefined; + tenantId: string; + userId: string; + spaceId: string; +}): Promise { + const resolver = new CredentialResolver({ loader: createProviderCredentialDbLoader(input.db) }); + const resolved = await resolver + .resolve('ollama', { + tenantId: input.tenantId, + credentialOwnerId: input.userId, + spaceId: input.spaceId, + }) + .catch(() => null); + if (!resolved) return null; + const machine = input.redis + ? await readLocalMachine(input.redis).catch(() => undefined) + : undefined; + const config = providerConfigFromCredential('ollama', resolved, { machine }); + return { config, client: localRuntimeClient(config) }; +} diff --git a/packages/server-runtime/src/routes/localModels.test.ts b/packages/server-runtime/src/routes/localModels.test.ts new file mode 100644 index 00000000..68264336 --- /dev/null +++ b/packages/server-runtime/src/routes/localModels.test.ts @@ -0,0 +1,42 @@ +/** + * Loading and unloading a model on this computer is the local edition's, and + * only for a model the catalog says runs there. + */ +import { describe, expect, it } from 'vitest'; +import Fastify from 'fastify'; +import { serializerCompiler, validatorCompiler } from 'fastify-type-provider-zod'; +import type { EditionId } from '@aflow/schemas'; +import { localModelRoutes } from './localModels.js'; + +const SPACE = '00000000-0000-0000-0000-0000000000a1'; + +async function post(edition: EditionId, model: string) { + const app = Fastify({ logger: false }); + app.setValidatorCompiler(validatorCompiler); + app.setSerializerCompiler(serializerCompiler); + app.decorate('edition', { edition } as never); + (app as unknown as { appContext: unknown }).appContext = { db: {}, redis: undefined }; + app.decorateRequest('requireTenant', async () => ({ tenantId: 't' }) as never); + app.decorateRequest('requireSpace', async () => ({ spaceId: SPACE }) as never); + await app.register(localModelRoutes); + const res = await app.inject({ + method: 'POST', + url: `/${SPACE}/local-models/warm`, + payload: { model }, + }); + await app.close(); + return res; +} + +describe('POST /spaces/:spaceId/local-models/:action', () => { + it('is refused in the hosted edition', async () => { + const res = await post('enterprise', 'gemma-local'); + expect(res.statusCode).toBe(404); + expect(res.json().message).toContain('local edition'); + }); + + it('refuses a model that does not run on this computer', async () => { + const res = await post('community-local', 'glm-pro'); + expect(res.statusCode).toBe(400); + }); +}); diff --git a/packages/server-runtime/src/routes/localModels.ts b/packages/server-runtime/src/routes/localModels.ts new file mode 100644 index 00000000..229c2f6a --- /dev/null +++ b/packages/server-runtime/src/routes/localModels.ts @@ -0,0 +1,100 @@ +/** + * The operator's actions on the space's models on this computer: load one + * before it is first needed, or unload one to give the machine its memory + * back (Plan 323 D10). Operator-only REST, never an operation: no agent loads + * gigabytes or evicts the model a run is on. + * + * Registered in every edition and refused outside the local one, where no + * model runs on the operator's machine. + */ +import type { FastifyPluginAsync } from 'fastify'; +import { z } from 'zod'; +import type { ZodTypeProvider } from 'fastify-type-provider-zod'; +import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'; +import { contextWindowForModel, createDefaultModelCatalog } from '@aflow/ai-client'; +import type { Redis } from '@aflow/redis'; +import { resolveSpaceLocalRuntime } from '../lib/localRuntime.js'; + +const ErrorSchema = z.object({ error: z.string(), message: z.string() }); + +export const localModelRoutes: FastifyPluginAsync = async (fastify) => { + const app = fastify.withTypeProvider(); + + app.post( + '/:spaceId/local-models/:action', + { + config: { + authz: { + resource: 'space', + action: 'admin', + spaceIdFrom: 'param', + resourceIdFrom: 'param', + resourceIdParam: 'spaceId', + }, + }, + schema: { + tags: ['Spaces'], + summary: 'Load or unload a model on this computer', + params: z.object({ spaceId: z.string().uuid(), action: z.enum(['warm', 'unload']) }), + body: z.object({ model: z.string().min(1) }), + response: { + 200: z.object({ + model: z.string(), + action: z.enum(['warm', 'unload']), + window: z.number().nullable(), + }), + 400: ErrorSchema, + 404: ErrorSchema, + 409: ErrorSchema, + 502: ErrorSchema, + }, + }, + }, + async (request, reply) => { + if (fastify.edition.edition !== 'community-local') { + return reply.status(404).send({ + error: 'NotFound', + message: + 'Models on this computer belong to the local edition, which runs on the machine they run on.', + }); + } + const tenant = await request.requireTenant(); + await request.requireSpace(); + const { spaceId, action } = request.params; + const model = createDefaultModelCatalog().getModel(request.body.model); + if (model?.provider !== 'ollama') { + return reply.status(400).send({ + error: 'NotLocal', + message: `${request.body.model} is not a model on this computer.`, + }); + } + const runtime = await resolveSpaceLocalRuntime({ + db: fastify.appContext.db as PostgresJsDatabase, + redis: fastify.appContext.redis as Redis | undefined, + tenantId: tenant.tenantId as string, + userId: request.authUser?.userId as string, + spaceId, + }); + if (!runtime) { + return reply.status(409).send({ + error: 'NoConnection', + message: 'This space has no Ollama connection. Add one under Settings → Credentials.', + }); + } + const tag = model.providerModelId ?? model.id; + const window = action === 'warm' ? contextWindowForModel(model, runtime.config) : null; + try { + if (window !== null) await runtime.client.warm(tag, window); + else await runtime.client.unload(tag); + } catch (error) { + return reply.status(502).send({ + error: 'RuntimeError', + message: `Ollama did not ${action === 'warm' ? 'load' : 'unload'} ${tag}: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } + return reply.send({ model: model.id, action, window }); + }, + ); +}; diff --git a/packages/server-runtime/src/routes/spaceLlmReadiness.ts b/packages/server-runtime/src/routes/spaceLlmReadiness.ts index fdfc722d..20aad82d 100644 --- a/packages/server-runtime/src/routes/spaceLlmReadiness.ts +++ b/packages/server-runtime/src/routes/spaceLlmReadiness.ts @@ -16,10 +16,16 @@ import { providerCredentials, } from '@aflow/database'; import { + contextWindowForModel, createDefaultModelCatalog, inferProviderForModelRef, isVerifiableProvider, + localModelFit, + type ModelDefinition, } from '@aflow/ai-client'; +import type { Redis } from '@aflow/redis'; +import { resolveSpaceLocalRuntime, type ResolvedLocalRuntime } from '../lib/localRuntime.js'; +import type { LoadedLocalModel } from '@aflow/ai-client'; import { DEFAULT_CYBERNETIC_MODEL, resolveRoleModel, @@ -44,8 +50,53 @@ const RoleReadinessSchema = z.object({ * check that has not run — one can never be satisfied and the other can. */ verified: z.boolean().nullable(), + /** + * For a model on this computer: the window its requests get and whether the + * model fits the machine at it (Plan 323 D8). Informs and never counts + * against `ready` — a model that spills still answers, slowly. + */ + localFit: z + .object({ + status: z.enum(['fits', 'tight', 'does_not_fit', 'unknown']), + window: z.number().int(), + largestFittingWindow: z.number().int().nullable(), + bytesAtWindow: z.number().nullable(), + gpuWorkingSetBytes: z.number().nullable(), + /** Whether the runtime answered when asked what it holds. */ + runtimeReachable: z.boolean(), + /** The model's size as loaded now — its weights and every conversation's cache — or null when not loaded. */ + loadedBytes: z.number().nullable(), + }) + .nullable(), }); +function localFitFor( + model: ModelDefinition | undefined, + runtime: ResolvedLocalRuntime | null, + loaded: LoadedLocalModel[] | null, +): z.infer['localFit'] { + if (!model?.local || !runtime) return null; + const parallel = runtime.config.parallel ?? 1; + const fit = localModelFit({ + footprint: model.local.footprint, + machine: runtime.config.machine, + catalogWindow: model.contextWindow, + parallel, + window: contextWindowForModel(model, runtime.config), + }); + const tag = model.providerModelId ?? model.id; + const held = loaded?.find((m) => m.tag === tag || m.tag === `${tag}:latest`); + return { + status: fit.status, + window: fit.window, + largestFittingWindow: fit.largestFittingWindow ?? null, + bytesAtWindow: fit.bytesAtWindow ?? null, + gpuWorkingSetBytes: runtime.config.machine?.gpuWorkingSetBytes ?? null, + runtimeReachable: loaded !== null, + loadedBytes: held?.sizeBytes ?? null, + }; +} + /** * What the Clerk resolves to, and whether anything does. * @@ -221,6 +272,17 @@ export const spaceLlmReadinessRoutes: FastifyPluginAsync = async (fastify) => { const errored = new Map(); let ready = true; + const runtime = roleRefs.some((r) => r.providerId === 'ollama') + ? await resolveSpaceLocalRuntime({ + db, + redis: fastify.appContext.redis as Redis | undefined, + tenantId: tenant.tenantId as string, + userId: authUser.userId as string, + spaceId, + }) + : null; + const loaded = runtime ? await runtime.client.loaded() : null; + for (const { role, ref, modelId, providerId, unknownModel } of roleRefs) { const preview = providerId ? previews.get(providerId) : undefined; const resolved = preview?.resolved ?? false; @@ -237,6 +299,7 @@ export const spaceLlmReadinessRoutes: FastifyPluginAsync = async (fastify) => { providerId, resolved, resolvedScope: preview?.resolvedScope ?? null, + localFit: localFitFor(modelId ? catalog.getModel(modelId) : undefined, runtime, loaded), status, verified, }; diff --git a/packages/web-product/src/ui/components/cybernetic/AgentModelSettings.tsx b/packages/web-product/src/ui/components/cybernetic/AgentModelSettings.tsx index aa5851ba..8cc8efa1 100644 --- a/packages/web-product/src/ui/components/cybernetic/AgentModelSettings.tsx +++ b/packages/web-product/src/ui/components/cybernetic/AgentModelSettings.tsx @@ -8,7 +8,8 @@ import { type EntityDirectives, } from '@aflow/schemas'; import { useChatModelOptions } from '../../hooks/useChatModelOptions.js'; -import { useSpaceLlmReadiness } from '../../hooks/useSpaceLlmReadiness.js'; +import { localFitForDraft, useSpaceLlmReadiness } from '../../hooks/useSpaceLlmReadiness.js'; +import { LocalModelStatus } from './LocalModelStatus.js'; import { useAgentDirectives } from '../../hooks/useAgentDirectives.js'; import { ConnectProviderDialog } from '../credentials/ConnectProviderDialog.js'; import { @@ -109,6 +110,10 @@ export function AgentModelSettings({ return unready ? provider : null; }} onConnectProvider={setConnectProvider} + renderLocalStatus={(role) => { + const local = localFitForDraft(readiness, directives.modelDefaults)(role); + return local ? : null; + }} /> )} diff --git a/packages/web-product/src/ui/components/cybernetic/LocalModelStatus.tsx b/packages/web-product/src/ui/components/cybernetic/LocalModelStatus.tsx new file mode 100644 index 00000000..056269c3 --- /dev/null +++ b/packages/web-product/src/ui/components/cybernetic/LocalModelStatus.tsx @@ -0,0 +1,78 @@ +'use client'; + +import { Button, Row, Text } from '@aflow/design-system'; +import { useApiMutation } from '../../hooks/useApiQuery.js'; +import { llmReadinessQueryKey, type LocalFit } from '../../hooks/useSpaceLlmReadiness.js'; +import { describeLocalFit } from './ModelDefaultsEditor.js'; + +const gb = (bytes: number): string => `${(bytes / 1e9).toFixed(1)} GB`; + +/** + * What a model on this computer is doing right now, and the operator's two + * actions on it: load it before the first request has to, or unload it to give + * the machine its memory back (Plan 323 D8, D10). A loaded model holds every + * conversation's cache it has read, so the loaded size is shown beside what + * the GPU allows — past it the model is spilling, which is the slowness. + */ +export function LocalModelStatus({ + spaceId, + model, + fit, +}: { + spaceId: string; + model: string; + fit: LocalFit; +}) { + const action = useApiMutation<{ action: 'warm' | 'unload' }>({ + path: ({ action: which }) => `/spaces/${spaceId}/local-models/${which}`, + serialize: () => JSON.stringify({ model }), + spaceId, + invalidate: [llmReadinessQueryKey(spaceId)], + }); + const spilling = + fit.loadedBytes !== null && + fit.gpuWorkingSetBytes !== null && + fit.loadedBytes > fit.gpuWorkingSetBytes; + + return ( +
+ + {describeLocalFit(fit)} + + + + {!fit.runtimeReachable + ? 'Ollama is not answering on this computer.' + : fit.loadedBytes === null + ? 'Not loaded — the first request loads it, which takes a minute or more.' + : spilling + ? `Loaded at ${gb(fit.loadedBytes)}, past the ${gb(fit.gpuWorkingSetBytes ?? 0)} the GPU allows: it holds the cache of every conversation it has read, and is spilling. Unloading frees it.` + : `Loaded, ${gb(fit.loadedBytes)}.`} + + {fit.runtimeReachable && ( + + )} + +
+ ); +} diff --git a/packages/web-product/src/ui/components/cybernetic/ModelDefaultsEditor.tsx b/packages/web-product/src/ui/components/cybernetic/ModelDefaultsEditor.tsx index 1d538ae8..0c2cea1b 100644 --- a/packages/web-product/src/ui/components/cybernetic/ModelDefaultsEditor.tsx +++ b/packages/web-product/src/ui/components/cybernetic/ModelDefaultsEditor.tsx @@ -1,6 +1,6 @@ 'use client'; -import type { CSSProperties } from 'react'; +import type { CSSProperties, ReactNode } from 'react'; import { Checkbox, Column, @@ -18,7 +18,7 @@ import type { DirectiveReasoningEffort, } from '@aflow/schemas'; import type { ModelOption } from '../../hooks/useChatModelOptions.js'; -import type { ClerkReadiness } from '../../hooks/useSpaceLlmReadiness.js'; +import type { ClerkReadiness, LocalFit } from '../../hooks/useSpaceLlmReadiness.js'; export type ModelRole = 'default' | 'helmsman' | 'runner' | 'coach' | 'judge' | 'clerk'; @@ -127,6 +127,36 @@ interface ModelDefaultsEditorProps { missingProviderForRef?: (ref: string) => string | null; /** Opens key entry for a provider surfaced by `missingProviderForRef`. */ onConnectProvider?: (providerId: string) => void; + /** For a role on a model on this computer: its window, how it fits, and what it holds now. */ + renderLocalStatus?: (role: ModelRole) => ReactNode; +} + +const gb = (bytes: number): string => `${(bytes / 1e9).toFixed(1)} GB`; +/** The window the platform's agents need (`MIN_USEFUL_WINDOW_TOKENS` in @aflow/ai-client). */ +const AGENT_WINDOW_TOKENS = 32_768; +const tokens = (n: number): string => `${Math.round(n / 1024)}K`; + +/** What a role on a model on this computer should expect, in the operator's terms. */ +export function describeLocalFit(fit: LocalFit): string { + const gpu = fit.gpuWorkingSetBytes; + switch (fit.status) { + case 'fits': + return `Runs on this computer with a ${tokens(fit.window)} window.`; + case 'tight': + return `${tokens(fit.window)} window, ${gb(fit.bytesAtWindow ?? 0)} of this computer's ${gb(gpu ?? 0)} of GPU memory — expect it to slow down while other apps need memory.`; + case 'does_not_fit': + return `Needs ${gb(fit.bytesAtWindow ?? 0)} at a ${tokens(fit.window)} window, more than this computer's ${gb(gpu ?? 0)} of GPU memory, so it will spill and run slowly.${ + fit.largestFittingWindow + ? ` This computer fits a ${tokens(fit.largestFittingWindow)} window${ + fit.largestFittingWindow < AGENT_WINDOW_TOKENS + ? ', smaller than the platform’s agents need; a machine with more memory, or a smaller model, runs them at full speed.' + : '.' + }` + : '' + }`; + case 'unknown': + return `${tokens(fit.window)} window — this computer's GPU memory has not been read yet, so the window is not sized to it.`; + } } /** @@ -151,6 +181,7 @@ export function ModelDefaultsEditor({ disabled = false, missingProviderForRef, onConnectProvider, + renderLocalStatus, }: ModelDefaultsEditorProps) { if (loading) { return Loading models…; @@ -226,6 +257,7 @@ export function ModelDefaultsEditor({ Needs key — connect )} + {renderLocalStatus?.(role)} ); }; diff --git a/packages/web-product/src/ui/components/cybernetic/localFit.test.ts b/packages/web-product/src/ui/components/cybernetic/localFit.test.ts new file mode 100644 index 00000000..57f913b2 --- /dev/null +++ b/packages/web-product/src/ui/components/cybernetic/localFit.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest'; +import { describeLocalFit } from './ModelDefaultsEditor.js'; +import { localFitForDraft, type SpaceLlmReadiness } from '../../hooks/useSpaceLlmReadiness.js'; + +const TIGHT = { + status: 'tight' as const, + window: 16_384, + largestFittingWindow: 16_384, + bytesAtWindow: 18_146_750_828, + gpuWorkingSetBytes: 19_069_665_280, + runtimeReachable: true, + loadedBytes: null, +}; + +describe('describeLocalFit', () => { + it('names the window and the memory it takes against what the computer has', () => { + expect(describeLocalFit(TIGHT)).toBe( + "16K window, 18.1 GB of this computer's 19.1 GB of GPU memory — expect it to slow down while other apps need memory.", + ); + }); + + it('says what the computer fits when the window in use spills, and that the agents need more', () => { + const text = describeLocalFit({ + ...TIGHT, + status: 'does_not_fit', + window: 32_768, + bytesAtWindow: 20_112_910_828, + }); + expect(text).toContain('so it will spill and run slowly'); + expect(text).toContain('fits a 16K window, smaller than the platform’s agents need'); + }); +}); + +describe('localFitForDraft', () => { + const readiness = { + roles: { default: { model: 'gemma-local', localFit: TIGHT } }, + } as unknown as SpaceLlmReadiness; + + it('describes the model readiness measured', () => { + expect(localFitForDraft(readiness, { default: 'gemma-local' })('default')).toEqual({ + model: 'gemma-local', + fit: TIGHT, + }); + }); + + it('says nothing about a model the draft has just replaced', () => { + expect(localFitForDraft(readiness, { default: 'sonnet' })('default')).toBeNull(); + }); +}); diff --git a/packages/web-product/src/ui/hooks/useSpaceLlmReadiness.ts b/packages/web-product/src/ui/hooks/useSpaceLlmReadiness.ts index 8377bbb4..3b0b69be 100644 --- a/packages/web-product/src/ui/hooks/useSpaceLlmReadiness.ts +++ b/packages/web-product/src/ui/hooks/useSpaceLlmReadiness.ts @@ -11,6 +11,20 @@ export interface RoleReadiness { status: 'active' | 'error' | null; /** `null` where the provider offers no live check. */ verified: boolean | null; + /** For a model on this computer, the window its requests get and how it fits the machine. */ + localFit: LocalFit | null; +} + +export interface LocalFit { + status: 'fits' | 'tight' | 'does_not_fit' | 'unknown'; + window: number; + largestFittingWindow: number | null; + bytesAtWindow: number | null; + gpuWorkingSetBytes: number | null; + /** Whether the runtime answered when asked what it holds. */ + runtimeReachable: boolean; + /** The model's size as loaded now, or null when it is not loaded. */ + loadedBytes: number | null; } /** @@ -56,6 +70,24 @@ export function llmReadinessQueryKey(spaceId: string): readonly unknown[] { * Standing LLM readiness for a space, recomputed server-side at read for the * calling user (user-scoped credentials participate in resolution). */ +/** + * A role's local fit, as long as the model being edited is the one readiness + * measured: readiness describes the saved assignment, and a note about the + * model a draft has just replaced would describe the wrong one. + */ +export function localFitForDraft( + readiness: SpaceLlmReadiness | null | undefined, + modelDefaults: { default?: string | undefined } & Record, +): (role: string) => { model: string; fit: LocalFit } | null { + return (role) => { + const measured = readiness?.roles[role]; + const editing = modelDefaults[role] ?? modelDefaults.default; + return measured?.localFit && measured.model === editing + ? { model: measured.model, fit: measured.localFit } + : null; + }; +} + export function useSpaceLlmReadiness(spaceId: string | null): { readiness: SpaceLlmReadiness | null; isLoading: boolean; diff --git a/packages/web-product/src/ui/screens/space-settings-agent.tsx b/packages/web-product/src/ui/screens/space-settings-agent.tsx index 91233939..484837a3 100644 --- a/packages/web-product/src/ui/screens/space-settings-agent.tsx +++ b/packages/web-product/src/ui/screens/space-settings-agent.tsx @@ -53,7 +53,8 @@ import { withHelmsmanOperations, } from '../components/cybernetic/HelmsmanDiscoveryEditor.js'; import { spaceRoute } from '../lib/space-routes.js'; -import { useSpaceLlmReadiness } from '../hooks/useSpaceLlmReadiness.js'; +import { localFitForDraft, useSpaceLlmReadiness } from '../hooks/useSpaceLlmReadiness.js'; +import { LocalModelStatus } from '../components/cybernetic/LocalModelStatus.js'; // ============================================================================ // Constants @@ -940,6 +941,10 @@ function ModelDefaultsSection({ onReasoningChange={onReasoningChange} variant="page" labelStyle={formLabelStyle} + renderLocalStatus={(role) => { + const local = localFitForDraft(readiness, draft.modelDefaults)(role); + return local ? : null; + }} /> ); } diff --git a/scripts/verify-local-models.ts b/scripts/verify-local-models.ts index 39363dc0..3ef4d63a 100644 --- a/scripts/verify-local-models.ts +++ b/scripts/verify-local-models.ts @@ -18,13 +18,24 @@ import { createDefaultModelCatalog, type LocalModelBuild, + type LocalModelFootprint, type ModelDefinition, } from '@aflow/ai-client'; import { OLLAMA_DEFAULT_BASE_URL } from '@aflow/schemas'; const BASE = (process.env['OLLAMA_URL'] ?? OLLAMA_DEFAULT_BASE_URL).replace(/\/+$/, ''); /** Bumped whenever a probe below changes what it measures. */ -const PROBE_VERSION = 1; +const PROBE_VERSION = 2; +/** Weights are read off the runtime's own report, so they repeat to within a rounding. */ +const WEIGHTS_TOLERANCE = 0.02; +/** + * The cache is allocated in blocks, so the per-token slope between two prompt + * sizes moves a few percent from run to run (§9 G6). A change past this is a + * different cache type or runner, not noise. + */ +const KV_TOLERANCE = 0.15; +/** Long enough that the slope between them is the cache and not its first block. */ +const FOOTPRINT_PROMPTS = [300, 1200]; const WEATHER_TOOL = { type: 'function', @@ -41,6 +52,7 @@ const WEATHER_TOOL = { interface Measured { build: LocalModelBuild; + footprint: LocalModelFootprint | null; capabilities: { vision: boolean; functionCalling: boolean; @@ -110,6 +122,48 @@ async function streamedToolCall(tag: string): Promise { }); } +async function loadedSize(tag: string): Promise { + const ps = (await (await fetch(`${BASE}/api/ps`)).json()) as { + models: Array<{ name: string; size: number }>; + }; + return ps.models.find((m) => m.name === tag || m.name === `${tag}:latest`)?.size ?? null; +} + +/** + * Weights from the loaded size before any context, and the per-token cost as + * the slope between two prompt sizes: the runtime reports a loaded model's + * size as its cache grows, and the cache grows with what it has read. + */ +async function measureFootprint(tag: string): Promise { + await post('/api/generate', { model: tag, keep_alive: 0 }); + const ask = async (items: number) => + chat(tag, { + options: { num_ctx: 32_768, num_predict: 1, temperature: 0 }, + messages: [ + { + role: 'user', + content: `${Array.from({ length: items }, (_, i) => `Item ${i} is a blue marble.`).join(' ')} Say ok.`, + }, + ], + }); + await ask(1); + const weights = await loadedSize(tag); + const points: Array<{ tokens: number; size: number }> = []; + for (const items of FOOTPRINT_PROMPTS) { + const res = await ask(items); + const size = await loadedSize(tag); + const tokens = res.json['prompt_eval_count']; + if (size === null || typeof tokens !== 'number') return null; + points.push({ tokens, size }); + } + const [a, b] = points; + if (weights === null || !a || !b || b.tokens <= a.tokens) return null; + return { + weightsBytes: weights, + kvBytesPerToken: Math.round((b.size - a.size) / (b.tokens - a.tokens)), + }; +} + async function measure(tag: string): Promise { const tags = (await (await fetch(`${BASE}/api/tags`)).json()) as { models: Array<{ name: string; digest: string; remote_host?: string }>; @@ -161,6 +215,7 @@ async function measure(tag: string): Promise { probeVersion: PROBE_VERSION, measuredAt: new Date().toISOString().slice(0, 10), }, + footprint: await measureFootprint(tag), capabilities: { vision: caps.has('vision'), functionCalling: toolCalls(single.json).some((c) => c.function?.name === 'get_weather'), @@ -205,6 +260,34 @@ function drift(entry: ModelDefinition, m: Measured): string[] { expect('local.quantization', entry.local?.quantization, m.build.quantization); expect('local.runtimeVersion', entry.local?.runtimeVersion, m.build.runtimeVersion); expect('local.probeVersion', entry.local?.probeVersion, m.build.probeVersion); + const near = ( + label: string, + catalog: number | undefined, + measured: number | undefined, + tolerance: number, + ) => { + if ( + catalog === undefined || + measured === undefined || + Math.abs(measured - catalog) > catalog * tolerance + ) { + found.push( + `${label}: catalog ${String(catalog)}, measured ${String(measured)} (±${tolerance * 100}%)`, + ); + } + }; + near( + 'local.footprint.weightsBytes', + entry.local?.footprint?.weightsBytes, + m.footprint?.weightsBytes, + WEIGHTS_TOLERANCE, + ); + near( + 'local.footprint.kvBytesPerToken', + entry.local?.footprint?.kvBytesPerToken, + m.footprint?.kvBytesPerToken, + KV_TOLERANCE, + ); return found; } @@ -216,6 +299,8 @@ async function main(): Promise { let failed = false; for (const tag of tags) { const measured = await measure(tag); + // Unloaded after, so the probe leaves the machine's memory as it found it. + if (measured) await post('/api/generate', { model: tag, keep_alive: 0 }); if (!measured) { console.log(`SKIPPED ${tag}: not installed (ollama pull ${tag})`); continue;