diff --git a/docs/plans/323-models-that-run-on-this-computer.md b/docs/plans/323-models-that-run-on-this-computer.md index e9afcb59..a24ac676 100644 --- a/docs/plans/323-models-that-run-on-this-computer.md +++ b/docs/plans/323-models-that-run-on-this-computer.md @@ -330,6 +330,8 @@ Each lineup entry (§4) is qualified on its own: both probe tiers, the build ide _Exit for each entry:_ its usefulness results, repeated runs included, are in the entry, and its recommended roles follow from them. +_Entries so far:_ `ollama-qwen3.5-9b` (`qwen-local`) is in the catalog from its compatibility tier alone — native single, parallel and streamed tool calls, vision, `off`/`high` reasoning, no `format` on MLX, 7.7 GB of weights and about 69 KB of cache per token at `q8_0`, so about 10 GB at the agents' 32K window. Like every local entry it is assignable under D18 and recommended for no role; the usefulness tier, which is not built, decides that. Both entries' probes run through `localRuntimeFetch`: Gemma's took 8½ minutes, past the transport's own five. + ### P3a — The window fits the machine (D8) **Build:** diff --git a/packages/ai-client/src/catalogModels.ts b/packages/ai-client/src/catalogModels.ts index 86fa9f9e..21772fb7 100644 --- a/packages/ai-client/src/catalogModels.ts +++ b/packages/ai-client/src/catalogModels.ts @@ -98,6 +98,41 @@ export const builtInModels: ModelDefinition[] = [ downloadBytes: 16_146_867_483, }, }, + { + id: 'ollama-qwen3.5-9b', + provider: 'ollama', + providerModelId: 'qwen3.5:9b', + displayName: 'Qwen3.5 9B (this computer)', + description: + "Alibaba's Qwen3.5 9B, dense, served by Ollama on this computer. Vision, native tool calls, on/off thinking; fits a 16 GB machine at the agents' window. Apache 2.0.", + contextWindow: 262_144, + maxOutputTokens: 8192, + capabilities: { + ...defaultCapabilities, + vision: true, + reasoning: true, + // The MLX runner refuses `format`; JSON rides a tool call instead. + structuredOutputs: false, + // The runtime has no tool_choice. + forcedToolChoice: false, + }, + pricing: { promptPer1M: 0, completionPer1M: 0, currency: 'USD' }, + traits: { speed: 2, cost: 1, intelligence: 2, outputType: 'text' }, + aliases: ['qwen-local'], + reasoning: { supported: ['off', 'high'] }, + local: { + runtimeTag: 'qwen3.5:9b', + digest: 'd63f830f28d515fefea546f213fca7a75e9dd5176ef51a6deabc952d1c99a911', + runner: 'mlx', + quantization: 'nvfp4', + runtimeVersion: '0.40.5', + probeVersion: 2, + measuredAt: '2026-10-17', + // Measured at OLLAMA_KV_CACHE_TYPE=q8_0 (§9 G6). + footprint: { weightsBytes: 7_735_695_816, kvBytesPerToken: 68_742 }, + downloadBytes: 7_639_229_431, + }, + }, // ============================================================================ // OpenAI Text Models diff --git a/packages/ai-client/src/index.ts b/packages/ai-client/src/index.ts index db817109..aea27d4e 100644 --- a/packages/ai-client/src/index.ts +++ b/packages/ai-client/src/index.ts @@ -205,3 +205,4 @@ export { type LocalRuntimeClient, type LoadedLocalModel, } from './providers/ollamaRuntime.js'; +export { localRuntimeFetch } from './providers/localRuntimeFetch.js'; diff --git a/packages/ai-client/src/providers/localRuntimeFetch.ts b/packages/ai-client/src/providers/localRuntimeFetch.ts index 5b17ccc8..9bf3e0dd 100644 --- a/packages/ai-client/src/providers/localRuntimeFetch.ts +++ b/packages/ai-client/src/providers/localRuntimeFetch.ts @@ -14,6 +14,6 @@ import { Agent } from 'undici'; const NO_TRANSPORT_LIMIT = new Agent({ headersTimeout: 0, bodyTimeout: 0 }); -export function localRuntimeFetch(url: string, init: RequestInit): Promise { +export function localRuntimeFetch(url: string, init: RequestInit = {}): Promise { return fetch(url, { ...init, dispatcher: NO_TRANSPORT_LIMIT } as RequestInit); } diff --git a/scripts/local-model-gates.ts b/scripts/local-model-gates.ts index 01719dda..12b8b77f 100644 --- a/scripts/local-model-gates.ts +++ b/scripts/local-model-gates.ts @@ -19,6 +19,7 @@ * `lifecycle` and `embeddings` pull small models (about 1–3 GB) and delete * what they pulled. `machine` loads every model it is given, one at a time. */ +import { localRuntimeFetch } from '@aflow/ai-client'; import { writeFileSync } from 'node:fs'; import { execFileSync } from 'node:child_process'; import os from 'node:os'; @@ -51,7 +52,7 @@ const describe = (e: unknown): string => const seconds = (ns: unknown): number => (typeof ns === 'number' ? Math.round(ns / 1e7) / 100 : 0); async function api(path: string, body?: unknown, signal?: AbortSignal): Promise { - return fetch(`${BASE}${path}`, { + return localRuntimeFetch(`${BASE}${path}`, { method: body === undefined ? 'GET' : 'POST', headers: { 'content-type': 'application/json' }, ...(body === undefined ? {} : { body: JSON.stringify(body) }), @@ -161,7 +162,7 @@ async function pull( } async function remove(model: string): Promise { - const res = await fetch(`${BASE}/api/delete`, { + const res = await localRuntimeFetch(`${BASE}/api/delete`, { method: 'DELETE', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ model }), diff --git a/scripts/verify-local-models.ts b/scripts/verify-local-models.ts index 3ef4d63a..694ba617 100644 --- a/scripts/verify-local-models.ts +++ b/scripts/verify-local-models.ts @@ -17,6 +17,7 @@ */ import { createDefaultModelCatalog, + localRuntimeFetch, type LocalModelBuild, type LocalModelFootprint, type ModelDefinition, @@ -69,7 +70,7 @@ async function post( path: string, body: unknown, ): Promise<{ status: number; json: Record }> { - const res = await fetch(`${BASE}${path}`, { + const res = await localRuntimeFetch(`${BASE}${path}`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body), @@ -103,7 +104,7 @@ function toolCalls( } async function streamedToolCall(tag: string): Promise { - const res = await fetch(`${BASE}/api/chat`, { + const res = await localRuntimeFetch(`${BASE}/api/chat`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ @@ -123,7 +124,7 @@ async function streamedToolCall(tag: string): Promise { } async function loadedSize(tag: string): Promise { - const ps = (await (await fetch(`${BASE}/api/ps`)).json()) as { + const ps = (await (await localRuntimeFetch(`${BASE}/api/ps`)).json()) as { models: Array<{ name: string; size: number }>; }; return ps.models.find((m) => m.name === tag || m.name === `${tag}:latest`)?.size ?? null; @@ -165,14 +166,16 @@ async function measureFootprint(tag: string): Promise { - const tags = (await (await fetch(`${BASE}/api/tags`)).json()) as { + const tags = (await (await localRuntimeFetch(`${BASE}/api/tags`)).json()) as { models: Array<{ name: string; digest: string; remote_host?: string }>; }; const installed = tags.models.find((m) => m.name === tag || m.name === `${tag}:latest`); if (!installed) return null; if (installed.remote_host) throw new Error(`${tag} is served by ${installed.remote_host}, not this computer`); - const version = (await (await fetch(`${BASE}/api/version`)).json()) as { version: string }; + const version = (await (await localRuntimeFetch(`${BASE}/api/version`)).json()) as { + version: string; + }; const show = (await post('/api/show', { model: tag })).json; const details = (show['details'] ?? {}) as Record; const caps = new Set((show['capabilities'] as string[] | undefined) ?? []);