From 899e39ce604cae3f5ccf4e94b0972c3c7527dd3f Mon Sep 17 00:00:00 2001 From: karim Date: Sat, 17 Oct 2026 13:44:20 +0200 Subject: [PATCH] fix(323): a long prompt on a local model no longer reads as an unreachable runtime Node's fetch abandons a response whose headers take five minutes. Ollama sends none until it has read the whole prompt, and Helmsman's prompt on Gemma took longer than that on a 24 GB machine: each attempt failed at 300 s as "Ollama is not reachable", three times, well inside the 15-minute idle window the platform gives a local model. - Every request to the local runtime goes through localRuntimeFetch, an undici Agent with no header or body limit, so the caller's signal is the only limit. The pull stream needed the same: the runtime is silent while it checks a large layer's digest. - "Not reachable" only for causes that mean nothing answers at the URL; any other broken request names its cause. - Silence through the whole idle window fails without a retry, naming the remedy (a smaller window or model), since a retry re-reads the prompt. - The spill estimate says the model will spill as its conversations fill the window, not that it is spilling. Co-Authored-By: Claude Opus 5.5 --- .../src/handlers/ai/handlers/agentTurn.ts | 2 +- .../323-models-that-run-on-this-computer.md | 2 +- packages/ai-client/package.json | 1 + .../ai-client/src/localRuntimeDiagnostics.ts | 2 +- .../src/providers/localRuntimeFetch.ts | 19 +++++ .../ai-client/src/providers/ollama.test.ts | 72 +++++++++++++++++-- packages/ai-client/src/providers/ollama.ts | 64 +++++++++++++++-- .../ai-client/src/providers/ollamaRuntime.ts | 9 +-- yarn.lock | 8 +++ 9 files changed, 159 insertions(+), 20 deletions(-) create mode 100644 packages/ai-client/src/providers/localRuntimeFetch.ts diff --git a/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.ts b/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.ts index c3cb275f..502f050e 100644 --- a/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.ts +++ b/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.ts @@ -472,7 +472,7 @@ export async function handleAgentTurn( // provider's fault — shortening the turn is the wrong medicine and wastes // a retry fighting a transient. Only a ceiling (or flat-clock) cut means // the turn itself was too large. - if (error instanceof AIClientError && error.code === 'timeout') { + if (error instanceof AIClientError && error.code === 'timeout' && error.retryable) { error.message += error.message.includes('stream stalled') ? ' The provider stopped sending chunks — this is transient; simply retry the same turn.' : ' The generation was cut mid-turn. On retry, keep this turn SHORT: brief reasoning,' + diff --git a/docs/plans/323-models-that-run-on-this-computer.md b/docs/plans/323-models-that-run-on-this-computer.md index d8ebb960..e9afcb59 100644 --- a/docs/plans/323-models-that-run-on-this-computer.md +++ b/docs/plans/323-models-that-run-on-this-computer.md @@ -61,7 +61,7 @@ Using one should look like using any other model. You choose it in the same pick - **Leases:** each lease has an expiry that its holder heartbeats. A worker that dies frees its lease when the lease expires, with no sweep. A cancelled step leaves the queue and releases what it holds. - **A step waits before it starts, by the rules of an operation slot.** The executor runtime gains an optional `StepHandler.admit`: the handler names what the step must hold from its own input, after the context is built and before the step is marked started. While it waits the step stays SCHEDULED, gives back its executor slot, and keeps its in-flight record alive; a cancel ends the wait; an executor that stops claiming gives the step back to its stream; and a step that waited is checked again for supersession and cancellation once admitted. Its timeout starts only after admission, so queued local work is never charged to the work and never holds up cloud work. The AI handler's `admit` takes the runtime lease when the step's model is served by Ollama, and asks nothing otherwise. _Rejected:_ `admitOperation` as it stands. Its limiter is keyed by operation id and is in-process, while the runtime a step needs is known only from its model and credential, and its capacity is shared across processes. - **Callers outside a step** take the same lease through the client: `createByokAiClientFactory(db, admission)` wraps a client for an Ollama model so every request holds a lease while it runs. Session titles pass `wait: false` and decline when the runtime is busy, since the conversation is due again on the next cycle; post-run judging waits. The eval-batch engine, the UI and API executors' generation and the memory embedder are not yet admitted, and are P5's coverage list. -- **The idle clock:** every AI operation on a local model runs under an idle window of `AI_LOCAL_MODEL_IDLE_TIMEOUT_MS` (15 min) and the agent turn's ceiling, in place of the flat 120 s or the 300 s stream idle window. Prefill is a silent stretch on every local request, and at 250 tokens a second a 75K-token prompt outlasts the cloud window (§9 G5). _Rejected:_ a per-connection first-token timeout. The timeout is chosen before any credential is read, so a connection field would be a setting nothing reads. +- **The idle clock:** every AI operation on a local model runs under an idle window of `AI_LOCAL_MODEL_IDLE_TIMEOUT_MS` (15 min) and the agent turn's ceiling, in place of the flat 120 s or the 300 s stream idle window. Prefill is a silent stretch on every local request, and at 250 tokens a second a 75K-token prompt outlasts the cloud window (§9 G5). _Rejected:_ a per-connection first-token timeout. The timeout is chosen before any credential is read, so a connection field would be a setting nothing reads. Node's fetch carries limits of its own under that window — five minutes for a response's headers and five of silence in its body — and a local runtime sends no headers until it has read the prompt, so every request to it goes through a dispatcher with neither (`localRuntimeFetch`), and the caller's signal is the only limit. Silence through the whole idle window fails the turn without a retry: the runtime was still reading the prompt, and a retry reads it again from the start. **D8 — The window is sized from the machine, and fit is an estimate until the model is loaded.** diff --git a/packages/ai-client/package.json b/packages/ai-client/package.json index 818c7160..67f93b87 100644 --- a/packages/ai-client/package.json +++ b/packages/ai-client/package.json @@ -29,6 +29,7 @@ "@anthropic-ai/sdk": "^0.133.0", "@google/genai": "2.28.0", "openai": "^6.49.0", + "undici": "^7.30.1", "zod": "^3.24.4", "zod-to-json-schema": "^3.25.2" }, diff --git a/packages/ai-client/src/localRuntimeDiagnostics.ts b/packages/ai-client/src/localRuntimeDiagnostics.ts index 3fde1f68..6f0e116d 100644 --- a/packages/ai-client/src/localRuntimeDiagnostics.ts +++ b/packages/ai-client/src/localRuntimeDiagnostics.ts @@ -233,7 +233,7 @@ function issuesFor( m.fit.largestFittingWindow !== null && m.fit.largestFittingWindow >= MIN_USEFUL_WINDOW_TOKENS ? `At a ${k(m.fit.window)} window ${m.displayName} takes ${gb(m.fit.bytesAtWindow ?? 0)}, past the ${gb(gpu ?? 0)} the GPU allows; it fits ${k(m.fit.largestFittingWindow)}.` - : `${m.displayName} takes ${gb(m.fit.bytesAtWindow ?? 0)} at the ${k(m.fit.window)} window the agents need, past the ${gb(gpu ?? 0)} this computer's GPU allows, so it spills and runs slowly.`, + : `${m.displayName} takes ${gb(m.fit.bytesAtWindow ?? 0)} at the ${k(m.fit.window)} window the agents need, past the ${gb(gpu ?? 0)} this computer's GPU allows, so it will spill and run slowly as its conversations fill the window.`, remedy: automaticWindow(m) ?? (smaller diff --git a/packages/ai-client/src/providers/localRuntimeFetch.ts b/packages/ai-client/src/providers/localRuntimeFetch.ts new file mode 100644 index 00000000..5b17ccc8 --- /dev/null +++ b/packages/ai-client/src/providers/localRuntimeFetch.ts @@ -0,0 +1,19 @@ +/** + * Every request to a model runtime on the operator's machine goes through this + * fetch (Plan 323 §9 G5). + * + * Node's fetch abandons a response whose headers take five minutes and a body + * silent for five. A local runtime sends no headers until it has read the + * whole prompt, and nothing while it checks a downloaded layer's digest, and + * on a laptop either outlasts five minutes — the cut surfaced as an + * unreachable runtime, and every retry read the prompt again from the start. + * The caller's signal is the only limit: the platform's idle window for a + * local model on a turn, the lease on a download. + */ +import { Agent } from 'undici'; + +const NO_TRANSPORT_LIMIT = new Agent({ headersTimeout: 0, bodyTimeout: 0 }); + +export function localRuntimeFetch(url: string, init: RequestInit): Promise { + return fetch(url, { ...init, dispatcher: NO_TRANSPORT_LIMIT } as RequestInit); +} diff --git a/packages/ai-client/src/providers/ollama.test.ts b/packages/ai-client/src/providers/ollama.test.ts index 6fe763f2..20d91338 100644 --- a/packages/ai-client/src/providers/ollama.test.ts +++ b/packages/ai-client/src/providers/ollama.test.ts @@ -434,22 +434,82 @@ describe('embeddings', () => { }); describe('failures', () => { - it('names the URL when the runtime is not answering', async () => { + const failWith = (error: unknown) => vi.stubGlobal( 'fetch', vi.fn(async () => { - throw new TypeError('fetch failed'); + throw error; + }), + ); + const ask = ( + adapter = createOllamaAdapter({ baseUrl: 'http://127.0.0.1:11434/' }), + signal?: AbortSignal, + ) => + rejection( + adapter.generateText({ + ...context, + model: 'm', + messages: [{ role: 'user', content: 'x' }], + ...(signal ? { signal } : {}), }), ); - const adapter = createOllamaAdapter({ baseUrl: 'http://127.0.0.1:11434/' }); - const error = await rejection( - adapter.generateText({ ...context, model: 'm', messages: [{ role: 'user', content: 'x' }] }), + it('names the URL when nothing answers there', async () => { + failWith( + new TypeError('fetch failed', { + cause: { code: 'ECONNREFUSED', message: 'connect ECONNREFUSED' }, + }), ); + const error = await ask(); + expect(error.code).toBe('network_error'); expect(error.retryable).toBe(true); - expect(error.message).toContain('http://127.0.0.1:11434'); + expect(error.message).toContain('not reachable at http://127.0.0.1:11434'); + }); + + it('reports a request that broke off by its cause, not as an unreachable runtime', async () => { + failWith( + new TypeError('fetch failed', { + cause: { code: 'ECONNRESET', message: 'other side closed' }, + }), + ); + + const error = await ask(); + + expect(error.message).not.toContain('not reachable'); + expect(error.message).toContain('other side closed (ECONNRESET)'); + }); + + it('waits on the runtime with no transport limit of its own, so only the caller’s signal ends a long prompt', async () => { + const seen: RequestInit[] = []; + vi.stubGlobal( + 'fetch', + vi.fn(async (_url: string, init: RequestInit) => { + seen.push(init); + return done({ content: 'ok' }, { prompt: 5, eval: 1 })(); + }), + ); + + await createOllamaAdapter({}).generateText({ + ...context, + model: 'm', + messages: [{ role: 'user', content: 'x' }], + }); + + expect((seen[0] as { dispatcher?: unknown }).dispatcher).toBeDefined(); + }); + + it('does not retry a runtime that stayed silent through the idle window: a retry reads the prompt again', async () => { + const controller = new AbortController(); + controller.abort({ marker: 'phoenix.executor.timeout', timeoutMs: 900_000, kind: 'idle' }); + failWith(new DOMException('This operation was aborted', 'AbortError')); + + const error = await ask(createOllamaAdapter({}), controller.signal); + + expect(error.code).toBe('timeout'); + expect(error.retryable).toBe(false); + expect(error.message).toContain('sent nothing for 15 minutes'); }); it('tells the operator how to download a model the runtime does not have', async () => { diff --git a/packages/ai-client/src/providers/ollama.ts b/packages/ai-client/src/providers/ollama.ts index dfa5b335..6e4fc76f 100644 --- a/packages/ai-client/src/providers/ollama.ts +++ b/packages/ai-client/src/providers/ollama.ts @@ -30,6 +30,7 @@ import { moveToolImagesToUserMessages, toolResultText } from './toolResultConten import { parseJsonResponse } from './jsonResponseParse.js'; import { MIN_ANSWER_TOKENS, sizePrompt, type PromptSizing } from './ollamaPromptSize.js'; import { localRuntimeClient, servesWithoutLoading } from './ollamaRuntime.js'; +import { localRuntimeFetch } from './localRuntimeFetch.js'; interface Fit { sizing: PromptSizing; @@ -183,26 +184,63 @@ function thinkFor(reasoning: AIRequestOptions['reasoning']): boolean | string | return effort === 'off' ? false : effort; } -function isPlatformTimeoutAbort(signal: AbortSignal | undefined): { timeoutMs: number } | null { +function isPlatformTimeoutAbort( + signal: AbortSignal | undefined, +): { timeoutMs: number; kind: string | undefined } | null { if (!signal?.aborted) return null; - const reason = signal.reason as { marker?: unknown; timeoutMs?: unknown } | undefined; + const reason = signal.reason as + { marker?: unknown; timeoutMs?: unknown; kind?: unknown } | undefined; if (reason?.marker === 'phoenix.executor.timeout' && typeof reason.timeoutMs === 'number') { - return { timeoutMs: reason.timeoutMs }; + return { + timeoutMs: reason.timeoutMs, + kind: typeof reason.kind === 'string' ? reason.kind : undefined, + }; } return null; } +/** The causes that mean nothing answers at the URL, as opposed to a request that broke. */ +const UNREACHABLE_CODES = new Set([ + 'ECONNREFUSED', + 'ENOTFOUND', + 'EAI_AGAIN', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'UND_ERR_CONNECT_TIMEOUT', +]); + +function causeOf(error: unknown): { code: string | undefined; message: string | undefined } { + const cause = (error as { cause?: { code?: unknown; message?: unknown } } | null)?.cause; + return { + code: typeof cause?.code === 'string' ? cause.code : undefined, + message: typeof cause?.message === 'string' ? cause.message : undefined, + }; +} + export function normalizeOllamaError( error: unknown, ctx: { baseUrl: string; model?: string; signal?: AbortSignal | undefined }, ): AIClientError { if (error instanceof AIClientError) return error; + const model = ctx.model ?? 'unknown'; if (error instanceof Error && error.name === 'AbortError') { const platformTimeout = isPlatformTimeoutAbort(ctx.signal); + // A local runtime that stays silent through the whole idle window is + // still reading the prompt, and a retry reads it again from the start. + if (platformTimeout?.kind === 'idle') { + const minutes = Math.round(platformTimeout.timeoutMs / 60_000); + return new AIClientError( + `Ollama sent nothing for ${minutes} minutes (model=${model}). A model on this computer reads the whole prompt before it writes, and this one took longer than that; a smaller context window under This Computer, or a smaller model, reads it faster.`, + 'timeout', + 'ollama', + false, + { cause: error }, + ); + } return new AIClientError( platformTimeout - ? `Platform timeout exceeded after ${platformTimeout.timeoutMs}ms (provider=ollama model=${ctx.model ?? 'unknown'}).` - : `Ollama request aborted (model=${ctx.model ?? 'unknown'}).`, + ? `Platform timeout exceeded after ${platformTimeout.timeoutMs}ms (provider=ollama model=${model}).` + : `Ollama request aborted (model=${model}).`, 'timeout', 'ollama', true, @@ -210,8 +248,20 @@ export function normalizeOllamaError( ); } const message = error instanceof Error ? error.message : String(error); + const cause = causeOf(error); + if (cause.code !== undefined && UNREACHABLE_CODES.has(cause.code)) { + return new AIClientError( + `Ollama is not reachable at ${ctx.baseUrl}: ${cause.message ?? message}. Start Ollama on this computer, or correct its URL in Settings → Credentials.`, + 'network_error', + 'ollama', + true, + { cause: error }, + ); + } return new AIClientError( - `Ollama is not reachable at ${ctx.baseUrl}: ${message}. Start Ollama on this computer, or correct its URL in Settings → Credentials.`, + `The request to Ollama at ${ctx.baseUrl} broke off: ${cause.message ?? message}${ + cause.code ? ` (${cause.code})` : '' + } (model=${model}).`, 'network_error', 'ollama', true, @@ -306,7 +356,7 @@ export function createOllamaAdapter(config: ProviderConfig): AIProviderAdapter { body: unknown, signal: AbortSignal | undefined, ): Promise { - return fetch(`${baseUrl}${path}`, { + return localRuntimeFetch(`${baseUrl}${path}`, { method: 'POST', headers, body: JSON.stringify(body), diff --git a/packages/ai-client/src/providers/ollamaRuntime.ts b/packages/ai-client/src/providers/ollamaRuntime.ts index edc2029d..1cb5d317 100644 --- a/packages/ai-client/src/providers/ollamaRuntime.ts +++ b/packages/ai-client/src/providers/ollamaRuntime.ts @@ -5,6 +5,7 @@ */ import { OLLAMA_DEFAULT_BASE_URL } from '@aflow/schemas'; import type { ProviderConfig } from '../types.js'; +import { localRuntimeFetch } from './localRuntimeFetch.js'; /** Readiness asks this on every read; a runtime slower than this to say what it holds is reported unreachable. */ const STATUS_TIMEOUT_MS = 2_000; @@ -76,7 +77,7 @@ export function localRuntimeClient( signal?: AbortSignal, ): Promise => { const timeout = AbortSignal.timeout(timeoutMs); - const res = await fetch(`${base}/api/generate`, { + const res = await localRuntimeFetch(`${base}/api/generate`, { method: 'POST', headers, body: JSON.stringify(body), @@ -87,7 +88,7 @@ export function localRuntimeClient( }; const read = async (path: string): Promise => { try { - const res = await fetch(`${base}${path}`, { + const res = await localRuntimeFetch(`${base}${path}`, { headers, signal: AbortSignal.timeout(STATUS_TIMEOUT_MS), }); @@ -113,7 +114,7 @@ export function localRuntimeClient( })); }, async pull(tag, onProgress, signal) { - const res = await fetch(`${base}/api/pull`, { + const res = await localRuntimeFetch(`${base}/api/pull`, { method: 'POST', headers, body: JSON.stringify({ model: tag, stream: true }), @@ -155,7 +156,7 @@ export function localRuntimeClient( } }, async remove(tag) { - const res = await fetch(`${base}/api/delete`, { + const res = await localRuntimeFetch(`${base}/api/delete`, { method: 'DELETE', headers, body: JSON.stringify({ model: tag }), diff --git a/yarn.lock b/yarn.lock index 94c3e894..012bbf9e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -269,6 +269,7 @@ __metadata: openai: "npm:^6.49.0" tsup: "npm:^8.5.0" typescript: "npm:^5.8.3" + undici: "npm:^7.30.1" vitest: "npm:^4.1.11" zod: "npm:^3.24.4" zod-to-json-schema: "npm:^3.25.2" @@ -10397,6 +10398,13 @@ __metadata: languageName: node linkType: hard +"undici@npm:^7.30.1": + version: 7.30.1 + resolution: "undici@npm:7.30.1" + checksum: 10c0/c838089b010e8e28a9a5358d4bf4621e0c8d7bb70058b23e819667349c4c0b6791a7c8bc9bbe94ff0923db132faae0d9d9592b9743eda4f2b49466fa57482213 + languageName: node + linkType: hard + "undici@npm:^8.11.2": version: 8.11.2 resolution: "undici@npm:8.11.2"