diff --git a/apps/aflow-executor-ai/src/handlers/agentTurnTimeoutSpec.test.ts b/apps/aflow-executor-ai/src/handlers/agentTurnTimeoutSpec.test.ts index 6587b074..228c987c 100644 --- a/apps/aflow-executor-ai/src/handlers/agentTurnTimeoutSpec.test.ts +++ b/apps/aflow-executor-ai/src/handlers/agentTurnTimeoutSpec.test.ts @@ -4,9 +4,19 @@ import { AiHandler } from './ai/AiHandler.js'; const handler = new AiHandler({ payloadStore: {} as never }); -function ctx(operationId: string, executionTimeoutMs?: number): ExecutorContext { +function ctx( + operationId: string, + executionTimeoutMs?: number, + input: Record = {}, +): ExecutorContext { return { operationId, + job: { + stepExecutionId: `step-${operationId}-${String(input['model'])}`, + attempt: 1, + inputRef: 'inline:e30=', + }, + readPayload: () => Promise.resolve(input), stepDefinition: executionTimeoutMs !== undefined ? { timeout: { executionTimeoutMs } } : undefined, } as unknown as ExecutorContext; @@ -32,4 +42,31 @@ describe('AiHandler.resolveTimeoutMs', () => { it('non-agent operations keep the executor default', async () => { expect(await handler.resolveTimeoutMs(ctx('ai.text.generate'))).toBeUndefined(); }); + + it('a model on this computer may read its prompt for minutes before its first token', async () => { + const cloud = (await handler.resolveTimeoutMs( + ctx('ai.agent.turn', undefined, { model: 'glm-pro' }), + )) as { + idleMs: number; + }; + const local = (await handler.resolveTimeoutMs( + ctx('ai.text.generate', undefined, { model: 'gemma-local' }), + )) as { idleMs: number; maxMs: number }; + expect(local.idleMs).toBeGreaterThan(cloud.idleMs); + expect(local.maxMs).toBeGreaterThan(local.idleMs); + }); +}); + +describe('AiHandler.admit', () => { + it('asks nothing of a step whose model is served elsewhere', async () => { + const withRedis = new AiHandler({ payloadStore: {} as never, redis: {} as never }); + const waiting = { + signal: new AbortController().signal, + waiting: () => undefined, + stillWaiting: () => undefined, + }; + expect( + await withRedis.admit(ctx('ai.agent.turn', undefined, { model: 'glm-pro' }), waiting), + ).toBeUndefined(); + }); }); diff --git a/apps/aflow-executor-ai/src/handlers/ai/AiHandler.ts b/apps/aflow-executor-ai/src/handlers/ai/AiHandler.ts index d231f104..159eec0d 100644 --- a/apps/aflow-executor-ai/src/handlers/ai/AiHandler.ts +++ b/apps/aflow-executor-ai/src/handlers/ai/AiHandler.ts @@ -4,6 +4,8 @@ import type { ExecutorContext, TimeoutSpec, StepResult, + StepAdmission, + StepAdmissionWaiting, } from '@aflow/executor-runtime'; import { failureWithError, validationError, internalError } from '@aflow/executor-runtime'; import { @@ -29,10 +31,11 @@ import { AIClientError } from '@aflow/ai-client'; import { positiveMsEnv } from '@aflow/lib'; import type { ErrorObject } from 'ajv'; import type { PayloadStore } from '@aflow/payload-store'; -import type { Redis } from '@aflow/redis'; +import { acquireRuntimeLease, type Redis } from '@aflow/redis'; import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'; import { createAsyncJobRepository, createTenantContext } from '@aflow/database'; -import { AISetupError, checkBudgetMock } from './aiClient.js'; +import { AISetupError, checkBudgetMock, resolveLocalRuntime } from './aiClient.js'; +import { inferProviderForModelRef } from '@aflow/ai-client'; import { ajv } from './ajv.js'; import { AiGenerateInputSchema, @@ -83,28 +86,104 @@ type OperationHandler = (ctx: ExecutorContext, input: unknown) => Promise 0 ? model : undefined; +} + +function attemptKey(ctx: ExecutorContext): string { + return `${ctx.job.stepExecutionId}:${ctx.job.attempt}`; +} export class AiHandler implements StepHandler { readonly stepType = 'ai'; - resolveTimeoutMs(ctx: ExecutorContext): Promise { + async resolveTimeoutMs(ctx: ExecutorContext): Promise { // An explicit step-definition timeout wins in both directions and stays a // flat wall clock (an operator who set a number meant a number); this only // replaces the executor-wide flat default. if (ctx.stepDefinition?.timeout?.executionTimeoutMs !== undefined) { - return Promise.resolve(undefined); + return undefined; + } + const model = stepModel(await this.stepInput(ctx)); + if (model !== undefined && inferProviderForModelRef(model) === 'ollama') { + return { idleMs: LOCAL_MODEL_IDLE_MS, maxMs: AGENT_TURN_MAX_MS }; } if (ctx.operationId === 'ai.agent.turn') { - return Promise.resolve({ idleMs: AGENT_TURN_STREAM_IDLE_MS, maxMs: AGENT_TURN_MAX_MS }); + return { idleMs: AGENT_TURN_STREAM_IDLE_MS, maxMs: AGENT_TURN_MAX_MS }; } // A render outlasts the executor-wide default, and the step that dies // waiting was still billed for it. if (ASYNC_JOB_LIFECYCLE_OPERATIONS.has(ctx.operationId)) { - return Promise.resolve(resolveVideoBudget().stepBudgetMs); + return resolveVideoBudget().stepBudgetMs; + } + return undefined; + } + + /** + * A step whose model runs on the operator's machine waits for that runtime + * to have room (Plan 323 D7) — before its timeout starts, and without its + * executor slot. + */ + async admit( + ctx: ExecutorContext, + waiting: StepAdmissionWaiting, + ): Promise { + if (!this.redis) return undefined; + const model = stepModel(await this.stepInput(ctx)); + if (model === undefined) return undefined; + const runtime = await resolveLocalRuntime(ctx, model); + if (!runtime) return undefined; + const lease = await acquireRuntimeLease({ + redis: this.redis, + runtimeId: runtime.runtimeId, + limit: runtime.maxParallel, + holderId: attemptKey(ctx), + wait: true, + signal: waiting.signal, + onWaiting: () => { + waiting.waiting({ runtimeId: runtime.runtimeId, limit: runtime.maxParallel, model }); + }, + onStillWaiting: waiting.stillWaiting, + }); + return lease ?? undefined; + } + + /** + * The step's input, read once per attempt: admission and the timeout read + * its model before the handler runs, and an agent turn's input is large. + */ + private stepInput(ctx: ExecutorContext): Promise { + const key = attemptKey(ctx); + let input = this.inputs.get(key); + if (!input) { + input = ctx.readPayload(ctx.job.inputRef); + this.inputs.set(key, input); + // An attempt that never reaches `execute` (cancelled while it waited) + // must not hold its input for the life of the process. + if (this.inputs.size > MAX_CACHED_INPUTS) { + const oldest = this.inputs.keys().next().value; + if (oldest !== undefined) this.inputs.delete(oldest); + } } - return Promise.resolve(undefined); + return input; } + private readonly inputs = new Map>(); + + private readonly redis: Redis | undefined; + private readonly payloadStore: PayloadStore; private readonly operations: Record< @@ -117,6 +196,7 @@ export class AiHandler implements StepHandler { constructor(deps: { payloadStore: PayloadStore; db?: PostgresJsDatabase; redis?: Redis }) { this.payloadStore = deps.payloadStore; + this.redis = deps.redis; const db = deps.db; const depsForHandlers: HandlerDeps = { @@ -327,7 +407,8 @@ export class AiHandler implements StepHandler { async execute(ctx: ExecutorContext): Promise { const operationId = ctx.operationId; - const input = await ctx.readPayload(ctx.job.inputRef); + const input = await this.stepInput(ctx); + this.inputs.delete(attemptKey(ctx)); const typedInput = input as Record; const budgetResult = checkBudgetMock({ diff --git a/apps/aflow-executor-ai/src/handlers/ai/aiClient.ts b/apps/aflow-executor-ai/src/handlers/ai/aiClient.ts index 0e266108..cb36d9d1 100644 --- a/apps/aflow-executor-ai/src/handlers/ai/aiClient.ts +++ b/apps/aflow-executor-ai/src/handlers/ai/aiClient.ts @@ -2,15 +2,17 @@ import { createAIClient, inferProviderForModelRef, type AIClient, - type ProviderConfig, type BudgetCheckResult, type AIProvider, } from '@aflow/ai-client'; import type { ExecutorContext } from '@aflow/executor-runtime'; -import type { AflowError } from '@aflow/schemas'; +import { localRuntimeId, type AflowError } from '@aflow/schemas'; import { CredentialResolver, credentialMissingMessage, + localRuntimeSettings, + missingRequiredSecrets, + providerConfigFromCredential, type CredentialContext, type CredentialLoader, } from '@aflow/credential-resolver'; @@ -27,6 +29,7 @@ const AI_TO_CREDENTIAL_PROVIDER: Record = { xai: 'xai', runware: 'runware', typesafe: 'typesafe', + ollama: 'ollama', }; // --------------------------------------------------------------------------- @@ -284,10 +287,10 @@ export async function getAIClientForContext( ); } - const apiKey = resolved.secrets['api_key']; - if (!apiKey) { + const missingSecrets = missingRequiredSecrets(credentialProviderId, resolved); + if (missingSecrets.length > 0) { throw new AISetupError( - `Credential for ${credentialProviderId} exists but has no api_key. ` + + `Credential for ${credentialProviderId} exists but has no ${missingSecrets.join(', ')}. ` + 'Update your credentials in Settings → Credentials.', 'AI_PROVIDER_API_KEY_MISSING', 'configuration', @@ -305,16 +308,10 @@ export async function getAIClientForContext( } // 5. Build single-provider AIClient - const providerConfig: ProviderConfig = { apiKey }; - if (aiProvider === 'openai' && resolved.config['org_id']) { - providerConfig.organization = resolved.config['org_id']; - } - - const providers: Partial> = { - [aiProvider]: providerConfig, - }; - - const client = createAIClient({ providers, defaultProvider: aiProvider }); + const client = createAIClient({ + providers: { [aiProvider]: providerConfigFromCredential(credentialProviderId, resolved) }, + defaultProvider: aiProvider, + }); CLIENT_CACHE.set(cacheKey, { client, @@ -323,3 +320,26 @@ export async function getAIClientForContext( return client; } + +/** + * The runtime on the operator's machine a model is served by, and how many + * requests it takes at once, or null for a model served anywhere else. + * A local model with no connection resolves to null too: the request then + * fails on the missing credential, which says what to do. + */ +export async function resolveLocalRuntime( + ctx: ExecutorContext, + model: string, +): Promise<{ runtimeId: string; maxParallel: number } | null> { + if (inferProviderForModelRef(model) !== 'ollama') return null; + const { tenantId, credentialOwnerId, spaceId } = ctx.job; + if (!credentialResolver || !credentialOwnerId || !spaceId) return null; + const resolved = await credentialResolver + .resolve('ollama', { tenantId, credentialOwnerId, spaceId }) + .catch(() => null); + if (!resolved) return null; + return { + runtimeId: localRuntimeId(providerConfigFromCredential('ollama', resolved).baseUrl), + maxParallel: localRuntimeSettings('ollama', resolved).maxParallel, + }; +} diff --git a/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.test.ts b/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.test.ts index 0a3a13f0..ae5ba0b1 100644 --- a/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.test.ts +++ b/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.test.ts @@ -605,6 +605,8 @@ describe('handleAgentTurn (integration seams)', () => { }; vi.mocked(getAIClientForContext).mockResolvedValue({ + contextWindowFor: () => undefined, + getModel: () => ({ id: model, provider, @@ -638,6 +640,8 @@ describe('handleAgentTurn (integration seams)', () => { }; vi.mocked(getAIClientForContext).mockResolvedValue({ + contextWindowFor: () => undefined, + getModel: () => ({ id: model, provider, @@ -672,6 +676,8 @@ describe('handleAgentTurn (integration seams)', () => { let call = 0; vi.mocked(getAIClientForContext).mockResolvedValue({ + contextWindowFor: () => undefined, + getModel: () => ({ id: model, provider, @@ -719,6 +725,8 @@ describe('handleAgentTurn (integration seams)', () => { let call = 0; vi.mocked(getAIClientForContext).mockResolvedValue({ + contextWindowFor: () => undefined, + getModel: () => ({ id: model, provider, @@ -787,6 +795,8 @@ describe('handleAgentTurn (integration seams)', () => { usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, }); vi.mocked(getAIClientForContext).mockResolvedValue({ + contextWindowFor: () => undefined, + getModel: () => ({ id: model, provider, @@ -834,6 +844,8 @@ describe('handleAgentTurn (integration seams)', () => { } vi.mocked(getAIClientForContext).mockResolvedValue({ + contextWindowFor: () => undefined, + getModel: () => ({ id: 'claude-3-test', provider: 'anthropic', @@ -889,6 +901,8 @@ describe('handleAgentTurn (integration seams)', () => { const generateJson = vi.fn(); vi.mocked(getAIClientForContext).mockResolvedValue({ + contextWindowFor: () => undefined, + getModel: () => ({ id: 'gpt-6.1-sol', provider: 'openai', @@ -938,6 +952,8 @@ describe('handleAgentTurn (integration seams)', () => { const generateJson = vi.fn(); vi.mocked(getAIClientForContext).mockResolvedValue({ + contextWindowFor: () => undefined, + getModel: () => ({ id: 'grok-4.7', provider: 'xai', @@ -1002,6 +1018,8 @@ describe('handleAgentTurn (integration seams)', () => { const generateJson = vi.fn(); vi.mocked(getAIClientForContext).mockResolvedValue({ + contextWindowFor: () => undefined, + getModel: () => ({ id: 'gpt-6.1-sol', provider: 'openai', diff --git a/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.ts b/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.ts index 17d001ce..c3cb275f 100644 --- a/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.ts +++ b/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurn.ts @@ -214,10 +214,10 @@ export async function handleAgentTurn( try { const client = await getAIClientForContext(ctx, model); const modelDef = client.getModel(model); - const nativeFCProviders = new Set(['google', 'anthropic', 'openai', 'fireworks', 'xai']); - const useNativeFC = - nativeFCProviders.has(modelDef?.provider ?? '') && - modelDef?.capabilities.functionCalling === true; + // A capability, measured per model, rather than a list of providers: a + // provider list has no way to say that one local build calls tools and the + // next does not (Plan 323 D6). + const useNativeFC = modelDef?.capabilities.functionCalling === true; // Provider-native reasoning continuity (Plan 259). Resolve + capability-gate // BEFORE any network I/O — an unsupported authored mode fails loud, never @@ -455,7 +455,7 @@ export async function handleAgentTurn( modelResult.toolCallSignatures, attemptSummary.length > 0 ? attemptSummary : undefined, prepared.tokenBreakdown, - params.contextWindowOverride ?? modelDef?.contextWindow, + params.contextWindowOverride ?? client.contextWindowFor(model) ?? modelDef?.contextWindow, continuityInfo, prepared.toolSurface, ); @@ -599,11 +599,9 @@ async function finishAgentTurn( // §4.8: this function is the single compaction trigger authority — it calls // triggerCompaction only when the post-clearing ledger (or the Tier-4 // ladder) demands it; the module itself has no gate. + const compactionModel = params.compactionModel ?? RETENTION_POLICY.compactionSummaryModel; const runCompaction = async () => { - const compactionClient = await getAIClientForContext( - ctx, - RETENTION_POLICY.compactionSummaryModel, - ); + const compactionClient = await getAIClientForContext(ctx, compactionModel); // Filter hydrated atoms to only those still in the conversation state — // clearing may have removed atoms since assembly time. @@ -620,6 +618,7 @@ async function finishAgentTurn( stepExecutionId: ctx.job.stepExecutionId, attempt: ctx.job.attempt, availableReadOpId, + summaryModel: compactionModel, }, currentState, currentAtoms, @@ -692,7 +691,7 @@ async function finishAgentTurn( compactionSummary = { compactionNumber: compactResult.compactionNumber, tokensSaved: compactResult.tokensSaved, - summaryModel: RETENTION_POLICY.compactionSummaryModel, + summaryModel: compactionModel, }; ctx.log.info('agent_turn_compaction', { turnNumber: params.turnNumber, @@ -757,7 +756,7 @@ async function finishAgentTurn( compactionSummary = { compactionNumber: compactResult.compactionNumber, tokensSaved: (compactionSummary?.tokensSaved ?? 0) + compactResult.tokensSaved, - summaryModel: RETENTION_POLICY.compactionSummaryModel, + summaryModel: compactionModel, }; ctx.log.info('agent_turn_forced_compaction', { turnNumber: params.turnNumber, diff --git a/apps/aflow-executor-ai/src/handlers/ai/handlers/mediaBudget.test.ts b/apps/aflow-executor-ai/src/handlers/ai/handlers/mediaBudget.test.ts index 5c5983cb..9cc3277d 100644 --- a/apps/aflow-executor-ai/src/handlers/ai/handlers/mediaBudget.test.ts +++ b/apps/aflow-executor-ai/src/handlers/ai/handlers/mediaBudget.test.ts @@ -88,6 +88,8 @@ function operationsDrivingTheVideoJobRunner(): Set { function ctx(operationId: string, executionTimeoutMs?: number): ExecutorContext { return { operationId, + job: { stepExecutionId: `step-${operationId}`, attempt: 1, inputRef: 'inline:e30=' }, + readPayload: () => Promise.resolve({}), stepDefinition: executionTimeoutMs !== undefined ? { timeout: { executionTimeoutMs } } : undefined, } as unknown as ExecutorContext; diff --git a/apps/aflow-executor-ai/src/handlers/ai/schema.ts b/apps/aflow-executor-ai/src/handlers/ai/schema.ts index 4e7066bf..179f4fb6 100644 --- a/apps/aflow-executor-ai/src/handlers/ai/schema.ts +++ b/apps/aflow-executor-ai/src/handlers/ai/schema.ts @@ -206,6 +206,8 @@ export const AgentTurnInputSchema = z.object({ turnTimestamp: z.string().optional(), lastTurnCompletedAtMs: z.number().optional(), contextWindowOverride: z.number().int().positive().optional(), + /** The model compaction summarizes with, resolved from the space's Clerk. */ + compactionModel: z.string().optional(), summaryTemplate: z .object({ sections: z diff --git a/apps/aflow-executor-ai/src/handlers/compaction.ts b/apps/aflow-executor-ai/src/handlers/compaction.ts index f0b9b863..223057e8 100644 --- a/apps/aflow-executor-ai/src/handlers/compaction.ts +++ b/apps/aflow-executor-ai/src/handlers/compaction.ts @@ -36,6 +36,8 @@ export interface CompactionConfig { attempt: number; /** The read op on this turn's tool surface (§4.9) — teaches the restore-atom pointers. */ availableReadOpId?: string | undefined; + /** The model that writes the summary; the retention policy's when absent. */ + summaryModel?: string | undefined; } export interface CompactionResult { @@ -487,7 +489,7 @@ export async function triggerCompaction( const template: SummaryTemplate = SummaryTemplateSchema.parse(summaryTemplate ?? {}); // 6. Call summarizer LLM - const summaryModel = RETENTION_POLICY.compactionSummaryModel; + const summaryModel = config.summaryModel ?? RETENTION_POLICY.compactionSummaryModel; const summaryMaxTokens = template.maxTokens; const prompt = previousSummary diff --git a/apps/aflow-executor-ai/src/handlers/reasoningContinuity.test.ts b/apps/aflow-executor-ai/src/handlers/reasoningContinuity.test.ts index f38c2320..62c92de0 100644 --- a/apps/aflow-executor-ai/src/handlers/reasoningContinuity.test.ts +++ b/apps/aflow-executor-ai/src/handlers/reasoningContinuity.test.ts @@ -146,6 +146,7 @@ describe('supportsToolLoopContinuity', () => { expect(supportsToolLoopContinuity('openrouter', true)).toBe(false); expect(supportsToolLoopContinuity('google', true)).toBe(false); expect(supportsToolLoopContinuity(undefined, true)).toBe(false); - expect(supportsToolLoopContinuity('local', true)).toBe(false); + // Ollama's chat templates drop replayed thinking, so there is nothing to continue. + expect(supportsToolLoopContinuity('ollama', true)).toBe(false); }); }); diff --git a/apps/aflow-executor-ai/src/handlers/resurrectionSignal.test.ts b/apps/aflow-executor-ai/src/handlers/resurrectionSignal.test.ts index 388759dd..5eb1fadd 100644 --- a/apps/aflow-executor-ai/src/handlers/resurrectionSignal.test.ts +++ b/apps/aflow-executor-ai/src/handlers/resurrectionSignal.test.ts @@ -434,6 +434,8 @@ function makeCtx(writes: Array<{ kind: string; data: unknown }>): ExecutorContex function mockClient(promptTokens: number): AIClient { return { + contextWindowFor: () => undefined, + getModel: (m: string) => ({ id: m, provider: 'openai', diff --git a/apps/aflow-executor-ai/src/handlers/retentionPolicy.ts b/apps/aflow-executor-ai/src/handlers/retentionPolicy.ts index edd785cb..e4d6615c 100644 --- a/apps/aflow-executor-ai/src/handlers/retentionPolicy.ts +++ b/apps/aflow-executor-ai/src/handlers/retentionPolicy.ts @@ -1,9 +1,14 @@ +import { MIN_ANSWER_TOKENS } from '@aflow/ai-client'; import { MAX_ATOMS_STRUCTURAL } from '@aflow/schemas'; /** Assumed when the model catalog reports no context window for the turn's model. */ export const DEFAULT_MODEL_WINDOW_TOKENS = 200_000; -/** Held back from the prompt for the model's answer. */ -export const COMPLETION_RESERVE_TOKENS = 4096; +/** + * Held back from the prompt for the model's answer. The same floor a local + * runtime refuses a prompt below, so a prompt this budget admits is never + * refused there. + */ +export const COMPLETION_RESERVE_TOKENS = MIN_ANSWER_TOKENS; /** Share of the window held back because prompt tokens are estimated before the call, not counted. */ export const SAFETY_MARGIN_FRACTION = 0.05; /** The window whose hard budget caps the working budget, so a model of this window or smaller keeps the trigger points it always had. */ diff --git a/apps/aflow-executor-ai/src/handlers/retentionPressure.test.ts b/apps/aflow-executor-ai/src/handlers/retentionPressure.test.ts index b9db723f..8f29dd01 100644 --- a/apps/aflow-executor-ai/src/handlers/retentionPressure.test.ts +++ b/apps/aflow-executor-ai/src/handlers/retentionPressure.test.ts @@ -358,6 +358,8 @@ function makeCtx(writes: Array<{ kind: string; data: unknown }>): ExecutorContex function mockClient(promptTokens: number): AIClient { return { + contextWindowFor: () => undefined, + getModel: (m: string) => ({ id: m, provider: 'openai', diff --git a/apps/aflow-executor-ai/src/handlers/retentionSafetyBound.test.ts b/apps/aflow-executor-ai/src/handlers/retentionSafetyBound.test.ts index f5ac3d00..42a073b3 100644 --- a/apps/aflow-executor-ai/src/handlers/retentionSafetyBound.test.ts +++ b/apps/aflow-executor-ai/src/handlers/retentionSafetyBound.test.ts @@ -412,6 +412,8 @@ function makeCtx(writes: Array<{ kind: string; data: unknown }>): ExecutorContex function mockClient(): AIClient { return { + contextWindowFor: () => undefined, + getModel: (m: string) => ({ id: m, provider: 'openai', diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/helpers/agentTurn.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/helpers/agentTurn.ts index af38f403..b8200c2a 100644 --- a/apps/aflow-orchestrator/src/services/SessionOrchestrator/helpers/agentTurn.ts +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/helpers/agentTurn.ts @@ -73,6 +73,8 @@ import { resolveSpaceContextRole, isCyberneticCoach, buildCyberneticTurnOverrides, + loadTenantAgentModelAllowlist, + resolveClerkModel, type CyberneticTurnOverrides, } from '@aflow/cybernetic-runtime'; import { @@ -1475,6 +1477,19 @@ export async function buildAgentTurnInput( } } + // Summarizing a conversation is the Clerk's work, so compaction takes the + // Clerk's model: in a space whose only provider is this computer, a fixed + // cloud summarizer would fail the first long conversation. + let compactionModel: string | undefined; + if (spaceDirectives && flowContextDetails?.cyberneticHandles?.db) { + const allowlist = await loadTenantAgentModelAllowlist( + flowContextDetails.cyberneticHandles.db as PostgresJsDatabase, + tenantId, + ).catch(() => null); + const clerk = resolveClerkModel((spaceDirectives as EntityDirectives).modelDefaults, allowlist); + if (clerk.resolved) compactionModel = clerk.modelRef; + } + // Resolve optional finalOutputSchema and completionPrompt from config. let finalOutputSchema: Record | undefined; if (finalOutputSchemaOverrideJson) { @@ -1566,6 +1581,7 @@ export async function buildAgentTurnInput( ...(rc['contextWindowOverride'] ? { contextWindowOverride: Number(rc['contextWindowOverride']) } : {}), + ...(compactionModel !== undefined ? { compactionModel } : {}), ...(rc['summaryTemplate'] && typeof rc['summaryTemplate'] === 'object' ? { summaryTemplate: rc['summaryTemplate'] } : {}), diff --git a/apps/aflow-orchestrator/src/services/cybernetic/__tests__/postRunHooks.test.ts b/apps/aflow-orchestrator/src/services/cybernetic/__tests__/postRunHooks.test.ts index 9c37c30c..0292e997 100644 --- a/apps/aflow-orchestrator/src/services/cybernetic/__tests__/postRunHooks.test.ts +++ b/apps/aflow-orchestrator/src/services/cybernetic/__tests__/postRunHooks.test.ts @@ -65,6 +65,7 @@ vi.mock('@aflow/database', () => ({ const mockAppendEntityEvent = vi.fn(); vi.mock('@aflow/redis', () => ({ + runtimeAdmission: () => async () => ({ release: async () => undefined }), appendEntityEvent: (...args: unknown[]) => mockAppendEntityEvent(...args), })); diff --git a/apps/aflow-orchestrator/src/services/cybernetic/postRunHooks.ts b/apps/aflow-orchestrator/src/services/cybernetic/postRunHooks.ts index cf6613d2..b65f00e2 100644 --- a/apps/aflow-orchestrator/src/services/cybernetic/postRunHooks.ts +++ b/apps/aflow-orchestrator/src/services/cybernetic/postRunHooks.ts @@ -41,7 +41,7 @@ import { createTenantContext, withTenantSchema, spaces } from '@aflow/database'; import { createByokAiClientFactory } from '@aflow/credential-resolver'; import { eq } from 'drizzle-orm'; import { EntityDirectivesSchema } from '@aflow/schemas'; -import { appendEntityEvent } from '@aflow/redis'; +import { appendEntityEvent, runtimeAdmission } from '@aflow/redis'; import type { PayloadStore } from '@aflow/payload-store'; import { randomUUID } from 'node:crypto'; @@ -149,7 +149,7 @@ export async function fireCyberneticPostRunHooksStandalone( // Step 1: Run eval — unconditional when a suite exists; the outcome is // recorded on the run as a typed evaluation envelope either way. - const byokFactory = createByokAiClientFactory(db); + const byokFactory = createByokAiClientFactory(db, runtimeAdmission(redis, { wait: true })); let evalOutcome: EvalRunOutcome; try { evalOutcome = await runEvaluation({ diff --git a/apps/aflow-orchestrator/src/services/sessionMetadataTask.test.ts b/apps/aflow-orchestrator/src/services/sessionMetadataTask.test.ts index e9bbdeb1..a274c16b 100644 --- a/apps/aflow-orchestrator/src/services/sessionMetadataTask.test.ts +++ b/apps/aflow-orchestrator/src/services/sessionMetadataTask.test.ts @@ -16,6 +16,7 @@ const mocks = vi.hoisted(() => ({ })); vi.mock('@aflow/redis', () => ({ + runtimeAdmission: () => async () => ({ release: async () => undefined }), claimSessionMetadataCandidates: async () => [ { tenantId: '00000000-0000-0000-0000-000000000001', diff --git a/apps/aflow-orchestrator/src/services/sessionMetadataTask.ts b/apps/aflow-orchestrator/src/services/sessionMetadataTask.ts index dc0c8813..37cbe1f7 100644 --- a/apps/aflow-orchestrator/src/services/sessionMetadataTask.ts +++ b/apps/aflow-orchestrator/src/services/sessionMetadataTask.ts @@ -20,6 +20,7 @@ import { settleSessionMetadata, sessionMetadataLeaseHeld, appendEntityEvent, + runtimeAdmission, type SessionMetadataCandidate, } from '@aflow/redis'; import { @@ -97,7 +98,12 @@ export function createSessionMetadataTask(deps: SessionMetadataTaskDeps): Backgr const claimed = await claimSessionMetadataCandidates(deps.redis, ctx.maxBatch); if (claimed.length === 0) return { candidates: 0 }; - byokFactory ??= createByokAiClientFactory(deps.db); + // A title is not worth queueing behind an agent turn on this computer's + // runtime; a busy one fails this conversation's cycle, and it is due again. + byokFactory ??= createByokAiClientFactory( + deps.db, + runtimeAdmission(deps.redis, { wait: false }), + ); let processed = 0; let failed = 0; diff --git a/docs/architecture/background-work.md b/docs/architecture/background-work.md index d9a31926..0ad3601c 100644 --- a/docs/architecture/background-work.md +++ b/docs/architecture/background-work.md @@ -9,7 +9,7 @@ Production background discovery must be event- or candidate-driven. A task may n work by scanning the Redis keyspace, reading a whole dirty set, or enumerating tenant schemas; idle cost may not grow with logical shards, tenants, stored keys, or connected subscribers. -**46 registered tasks** across 8 services. +**47 registered tasks** across 8 services. 2 task(s) still carry a residual poll — a periodic datastore read that exists only because an event or candidate path is incomplete. @@ -28,6 +28,7 @@ Datastore operations per minute with zero due work, grouped by what each scope m | Task | Service | Criticality | Trigger | Scope | Cadence | Idle ops/min | Disable | | ---- | ------- | ----------- | ------- | ----- | ------- | -----------: | ------- | +| `ai.local_runtime_lease_renewal` | shared-runtime | correctness | active-resource | per_instance | 10s | 0 | never | | `executor.compute.session_reaper` | executor-compute | feature | active-resource | per_instance | 1m | 0 | safe | | `executor.heartbeat` | shared-runtime | correctness | heartbeat | per_instance | 10s | 6 | never | | `executor.job_consumer` | shared-runtime | correctness | blocking | per_instance | event-driven | 60 | never | @@ -77,6 +78,32 @@ Datastore operations per minute with zero due work, grouped by what each scope m ## Tasks +### `ai.local_runtime_lease_renewal` + +**Purpose.** Renew the lease each in-flight request holds on a model runtime on the operator machine, for as long as the request runs. + +**Invariant.** A runtime on the operator machine serves no more requests at once than its connection admits, and a holder that died frees its place. + +**Recovery.** An unrenewed lease lapses after its TTL and the next waiter takes the place. + +| Field | Value | +| ----- | ----- | +| Service | shared-runtime | +| Owner domain | model-execution | +| Criticality | correctness | +| Trigger | active-resource | +| Execution scope | per_instance | +| Substrate | redis-lease | +| Base cadence | 10s | +| Max batch | 1 | +| Max cycle | 5000 ms | +| Idle datastore ops/min | 0 | +| Hot-path producer budget | 0 added RTT — Scoped to one request holding a lease; no timer exists at idle. | +| Feature gate | — | +| Disable policy | never | +| Residual poll | — | +| Source | `packages/redis/src/localRuntimeAdmission.ts` | + ### `executor.compute.session_reaper` **Purpose.** Tear down sandbox containers whose session expired. diff --git a/docs/plans/323-models-that-run-on-this-computer.md b/docs/plans/323-models-that-run-on-this-computer.md index 47ea14f5..3b70230f 100644 --- a/docs/plans/323-models-that-run-on-this-computer.md +++ b/docs/plans/323-models-that-run-on-this-computer.md @@ -1,6 +1,6 @@ # Plan 323 — Models that run on this computer -**Status:** 📋 Planned. Nothing is built. P0 is a set of feasibility gates and changes no product code; P0 and P1 together are the first complete slice: one model on one machine. +**Status:** 🔨 P0 met on one machine (§9): every gate answered on an Apple M3 with 24 GB and Ollama 0.40.5, and the decisions they served revised. P1 built and unit-tested, and its adapter run against the live runtime; its exit, a Helmsman conversation through the web UI on the dev stack, has not been run. P0 and P1 together are the first complete slice: one model on one machine. **Supersedes for the local edition:** Tier 0 of archive plan 98 (open-weight inference). Plan 98 put a platform-managed `selfhosted` provider behind Ollama's OpenAI-compatible `/v1` endpoint. This plan uses Ollama's native API instead (D3) and makes the runtime the operator's own (D2). Plan 98's hosted tiers (Cloud Run GPU, GKE/vLLM) are untouched; the OpenAI-compatible adapter in P6 is the piece they would share. @@ -31,25 +31,27 @@ Using one should look like using any other model. You choose it in the same pick **D2 — Aflow does not ship or supervise the runtime.** The operator installs Ollama, and its own app or service keeps it running. Aflow detects it, says how to install it when it is absent, and drives it over its API: list, pull, delete, load, unload. On a paired Mac, the UI may also offer a one-time install through the host lane (`brew install ollama`) after the operator approves it. This is a command run once, not supervision. _Rejected:_ bundling the runtime in the image. On macOS it would run without the GPU. _Rejected:_ the host executor supervising a model server. That is a second process supervisor beside launchd, systemd and the runtime's own, for something the runtime already does. The exception is the Linux appliance (D11). -**D3 — The Ollama adapter speaks the native `/api/chat` and `/api/embed`, not `/v1`.** Through `/v1` you cannot set the context length, cannot pass `tool_choice`, and get no `think` control. The context length is the decisive one (D5). The adapter implements `generateText`, `generateTextStream`, `generateJson` (`format` = the JSON schema) and `generateEmbedding`. It maps the catalog's reasoning ladder onto `think`. It reads `thinking` from responses and replays it for continuity as the Fireworks adapter does with `reasoning_content`. It collects a streamed turn's `thinking`, `content` and `tool_calls` and sends them back together with the tool results, which is how the runtime expects a turn to be returned. +**D3 — The Ollama adapter speaks the native `/api/chat` and `/api/embed`, not `/v1`.** Through `/v1` you cannot set the context length, cannot pass `tool_choice`, and get no `think` control. The context length is the decisive one (D5). The adapter implements `generateText`, `generateTextStream`, `generateJson` (`format` = the JSON schema) and `generateEmbedding`. It maps the catalog's reasoning ladder onto `think`, whose accepted values the runtime publishes per model in `/api/show` (`thinking.values`: `[false, true]` for Gemma 4 and Qwen3.5, `low | medium | high` for gpt-oss). It reads `thinking` from responses for the run view. It does not replay it: the runtime's chat templates drop `thinking` from history, so a replayed reasoning trace costs no tokens and reaches no model (§9 G1). It collects a streamed turn's `content` and `tool_calls` and sends them back together with the tool results, which is how the runtime expects a turn to be returned. **Structured output follows the runner.** The MLX runner refuses `format` outright, a JSON schema and `"json"` alike, with a 501 `structured output is unavailable`, while the llama.cpp runner honours both (§9 G1). So `generateJson` sends `format` first and, once a model's runner has refused it, carries the schema as a tool whose arguments are the response, as the Anthropic adapter does. The refusal is remembered per model by the adapter, learned from the runtime's own answer rather than written into the catalog, so a runner that gains structured output is used as soon as it does. **D4 — A local runtime is a connection in the credential registry. Its provider id is `ollama`, and `local` is removed.** The credential is a URL, `base_url`, which defaults to `http://127.0.0.1:11434`. It has an optional token for a runtime behind a proxy and no required secret. Which secret a provider requires becomes registry data, not `api_key` hard-coded in the executor. Reusing the registry gives the connection the user → space → tenant resolution, the settings screens, the validation probe (`GET /api/version` plus `/api/tags`) and readiness, all unchanged. `base_url` origins are restricted with the existing `allowedOrigins` mechanism to loopback and `host.docker.internal`. A runtime on another machine is admitted only through `CREDENTIAL_EXTRA_BASE_URL_ORIGINS` (P6). _Rejected:_ an environment variable naming the runtime. It would bypass readiness and the UI, and it is exactly the "configured somewhere else" state that shows green and fails at run time. -**D5 — The context window a request gets is the machine's answer, sent on every request, and a prompt that might not fit is never sent.** Ollama's default context is sized by available memory and can be as small as 4K. Its documentation does not say what happens to a prompt longer than the window. If it is cut, the agent loses its instructions and carries on. +**D5 — The context window a request gets is the machine's answer, sent on every request, and a prompt that might not fit is never sent.** Ollama's default context is sized by available memory and can be as small as 4K. What an over-long prompt does depends on the runner (§9 G1): the llama.cpp runner silently cuts it from the front to about half the window, dropping the system message, and the MLX runner does not enforce `num_ctx` at all and grows its cache with the prompt. The first loses the agent's instructions; the second spends memory the fit estimate did not count. - **The window:** the connection carries `context_tokens`: chosen by the operator, offered from the fit estimate (D8), and never above the model's catalog `contextWindow`. The adapter sends it as `options.num_ctx` on every request. Prompt budgeting reads the effective window, `min(catalog, connection)`, rather than the catalog's. -- **Why counting after the fact is not enough:** a guarantee cannot rest on comparing `prompt_eval_count` with the prompt's size. Aflow's size is a characters-over-four estimate (`history.ts`), which knows nothing of the runtime's tokenizer, chat template, tool schemas or image tokens. And with prefix caching, `prompt_eval_count` counts only the tokens evaluated afresh, so it is smaller than the prompt even when nothing was cut. -- **Prevention comes first:** a request is sent only when its prompt, plus the output reserve (`maxTokens`, reasoning included), plus a measured safety margin, fits the window. -- **Which method is a P0 gate:** P0 establishes how the prompt is sized and how an overflow is caught, and P1 does not start until one method holds. The candidates, in order of preference: - 1. a runtime that refuses an overflow, if the current release does or can be configured to; - 2. an exact count from the runtime's own tokenizer and template; - 3. a conservative estimator whose ratio is calibrated per model. +- **The characters-over-four estimate cannot size a local prompt.** Measured against the runtime's own counts (§9 G1), English and German run at about 4.8 characters a token, code at 3.5–4.0, JSON at 2.8 and Chinese at 1.6. A tool result in JSON is undercounted by 40% and Chinese text by 2.5×. Eight tool schemas cost 570–890 tokens. +- **`prompt_eval_count` is exact.** On both runners it is the whole prompt's token count, cached prefix included: prefix caching shows in the time taken, never in the count. When the llama.cpp runner truncates, it reports the truncated count. +- **The sizing method** (P0's gate, met by the measurements in §9 G1; `packages/ai-client/src/providers/ollamaPromptSize.ts`): + 1. **Conservative size.** The prompt, tool schemas and tool calls are sized at the lowest measured ratio, 1.5 characters a token, so no script or format is undercounted. A prompt that fits at that size with its answer is sent as is. + 2. **Exact count when the estimate is not enough.** Any other prompt, and every prompt carrying an image, is counted exactly by a one-token evaluation before it is sent. The real request reuses the evaluated prefix from the runtime's cache, so the count costs a round trip and no second prefill: in §9 G1 a 6,422-token prompt counted this way was read back in 0.07 s. Images are always counted, because characters cannot size one: a 1280×800 image is 524 tokens on Gemma 4 and 1,002 on Qwen3.5, and a model that sizes images by resolution costs more for a larger one. + 3. **The answer gets what the window leaves.** A counted prompt is sent with `num_predict` set to the smaller of the requested `maxTokens` and what the window has left, so prompt and answer together never pass it. + 4. **Refusal.** A prompt that leaves less than `MIN_ANSWER_TOKENS` (4,096, the same reserve prompt budgeting holds back) fails before it is sent, as `context_length` with provider code `context_exceeded`, naming the size, the window and the remedy (D15). A count the runtime cut while counting is reported as _at least_ the prompt's lower bound. + 5. **Backstop.** A response whose `prompt_eval_count` is below the prompt's lower bound was truncated by the runtime. It fails with a message that says so, and its answer is discarded. The lower bound divides by the highest measured ratio, 5, and counts a run of one character (indentation, padding, a rule of dashes) once, since such a run can be one token however long it is. - Whichever is chosen must be correct on multilingual text, tool-heavy turns, reasoning history and images. A request the chosen method cannot rule safe fails before it is sent, as `context_exceeded`, naming the estimate, the window and the remedy (D15). + _Not built:_ a baseline carried from the previous response. With prefix caching the exact count is close to free, and a carried baseline would be a second source of truth for a number the runtime gives directly. -- **Streamed text is provisional:** what an agent turn streams is not the turn's result until the final chunk arrives and the response validates. A turn that fails validation is shown as discarded, not as an answer. P1 defines how the run view marks it. +- **Streamed text is provisional:** what an agent turn streams is not the turn's result until the final chunk arrives and the response validates. A turn the backstop rejects fails, so nothing it streamed is committed as an answer. How the run view marks the text it already showed is P3's. -**D6 — Native tool calling is chosen by measured capability, not by a provider list.** `nativeFCProviders` is deleted. An agent turn uses native function calling when the catalog entry says `functionCalling: true`, and that flag is written for local entries by the conformance probe (D9). The same probe writes `structuredOutputs`, `forcedToolChoice` and the reasoning ladder. The `generateJson` fallback streams for every provider, so a long local generation keeps the idle clock alive. +**D6 — Native tool calling is chosen by measured capability, not by a provider list.** `nativeFCProviders` is deleted. An agent turn uses native function calling when the catalog entry says `functionCalling: true`, and that flag is written for local entries by the conformance probe (D9). The same probe writes `structuredOutputs`, `forcedToolChoice` and the reasoning ladder. This also moves the three OpenRouter models, all declaring `functionCalling: true`, onto native tool calls; their route already accepts tool definitions in the streaming path, which `models:verify-reasoning` exercises. The `generateJson` fallback is reached only by a model without measured tool calling, and making it stream is P2's, with the first such model. **D7 — A runtime has a concurrency limit, and waiting for it is not idleness.** @@ -57,24 +59,20 @@ Using one should look like using any other model. You choose it in the same pick - **Runtime identity:** each local connection resolves to a `runtimeId`, the normalized base URL with every name for this machine's loopback (`localhost`, `127.0.0.1`, `::1`, `host.docker.internal`) folded into one. - **Admission:** a Redis-leased semaphore in `@aflow/redis` keyed by `runtimeId`, sized by `max_parallel` (default 1, the runtime's own default). Every caller goes through it. - **Leases:** each lease has an expiry that its holder heartbeats. A worker that dies frees its lease when the lease expires, with no sweep. A cancelled step leaves the queue and releases what it holds. -- **Waiting gives back the executor's slot.** Inside the AI executor, admission goes through `admitOperation` (`packages/executor-runtime/src/executor/operationAdmission.ts`). It already releases the executor-wide slot while a step waits, heartbeats the in-flight record, and starts the timeout only on admission, so queued local work never holds up cloud work. -- **Callers that cannot wait:** a call such as titling a session declines and retries later rather than queueing behind an agent turn for minutes. -- **Tests:** two credentials naming one runtime, a cancellation while queued, a worker killed while holding a lease, and a cloud call that is not held up by a local queue. -- **The idle clock:** a job waiting for admission heartbeats, and its idle window starts when its request is sent. Time in the queue never counts. -- **Prefill:** prefill is a silent stretch on every local request. A local connection therefore carries `first_token_timeout_ms` (default 15 min), which replaces the stream idle window until the first token. After that the ordinary idle window applies. -- **Non-agent ops:** the flat 120 s timeout becomes the same idle-plus-ceiling pair agent turns already use, for every AI op on a local connection. -- **SDK timeout:** the client's request timeout follows the step's budget instead of the SDK default. +- **A step waits before it starts, by the rules of an operation slot.** The executor runtime gains an optional `StepHandler.admit`: the handler names what the step must hold from its own input, after the context is built and before the step is marked started. While it waits the step stays SCHEDULED, gives back its executor slot, and keeps its in-flight record alive; a cancel ends the wait; an executor that stops claiming gives the step back to its stream; and a step that waited is checked again for supersession and cancellation once admitted. Its timeout starts only after admission, so queued local work is never charged to the work and never holds up cloud work. The AI handler's `admit` takes the runtime lease when the step's model is served by Ollama, and asks nothing otherwise. _Rejected:_ `admitOperation` as it stands. Its limiter is keyed by operation id and is in-process, while the runtime a step needs is known only from its model and credential, and its capacity is shared across processes. +- **Callers outside a step** take the same lease through the client: `createByokAiClientFactory(db, admission)` wraps a client for an Ollama model so every request holds a lease while it runs. Session titles pass `wait: false` and decline when the runtime is busy, since the conversation is due again on the next cycle; post-run judging waits. The eval-batch engine, the UI and API executors' generation and the memory embedder are not yet admitted, and are P5's coverage list. +- **The idle clock:** every AI operation on a local model runs under an idle window of `AI_LOCAL_MODEL_IDLE_TIMEOUT_MS` (15 min) and the agent turn's ceiling, in place of the flat 120 s or the 300 s stream idle window. Prefill is a silent stretch on every local request, and at 250 tokens a second a 75K-token prompt outlasts the cloud window (§9 G5). _Rejected:_ a per-connection first-token timeout. The timeout is chosen before any credential is read, so a connection field would be a setting nothing reads. **D8 — Fit is an estimate until the model is loaded, and both are labelled as what they are.** - **The estimate:** computed from: - - the catalog footprint, `local: { weightsBytes, kvBytesPerToken, measuredUnder: { contextTokens, parallel, kvCacheType, runtimeVersion } }`, written by the probe; - - the host inventory, which gains `machine: { totalMemoryBytes, availableMemoryBytes, gpuMemoryLimitBytes, chip }`. On macOS the GPU limit is the wired-memory cap, which is below total RAM. On Linux it is VRAM per device; + - the catalog footprint, `local: { weightsBytes, kvBytesPerToken, measuredUnder: { contextTokens, parallel, kvCacheType, runtimeVersion } }`, written by the probe. `/api/ps` cannot measure `kvBytesPerToken` on the MLX runner: it reports the same size and a 4,096-token context whatever `num_ctx` asks for, because that runner grows its cache with the prompt (§9 G5). The probe measures it from the runner process's resident memory before and after a long prompt; + - the host inventory, which gains `machine: { totalMemoryBytes, availableMemoryBytes, gpuMemoryLimitBytes, chip }`. On macOS the GPU limit is Metal's `recommendedMaxWorkingSetSize`, about 74% of RAM on the measured machine (19.07 GB of 24), which no `sysctl` reports. The host executor reads it through Metal. On Linux it is VRAM per device; - the models already resident (`/api/ps`); - the connection's `max_parallel`. Parallel requests multiply the KV cache, which grows with context × parallel. - **The headroom:** the estimate keeps headroom for the stack, the operating system and a browser, and reports the conditions it assumes. - **The four answers:** _fits_, _tight_, _does not fit_, or _unknown_. The answer is _unknown_ when the inputs are missing: an unpaired machine, a KV cache type the footprint was not measured with, or a build that differs from the one measured (D9). -- **Verified state:** once the model is loaded, `/api/ps` reports its actual size, its context and its GPU/CPU split. That is shown as verified and replaces the estimate. +- **Verified state:** once the model is loaded, `/api/ps` reports its actual size and its GPU/CPU split, and the host executor reads the runner process's memory as it grows. That is shown as verified and replaces the estimate. Free memory and swap are part of it: on the measured machine a 16 GB model left about 80 MB free and 6.6 GB in swap with the stack's databases running, which is _tight_ however the model's own numbers read. - **A trial load is an action:** on an unpaired machine, the operator starts a trial load explicitly ("load to check"). Reading diagnostics never loads a model. - **What over-sizing does:** a model that does not fit may load partly on CPU and run slowly, or fail to load, or press the whole machine into swap. Fit informs and never blocks a choice the operator makes. @@ -120,11 +118,11 @@ Which local models are _recommended_ for which role follows from the usefulness - **No specialized agent.** The work is reading one record and proposing one of its remedies, and Helmsman with these operations does both. A separate system agent would be a second surface whose guidance could drift from the UI's. - **Runner holds none of these operations.** They are Helmsman's alone, like the eval plane. A skill's run does not reconfigure the machine it runs on. -- **One GPU, several roles.** Helmsman, clerk, judge, titles and embeddings all compete for one runtime, and each model switch costs a load. The default for a local space is therefore a preset that assigns one chat model to every role (plus the embedding model). Mixing models per role is the operator's choice, and the models page shows its cost: total resident memory, and the load time per switch as P0 measures it. Only the model a role is about to use is warmed. `keep_alive` comes from the connection, default 30 min. +- **One GPU, several roles.** Helmsman, clerk, judge, titles and embeddings all compete for one runtime, and each model switch costs a load. The default for a local space is therefore a preset that assigns one chat model to every role (plus the embedding model). Mixing models per role is the operator's choice, and the models page shows its cost: total resident memory, and the load time per switch. On the measured machine a switch from a 9B to a 26B model took 49 s, and the MLX runner's first prompt after a load took 25–56 s whatever its length (§9 G5), so a switch in the middle of a run is a pause the operator can see. Only the model a role is about to use is warmed. `keep_alive` comes from the connection, default 30 min. - **Lifecycle actions are durable.** - - **Pull:** one pull per runtime and tag, observed and not duplicated. A second request joins the first. + - **Pull:** one pull per runtime and tag, observed and not duplicated. A second request joins the first; the runtime already joins two pulls of one tag, reporting the same progress to both (§9 G3). - **Disk space:** a pull is refused before it starts when the host inventory's free disk cannot hold the download. - - **Cancel and resume:** a pull can be cancelled. Whether an interrupted pull resumes is the runtime's behaviour, which P0 verifies. The UI says which one happens. + - **Cancel and resume:** a pull is cancelled by closing its request, which installs nothing. The next pull of the tag resumes from what was already downloaded (§9 G3), and the UI says so. - **Stale approvals:** an approved action re-checks its impact when it executes. When the set of runs it would end, or the space it needs, has changed since the card was shown, it parks again with the new impact rather than acting on the old one. - **Edition gating:** the provider, the routes, the operations and the "On this computer" surface exist only in the local edition. The UI gates on `useEdition().id === 'community-local'`, the server on the edition descriptor. A hosted tenant pointing a URL at loopback would be reaching the platform's own machine. @@ -141,8 +139,8 @@ Which local models are _recommended_ for which role follows from the usefulness - **Coverage:** every model call the platform makes on a space's behalf follows that space's choice. That includes session titles, the clerk, the judge, eval batches and memory embedding. P0 lists every call site that resolves a model outside role resolution. - **What counts as local:** a call is local only when both of these hold: - **The endpoint is this machine:** the connection's `runtimeId` (D7) is this machine's loopback or the appliance's `ollama` service. A runtime on the network (D14) is not local, even on the operator's own LAN. - - **The model runs on the runtime:** Ollama can forward a request to its own cloud models. A model the runtime marks as remote, or a cloud-suffixed tag, is not local. -- **The runtime's cloud setting:** diagnostics report whether cloud features are disabled (`OLLAMA_NO_CLOUD=1` or `disable_ollama_cloud` in `~/.ollama/server.json`), when the runtime exposes it. The remedy is the setting itself. The Linux appliance's `ollama` service sets it. + - **The model runs on the runtime:** Ollama can forward a request to its own cloud models. A model `/api/tags` lists with a `remote_host` is not local. `/api/show` carries no such marker (§9 G2), so the test reads the listing. +- **The runtime's cloud setting:** diagnostics report whether cloud features are disabled (`OLLAMA_NO_CLOUD=1` or `disable_ollama_cloud` in `~/.ollama/server.json`), read from `GET /api/status` (`{ cloud: { disabled, source } }`, §9 G2). The remedy is the setting itself. The Linux appliance's `ollama` service sets it. - **Discovered models (P6)** are classified by the same two tests. - **Refusal:** under `local_only`, a non-local model is refused at resolution with a `permission` error naming the role and the model. It is not silently swapped. - **Embeddings:** under `local_only` they need a local embedding model (D16). @@ -154,7 +152,7 @@ Which local models are _recommended_ for which role follows from the usefulness - **Changing the model** creates a new identity and a rebuild. The rebuild is a registered, candidate-driven background task. Search keeps using the old identity until the rebuild has covered the scope, then switches, and the old vectors are dropped. A search never runs against a partially rebuilt index. - **Jobs in flight:** a queued embedding job carries its identity and is discarded if the identity changed while it waited. - **No silent truncation:** the adapter always sends `truncate: false` to `/api/embed` (the runtime defaults to `true`). An input over the window fails, and the chunker splits by the model's window. -- **Dimensions:** local models emit 768 or 1024 (or 1536 through the `dimensions` option, for models trained to be truncated). Which of the two becomes the stored form is settled in P0 (§8). Either way, the identity rule above holds. +- **Dimensions:** the first local embedding model is one whose native size is at least 1,536, served at 1,536 through `dimensions`, so P5 needs no new vector column. Qwen3-Embedding 4B (2,560 native) does this; EmbeddingGemma (768 native) refuses it (§9 G4). A model below 1,536 needs its own column and is left until one is wanted. Either way, the identity rule above holds. **D13 — Usage is recorded as for any model, and cost shows "local", not "$0.000".** @@ -222,7 +220,7 @@ Not in the lineup: **Muse Spark Open**, which needs hundreds of gigabytes of mem - `diagnoseLocalRuntime`. - `packages/schemas`: - the credential registry entry `ollama`, with `base_url`, an optional token and the per-provider required secrets; - - the connection config fields (`context_tokens`, `max_parallel`, `first_token_timeout_ms`, `keep_alive`); + - the connection config fields (`context_tokens`, `max_parallel`, `keep_alive`); - `HostInventory.machine`, including free disk; - `LocalRuntimeDiagnostics` and its issue codes; - the local-model status and acting operations, the `local_model` target in the approval payload union (`requestedInput.ts`, `sessionBlockedOn.ts` and `actionCenter.ts`, beside `browser`), a new capability group with its tenant migration for the system profiles, and the Helmsman preset entries (Runner excluded); @@ -236,7 +234,8 @@ Not in the lineup: **Muse Spark Open**, which needs hundreds of gigabytes of mem - provisional streamed text until the response validates; - local timeouts; - a streaming `generateJson` fallback. -- `packages/redis`: the leased semaphore keyed by `runtimeId`, used by every caller. +- `packages/redis`: the leased semaphore keyed by `runtimeId`, and `runtimeAdmission` for callers outside a step. +- `packages/executor-runtime`: `StepHandler.admit`, waited on before a step starts. - `packages/server-runtime`: - readiness for `ollama`, with no `providerId = null` special case; - model-lifecycle routes (local edition only); @@ -291,14 +290,14 @@ _Exit:_ - the Ollama adapter; - the `ollama` connection, set up through the existing credentials screens; -- one curated catalog entry, for the model P0 ran; +- one curated catalog entry, `gemma4:26b`, the model P0 measured on the 24 GB machine; - `models:verify-local` with its compatibility tier and build identity; - native tool calling decided by capability; - the effective context window with overflow prevention; - streamed text that stays provisional until the response validates; -- runtime admission (`runtimeId`, leased semaphore, `admitOperation`, cancellation); -- the first-token window and idle-plus-ceiling timeouts; -- the streaming JSON fallback; +- runtime admission (`runtimeId`, the leased semaphore, `StepHandler.admit`, cancellation, the client wrapper for titles and judging); +- the local idle window and idle-plus-ceiling timeouts for every AI operation on a local model; +- compaction's summary model and the clerk's `auto` choice resolved through the space's roles, so a space whose only provider is the local runtime can compact a long conversation and title a session (§9, call sites 2 and 11); - readiness. The dev stack only. @@ -309,10 +308,12 @@ _Exit:_ - three concurrent sessions on it run one at a time and finish with no timeout; - a prompt near the window survives its prefill. +_Built and tested:_ everything in the list above, with unit tests for each proof below, and the adapter run against Ollama 0.40.5 on the 24 GB machine: plain text, two parallel native tool calls, streaming with thinking, JSON through the tool fallback after the MLX runner's 501, and a refusal before sending of a prompt the llama.cpp runner would have cut. `yarn models:verify-local` reports the `gemma4:26b` entry as matching its build. _Not yet run:_ the exit itself, which needs the dev stack started from this branch. + _Proof:_ - **Overflow:** a prompt the sizing method cannot rule safe is refused before it is sent, as `context_exceeded`. -- **Tool calling:** a model the probe marks `functionCalling: false` takes the JSON path. +- **Tool calling:** native tool calling follows `capabilities.functionCalling`, not the provider. - **Readiness:** a connection with no runtime listening is red in readiness, not green. - **Admission:** two credentials naming one runtime share one admission. A step cancelled while queued leaves the queue. A worker killed while holding a lease frees it when the lease expires. A cloud call is not held up by a local queue. - **Prefill:** a runtime that takes longer than the old 300 s idle window to send its first byte does not fail the turn. @@ -432,7 +433,74 @@ Each is a test in the phase that introduces it. The P1, P3, P3b and P5 exits are ## 8. Open questions -- **Embedding dimensions.** Several local embedding models are Matryoshka-trained and can emit 1536 dimensions through `dimensions`, which would avoid a migration. G4 measures what that costs in retrieval quality. Identity (D16) is required either way. +- **Retrieval at reduced dimensions.** G4 showed that 1,536-dimension output is available, not what it costs: its twenty-pair set scores every configuration perfectly. P5 measures recall at 1,536 against native size on a real memory corpus before the first local embedding model is recommended. +- **Image tokens.** G1 measured an 8×8 image at 11 tokens (llama.cpp) and 66 (MLX). P1 measures the image sizes the platform actually sends before images are sized by the method in D5. - **Discovered models earlier?** If the curated lineup turns out too narrow for testing, the P6 overlay could move up to P3. It costs a table and an on-demand probe run. - **Runtime decision models.** The current Ollama release lists models that return choices with probabilities. Whether they could back `ai.decision.decide` (plan 316) locally is a question for after P1. - **Windows.** Ollama runs there and the dev stack's host-side executors would reach it, but nobody has measured it. + +## 9. P0 results + +Measured on an Apple M3 with 24 GB of unified memory against Ollama 0.40.5, running as a Homebrew service with `OLLAMA_KV_CACHE_TYPE=q8_0`, flash attention on and default parallelism. Raw output: [`323-p0/apple-m3-24gb.json`](323-p0/apple-m3-24gb.json), produced by `yarn models:local-gates `. Models: `gemma4:26b` (MLX, NVFP4), `qwen3.5:9b` (MLX, NVFP4), `qwen3.5:4b-q4_K_M` (llama.cpp, Q4_K_M). On Apple Silicon this release serves a model through MLX unless its tag names a GGUF quantisation. + +### G1 — Overflow + +| | llama.cpp runner | MLX runner | +| ---------------------------------------------------------- | ------------------------------------------------------------------------------- | ---------------------------------------- | +| 6.4K-token prompt, `num_ctx` 4096 | cut from the front to 2,050 tokens, system message lost, answer wrong, no error | all 6,437 tokens evaluated, answer right | +| `/api/ps` context after a 16K request | 16,384 | 4,096 | +| `prompt_eval_count` on a cached repeat | full count (6,437), prompt read in 0.06 s | full count (6,437), 0.08 s | +| Characters per token: English, German, code, JSON, Chinese | 4.8, 4.8, 4.0, 2.9, 1.7 | 4.8, 4.8, 3.5, 2.8, 1.6 | +| Eight tool schemas | 889 tokens | 573 tokens | +| An 8×8 image | 11 tokens | 66 tokens | +| 1,800 characters of replayed `thinking` | 0 tokens | 0 tokens | + +A one-token evaluation counts a prompt exactly and leaves it cached: the request after it read 6,422 tokens in 0.07 s. `num_predict: 0` is not that: it means no limit. There is no tokenize endpoint. The chat template `/api/show` returns is `{{ .Prompt }}` on both runners, because the runner applies the model's own template, so a template fingerprint (D9) is taken from the model's digest and parameters rather than from that field. + +### G2 — Locality + +`GET /api/status` returns `{"cloud":{"disabled":false,"source":"none"}}`. A cloud model, once pulled, is a 307-byte manifest that `/api/tags` lists with `remote_host: "https://ollama.com"` and `remote_model`. `/api/show` for it has no remote field, and describes a cloud tag even before it is pulled. + +### G3 — Lifecycle + +Closing a pull's request cancels it and installs nothing. The next pull resumes from the bytes already downloaded. Two concurrent pulls of one tag both succeed with identical progress. A tag that does not exist fails at once with `pull model manifest: file does not exist`. A full disk was not exercised. + +### G4 — Embeddings + +| | EmbeddingGemma | Qwen3-Embedding 4B | +| ---------------------------------- | ----------------------------------------- | ----------------------------------------- | +| Native dimensions | 768 | 2,560 | +| `dimensions: 1536` | refused: exceeds embedding length | 1,536 | +| Over-long input, `truncate: false` | refused: input exceeds the context length | refused: input exceeds the context length | +| Over-long input, default | cut silently to 2,048 tokens of ~39,000 | cut silently to 4,095 tokens | + +The built-in retrieval set scored 1.0 recall in every configuration, so it separates nothing: what reduced dimensions cost is still open (§8). + +### G5 — The machine + +| | `qwen3.5:9b` | `gemma4:26b` | +| ------------------------------------- | ------------- | --------------- | +| Resident size, all on the GPU | 7.8 GB | 16.2 GB | +| Cold load: weights, then first prompt | 3.9 s + 0.9 s | 10.8 s + 24.7 s | +| Switch from the other model | — | 48.8 s | +| Prefill, 12K-token prompt | 157 tokens/s | 253 tokens/s | +| Decode | 25 tokens/s | 20 tokens/s | + +Metal's working-set limit is 19.07 GB. With `gemma4:26b` loaded and the stack's Postgres and Redis running, free memory was about 80 MB and swap held 6.6 GB. At 253 tokens a second, a prompt of about 75,000 tokens takes longer to read than the 300 s stream idle window allows (D7). + +### Call sites outside role resolution + +Every production model call was inventoried. The ones a `local_only` space (D12) would miss today: + +| # | Call | Process | How the model is chosen | Credentials | +| ----- | ------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | +| 1 | Agent turn, for an agent that is not a platform role, or a delegated Runner or Coach given `runner_model` / `coach_model` | AI executor | the agent's own config or the caller's argument, else `haiku` | space's | +| 2 | Compaction summary | AI executor | constant `RETENTION_POLICY.compactionSummaryModel` (`flash-lite`) | space's | +| 3–7 | `ai.*` text, JSON, stream, decide, embed, image and video operations | AI executor | step input, else a constant per operation (`haiku`, `jev`, `text-embedding-3-small`, `flash-image`, `veo`) | space's | +| 8–9 | UI surface and applet generation, and their repair | UI executor | step input, the calling agent's model, `UI_GEN_MODEL`, a constant, then the first entry of `TEXT_MODELS` with a resolvable credential | space's | +| 10 | API-simulation answers | API executor | the simulation's model, the caller's, `SIMULATION_GEN_MODEL`, a constant, then the same sweep | space's | +| 11 | Session titles and metadata (clerk) | orchestrator | the clerk role; `auto` picks a catalog candidate on the default's provider and refuses a `local` one | space's | +| 12–14 | Post-run judge, eval-batch judge, re-judge | orchestrator, API server | a criterion's `model` wins over the judge role | space's | +| 15–17 | Memory document, memory query and memory entry embedding | memory executor, AI executor | `memory_embed_config`, else `text-embedding-3-small` | **platform environment keys** | + +Rows 1–14 use the space's credentials, so a space holding only a local connection fails on them rather than leaking; a space or tenant that also holds a cloud key routes them to the cloud. Rows 15–17 never consult the space: its memory text goes to whichever provider the platform's environment keys name. Row 2 and row 11 are moved into P1, because a local-only Helmsman needs both. The rest are P5's coverage list. diff --git a/docs/plans/323-p0/apple-m3-24gb.json b/docs/plans/323-p0/apple-m3-24gb.json new file mode 100644 index 00000000..5265748e --- /dev/null +++ b/docs/plans/323-p0/apple-m3-24gb.json @@ -0,0 +1,349 @@ +{ + "machine": "Apple M3, 24 GB unified memory (MacBook Air), macOS 26.3", + "runtime": "Ollama 0.40.5, Homebrew service: OLLAMA_KV_CACHE_TYPE=q8_0, OLLAMA_FLASH_ATTENTION=1, default parallelism", + "gates": { + "g1": { + "gate": "overflow", + "runtime": { + "url": "http://127.0.0.1:11434", + "version": "0.40.5" + }, + "measuredAt": "2026-10-16T12:29:40.176Z", + "result": { + "qwen3.5:4b-q4_K_M": { + "runner": "llamacpp", + "format": "gguf", + "quantization": "Q4_K_M", + "digest": "d8b0f5e9760cd1682034f292d7ef72ec46f432149be0df7574bf2d6e92e38c04", + "overflow": { + "small": { + "promptEvalCount": 47, + "evalCount": 7, + "doneReason": "stop", + "content": "PELICAN-42", + "toolCalls": null, + "loadS": 1.34, + "promptS": 0.26, + "evalS": 0.23, + "wallS": 1.83 + }, + "overWindow": { + "promptEvalCount": 2050, + "evalCount": 40, + "doneReason": "length", + "content": "Based on the pattern provided in your text, here is the analysis:\n\n1. **Observation**: Every item listed from 447 to 649 is described as \"a", + "toolCalls": null, + "loadS": 0, + "promptS": 6.37, + "evalS": 1.67, + "wallS": 8.06, + "recalledSecret": false + }, + "fitsWindow": { + "promptEvalCount": 6437, + "evalCount": 7, + "doneReason": "stop", + "content": "PELICAN-42", + "toolCalls": null, + "loadS": 1.6, + "promptS": 21.21, + "evalS": 0.24, + "wallS": 23.07, + "recalledSecret": true + }, + "repeatedPrefix": { + "promptEvalCount": 6437, + "evalCount": 7, + "doneReason": "stop", + "content": "PELICAN-42", + "toolCalls": null, + "loadS": 0, + "promptS": 0.06, + "evalS": 0.24, + "wallS": 0.32 + }, + "truncatedSilently": true + }, + "charsPerToken": { + "english": { + "chars": 5160, + "tokens": 1079, + "charsPerToken": 4.78 + }, + "german": { + "chars": 5560, + "tokens": 1159, + "charsPerToken": 4.8 + }, + "chinese": { + "chars": 1520, + "tokens": 919, + "charsPerToken": 1.65 + }, + "code": { + "chars": 4825, + "tokens": 1198, + "charsPerToken": 4.03 + }, + "json": { + "chars": 4121, + "tokens": 1421, + "charsPerToken": 2.9 + } + }, + "overheadTokens": { + "eightTools": 889, + "oneTinyImage": 11, + "replayedThinking": 0, + "replayedThinkingChars": 1800 + } + }, + "gemma4:26b": { + "runner": "mlx", + "format": "safetensors", + "quantization": "nvfp4", + "digest": "48eb98ec778c163d87f69d735fcdd79e345063af9af4685613fd018ae5debf5e", + "overflow": { + "small": { + "promptEvalCount": 47, + "evalCount": 5, + "doneReason": "stop", + "content": "PELICAN-42", + "toolCalls": null, + "loadS": 20.58, + "promptS": 55.86, + "evalS": 0.23, + "wallS": 76.78 + }, + "overWindow": { + "promptEvalCount": 6437, + "evalCount": 5, + "doneReason": "stop", + "content": "PELICAN-42", + "toolCalls": null, + "loadS": 0.02, + "promptS": 30.35, + "evalS": 0.5, + "wallS": 31, + "recalledSecret": true + }, + "fitsWindow": { + "promptEvalCount": 6437, + "evalCount": 5, + "doneReason": "stop", + "content": "PELICAN-42", + "toolCalls": null, + "loadS": 0.05, + "promptS": 0.11, + "evalS": 0.35, + "wallS": 0.69, + "recalledSecret": true + }, + "repeatedPrefix": { + "promptEvalCount": 6437, + "evalCount": 5, + "doneReason": "stop", + "content": "PELICAN-42", + "toolCalls": null, + "loadS": 0, + "promptS": 0.08, + "evalS": 0.37, + "wallS": 0.47 + }, + "truncatedSilently": false + }, + "charsPerToken": { + "english": { + "chars": 5160, + "tokens": 1079, + "charsPerToken": 4.78 + }, + "german": { + "chars": 5560, + "tokens": 1159, + "charsPerToken": 4.8 + }, + "chinese": { + "chars": 1520, + "tokens": 959, + "charsPerToken": 1.58 + }, + "code": { + "chars": 4825, + "tokens": 1398, + "charsPerToken": 3.45 + }, + "json": { + "chars": 4121, + "tokens": 1480, + "charsPerToken": 2.78 + } + }, + "overheadTokens": { + "eightTools": 573, + "oneTinyImage": 66, + "replayedThinking": 0, + "replayedThinkingChars": 1800 + } + } + } + }, + "g2": { + "gate": "locality", + "runtime": { + "url": "http://127.0.0.1:11434", + "version": "0.40.5" + }, + "measuredAt": "2026-10-16T12:29:30.776Z", + "result": { + "status": { + "cloud": { + "disabled": false, + "source": "none" + } + }, + "remoteModels": [], + "localModels": ["qwen3.5:4b-q4_K_M", "qwen3.5:9b", "gemma4:26b"] + } + }, + "g3": { + "gate": "lifecycle", + "runtime": { + "url": "http://127.0.0.1:11434", + "version": "0.40.5" + }, + "measuredAt": "2026-10-16T12:47:23.963Z", + "result": { + "tag": "qwen3.5:0.8b-q8_0", + "cancelMidPull": { + "finished": false, + "error": "AbortError: This operation was aborted", + "cancelledAtBytes": 22021561, + "installedAfterCancel": false + }, + "resumedAndConcurrent": { + "wallS": 15, + "first": { + "events": 229, + "lastStatus": "success", + "firstCompleted": 26064741 + }, + "second": { + "events": 225, + "lastStatus": "success", + "firstCompleted": 26064741 + }, + "resumedFromPartial": true + }, + "missingTag": "Error: pull aflow-gate/does-not-exist:latest: pull model manifest: file does not exist", + "diskSpaceRefusal": "not exercised: needs a full disk; P3 checks free space before pulling" + } + }, + "g4": { + "gate": "embeddings", + "runtime": { + "url": "http://127.0.0.1:11434", + "version": "0.40.5" + }, + "measuredAt": "2026-10-16T12:33:44.044Z", + "result": { + "embeddinggemma": { + "nativeDims": 768, + "requested1536": "400 {\"error\":\"dimensions 1536 exceeds embedding length 768\"}", + "overWindow": { + "truncateFalse": "400 {\"error\":\"the input length exceeds the context length\"}", + "default": "accepted silently (2048 tokens of ~39000)" + }, + "recallAt1": { + "native": 1 + }, + "retrievalSet": "20 pairs \u2014 indicative only" + }, + "qwen3-embedding:4b": { + "nativeDims": 2560, + "requested1536": 1536, + "overWindow": { + "truncateFalse": "400 {\"error\":\"the input length exceeds the context length\"}", + "default": "accepted silently (4095 tokens of ~39000)" + }, + "recallAt1": { + "native": 1, + "at1536": 1, + "at768": 1 + }, + "retrievalSet": "20 pairs \u2014 indicative only" + } + } + }, + "g5": { + "gate": "machine", + "runtime": { + "url": "http://127.0.0.1:11434", + "version": "0.40.5" + }, + "measuredAt": "2026-10-16T12:35:38.157Z", + "result": { + "host": { + "platform": "darwin", + "arch": "arm64", + "cpu": "Apple M3", + "totalMemoryBytes": 25769803776, + "freeMemoryBytes": 81182720 + }, + "gpuWorkingSetBytes": 19069665280, + "serverConditions": "KV cache type, flash attention and parallelism are server environment, not visible over the API; record them beside the result", + "models": { + "qwen3.5:9b": { + "coldLoad": { + "wallS": 5.1, + "loadS": 3.93, + "firstPromptS": 0.85 + }, + "footprint": { + "ctx4096": { + "size": 7787985608, + "sizeVram": 7787985608, + "contextLength": 4096 + }, + "ctx32768": { + "size": 7787985608, + "sizeVram": 7787985608, + "contextLength": 4096 + } + }, + "prefillTokensPerS": 157, + "prefillTokens": 12105, + "decodeTokensPerS": 25, + "switchFromPrevious": null + }, + "gemma4:26b": { + "coldLoad": { + "wallS": 35.7, + "loadS": 10.77, + "firstPromptS": 24.66 + }, + "footprint": { + "ctx4096": { + "size": 16203594076, + "sizeVram": 16203594076, + "contextLength": 4096 + }, + "ctx32768": { + "size": 16203594076, + "sizeVram": 16203594076, + "contextLength": 4096 + } + }, + "prefillTokensPerS": 253, + "prefillTokens": 12106, + "decodeTokensPerS": 20, + "switchFromPrevious": { + "from": "qwen3.5:9b", + "wallS": 48.8 + } + } + } + } + } + } +} diff --git a/package.json b/package.json index 2926da2d..0ab7cc54 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,8 @@ "local:up": "docker compose -f docker-compose.local.yml up -d --build", "mcp:dev": "NODE_OPTIONS='--conditions=ts-source' node scripts/with-stack-env.mjs node scripts/watch-service.mjs apps/aflow-mcp/src/index.ts apps/aflow-mcp packages", "mcp:setup": "node scripts/mcp-local-setup.mjs", + "models:local-gates": "NODE_OPTIONS='--conditions=ts-source' node scripts/with-stack-env.mjs tsx scripts/local-model-gates.ts", + "models:verify-local": "NODE_OPTIONS='--conditions=ts-source' node scripts/with-stack-env.mjs tsx scripts/verify-local-models.ts", "models:verify-reasoning": "NODE_OPTIONS='--conditions=ts-source' node scripts/with-stack-env.mjs tsx scripts/verify-reasoning-profiles.ts", "orchestrator:dev": "NODE_OPTIONS='--conditions=ts-source' node scripts/with-stack-env.mjs node scripts/watch-service.mjs apps/aflow-orchestrator/src/index.ts apps/aflow-orchestrator packages", "packages:watch": "yarn workspaces foreach -A --include '@aflow/schemas' --include '@aflow/redis' --include '@aflow/database' --include '@aflow/executor-runtime' --include '@aflow/payload-store' --include '@aflow/ai-client' --include '@aflow/input-resolution' --include '@aflow/observability' --include '@aflow/lib' -p run dev", diff --git a/packages/ai-client/src/catalogModels.ts b/packages/ai-client/src/catalogModels.ts index cd5a3bb2..4c790094 100644 --- a/packages/ai-client/src/catalogModels.ts +++ b/packages/ai-client/src/catalogModels.ts @@ -56,6 +56,46 @@ const embeddingCapabilities: ModelCapabilities = { }; export const builtInModels: ModelDefinition[] = [ + // ============================================================================ + // Models on this computer (Ollama, local edition) — Plan 323 + // + // Capabilities are the conformance probe's (`yarn models:verify-local`), for + // the build in `local`. Free: nothing is billed for inference on the + // operator's own machine. + // ============================================================================ + { + id: 'ollama-gemma4-26b', + provider: 'ollama', + providerModelId: 'gemma4:26b', + displayName: 'Gemma 4 26B (this computer)', + description: + "Google's Gemma 4 26B mixture-of-experts, 3.8B active, served by Ollama on this computer. Vision, native tool calls, on/off thinking. Apache 2.0.", + contextWindow: 262_144, + maxOutputTokens: 8192, + capabilities: { + ...defaultCapabilities, + vision: true, + reasoning: true, + // The MLX runner refuses `format`; JSON rides a tool call instead. + structuredOutputs: false, + // The runtime has no tool_choice. + forcedToolChoice: false, + }, + pricing: { promptPer1M: 0, completionPer1M: 0, currency: 'USD' }, + traits: { speed: 2, cost: 1, intelligence: 3, outputType: 'text' }, + aliases: ['gemma-local'], + reasoning: { supported: ['off', 'high'] }, + local: { + runtimeTag: 'gemma4:26b', + digest: '48eb98ec778c163d87f69d735fcdd79e345063af9af4685613fd018ae5debf5e', + runner: 'mlx', + quantization: 'nvfp4', + runtimeVersion: '0.40.5', + probeVersion: 1, + measuredAt: '2026-10-16', + }, + }, + // ============================================================================ // OpenAI Text Models // diff --git a/packages/ai-client/src/client.ts b/packages/ai-client/src/client.ts index e4755e9a..33c506d9 100644 --- a/packages/ai-client/src/client.ts +++ b/packages/ai-client/src/client.ts @@ -140,6 +140,14 @@ export interface AIClient { * to ensure the actual provider model ID is sent, not the alias. */ resolveModelId(modelKey: string, explicitProvider?: AIProvider): string; + + /** + * The context window a request for this model gets: the catalog's, narrowed + * by the connection's when the connection sets one. What prompt budgeting + * and compaction must size against — a local runtime's window is the + * machine's, and can be a tenth of what the model supports. + */ + contextWindowFor(modelKey: string): number | undefined; } // ============================================================================ @@ -215,13 +223,11 @@ export function createAIClient(config: AIClientConfig): AIClient { adapter = createTypeSafeAdapter(providerConfig); break; } - case 'local': - throw new AIClientError( - `Provider ${provider} is not yet supported`, - 'invalid_request', - provider, - false, - ); + case 'ollama': { + const { createOllamaAdapter } = await import('./providers/ollama.js'); + adapter = createOllamaAdapter(providerConfig); + break; + } } adapters.set(provider, adapter); @@ -252,6 +258,8 @@ export function createAIClient(config: AIClientConfig): AIClient { maxOutputTokens?: number | undefined; /** True once the catalog has been consulted, whether or not it had an entry. */ fromCatalog?: boolean; + /** The window this request may use: the catalog's, narrowed by the connection's. */ + contextWindow?: number | undefined; /** * Whether the model is shown tool images. Catalog data only: a model * outside the catalog gets each image's description instead. @@ -282,21 +290,21 @@ export function createAIClient(config: AIClientConfig): AIClient { maxOutputTokens: model.maxOutputTokens, fromCatalog: true, vision: model.capabilities.vision, + contextWindow: effectiveContextWindow( + model.contextWindow, + config.providers[explicitProvider ?? model.provider]?.contextTokens, + ), }; } // Model not in catalog - use key as-is with explicit or default provider - if (explicitProvider) { + const fallbackProvider = explicitProvider ?? config.defaultProvider; + if (fallbackProvider) { + const connectionWindow = config.providers[fallbackProvider]?.contextTokens; return { - provider: explicitProvider, - providerModelId: modelKey, - }; - } - - if (config.defaultProvider) { - return { - provider: config.defaultProvider, + provider: fallbackProvider, providerModelId: modelKey, + ...(connectionWindow !== undefined ? { contextWindow: connectionWindow } : {}), }; } @@ -374,6 +382,7 @@ export function createAIClient(config: AIClientConfig): AIClient { ...(messages ? { messages } : {}), model: resolved.providerModelId, ...(maxTokens !== undefined ? { maxTokens } : {}), + ...(resolved.contextWindow !== undefined ? { contextWindow: resolved.contextWindow } : {}), ...(resolvedReasoning.reasoning !== undefined ? { reasoning: resolvedReasoning.reasoning } : {}), @@ -739,5 +748,17 @@ export function createAIClient(config: AIClientConfig): AIClient { const resolved = resolveModel(modelKey, explicitProvider); return resolved.providerModelId; }, + + contextWindowFor(modelKey: string): number | undefined { + if (!catalog.getModel(modelKey) && !config.defaultProvider) return undefined; + return resolveModel(modelKey).contextWindow; + }, }; } + +function effectiveContextWindow( + catalogWindow: number, + connectionWindow: number | undefined, +): number { + return connectionWindow === undefined ? catalogWindow : Math.min(catalogWindow, connectionWindow); +} diff --git a/packages/ai-client/src/index.ts b/packages/ai-client/src/index.ts index 2ebf7510..98929d20 100644 --- a/packages/ai-client/src/index.ts +++ b/packages/ai-client/src/index.ts @@ -55,6 +55,7 @@ export type { ModelCapabilities, ModelPricing, ModelDefinition, + LocalModelBuild, ModelTraits, UsageRecord, AIErrorCode, @@ -178,3 +179,6 @@ export { createRunwareAdapter, runwareTaskUuid } from './providers/runware.js'; // Decisions export { resolveDecisionAnswers } from './decision.js'; + +// Plan 323: the answer floor prompt budgeting and a local runtime's sizing share +export { MIN_ANSWER_TOKENS } from './providers/ollamaPromptSize.js'; diff --git a/packages/ai-client/src/providers/ollama.test.ts b/packages/ai-client/src/providers/ollama.test.ts new file mode 100644 index 00000000..079a01bd --- /dev/null +++ b/packages/ai-client/src/providers/ollama.test.ts @@ -0,0 +1,427 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import type { SessionId, StepExecutionId, TenantId } from '@aflow/schemas'; +import { createAIClient } from '../client.js'; +import { AIClientError } from '../errors.js'; +import { createOllamaAdapter } from './ollama.js'; + +const context = { + tenantId: 'tenant' as TenantId, + runId: 'run' as SessionId, + stepExecutionId: 'step' as StepExecutionId, +}; + +interface Sent { + url: string; + method: string; + body: Record; +} + +/** Answers each request with the next responder's result and records what was sent. */ +function runtime(...responders: Array<(body: Record) => Response>) { + const sent: Sent[] = []; + const fetchMock = vi.fn(async (url: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? '{}')) as Record; + sent.push({ url: String(url), method: init?.method ?? 'GET', body }); + const next = responders[Math.min(sent.length - 1, responders.length - 1)]; + if (!next) throw new Error('no responder'); + return next(body); + }); + vi.stubGlobal('fetch', fetchMock); + return sent; +} + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); + +const done = (message: Record, counts: { prompt: number; eval: number }) => () => + json({ + model: 'gemma4:26b', + message: { role: 'assistant', ...message }, + done: true, + done_reason: 'stop', + prompt_eval_count: counts.prompt, + eval_count: counts.eval, + }); + +async function rejection(promise: Promise): Promise { + const error = await promise.then( + () => { + throw new Error('expected a rejection'); + }, + (e: unknown) => e, + ); + expect(error).toBeInstanceOf(AIClientError); + return error as AIClientError; +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('the context window', () => { + it('is the smaller of the connection and the catalog, and is sent on every request', async () => { + const sent = runtime(done({ content: 'ok' }, { prompt: 12, eval: 1 })); + const client = createAIClient({ + providers: { ollama: { contextTokens: 16_384 } }, + defaultProvider: 'ollama', + }); + + expect(client.contextWindowFor('ollama-gemma4-26b')).toBe(16_384); + await client.generateText({ + ...context, + model: 'ollama-gemma4-26b', + messages: [{ role: 'user', content: 'Say ok.' }], + }); + + expect(sent[0]?.body['model']).toBe('gemma4:26b'); + expect((sent[0]?.body['options'] as Record)['num_ctx']).toBe(16_384); + }); + + it('refuses a prompt that cannot fit before sending it, naming its size and the window', async () => { + const big = 'The orchestrator writes durable state. '.repeat(1000); + const sent = runtime(done({ content: '' }, { prompt: 9_000, eval: 1 })); + const adapter = createOllamaAdapter({ contextTokens: 8192 }); + + const error = await rejection( + adapter.generateText({ + ...context, + model: 'gemma4:26b', + messages: [{ role: 'user', content: big }], + }), + ); + + expect(error.code).toBe('context_length'); + expect(error.message).toContain('9000 prompt tokens'); + expect(error.message).toContain('8192-token window'); + // Only the one-token count went out; the request itself was never sent. + expect(sent).toHaveLength(1); + expect((sent[0]?.body['options'] as Record)['num_predict']).toBe(1); + }); + + it('gives the answer what the window has left once the prompt is counted exactly', async () => { + const prompt = 'Plain English prose for the model to read. '.repeat(400); + const sent = runtime( + done({ content: '' }, { prompt: 3_600, eval: 1 }), + done({ content: 'ok' }, { prompt: 3_600, eval: 2 }), + ); + const adapter = createOllamaAdapter({ contextTokens: 8192 }); + + await adapter.generateText({ + ...context, + model: 'gemma4:26b', + maxTokens: 8192, + messages: [{ role: 'user', content: prompt }], + }); + + expect(sent).toHaveLength(2); + expect((sent[1]?.body['options'] as Record)['num_predict']).toBe(8192 - 3_600); + }); + + it('fails a response whose runtime evaluated less than the prompt can hold', async () => { + // The llama.cpp runner cuts an over-long prompt from the front and says + // nothing; what it reports evaluating is the only trace. + const prompt = 'A sentence the model has to read in full. '.repeat(200); + runtime(done({ content: 'answer about marbles' }, { prompt: 40, eval: 4 })); + const adapter = createOllamaAdapter({ contextTokens: 32_768 }); + + const error = await rejection( + adapter.generateText({ + ...context, + model: 'qwen3.5:4b-q4_K_M', + messages: [{ role: 'user', content: prompt }], + }), + ); + + expect(error.code).toBe('context_length'); + expect(error.message).toContain('Ollama read 40 tokens'); + }); + + it('counts a prompt carrying an image exactly, since characters cannot size one', async () => { + const sent = runtime( + done({ content: '' }, { prompt: 540, eval: 1 }), + done({ content: 'a red square' }, { prompt: 540, eval: 3 }), + ); + const adapter = createOllamaAdapter({ contextTokens: 32_768 }); + + await adapter.generateText({ + ...context, + model: 'gemma4:26b', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'What is this?' }, + { + type: 'image', + source: { type: 'base64', mediaType: 'image/png', data: 'iVBORw0KGgo=' }, + }, + ], + }, + ], + }); + + expect(sent).toHaveLength(2); + expect((sent[0]?.body['options'] as Record)['num_predict']).toBe(1); + expect((sent[1]?.body['messages'] as Array>)[0]?.['images']).toEqual([ + 'iVBORw0KGgo=', + ]); + }); +}); + +describe('tool calls', () => { + it('carries arguments as JSON text out and as objects in, keeping each call id', async () => { + const sent = runtime( + done( + { + content: '', + tool_calls: [ + { id: 'call_a', function: { name: 'get_weather', arguments: { city: 'Berlin' } } }, + { id: 'call_b', function: { name: 'get_weather', arguments: { city: 'Paris' } } }, + ], + }, + { prompt: 71, eval: 28 }, + ), + ); + const adapter = createOllamaAdapter({}); + + const response = await adapter.generateText({ + ...context, + model: 'gemma4:26b', + messages: [ + { role: 'user', content: 'Weather?' }, + { + role: 'assistant', + content: null, + toolCalls: [ + { + id: 'prev', + type: 'function', + function: { name: 'get_weather', arguments: '{"city":"Rome"}' }, + }, + ], + }, + { role: 'tool', toolCallId: 'prev', name: 'get_weather', content: 'sunny' }, + ], + tools: [ + { type: 'function', function: { name: 'get_weather', parameters: { type: 'object' } } }, + ], + }); + + expect(response.finishReason).toBe('tool_calls'); + expect(response.toolCalls?.map((c) => [c.id, c.function.arguments])).toEqual([ + ['call_a', '{"city":"Berlin"}'], + ['call_b', '{"city":"Paris"}'], + ]); + const messages = sent[0]?.body['messages'] as Array>; + expect(messages[1]?.['tool_calls']).toEqual([ + { function: { name: 'get_weather', arguments: { city: 'Rome' } } }, + ]); + expect(messages[2]).toMatchObject({ role: 'tool', tool_name: 'get_weather', content: 'sunny' }); + }); +}); + +describe('streaming', () => { + it('yields deltas as they arrive and resolves with the final counts', async () => { + const lines = [ + { message: { thinking: 'Let me see. ' }, done: false }, + { message: { content: 'Hello' }, done: false }, + { message: { content: ' world' }, done: false }, + { + message: { content: '' }, + done: true, + done_reason: 'stop', + prompt_eval_count: 20, + eval_count: 3, + }, + ]; + runtime( + () => + new Response(lines.map((l) => JSON.stringify(l)).join('\n') + '\n', { + status: 200, + headers: { 'content-type': 'application/x-ndjson' }, + }), + ); + const adapter = createOllamaAdapter({}); + const progress = vi.fn(); + + const { stream, response } = adapter.generateTextStream({ + ...context, + model: 'gemma4:26b', + messages: [{ role: 'user', content: 'Hi' }], + onStreamProgress: progress, + }); + const deltas: string[] = []; + for await (const chunk of stream) { + if (chunk.type === 'text_delta' && chunk.delta) deltas.push(chunk.delta); + } + + expect(deltas).toEqual(['Hello', ' world']); + const final = await response; + expect(final.content).toBe('Hello world'); + expect(final.thinking).toBe('Let me see. '); + expect(final.usage).toEqual({ promptTokens: 20, completionTokens: 3, totalTokens: 23 }); + expect(progress).toHaveBeenCalledTimes(4); + }); +}); + +describe('structured output', () => { + const schema = z.object({ answer: z.number() }); + + it('rides a tool when the runner refuses `format`, and stops asking for it', async () => { + const sent = runtime( + () => json({ error: 'structured output is unavailable' }, 501), + done( + { + content: '', + tool_calls: [{ function: { name: 'response', arguments: { answer: 391 } } }], + }, + { prompt: 160, eval: 8 }, + ), + done( + { content: '', tool_calls: [{ function: { name: 'response', arguments: { answer: 7 } } }] }, + { prompt: 160, eval: 8 }, + ), + ); + const adapter = createOllamaAdapter({}); + const ask = { + ...context, + model: 'gemma4:26b', + schema, + messages: [{ role: 'user' as const, content: '17*23?' }], + }; + + const first = await adapter.generateJson(ask); + const second = await adapter.generateJson(ask); + + expect(first.data).toEqual({ answer: 391 }); + expect(second.data).toEqual({ answer: 7 }); + expect(sent[0]?.body['format']).toBeDefined(); + expect(sent[1]?.body['format']).toBeUndefined(); + expect(sent[1]?.body['tools']).toBeDefined(); + expect(sent[2]?.body['format']).toBeUndefined(); + }); + + it('uses `format` where the runner supports it', async () => { + const sent = runtime(done({ content: '{"answer": 391}' }, { prompt: 30, eval: 8 })); + const adapter = createOllamaAdapter({}); + + const result = await adapter.generateJson({ + ...context, + model: 'qwen3.5:4b-q4_K_M', + schema, + messages: [{ role: 'user', content: '17*23?' }], + }); + + expect(result.data).toEqual({ answer: 391 }); + expect(sent[0]?.body['format']).toMatchObject({ type: 'object' }); + }); +}); + +describe('reasoning', () => { + it('turns thinking off for `off` and passes any other effort as the level', async () => { + const sent = runtime(done({ content: 'ok' }, { prompt: 5, eval: 1 })); + const adapter = createOllamaAdapter({}); + + await adapter.generateText({ + ...context, + model: 'm', + reasoning: { effort: 'off' }, + messages: [{ role: 'user', content: 'x' }], + }); + await adapter.generateText({ + ...context, + model: 'm', + reasoning: { effort: 'high' }, + messages: [{ role: 'user', content: 'x' }], + }); + await adapter.generateText({ + ...context, + model: 'm', + messages: [{ role: 'user', content: 'x' }], + }); + + expect(sent.map((s) => s.body['think'])).toEqual([false, 'high', undefined]); + }); +}); + +describe('embeddings', () => { + it('refuses to have an over-long input cut silently', async () => { + const sent = runtime(() => + json({ model: 'qwen3-embedding:4b', embeddings: [[0.1, 0.2]], prompt_eval_count: 4 }), + ); + const adapter = createOllamaAdapter({}); + + const result = await adapter.generateEmbedding({ + ...context, + model: 'qwen3-embedding:4b', + input: 'hello', + dimensions: 1536, + }); + + expect(sent[0]?.url).toBe('http://127.0.0.1:11434/api/embed'); + expect(sent[0]?.body).toMatchObject({ truncate: false, dimensions: 1536 }); + expect(result.dimensions).toBe(2); + }); +}); + +describe('failures', () => { + it('names the URL when the runtime is not answering', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new TypeError('fetch failed'); + }), + ); + const adapter = createOllamaAdapter({ baseUrl: 'http://127.0.0.1:11434/' }); + + const error = await rejection( + adapter.generateText({ ...context, model: 'm', messages: [{ role: 'user', content: 'x' }] }), + ); + + expect(error.code).toBe('network_error'); + expect(error.retryable).toBe(true); + expect(error.message).toContain('http://127.0.0.1:11434'); + }); + + it('tells the operator how to download a model the runtime does not have', async () => { + runtime(() => json({ error: "model 'gemma4:26b' not found" }, 404)); + const adapter = createOllamaAdapter({}); + + const error = await rejection( + adapter.generateText({ + ...context, + model: 'gemma4:26b', + messages: [{ role: 'user', content: 'x' }], + }), + ); + + expect(error.code).toBe('model_not_found'); + expect(error.message).toContain('ollama pull gemma4:26b'); + }); + + it('sends the proxy token only when the connection has one', async () => { + const seen: Array> = []; + vi.stubGlobal( + 'fetch', + vi.fn(async (_url: string, init?: RequestInit) => { + seen.push(init?.headers as Record); + return done({ content: 'ok' }, { prompt: 5, eval: 1 })(); + }), + ); + + await createOllamaAdapter({ apiKey: '' }).generateText({ + ...context, + model: 'm', + messages: [{ role: 'user', content: 'x' }], + }); + await createOllamaAdapter({ apiKey: 'secret' }).generateText({ + ...context, + model: 'm', + messages: [{ role: 'user', content: 'x' }], + }); + + expect(seen[0]?.['authorization']).toBeUndefined(); + expect(seen[1]?.['authorization']).toBe('Bearer secret'); + }); +}); diff --git a/packages/ai-client/src/providers/ollama.ts b/packages/ai-client/src/providers/ollama.ts new file mode 100644 index 00000000..b1f4f850 --- /dev/null +++ b/packages/ai-client/src/providers/ollama.ts @@ -0,0 +1,699 @@ +/** + * Ollama provider adapter (Plan 323 D3). + * + * Speaks the native `/api/chat` and `/api/embed`, not the OpenAI-compatible + * `/v1`: only the native API takes a per-request context length, and a local + * runtime's window is a property of the machine that must be sent every time. + */ +import { zodToJsonSchema } from 'zod-to-json-schema'; +import { OLLAMA_DEFAULT_BASE_URL, OLLAMA_DEFAULT_CONTEXT_TOKENS } from '@aflow/schemas'; +import type { + GenerateTextRequest, + GenerateTextResponse, + GenerateJsonRequest, + GenerateJsonResponse, + GenerateEmbeddingRequest, + GenerateEmbeddingResponse, + StreamingResponse, + ProviderConfig, + ChatMessage, + ToolCall, + ToolDefinition, + TextStreamChunk, + TokenUsage, + FinishReason, + AIRequestOptions, +} from '../types.js'; +import type { AIProviderAdapter } from '../adapter.js'; +import { AIClientError, buildStreamTruncationError } from '../errors.js'; +import { moveToolImagesToUserMessages, toolResultText } from './toolResultContent.js'; +import { parseJsonResponse } from './jsonResponseParse.js'; +import { MIN_ANSWER_TOKENS, sizePrompt, type PromptSizing } from './ollamaPromptSize.js'; + +interface Fit { + sizing: PromptSizing; + /** Tokens the answer may take: what the caller asked for, or what the window has left. */ + numPredict: number; +} + +interface OllamaMessage { + role: 'system' | 'user' | 'assistant' | 'tool'; + content: string; + images?: string[]; + tool_calls?: Array<{ function: { name: string; arguments: Record } }>; + tool_name?: string; +} + +interface OllamaTool { + type: 'function'; + function: { name: string; description?: string; parameters: Record }; +} + +interface OllamaChatChunk { + model?: string; + message?: { + content?: string; + thinking?: string; + tool_calls?: Array<{ id?: string; function?: { name?: string; arguments?: unknown } }>; + }; + done?: boolean; + done_reason?: string; + prompt_eval_count?: number; + eval_count?: number; + error?: string; +} + +export function toOllamaMessages(messages: ChatMessage[]): OllamaMessage[] { + return moveToolImagesToUserMessages(messages, 'ollama').map((message): OllamaMessage => { + switch (message.role) { + case 'system': + return { role: 'system', content: message.content }; + case 'user': { + if (typeof message.content === 'string') return { role: 'user', content: message.content }; + const text: string[] = []; + const images: string[] = []; + for (const part of message.content) { + if (part.type === 'text') text.push(part.text); + else if (part.source.type === 'base64') images.push(part.source.data); + else { + throw new AIClientError( + 'Ollama takes images as bytes, not URLs; the image must be resolved before it is sent.', + 'invalid_request', + 'ollama', + false, + ); + } + } + return { + role: 'user', + content: text.join('\n'), + ...(images.length > 0 ? { images } : {}), + }; + } + case 'assistant': + return { + role: 'assistant', + content: message.content ?? '', + ...(message.toolCalls && message.toolCalls.length > 0 + ? { + tool_calls: message.toolCalls.map((tc) => ({ + function: { + name: tc.function.name, + arguments: parseArguments(tc.function.arguments), + }, + })), + } + : {}), + }; + case 'tool': + return { + role: 'tool', + content: toolResultText(message, 'ollama'), + ...(message.name ? { tool_name: message.name } : {}), + }; + default: { + const exhaustive: never = message; + throw new Error(`Unknown message role: ${JSON.stringify(exhaustive)}`); + } + } + }); +} + +/** Ollama carries tool arguments as an object; the platform carries them as JSON text. */ +function parseArguments(text: string): Record { + try { + const parsed: unknown = JSON.parse(text); + return parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + } catch { + return {}; + } +} + +function toOllamaTools(tools: ToolDefinition[] | undefined): OllamaTool[] | undefined { + if (!tools || tools.length === 0) return undefined; + return tools.map((tool) => ({ + type: 'function', + function: { + name: tool.function.name, + ...(tool.function.description ? { description: tool.function.description } : {}), + parameters: tool.function.parameters, + }, + })); +} + +function fromOllamaToolCalls( + calls: NonNullable['tool_calls'], + offset: number, +): ToolCall[] { + return (calls ?? []).map((call, i) => ({ + id: call.id ?? `call_${offset + i}`, + type: 'function', + function: { + name: call.function?.name ?? '', + arguments: + typeof call.function?.arguments === 'string' + ? call.function.arguments + : JSON.stringify(call.function?.arguments ?? {}), + }, + })); +} + +function toFinishReason(doneReason: string | undefined, toolCalls: number): FinishReason { + if (toolCalls > 0) return 'tool_calls'; + return doneReason === 'length' ? 'length' : 'stop'; +} + +function toUsage(chunk: OllamaChatChunk): TokenUsage { + const promptTokens = chunk.prompt_eval_count ?? 0; + const completionTokens = chunk.eval_count ?? 0; + return { promptTokens, completionTokens, totalTokens: promptTokens + completionTokens }; +} + +/** + * `think` takes `false`, `true` or a level. A model whose only levels are + * on and off treats any level as on (§9 G1), so an effort above `off` is + * passed as the level and every model reads it. + */ +function thinkFor(reasoning: AIRequestOptions['reasoning']): boolean | string | undefined { + const effort = reasoning?.effort; + if (effort === undefined) return undefined; + return effort === 'off' ? false : effort; +} + +function isPlatformTimeoutAbort(signal: AbortSignal | undefined): { timeoutMs: number } | null { + if (!signal?.aborted) return null; + const reason = signal.reason as { marker?: unknown; timeoutMs?: unknown } | undefined; + if (reason?.marker === 'phoenix.executor.timeout' && typeof reason.timeoutMs === 'number') { + return { timeoutMs: reason.timeoutMs }; + } + return null; +} + +export function normalizeOllamaError( + error: unknown, + ctx: { baseUrl: string; model?: string; signal?: AbortSignal | undefined }, +): AIClientError { + if (error instanceof AIClientError) return error; + if (error instanceof Error && error.name === 'AbortError') { + const platformTimeout = isPlatformTimeoutAbort(ctx.signal); + return new AIClientError( + platformTimeout + ? `Platform timeout exceeded after ${platformTimeout.timeoutMs}ms (provider=ollama model=${ctx.model ?? 'unknown'}).` + : `Ollama request aborted (model=${ctx.model ?? 'unknown'}).`, + 'timeout', + 'ollama', + true, + { cause: error }, + ); + } + const message = error instanceof Error ? error.message : String(error); + return new AIClientError( + `Ollama is not reachable at ${ctx.baseUrl}: ${message}. Start Ollama on this computer, or correct its URL in Settings → Credentials.`, + 'network_error', + 'ollama', + true, + { cause: error }, + ); +} + +async function httpError(res: Response, model: string): Promise { + const body = await res.text().catch(() => ''); + let detail = body; + try { + const parsed = JSON.parse(body) as { error?: unknown }; + if (typeof parsed.error === 'string') detail = parsed.error; + } catch { + // The body was not JSON; report it as text. + } + if (res.status === 404) { + return new AIClientError( + `Ollama has no model ${model} (${detail}). Download it on this computer first: ollama pull ${model}`, + 'model_not_found', + 'ollama', + false, + { providerErrorCode: String(res.status) }, + ); + } + if (res.status === 401 || res.status === 403) { + return new AIClientError(`Ollama refused the token: ${detail}`, 'auth', 'ollama', false, { + providerErrorCode: String(res.status), + }); + } + return new AIClientError( + `Ollama error ${res.status}: ${detail}`, + res.status >= 500 ? 'provider_error' : 'invalid_request', + 'ollama', + res.status >= 500 && res.status !== 501, + { providerErrorCode: String(res.status) }, + ); +} + +const STRUCTURED_OUTPUT_UNAVAILABLE = 'structured output is unavailable'; + +export function createOllamaAdapter(config: ProviderConfig): AIProviderAdapter { + const baseUrl = (config.baseUrl ?? OLLAMA_DEFAULT_BASE_URL).replace(/\/+$/, ''); + const headers: Record = { + 'content-type': 'application/json', + ...(config.apiKey ? { authorization: `Bearer ${config.apiKey}` } : {}), + }; + // Models whose runner refuses `format` (the MLX runner refuses it outright, + // §9 G1). Learned from the runtime's own answer rather than written down, so + // a runner that gains structured output is used the moment it does. + const refusesFormat = new Set(); + + function windowFor(request: AIRequestOptions): number { + return request.contextWindow ?? config.contextTokens ?? OLLAMA_DEFAULT_CONTEXT_TOKENS; + } + + function chatBody( + request: GenerateTextRequest | GenerateJsonRequest, + messages: OllamaMessage[], + extra: { + stream: boolean; + tools?: OllamaTool[] | undefined; + format?: unknown; + numPredict?: number; + }, + ): Record { + const think = thinkFor(request.reasoning); + return { + model: request.model, + messages, + stream: extra.stream, + ...(extra.tools ? { tools: extra.tools } : {}), + ...(extra.format !== undefined ? { format: extra.format } : {}), + ...(think !== undefined ? { think } : {}), + ...(config.keepAlive ? { keep_alive: config.keepAlive } : {}), + options: { + num_ctx: windowFor(request), + ...(extra.numPredict !== undefined + ? { num_predict: extra.numPredict } + : request.maxTokens + ? { num_predict: request.maxTokens } + : {}), + ...(request.temperature !== undefined ? { temperature: request.temperature } : {}), + ...(request.stopSequences ? { stop: request.stopSequences } : {}), + }, + }; + } + + async function post( + path: string, + body: unknown, + signal: AbortSignal | undefined, + ): Promise { + return fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: JSON.stringify(body), + ...(signal ? { signal } : {}), + }); + } + + async function chatOnce( + body: Record, + request: AIRequestOptions, + ): Promise { + let res: Response; + try { + res = await post('/api/chat', body, request.signal); + } catch (error) { + throw normalizeOllamaError(error, { baseUrl, model: request.model, signal: request.signal }); + } + if (!res.ok) throw await httpError(res, request.model); + const chunk = (await res.json()) as OllamaChatChunk; + if (chunk.error) + throw new AIClientError(`Ollama error: ${chunk.error}`, 'provider_error', 'ollama', true); + return chunk; + } + + /** + * Plan 323 D5: a request is sent only once it is known to fit. A prompt + * whose conservative size fits is sent as is; any other is counted exactly + * by a one-token evaluation, whose prefix the real request then reads from + * the runtime's cache. + */ + async function ensureFits( + request: GenerateTextRequest | GenerateJsonRequest, + messages: OllamaMessage[], + tools: OllamaTool[] | undefined, + ): Promise { + const window = windowFor(request); + const requested = request.maxTokens ?? MIN_ANSWER_TOKENS; + const sizing = sizePrompt(messages, tools); + if (!sizing.needsExactCount && sizing.conservativeTokens + requested <= window) { + return { sizing, numPredict: requested }; + } + const counted = await chatOnce( + chatBody(request, messages, { stream: false, tools, numPredict: 1 }), + request, + ); + const exact = counted.prompt_eval_count ?? 0; + // A counted prompt below the floor is one the runtime cut while counting. + const promptTokens = exact < sizing.lowerBoundTokens ? sizing.lowerBoundTokens : exact; + const room = window - promptTokens; + if (exact < sizing.lowerBoundTokens || room < Math.min(requested, MIN_ANSWER_TOKENS)) { + throw contextExceeded(request.model, { + promptTokens, + atLeast: exact < sizing.lowerBoundTokens, + reserve: Math.min(requested, MIN_ANSWER_TOKENS), + window, + }); + } + // The answer gets what the window has left, so prompt and answer together + // never pass it — a runner that enforces the window cuts there, and one + // that does not spends memory nothing accounted for. + return { sizing, numPredict: Math.min(requested, room) }; + } + + /** The runtime evaluated fewer tokens than the prompt can possibly hold: it truncated. */ + function assertNotTruncated( + sizing: PromptSizing, + chunk: OllamaChatChunk, + request: AIRequestOptions, + ): void { + const evaluated = chunk.prompt_eval_count; + if (evaluated !== undefined && evaluated < sizing.lowerBoundTokens) { + throw new AIClientError( + `Ollama read ${evaluated} tokens of a prompt that holds at least ${sizing.lowerBoundTokens}: the runtime cut it to fit its window, and the answer was discarded (model=${request.model}, window=${windowFor(request)}). ` + + 'Raise the context window on the Ollama connection if memory allows, or let the conversation compact.', + 'context_length', + 'ollama', + false, + { providerErrorCode: 'context_exceeded' }, + ); + } + } + + async function generateText(request: GenerateTextRequest): Promise { + const messages = toOllamaMessages(request.messages); + const tools = toOllamaTools(request.tools); + const fit = await ensureFits(request, messages, tools); + const chunk = await chatOnce( + chatBody(request, messages, { stream: false, tools, numPredict: fit.numPredict }), + request, + ); + assertNotTruncated(fit.sizing, chunk, request); + const toolCalls = fromOllamaToolCalls(chunk.message?.tool_calls, 0); + const thinking = chunk.message?.thinking; + return { + content: chunk.message?.content || null, + ...(toolCalls.length > 0 ? { toolCalls } : {}), + finishReason: toFinishReason(chunk.done_reason, toolCalls.length), + usage: toUsage(chunk), + model: chunk.model ?? request.model, + provider: 'ollama', + ...(thinking ? { thinking } : {}), + }; + } + + return { + provider: 'ollama', + + generateText, + + generateTextStream(request: GenerateTextRequest): StreamingResponse { + let resolveResponse!: (value: GenerateTextResponse) => void; + let rejectResponse!: (error: Error) => void; + const response = new Promise((resolve, reject) => { + resolveResponse = resolve; + rejectResponse = reject; + }); + + const stream = (async function* (): AsyncIterable { + const startMs = Date.now(); + try { + const messages = toOllamaMessages(request.messages); + const tools = toOllamaTools(request.tools); + const fit = await ensureFits(request, messages, tools); + let res: Response; + try { + res = await post( + '/api/chat', + chatBody(request, messages, { stream: true, tools, numPredict: fit.numPredict }), + request.signal, + ); + } catch (error) { + throw normalizeOllamaError(error, { + baseUrl, + model: request.model, + signal: request.signal, + }); + } + if (!res.ok || !res.body) throw await httpError(res, request.model); + + let content = ''; + let thinking = ''; + const toolCalls: ToolCall[] = []; + let final: OllamaChatChunk | undefined; + const decoder = new TextDecoder(); + let buffer = ''; + const reader = res.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + let newline; + while ((newline = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (!line) continue; + const chunk = JSON.parse(line) as OllamaChatChunk; + request.onStreamProgress?.(); + if (chunk.error) { + throw new AIClientError( + `Ollama error: ${chunk.error}`, + 'provider_error', + 'ollama', + true, + ); + } + const delta = chunk.message; + if (delta?.thinking) { + thinking += delta.thinking; + yield { type: 'thinking_delta', delta: delta.thinking }; + } + if (delta?.content) { + content += delta.content; + yield { type: 'text_delta', delta: delta.content }; + } + if (delta?.tool_calls && delta.tool_calls.length > 0) { + // A tool call arrives whole in one chunk rather than as argument deltas. + for (const call of fromOllamaToolCalls(delta.tool_calls, toolCalls.length)) { + toolCalls.push(call); + yield { + type: 'tool_call_delta', + toolCallId: call.id, + toolCallName: call.function.name, + toolCallArguments: call.function.arguments, + }; + } + } + if (chunk.done) final = chunk; + } + } + + if (!final) { + throw buildStreamTruncationError({ + provider: 'ollama', + model: request.model, + signal: request.signal, + startMs, + accumulatedChars: content.length, + toolCallCount: toolCalls.length, + }); + } + assertNotTruncated(fit.sizing, final, request); + const usage = toUsage(final); + const finishReason = toFinishReason(final.done_reason, toolCalls.length); + yield { type: 'usage', usage }; + yield { type: 'done', finishReason }; + resolveResponse({ + content: content || null, + ...(toolCalls.length > 0 ? { toolCalls } : {}), + finishReason, + usage, + model: final.model ?? request.model, + provider: 'ollama', + ...(thinking ? { thinking } : {}), + }); + } catch (error) { + const normalized = normalizeOllamaError(error, { + baseUrl, + model: request.model, + signal: request.signal, + }); + rejectResponse(normalized); + throw normalized; + } + })(); + + return { stream, response }; + }, + + async generateJson(request: GenerateJsonRequest): Promise> { + const schema = + request.rawJsonSchema ?? + (zodToJsonSchema(request.schema, { $refStrategy: 'none' }) as Record); + const messages = toOllamaMessages(request.messages); + + let chunk: OllamaChatChunk | undefined; + let rawContent: string | undefined; + if (!refusesFormat.has(request.model)) { + try { + const fit = await ensureFits(request, messages, undefined); + chunk = await chatOnce( + chatBody(request, messages, { + stream: false, + format: schema, + numPredict: fit.numPredict, + }), + request, + ); + assertNotTruncated(fit.sizing, chunk, request); + rawContent = chunk.message?.content ?? ''; + } catch (error) { + if (!( + error instanceof AIClientError && error.message.includes(STRUCTURED_OUTPUT_UNAVAILABLE) + )) { + throw error; + } + refusesFormat.add(request.model); + } + } + if (rawContent === undefined) { + // The runner cannot constrain output, so the schema rides a tool, as + // the Anthropic adapter carries it: the model fills the tool's + // arguments, and those are the response. + const toolName = request.schemaName ?? 'response'; + const tools: OllamaTool[] = [ + { + type: 'function', + function: { + name: toolName, + description: request.schemaDescription ?? 'Structured response output', + parameters: schema, + }, + }, + ]; + const withInstruction: OllamaMessage[] = [ + ...messages, + { + role: 'system', + content: `Respond by calling the "${toolName}" tool. Do not answer outside the tool call.`, + }, + ]; + const fit = await ensureFits(request, withInstruction, tools); + chunk = await chatOnce( + chatBody(request, withInstruction, { stream: false, tools, numPredict: fit.numPredict }), + request, + ); + assertNotTruncated(fit.sizing, chunk, request); + const call = chunk.message?.tool_calls?.find((c) => c.function?.name === toolName); + rawContent = + call !== undefined + ? typeof call.function?.arguments === 'string' + ? call.function.arguments + : JSON.stringify(call.function?.arguments ?? {}) + : (chunk.message?.content ?? ''); + } + + if (chunk?.done_reason === 'length') { + throw new AIClientError( + `Ollama response truncated at the output limit (done_reason='length', eval_count=${chunk.eval_count ?? 0}, model=${request.model}). Increase maxTokens or simplify the requested output.`, + 'output_truncated', + 'ollama', + true, + ); + } + let parsedUnknown: unknown; + try { + parsedUnknown = parseJsonResponse(rawContent).parsed; + } catch (err) { + throw new AIClientError( + `Ollama returned invalid JSON: ${err instanceof Error ? err.message : String(err)}`, + 'invalid_request', + 'ollama', + false, + { cause: err }, + ); + } + const parsed = request.schema.safeParse(parsedUnknown); + if (!parsed.success) { + throw new AIClientError( + `Ollama returned JSON that failed schema validation: ${parsed.error.message}`, + 'invalid_request', + 'ollama', + false, + { cause: parsed.error }, + ); + } + const thinking = chunk?.message?.thinking; + return { + data: parsed.data, + rawContent, + finishReason: 'stop', + usage: chunk ? toUsage(chunk) : { promptTokens: 0, completionTokens: 0, totalTokens: 0 }, + model: chunk?.model ?? request.model, + provider: 'ollama', + ...(thinking ? { thinking } : {}), + }; + }, + + async generateEmbedding(request: GenerateEmbeddingRequest): Promise { + let res: Response; + try { + res = await post( + '/api/embed', + { + model: request.model, + input: request.input, + // The runtime's default cuts an over-long input to its window + // without a word (§9 G4); refusing it lets the chunker split it. + truncate: false, + ...(request.dimensions ? { dimensions: request.dimensions } : {}), + ...(config.keepAlive ? { keep_alive: config.keepAlive } : {}), + }, + request.signal, + ); + } catch (error) { + throw normalizeOllamaError(error, { + baseUrl, + model: request.model, + signal: request.signal, + }); + } + if (!res.ok) throw await httpError(res, request.model); + const body = (await res.json()) as { + embeddings: number[][]; + model?: string; + prompt_eval_count?: number; + }; + const promptTokens = body.prompt_eval_count ?? 0; + return { + embeddings: body.embeddings, + usage: { promptTokens, completionTokens: 0, totalTokens: promptTokens }, + model: body.model ?? request.model, + provider: 'ollama', + dimensions: body.embeddings[0]?.length ?? 0, + }; + }, + }; +} + +function contextExceeded( + model: string, + size: { promptTokens: number; atLeast: boolean; reserve: number; window: number }, +): AIClientError { + const prompt = `${size.atLeast ? 'at least ' : ''}${size.promptTokens} prompt tokens`; + return new AIClientError( + `The prompt does not fit this computer's context window: ${prompt} plus ${size.reserve} reserved for the answer exceed the ${size.window}-token window (model=${model}). ` + + 'Raise the context window on the Ollama connection if memory allows, or let the conversation compact.', + 'context_length', + 'ollama', + false, + { providerErrorCode: 'context_exceeded' }, + ); +} diff --git a/packages/ai-client/src/providers/ollamaPromptSize.test.ts b/packages/ai-client/src/providers/ollamaPromptSize.test.ts new file mode 100644 index 00000000..c6641bbc --- /dev/null +++ b/packages/ai-client/src/providers/ollamaPromptSize.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest'; +import { sizePrompt } from './ollamaPromptSize.js'; + +describe('sizePrompt', () => { + it('bounds a prompt between the lowest and highest measured characters per token', () => { + const text = 'The orchestrator schedules each tool call as its own step. '.repeat(20); + const sizing = sizePrompt([{ content: text }]); + + // English runs at about 4.8 characters a token, so the true count sits between the two. + const actual = text.length / 4.8; + expect(sizing.conservativeTokens).toBeGreaterThan(actual); + expect(sizing.lowerBoundTokens).toBeLessThan(actual); + expect(sizing.needsExactCount).toBe(false); + }); + + it('never overstates the floor for padding, indentation or rules of one character', () => { + // A run of one character can be a single token however long it is; a + // floor that counted it would read a legitimate prompt as a truncated one. + const padded = `${' '.repeat(5000)}${'-'.repeat(5000)}x`; + expect(sizePrompt([{ content: padded }]).lowerBoundTokens).toBe(0); + }); + + it('counts tool schemas and tool calls toward both bounds', () => { + const bare = sizePrompt([{ content: 'Say ok.' }]); + const withTools = sizePrompt( + [ + { + content: 'Say ok.', + tool_calls: [{ function: { name: 'read', arguments: { path: '/a' } } }], + }, + ], + [{ type: 'function', function: { name: 'read', parameters: { type: 'object' } } }], + ); + expect(withTools.conservativeTokens).toBeGreaterThan(bare.conservativeTokens); + expect(withTools.lowerBoundTokens).toBeGreaterThan(bare.lowerBoundTokens); + }); + + it('asks for an exact count whenever an image is in the prompt', () => { + expect(sizePrompt([{ content: 'What is this?', images: ['iVBOR'] }]).needsExactCount).toBe( + true, + ); + }); +}); diff --git a/packages/ai-client/src/providers/ollamaPromptSize.ts b/packages/ai-client/src/providers/ollamaPromptSize.ts new file mode 100644 index 00000000..b73fc9d9 --- /dev/null +++ b/packages/ai-client/src/providers/ollamaPromptSize.ts @@ -0,0 +1,76 @@ +/** + * How large a prompt is, in a runtime's tokens, without its tokenizer + * (Plan 323 D5). + * + * Characters per token, measured against the runtime's own counts (§9 G1), + * run from 1.6 (Chinese) through 2.8 (JSON) and 3.5–4.0 (code) to 4.8 + * (English and German). Dividing by the lowest gives a size no prompt + * exceeds; dividing by the highest gives one every prompt reaches. The first + * decides whether a prompt may be sent without counting it; the second is + * how a response reveals that the runtime cut its prompt. + */ + +/** + * The smallest answer a prompt must leave room for. Prompt budgeting holds the + * same reserve back (the executor's retention policy reads this constant), so + * a prompt the budget let through is never refused here for want of room. + */ +export const MIN_ANSWER_TOKENS = 4096; + +/** Below the lowest measured ratio, so no script or format is undercounted. */ +const CHARS_PER_TOKEN_FLOOR = 1.5; +/** Above the highest measured ratio, so every prompt is at least this many tokens. */ +const CHARS_PER_TOKEN_CEILING = 5; + +interface SizedMessage { + content: string; + images?: string[]; + tool_calls?: unknown; + tool_name?: string; +} + +export interface PromptSizing { + /** Tokens the prompt cannot exceed. */ + conservativeTokens: number; + /** Tokens the prompt cannot fall below. */ + lowerBoundTokens: number; + /** + * The prompt carries something characters cannot size. Images are one: a + * 1280×800 image costs 524 tokens on one model and 1,002 on another, and + * a model that sizes images by resolution costs more for a larger one. + */ + needsExactCount: boolean; +} + +export function sizePrompt( + messages: readonly SizedMessage[], + tools?: readonly unknown[], +): PromptSizing { + const texts: string[] = tools && tools.length > 0 ? [JSON.stringify(tools)] : []; + let needsExactCount = false; + for (const message of messages) { + texts.push(message.content); + if (message.tool_calls !== undefined) texts.push(JSON.stringify(message.tool_calls)); + if (message.tool_name !== undefined) texts.push(message.tool_name); + if (message.images && message.images.length > 0) needsExactCount = true; + } + let chars = 0; + let incompressibleChars = 0; + for (const text of texts) { + chars += text.length; + incompressibleChars += collapseRuns(text).length; + } + return { + conservativeTokens: Math.ceil(chars / CHARS_PER_TOKEN_FLOOR), + lowerBoundTokens: Math.floor(incompressibleChars / CHARS_PER_TOKEN_CEILING), + needsExactCount, + }; +} + +/** + * A run of one character — indentation, padding, a rule of dashes — can be a + * single token however long it is, so the floor counts each run once. + */ +function collapseRuns(text: string): string { + return text.replace(/(\s)\s+/g, '$1').replace(/(.)\1+/gs, '$1'); +} diff --git a/packages/ai-client/src/types.ts b/packages/ai-client/src/types.ts index 964bd622..34047ae8 100644 --- a/packages/ai-client/src/types.ts +++ b/packages/ai-client/src/types.ts @@ -344,6 +344,13 @@ export interface AIRequestOptions { * effect on models without `capabilities.reasoning`. */ reasoning?: ReasoningConfig | undefined; + + /** + * The context window this request may use, set by the client from the + * catalog and the connection. An adapter whose runtime sizes its window per + * request (a local runtime) sends it; the rest ignore it. + */ + contextWindow?: number | undefined; } /** @@ -760,7 +767,7 @@ export const AIProviderSchema = z.enum([ 'xai', 'runware', 'typesafe', - 'local', + 'ollama', ]); export type AIProvider = z.infer; @@ -775,6 +782,14 @@ export interface ProviderConfig { defaultModel?: string | undefined; maxRetries?: number | undefined; timeoutMs?: number | undefined; + /** + * The context window this connection serves, for a runtime whose window is a + * property of the machine rather than the model. The client sends each + * request the smaller of this and the model's catalog window. + */ + contextTokens?: number | undefined; + /** How long a local runtime keeps a model loaded after a request, in its own notation. */ + keepAlive?: string | undefined; } // ============================================================================ @@ -945,6 +960,26 @@ export interface ModelDefinition { * is a text-generation control that no image or embedding path reads. */ reasoning?: ModelReasoningProfile; + /** + * For a model served by a runtime on the operator's machine: the build its + * capabilities were measured on (Plan 323 D9). A runtime serving another + * digest serves a model these flags were not measured for. + */ + local?: LocalModelBuild; +} + +/** The build of a local model that `yarn models:verify-local` measured. */ +export interface LocalModelBuild { + /** The tag the runtime serves it under. */ + runtimeTag: string; + /** The runtime's content digest for that tag. */ + digest: string; + /** Which runner served it (`mlx`, `llamacpp`), and so which features it has. */ + runner: string; + quantization: string; + runtimeVersion: string; + probeVersion: number; + measuredAt: string; } // ============================================================================ diff --git a/packages/ai-client/src/usage.ts b/packages/ai-client/src/usage.ts index a4698e2d..89927c9a 100644 --- a/packages/ai-client/src/usage.ts +++ b/packages/ai-client/src/usage.ts @@ -23,7 +23,7 @@ export const UsageRecordSchema = z.object({ /** Attempt number */ attempt: z.number().int().positive(), /** Provider used */ - provider: z.enum(['openai', 'anthropic', 'google', 'openrouter', 'fireworks', 'xai', 'local']), + provider: z.enum(['openai', 'anthropic', 'google', 'openrouter', 'fireworks', 'xai', 'ollama']), /** Model used */ model: z.string(), /** Token usage */ diff --git a/packages/ai-client/src/verifyCredential.test.ts b/packages/ai-client/src/verifyCredential.test.ts index b0825756..879c298c 100644 --- a/packages/ai-client/src/verifyCredential.test.ts +++ b/packages/ai-client/src/verifyCredential.test.ts @@ -20,6 +20,39 @@ describe('verifyProviderKey', () => { } }); + it('asks a local runtime for its version at the configured URL, with no key', async () => { + const seen: Array<{ url: string; headers: Record }> = []; + const spyFetch = (async (url: RequestInfo | URL, init?: RequestInit) => { + seen.push({ url: String(url), headers: (init?.headers ?? {}) as Record }); + return { ok: true, status: 200 }; + }) as unknown as typeof fetch; + + expect(await verifyProviderKey('ollama', {}, {}, spyFetch)).toEqual({ + supported: true, + ok: true, + }); + await verifyProviderKey( + 'ollama', + { token: 't' }, + { base_url: 'http://host.docker.internal:11434/' }, + spyFetch, + ); + + expect(seen[0]).toEqual({ url: 'http://127.0.0.1:11434/api/version', headers: {} }); + expect(seen[1]).toEqual({ + url: 'http://host.docker.internal:11434/api/version', + headers: { Authorization: 'Bearer t' }, + }); + }); + + it('tells the operator to start a local runtime that is not answering', async () => { + const outcome = await verifyProviderKey('ollama', {}, {}, failingFetch); + expect(outcome).toMatchObject({ supported: true, ok: false, errorCode: 'network_error' }); + expect(outcome.supported && !outcome.ok && outcome.message).toContain( + 'Start it on this computer', + ); + }); + it('returns ok on a 200 probe', async () => { const outcome = await verifyProviderKey('openai', { api_key: 'sk-x' }, {}, okFetch); expect(outcome).toEqual({ supported: true, ok: true }); diff --git a/packages/ai-client/src/verifyCredential.ts b/packages/ai-client/src/verifyCredential.ts index 510a3a23..4529ce3c 100644 --- a/packages/ai-client/src/verifyCredential.ts +++ b/packages/ai-client/src/verifyCredential.ts @@ -2,6 +2,7 @@ * On-demand provider key verification — a cheap authenticated read per * provider (models list / key info), never a token-spending generation. */ +import { OLLAMA_DEFAULT_BASE_URL } from '@aflow/schemas'; export const VERIFIABLE_PROVIDERS = [ 'openai', @@ -11,6 +12,7 @@ export const VERIFIABLE_PROVIDERS = [ 'openrouter', 'xai', 'typesafe', + 'ollama', ] as const; export type VerifiableProvider = (typeof VERIFIABLE_PROVIDERS)[number]; @@ -31,6 +33,12 @@ interface Probe { secrets: Record, config: Record, ) => Record; + /** What to tell the operator when the probe gets no answer at all. */ + unreachable?: (config: Record) => string; +} + +function ollamaBase(config: Record): string { + return (config['base_url'] || OLLAMA_DEFAULT_BASE_URL).replace(/\/+$/, ''); } const PROBES: Record = { @@ -69,6 +77,12 @@ const PROBES: Record = { url: () => 'https://api.typesafe.ai/v1/models', headers: (secrets) => ({ Authorization: `Bearer ${secrets['api_key'] ?? ''}` }), }, + ollama: { + url: (_secrets, config) => `${ollamaBase(config)}/api/version`, + headers: (secrets) => (secrets['token'] ? { Authorization: `Bearer ${secrets['token']}` } : {}), + unreachable: (config) => + `Ollama is not answering at ${ollamaBase(config)}. Start it on this computer (brew services start ollama, or open the Ollama app).`, + }, }; export async function verifyProviderKey( @@ -105,7 +119,7 @@ export async function verifyProviderKey( supported: true, ok: false, errorCode: err instanceof Error && err.name === 'AbortError' ? 'timeout' : 'network_error', - message: 'Could not reach the provider to verify the key.', + message: probe.unreachable?.(config) ?? 'Could not reach the provider to verify the key.', }; } finally { clearTimeout(timer); diff --git a/packages/credential-resolver/src/byokAiClient.ts b/packages/credential-resolver/src/byokAiClient.ts index 4ac46d3b..a483a54c 100644 --- a/packages/credential-resolver/src/byokAiClient.ts +++ b/packages/credential-resolver/src/byokAiClient.ts @@ -9,11 +9,13 @@ import { createAIClient, inferProviderForModelRef, type AIClient, - type ProviderConfig, + type GenerateTextResponse, } from '@aflow/ai-client'; +import { localRuntimeId } from '@aflow/schemas'; import { CredentialResolver } from './resolver.js'; import { createProviderCredentialDbLoader } from './dbLoader.js'; import { credentialMissingMessage } from './errors.js'; +import { localRuntimeSettings, providerConfigFromCredential } from './providerConfig.js'; const CLIENT_CACHE_TTL_MS = 300_000; @@ -25,6 +27,7 @@ const BYOK_PROVIDERS = new Set([ 'fireworks', 'xai', 'typesafe', + 'ollama', ]); /** @@ -61,10 +64,83 @@ export class ByokCredentialError extends Error { } } +/** + * Takes a lease on a model runtime on the operator's machine (Plan 323 D7), + * or answers null when the runtime is busy and the caller does not wait. + */ +export type RuntimeAdmission = ( + runtimeId: string, + limit: number, +) => Promise<{ release(): Promise } | null>; + +export class LocalRuntimeBusyError extends Error { + constructor(readonly runtimeId: string) { + super(`The model runtime on this computer (${runtimeId}) is busy; try again once it is free.`); + this.name = 'LocalRuntimeBusyError'; + } +} + +/** A client whose every request to the runtime holds a lease on it for as long as it runs. */ +function admitted( + client: AIClient, + runtimeId: string, + limit: number, + admission: RuntimeAdmission, +): AIClient { + const take = async () => { + const lease = await admission(runtimeId, limit); + if (!lease) throw new LocalRuntimeBusyError(runtimeId); + return lease; + }; + const holding = + (call: (...args: A) => Promise) => + async (...args: A): Promise => { + const lease = await take(); + try { + return await call(...args); + } finally { + await lease.release(); + } + }; + return { + ...client, + generateText: holding(client.generateText.bind(client)), + generateJson: holding(client.generateJson.bind(client)) as AIClient['generateJson'], + generateEmbedding: holding(client.generateEmbedding.bind(client)), + decide: holding(client.decide.bind(client)), + generateTextStream(request) { + let resolveResponse!: (value: GenerateTextResponse) => void; + let rejectResponse!: (error: unknown) => void; + const response = new Promise((resolve, reject) => { + resolveResponse = resolve; + rejectResponse = reject; + }); + const stream = (async function* () { + const lease = await take().catch((error: unknown) => { + rejectResponse(error); + throw error; + }); + try { + const inner = client.generateTextStream(request); + inner.response.then(resolveResponse, rejectResponse); + yield* inner.stream; + await inner.response; + } finally { + await lease.release(); + } + })(); + return { stream, response }; + }, + }; +} + export class ByokAiClientFactory { private cache = new Map(); - constructor(private resolver: CredentialResolver) {} + constructor( + private resolver: CredentialResolver, + private admission?: RuntimeAdmission, + ) {} /** Whether `model`'s provider resolves a credential in this context. */ async canResolveModel(model: string, ctx: ByokClientContext): Promise { @@ -103,21 +179,31 @@ export class ByokAiClientFactory { const hit = this.cache.get(cacheKey); if (hit && hit.expiresAt > Date.now()) return { client: hit.client, providerId }; - const providerConfig: ProviderConfig = { apiKey: resolved.secrets['api_key'] ?? '' }; - if (providerId === 'openai' && resolved.config['org_id']) { - providerConfig.organization = resolved.config['org_id']; - } - const client = createAIClient({ + const providerConfig = providerConfigFromCredential(providerId, resolved); + const plain = createAIClient({ providers: { [providerId]: providerConfig }, defaultProvider: providerId as 'openai', }); + const client = + providerId === 'ollama' && this.admission + ? admitted( + plain, + localRuntimeId(providerConfig.baseUrl), + localRuntimeSettings(providerId, resolved).maxParallel, + this.admission, + ) + : plain; this.cache.set(cacheKey, { client, expiresAt: Date.now() + CLIENT_CACHE_TTL_MS }); return { client, providerId }; } } -export function createByokAiClientFactory(db: PostgresJsDatabase): ByokAiClientFactory { +export function createByokAiClientFactory( + db: PostgresJsDatabase, + admission?: RuntimeAdmission, +): ByokAiClientFactory { return new ByokAiClientFactory( new CredentialResolver({ loader: createProviderCredentialDbLoader(db) }), + admission, ); } diff --git a/packages/credential-resolver/src/index.ts b/packages/credential-resolver/src/index.ts index e783f4c7..5f9b7255 100644 --- a/packages/credential-resolver/src/index.ts +++ b/packages/credential-resolver/src/index.ts @@ -13,5 +13,12 @@ export { ByokCredentialError, createByokAiClientFactory, byokProviderForModelRef, + LocalRuntimeBusyError, type ByokClientContext, + type RuntimeAdmission, } from './byokAiClient.js'; +export { + providerConfigFromCredential, + missingRequiredSecrets, + localRuntimeSettings, +} from './providerConfig.js'; diff --git a/packages/credential-resolver/src/providerConfig.test.ts b/packages/credential-resolver/src/providerConfig.test.ts new file mode 100644 index 00000000..630902d0 --- /dev/null +++ b/packages/credential-resolver/src/providerConfig.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest'; +import { + localRuntimeSettings, + missingRequiredSecrets, + providerConfigFromCredential, +} from './providerConfig.js'; +import type { ResolvedProvider } from './types.js'; + +const resolved = ( + secrets: Record, + config: Record = {}, +): ResolvedProvider => ({ + secrets, + config, + scope: 'tenant', + scopeId: 'tenant', + credentialId: 'cred', + updatedAt: '2026-10-16T00:00:00Z', +}); + +describe('providerConfigFromCredential', () => { + it('reaches the runtime the registry describes when the connection sets nothing', () => { + expect(providerConfigFromCredential('ollama', resolved({}))).toEqual({ + apiKey: '', + baseUrl: 'http://127.0.0.1:11434', + contextTokens: 32_768, + keepAlive: '30m', + }); + }); + + it('takes what the connection sets over the defaults', () => { + const config = providerConfigFromCredential( + 'ollama', + resolved( + { token: 'proxy' }, + { base_url: 'http://host.docker.internal:11434', context_tokens: '65536' }, + ), + ); + expect(config).toMatchObject({ + apiKey: 'proxy', + baseUrl: 'http://host.docker.internal:11434', + contextTokens: 65_536, + }); + }); + + it('always sets a key, so no SDK falls back to the platform key in its environment', () => { + expect(providerConfigFromCredential('openai', resolved({})).apiKey).toBe(''); + }); + + it('carries the OpenAI organization only for OpenAI', () => { + expect( + providerConfigFromCredential('openai', resolved({ api_key: 'sk' }, { org_id: 'org-1' })), + ).toEqual({ + apiKey: 'sk', + organization: 'org-1', + }); + }); +}); + +describe('missingRequiredSecrets', () => { + it('asks a cloud provider for its key and a local runtime for nothing', () => { + expect(missingRequiredSecrets('openai', resolved({}))).toEqual(['api_key']); + expect(missingRequiredSecrets('ollama', resolved({}))).toEqual([]); + }); +}); + +describe('localRuntimeSettings', () => { + it('admits one request at a time unless the connection says otherwise', () => { + expect(localRuntimeSettings('ollama', resolved({}))).toEqual({ maxParallel: 1 }); + expect(localRuntimeSettings('ollama', resolved({}, { max_parallel: '2' })).maxParallel).toBe(2); + }); +}); diff --git a/packages/credential-resolver/src/providerConfig.ts b/packages/credential-resolver/src/providerConfig.ts new file mode 100644 index 00000000..d90f07e0 --- /dev/null +++ b/packages/credential-resolver/src/providerConfig.ts @@ -0,0 +1,63 @@ +/** + * The client configuration a resolved credential stands for — one reading, + * shared by every process that builds an AI client from a credential, so the + * executor and the API server cannot disagree about where a request goes. + */ +import type { ProviderConfig } from '@aflow/ai-client'; +import { getProviderDefinition, getRequiredSecretFieldIds } from '@aflow/schemas'; +import type { ResolvedProvider } from './types.js'; + +/** + * A field the credential leaves unset takes the registry's default, so a + * connection saved with nothing but its name (a local runtime) reaches the + * runtime the registry describes. + */ +function configValue( + providerId: string, + resolved: ResolvedProvider, + fieldId: string, +): string | undefined { + const value = resolved.config[fieldId]; + if (value !== undefined && value !== '') return value; + return getProviderDefinition(providerId)?.fields.find((f) => f.fieldId === fieldId)?.defaultValue; +} + +function positiveInteger(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; +} + +export function providerConfigFromCredential( + providerId: string, + resolved: ResolvedProvider, +): ProviderConfig { + // Always set, empty when the credential carries none: an SDK handed no key + // reads the platform's own from the environment, and a space's request must + // never be paid for with it. + const config: ProviderConfig = { + apiKey: resolved.secrets['api_key'] || resolved.secrets['token'] || '', + }; + const orgId = configValue(providerId, resolved, 'org_id'); + if (providerId === 'openai' && orgId) config.organization = orgId; + const baseUrl = configValue(providerId, resolved, 'base_url'); + if (baseUrl) config.baseUrl = baseUrl; + const contextTokens = positiveInteger(configValue(providerId, resolved, 'context_tokens')); + if (contextTokens !== undefined) config.contextTokens = contextTokens; + const keepAlive = configValue(providerId, resolved, 'keep_alive'); + if (keepAlive) config.keepAlive = keepAlive; + return config; +} + +/** The secrets a credential must carry and does not, as the registry declares them. */ +export function missingRequiredSecrets(providerId: string, resolved: ResolvedProvider): string[] { + return getRequiredSecretFieldIds(providerId).filter((fieldId) => !resolved.secrets[fieldId]); +} + +/** How many requests at once a local runtime's connection admits (Plan 323 D7). */ +export function localRuntimeSettings( + providerId: string, + resolved: ResolvedProvider, +): { maxParallel: number } { + return { maxParallel: positiveInteger(configValue(providerId, resolved, 'max_parallel')) ?? 1 }; +} diff --git a/packages/cybernetic-runtime/src/__tests__/sessionMetadata.test.ts b/packages/cybernetic-runtime/src/__tests__/sessionMetadata.test.ts index f41bd728..fe7269d9 100644 --- a/packages/cybernetic-runtime/src/__tests__/sessionMetadata.test.ts +++ b/packages/cybernetic-runtime/src/__tests__/sessionMetadata.test.ts @@ -24,6 +24,22 @@ describe('resolveClerkModel', () => { expect(resolved.modelId).toContain('glm-5p3-flash'); }); + it('takes a free space default itself, since the reason to avoid it is the bill', () => { + const resolved = resolveClerkModel({ default: 'gemma-local' }, null); + expect(resolved).toMatchObject({ + resolved: true, + modelRef: 'gemma-local', + modelId: 'ollama-gemma4-26b', + providerId: 'ollama', + mode: 'auto', + }); + }); + + it('does not take a free default the tenant has not permitted', () => { + const resolved = resolveClerkModel({ default: 'gemma-local' }, ['sonnet']); + expect(resolved.resolved).toBe(false); + }); + it('moves with the space default when its provider changes', () => { const anthropic = resolveClerkModel({ default: 'sonnet' }, null); expect(anthropic.resolved && anthropic.providerId).toBe('anthropic'); diff --git a/packages/cybernetic-runtime/src/clerkModel.ts b/packages/cybernetic-runtime/src/clerkModel.ts index 3f015d16..03c9f0bd 100644 --- a/packages/cybernetic-runtime/src/clerkModel.ts +++ b/packages/cybernetic-runtime/src/clerkModel.ts @@ -22,7 +22,17 @@ import { type ClerkResolutionMode, type DirectiveModelDefaults, } from '@aflow/schemas'; -import { createDefaultModelCatalog, inferProviderForModelRef } from '@aflow/ai-client'; +import { + createDefaultModelCatalog, + inferProviderForModelRef, + type ModelPricing, +} from '@aflow/ai-client'; + +function isFree(pricing: ModelPricing): boolean { + return ( + pricing.promptPer1M === 0 && pricing.completionPer1M === 0 && pricing.scheduled === undefined + ); +} export type ClerkModelResolution = | { @@ -67,7 +77,7 @@ export function resolveClerkModel( const spaceDefault = defaults?.default ?? DEFAULT_CYBERNETIC_MODEL; const provider = catalog.getModel(spaceDefault)?.provider ?? inferProviderForModelRef(spaceDefault); - if (!provider || provider === 'local') { + if (!provider) { return { resolved: false, mode: 'auto', reason: 'no_candidate_for_provider' }; } // Compared as catalog ids, never as the strings either side happens to @@ -84,6 +94,26 @@ export function resolveClerkModel( ) : null; + // The reason not to fall back is the bill, and a model that bills nothing + // — one on the operator's own machine — has none. Taking the default there + // also keeps a single GPU on the model it already has loaded. + const defaultModel = catalog.getModel(spaceDefault); + if ( + defaultModel && + !defaultModel.deprecated && + defaultModel.capabilities.chat && + isFree(defaultModel.pricing) && + (!permittedIds || permittedIds.has(defaultModel.id)) + ) { + return { + resolved: true, + modelRef: spaceDefault, + modelId: defaultModel.id, + providerId: defaultModel.provider, + mode: 'auto', + }; + } + let liveCandidates = 0; for (const ref of clerkCandidateRefs(provider)) { const model = catalog.getModel(ref); diff --git a/packages/executor-runtime/src/__tests__/handlerAdmission.test.ts b/packages/executor-runtime/src/__tests__/handlerAdmission.test.ts new file mode 100644 index 00000000..d3a28705 --- /dev/null +++ b/packages/executor-runtime/src/__tests__/handlerAdmission.test.ts @@ -0,0 +1,191 @@ +/** + * A step whose handler must hold something before it runs — a lease on a + * model runtime — waits for it by the rules of an operation slot: it stays + * SCHEDULED, gives back its executor slot, keeps its claim alive, and starts + * its timeout only once admitted. Whatever it was admitted with is released + * when the attempt ends. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { redisMock } = vi.hoisted(() => ({ + redisMock: { + getStepState: vi.fn(), + updateStepState: vi.fn().mockResolvedValue(undefined), + appendSessionEvent: vi.fn().mockResolvedValue(undefined), + registerStepInFlight: vi.fn().mockResolvedValue(undefined), + extendStepInFlight: vi.fn().mockResolvedValue(undefined), + clearStepInFlight: vi.fn().mockResolvedValue(undefined), + wasStepCancelled: vi.fn().mockResolvedValue(false), + releaseStepJob: vi.fn().mockResolvedValue('2-0'), + StepJobNotPendingError: class StepJobNotPendingError extends Error {}, + }, +})); +vi.mock('@aflow/redis', () => redisMock); + +const { reportingMock } = vi.hoisted(() => ({ + reportingMock: { + acknowledgeJob: vi.fn().mockResolvedValue(undefined), + emitFailure: vi.fn().mockResolvedValue(undefined), + emitResult: vi.fn().mockResolvedValue(undefined), + emitSuccess: vi.fn().mockResolvedValue(undefined), + }, +})); +vi.mock('../executor/resultReporting.js', () => reportingMock); + +vi.mock('../executor/buildContext.js', () => ({ + buildExecutionContext: vi.fn((_deps: unknown, job: unknown) => + Promise.resolve({ + job, + outputExists: () => Promise.resolve(undefined), + stepDefinition: undefined, + }), + ), +})); + +import { processJob } from '../executor/processJob.js'; +import type { StepAdmissionWaiting } from '../types.js'; + +const SCHEDULED_AT = 1_800_000_000_000; +/** Longer than the step's own timeout, so a wait charged to it would fail it. */ +const ADMISSION_WAIT_MS = 10 * 60_000; +const STEP_TIMEOUT_MS = 60_000; + +const JOB = { + tenantId: 't1', + sessionId: 'session-1', + stepExecutionId: 'turn-queued', + stepType: 'ai', + stepId: 'turn', + attempt: 1, + operationId: 'ai.agent.turn', + inputRef: 'inline:e30=', +} as never; + +const startedWrites = (): unknown[] => + redisMock.updateStepState.mock.calls.filter( + ([, , , patch]: [unknown, unknown, unknown, { status?: string }]) => patch.status === 'STARTED', + ); + +describe('processJob — a step waiting for what its handler must hold', () => { + let execute: ReturnType; + let release: ReturnType; + let admitNow: () => void; + let seenWaiting: StepAdmissionWaiting | undefined; + let slot: { held: boolean; release: ReturnType; acquire: ReturnType }; + let claimingStopped: AbortController; + + function host() { + const handler = { + stepType: 'ai', + execute, + resolveTimeoutMs: () => Promise.resolve(STEP_TIMEOUT_MS), + admit: (_ctx: unknown, waiting: StepAdmissionWaiting) => { + seenWaiting = waiting; + waiting.waiting({ runtimeId: 'this-machine:11434' }); + return new Promise<{ release: () => Promise }>((resolve, reject) => { + const timer = setInterval(() => waiting.stillWaiting(), 10_000); + admitNow = () => { + clearInterval(timer); + resolve({ release }); + }; + waiting.signal.addEventListener('abort', () => { + clearInterval(timer); + reject(new Error('aborted')); + }); + }); + }, + }; + return { + config: { consumerName: 'ai-test', defaultTimeoutMs: STEP_TIMEOUT_MS }, + deps: { redis: {} }, + handlers: new Map([['ai', handler]]), + log: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + abortControllers: new Map(), + operationLimiters: new Map(), + claimingStopped: claimingStopped.signal, + stopped: new AbortController().signal, + stepStarted: vi.fn(), + stepRunning: vi.fn(), + } as never; + } + + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers({ now: SCHEDULED_AT }); + redisMock.getStepState.mockResolvedValue({ + status: 'SCHEDULED', + attempt: 1, + scheduledAt: SCHEDULED_AT, + }); + execute = vi.fn().mockResolvedValue({ status: 'SUCCEEDED', outputRef: 'inline:e30=' }); + release = vi.fn().mockResolvedValue(undefined); + slot = { held: true, release: vi.fn(), acquire: vi.fn().mockResolvedValue(undefined) }; + claimingStopped = new AbortController(); + seenWaiting = undefined; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('stays SCHEDULED while it waits, without its slot, and keeps its claim alive', async () => { + const processing = processJob(host(), '1-0', JOB, slot as never); + await vi.advanceTimersByTimeAsync(ADMISSION_WAIT_MS); + + expect(startedWrites()).toHaveLength(0); + expect(execute).not.toHaveBeenCalled(); + expect(slot.release).toHaveBeenCalledTimes(1); + expect(redisMock.registerStepInFlight.mock.calls.length).toBeGreaterThan(1); + + admitNow(); + await processing; + expect(slot.acquire).toHaveBeenCalledTimes(1); + }); + + it('starts its timeout on admission, so a wait longer than the timeout does not fail it', async () => { + const processing = processJob(host(), '1-0', JOB, slot as never); + await vi.advanceTimersByTimeAsync(ADMISSION_WAIT_MS); + admitNow(); + await processing; + + expect(execute).toHaveBeenCalledTimes(1); + expect(reportingMock.emitResult).toHaveBeenCalledTimes(1); + expect(reportingMock.emitFailure).not.toHaveBeenCalled(); + }); + + it('releases what it was admitted with when the attempt ends', async () => { + const processing = processJob(host(), '1-0', JOB, slot as never); + await vi.advanceTimersByTimeAsync(1_000); + admitNow(); + await processing; + + expect(release).toHaveBeenCalledTimes(1); + expect(release.mock.invocationCallOrder[0]).toBeGreaterThan( + execute.mock.invocationCallOrder[0] ?? 0, + ); + }); + + it('a cancelled wait never starts the step', async () => { + const h = host() as unknown as { abortControllers: Map }; + const processing = processJob(h as never, '1-0', JOB, slot as never); + await vi.advanceTimersByTimeAsync(1_000); + h.abortControllers.get('turn-queued')?.abort(); + await processing; + + expect(execute).not.toHaveBeenCalled(); + expect(startedWrites()).toHaveLength(0); + expect(reportingMock.acknowledgeJob).toHaveBeenCalledTimes(1); + expect(seenWaiting?.signal.aborted).toBe(true); + }); + + it('gives the step back to its stream when the executor stops claiming', async () => { + const processing = processJob(host(), '1-0', JOB, slot as never); + await vi.advanceTimersByTimeAsync(1_000); + claimingStopped.abort(); + await processing; + + expect(execute).not.toHaveBeenCalled(); + expect(startedWrites()).toHaveLength(0); + expect(redisMock.releaseStepJob).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/executor-runtime/src/executor/processJob.ts b/packages/executor-runtime/src/executor/processJob.ts index b766c757..b4636e4f 100644 --- a/packages/executor-runtime/src/executor/processJob.ts +++ b/packages/executor-runtime/src/executor/processJob.ts @@ -23,6 +23,7 @@ import type { ExecutorDependencies, ExecutorLogger, SlotController, + StepAdmission, StepHandler, StepResult, } from '../types.js'; @@ -183,6 +184,67 @@ async function admitLimitedStep(admission: { return after; } +/** + * Wait for what the handler says this step needs before it starts (a lease + * on a model runtime), by the same rules as an operation slot: the step stays + * SCHEDULED and gives back its executor slot while it waits, the wait ends + * when the step is cancelled or the executor stops claiming, and a step that + * waited is checked again once admitted. + */ +async function admitHandlerStep(admission: { + host: ProcessJobHost; + job: StepJobMessage; + jobLog: ExecutorLogger; + ctx: ExecutorContext; + admit: NonNullable; + slotController: SlotController; + externalAbort: AbortSignal; +}): Promise< + | { outcome: 'admitted'; lease: StepAdmission | undefined } + | { outcome: Exclude } +> { + const { host, job, jobLog, ctx, admit, slotController, externalAbort } = admission; + const refusal = (): Exclude | undefined => { + if (externalAbort.aborted) return 'cancelled'; + if (host.claimingStopped.aborted) return 'released'; + return undefined; + }; + + // Set from inside the admission's callback, so held where the flow analysis sees it change. + const wait = { happened: false }; + let lease: StepAdmission | undefined; + try { + lease = await admit(ctx, { + signal: AbortSignal.any([externalAbort, host.claimingStopped]), + waiting: (detail) => { + jobLog.info('Waiting for admission', { operationId: job.operationId, ...detail }); + slotController.release(); + wait.happened = true; + }, + stillWaiting: () => { + void refreshInFlight(host, job, jobLog); + }, + }); + } catch (error) { + const ended = refusal(); + if (ended !== undefined) return { outcome: ended }; + throw error; + } finally { + if (wait.happened) await slotController.acquire(); + } + + let after = refusal(); + if (after === undefined && wait.happened) { + if (await isSuperseded(host, job, jobLog)) after = 'superseded'; + else if (await wasStepCancelled(host.deps.redis, job.stepExecutionId, job.attempt)) { + after = 'cancelled'; + } + } + if (after === undefined) return { outcome: 'admitted', lease }; + await lease?.release().catch(() => undefined); + return { outcome: after }; +} + /** Ends a claimed step that never started, without emitting a result for it. */ async function setAsideUnstarted( host: ProcessJobHost, @@ -338,6 +400,7 @@ export async function processJob( simulatedFulfillment = report; }; let admittedBy: ConcurrencyLimiter | undefined; + let stepAdmission: StepAdmission | undefined; const externalAbort = new AbortController(); try { @@ -395,15 +458,6 @@ export async function processJob( admittedBy = operationLimiter; } - // Started from here on: a drain counts this step and waits for it, and its - // run shows it running, timed from now rather than from its claim. - startTime = Date.now(); - if (inFlight !== undefined) { - inFlight.deadlineRef = { current: startTime + host.config.defaultTimeoutMs }; - } - host.stepStarted(); - await markStarted(host, job, jobLog); - const handler = host.handlers.get(job.stepType); if (!handler) { jobLog.error(`No handler for step type: ${job.stepType}`); @@ -424,6 +478,32 @@ export async function processJob( const ctx = await buildExecutionContext(host.deps, job, jobLog, slotController); + if (handler.admit) { + const admission = await admitHandlerStep({ + host, + job, + jobLog, + ctx, + admit: handler.admit.bind(handler), + slotController, + externalAbort: externalAbort.signal, + }); + if (admission.outcome !== 'admitted') { + await setAsideUnstarted(host, job, messageId, jobLog, admission.outcome); + return; + } + stepAdmission = admission.lease; + } + + // Started from here on: a drain counts this step and waits for it, and its + // run shows it running, timed from now rather than from its claim. + startTime = Date.now(); + if (inFlight !== undefined) { + inFlight.deadlineRef = { current: startTime + host.config.defaultTimeoutMs }; + } + host.stepStarted(); + await markStarted(host, job, jobLog); + if (handler.validate) { const validationError = await handler.validate(ctx); if (validationError) { @@ -596,6 +676,7 @@ export async function processJob( await acknowledgeJob(host.deps, job.stepType, messageId); } finally { admittedBy?.release(); + await stepAdmission?.release().catch(() => undefined); if (host.abortControllers.get(job.stepExecutionId) === externalAbort) { host.abortControllers.delete(job.stepExecutionId); } diff --git a/packages/executor-runtime/src/index.ts b/packages/executor-runtime/src/index.ts index cfbffd2c..cbd31c26 100644 --- a/packages/executor-runtime/src/index.ts +++ b/packages/executor-runtime/src/index.ts @@ -16,6 +16,8 @@ export type { ExecutorDependencies, ResolvedInput, SlotController, + StepAdmission, + StepAdmissionWaiting, } from './types.js'; export { diff --git a/packages/executor-runtime/src/types.ts b/packages/executor-runtime/src/types.ts index 53eb297d..25ab4f1d 100644 --- a/packages/executor-runtime/src/types.ts +++ b/packages/executor-runtime/src/types.ts @@ -225,6 +225,19 @@ export interface ExecutorContext { readonly slotController?: SlotController; } +export interface StepAdmission { + release(): Promise; +} + +export interface StepAdmissionWaiting { + /** Ends the wait: the step was cancelled or the executor is stopping. */ + readonly signal: AbortSignal; + /** Call once, when the step first has to wait; gives back the executor slot. */ + readonly waiting: (detail: Record) => void; + /** Call while waiting, so the step is not taken for a stalled one. */ + readonly stillWaiting: () => void; +} + export interface SlotController { /** Release the slot if currently held. Idempotent. */ release(): void; @@ -328,6 +341,18 @@ export interface StepHandler { */ resolveTimeoutMs?(ctx: ExecutorContext): Promise; + /** + * Optional: wait for something this step must hold to run that only its + * input names — a lease on a model runtime the step's model is served by. + * + * Runs after the context is built and before the timeout starts, so the wait + * is never charged to the work. While it waits the step gives back its + * executor slot, so steps that need nothing of the kind are not held up + * behind it. The admission returned is released when the attempt ends, + * whatever its outcome. Undefined means the step needs nothing. + */ + admit?(ctx: ExecutorContext, waiting: StepAdmissionWaiting): Promise; + /** * Optional: Initialize handler (called once on startup). */ diff --git a/packages/redis/src/__tests__/localRuntimeAdmission.test.ts b/packages/redis/src/__tests__/localRuntimeAdmission.test.ts new file mode 100644 index 00000000..d8ea8475 --- /dev/null +++ b/packages/redis/src/__tests__/localRuntimeAdmission.test.ts @@ -0,0 +1,124 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import Redis from 'ioredis-mock'; +import type { Redis as RedisType } from 'ioredis'; +import { acquireRuntimeLease, RuntimeLeaseAbortedError } from '../localRuntimeAdmission.js'; + +const RUNTIME = 'this-machine:11434'; +let redis: RedisType; + +beforeEach(async () => { + redis = new Redis() as unknown as RedisType; + await redis.flushall(); +}); + +afterEach(() => { + redis.disconnect(); +}); + +const take = ( + holderId: string, + opts: { wait?: boolean; limit?: number; signal?: AbortSignal } = {}, +) => + acquireRuntimeLease({ + redis, + runtimeId: RUNTIME, + limit: opts.limit ?? 1, + holderId, + wait: opts.wait ?? true, + ...(opts.signal ? { signal: opts.signal } : {}), + }); + +describe('acquireRuntimeLease', () => { + it('admits no more requests than the runtime takes', async () => { + const a = await take('a'); + expect(a).not.toBeNull(); + expect(await take('b', { wait: false })).toBeNull(); + + await a?.release(); + const b = await take('b', { wait: false }); + expect(b).not.toBeNull(); + await b?.release(); + }); + + it('admits as many at once as the limit allows', async () => { + const leases = await Promise.all([take('a', { limit: 2 }), take('b', { limit: 2 })]); + expect(leases.every((l) => l !== null)).toBe(true); + expect(await take('c', { limit: 2, wait: false })).toBeNull(); + await Promise.all(leases.map((l) => l?.release())); + }); + + it('serves waiters in the order they arrived', async () => { + const a = await take('a'); + const order: string[] = []; + const b = take('b').then((lease) => { + order.push('b'); + return lease; + }); + await new Promise((r) => setTimeout(r, 20)); + const c = take('c').then((lease) => { + order.push('c'); + return lease; + }); + await new Promise((r) => setTimeout(r, 20)); + + await a?.release(); + const bLease = await b; + expect(order).toEqual(['b']); + await bLease?.release(); + const cLease = await c; + expect(order).toEqual(['b', 'c']); + await cLease?.release(); + }, 10_000); + + it('a cancelled wait leaves the queue, so the next waiter is not held behind it', async () => { + const a = await take('a'); + const controller = new AbortController(); + const b = take('b', { signal: controller.signal }); + await new Promise((r) => setTimeout(r, 20)); + const c = take('c'); + await new Promise((r) => setTimeout(r, 20)); + + controller.abort(); + await expect(b).rejects.toBeInstanceOf(RuntimeLeaseAbortedError); + await a?.release(); + const cLease = await c; + expect(cLease).not.toBeNull(); + await cLease?.release(); + }, 10_000); + + it('a holder that stopped renewing frees its place without a sweep', async () => { + // A worker killed while holding a lease leaves an entry that expired. + await redis.zadd(`aflow:local-runtime:{${RUNTIME}}:holders`, Date.now() - 1, 'dead-worker'); + const lease = await take('b', { wait: false }); + expect(lease).not.toBeNull(); + await lease?.release(); + }); + + it('two names for this machine share one runtime', async () => { + const a = await acquireRuntimeLease({ + redis, + runtimeId: RUNTIME, + limit: 1, + holderId: 'a', + wait: true, + }); + const sameRuntime = await acquireRuntimeLease({ + redis, + runtimeId: RUNTIME, + limit: 1, + holderId: 'b', + wait: false, + }); + const otherRuntime = await acquireRuntimeLease({ + redis, + runtimeId: 'gpu-box:11434', + limit: 1, + holderId: 'c', + wait: false, + }); + expect(sameRuntime).toBeNull(); + expect(otherRuntime).not.toBeNull(); + await a?.release(); + await otherRuntime?.release(); + }); +}); diff --git a/packages/redis/src/index.ts b/packages/redis/src/index.ts index 4ed1009d..8a047d33 100644 --- a/packages/redis/src/index.ts +++ b/packages/redis/src/index.ts @@ -530,3 +530,13 @@ export { export * from './hostInventory.js'; export * from './browserHandoff.js'; export * from './sessionResidue.js'; +export { + acquireRuntimeLease, + runtimeAdmission, + RuntimeLeaseAbortedError, + RUNTIME_LEASE_TTL_MS, + RUNTIME_LEASE_RENEW_MS, + RUNTIME_LEASE_POLL_MS, + type RuntimeLease, + type AcquireRuntimeLeaseInput, +} from './localRuntimeAdmission.js'; diff --git a/packages/redis/src/localRuntimeAdmission.ts b/packages/redis/src/localRuntimeAdmission.ts new file mode 100644 index 00000000..34ad5cc5 --- /dev/null +++ b/packages/redis/src/localRuntimeAdmission.ts @@ -0,0 +1,216 @@ +/** + * Admission to a model runtime on the operator's machine (Plan 323 D7). + * + * The runtime serves `limit` requests at once whoever sends them: the AI + * executor, the API server and the memory embedder each reach it, under + * whichever credential named it. So the limit lives here, keyed by the + * runtime rather than by a credential or a process, and every caller takes a + * lease from it. + * + * A lease is a holder with an expiry its holder renews. A holder that dies + * stops renewing and its lease lapses on the next attempt anyone makes — no + * sweep. Waiters queue in arrival order and keep their place only while they + * keep asking, so a waiter that died does not block the queue either. + */ +import { randomUUID } from 'node:crypto'; +import type { Redis } from 'ioredis'; + +const KEY_PREFIX = 'aflow:local-runtime'; +/** Long enough that a renewal missed to a GC pause or a slow round trip does not drop a live holder. */ +export const RUNTIME_LEASE_TTL_MS = 30_000; +/** A third of the lease, so two renewals can be lost before it lapses. */ +export const RUNTIME_LEASE_RENEW_MS = 10_000; +/** + * How often a waiter asks again. A local request runs for seconds to minutes, + * so half a second of slack after a release is noise next to it. + */ +export const RUNTIME_LEASE_POLL_MS = 500; + +function runtimeKeys(runtimeId: string): [string, string, string] { + // One hash tag, so the three keys a script touches share a cluster slot. + const base = `${KEY_PREFIX}:{${runtimeId}}`; + return [`${base}:holders`, `${base}:queue`, `${base}:waiting`]; +} + +/** + * KEYS: holders (zset id → expiry), queue (zset id → arrival), waiting (zset id → expiry) + * ARGV: id, limit, now, ttl, arrival + * Returns 1 when `id` holds a lease after the call, 0 when it waits. + */ +const ACQUIRE = ` +local holders, queue, waiting = KEYS[1], KEYS[2], KEYS[3] +local id, limit, now, ttl, arrival = ARGV[1], tonumber(ARGV[2]), tonumber(ARGV[3]), tonumber(ARGV[4]), tonumber(ARGV[5]) +redis.call('ZREMRANGEBYSCORE', holders, '-inf', now) +local dead = redis.call('ZRANGEBYSCORE', waiting, '-inf', now) +for _, d in ipairs(dead) do + redis.call('ZREM', queue, d) + redis.call('ZREM', waiting, d) +end +local expiry = now + ttl +if redis.call('ZSCORE', holders, id) then + redis.call('ZADD', holders, expiry, id) + return 1 +end +redis.call('ZADD', queue, 'NX', arrival, id) +redis.call('ZADD', waiting, expiry, id) +local free = limit - redis.call('ZCARD', holders) +local granted = 0 +if free > 0 then + local rank = redis.call('ZRANK', queue, id) + if rank and rank < free then + redis.call('ZREM', queue, id) + redis.call('ZREM', waiting, id) + redis.call('ZADD', holders, expiry, id) + granted = 1 + end +end +for _, k in ipairs({holders, queue, waiting}) do + redis.call('PEXPIRE', k, ttl * 2) +end +return granted +`; + +const RENEW = ` +if redis.call('ZSCORE', KEYS[1], ARGV[1]) then + redis.call('ZADD', KEYS[1], tonumber(ARGV[2]), ARGV[1]) + redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[3]) * 2) + return 1 +end +return 0 +`; + +const RELEASE = ` +redis.call('ZREM', KEYS[1], ARGV[1]) +redis.call('ZREM', KEYS[2], ARGV[1]) +redis.call('ZREM', KEYS[3], ARGV[1]) +return 1 +`; + +export interface RuntimeLease { + /** Give the lease back. Idempotent. */ + release(): Promise; +} + +export interface AcquireRuntimeLeaseInput { + redis: Redis; + runtimeId: string; + limit: number; + /** Unique per request, e.g. a step execution id with its attempt. */ + holderId: string; + /** Wait for a lease; without it a busy runtime answers null at once. */ + wait: boolean; + signal?: AbortSignal | undefined; + /** Called once, when the request first has to wait. */ + onWaiting?: (() => void) | undefined; + /** Called on every attempt while waiting, so the caller can show it is alive. */ + onStillWaiting?: (() => void) | undefined; +} + +export class RuntimeLeaseAbortedError extends Error { + constructor() { + super('Waiting for the local model runtime was cancelled.'); + this.name = 'RuntimeLeaseAbortedError'; + } +} + +async function attempt(input: AcquireRuntimeLeaseInput, arrival: number): Promise { + const granted = await input.redis.eval( + ACQUIRE, + 3, + ...runtimeKeys(input.runtimeId), + input.holderId, + String(input.limit), + String(Date.now()), + String(RUNTIME_LEASE_TTL_MS), + String(arrival), + ); + return granted === 1; +} + +function sleep(ms: number, signal: AbortSignal | undefined): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new RuntimeLeaseAbortedError()); + return; + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + reject(new RuntimeLeaseAbortedError()); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +async function release(redis: Redis, runtimeId: string, holderId: string): Promise { + await redis.eval(RELEASE, 3, ...runtimeKeys(runtimeId), holderId); +} + +/** + * Take a lease on a runtime, waiting in arrival order when it is busy and + * `wait` is set. Returns null only when `wait` is false and the runtime is + * busy. A cancelled wait leaves the queue and throws `RuntimeLeaseAbortedError`. + */ +export async function acquireRuntimeLease( + input: AcquireRuntimeLeaseInput, +): Promise { + const arrival = Date.now(); + let granted = await attempt(input, arrival); + if (!granted && !input.wait) { + await release(input.redis, input.runtimeId, input.holderId); + return null; + } + if (!granted) { + input.onWaiting?.(); + try { + while (!granted) { + await sleep(RUNTIME_LEASE_POLL_MS, input.signal); + input.onStillWaiting?.(); + granted = await attempt(input, arrival); + } + } catch (error) { + await release(input.redis, input.runtimeId, input.holderId).catch(() => undefined); + throw error; + } + } + + const renewal = setInterval(() => { + void input.redis + .eval( + RENEW, + 1, + runtimeKeys(input.runtimeId)[0], + input.holderId, + String(Date.now() + RUNTIME_LEASE_TTL_MS), + String(RUNTIME_LEASE_TTL_MS), + ) + .catch(() => undefined); + }, RUNTIME_LEASE_RENEW_MS); + renewal.unref(); + + let released = false; + return { + async release() { + if (released) return; + released = true; + clearInterval(renewal); + await release(input.redis, input.runtimeId, input.holderId).catch(() => undefined); + }, + }; +} + +/** + * Admission for a caller outside a step — a title, a judge — as a function + * of the runtime and its limit. `wait: false` is for work that would rather + * be retried later than queue behind an agent turn for minutes. + */ +export function runtimeAdmission( + redis: Redis, + options: { wait: boolean }, +): (runtimeId: string, limit: number) => Promise { + return (runtimeId, limit) => + acquireRuntimeLease({ redis, runtimeId, limit, holderId: randomUUID(), wait: options.wait }); +} diff --git a/packages/schemas/src/background/registry.ts b/packages/schemas/src/background/registry.ts index 56c4c1c8..0a4971aa 100644 --- a/packages/schemas/src/background/registry.ts +++ b/packages/schemas/src/background/registry.ts @@ -1168,6 +1168,30 @@ const DEFINITIONS: readonly BackgroundTaskDefinitionInput[] = [ }, ], }, + { + id: 'ai.local_runtime_lease_renewal', + service: 'shared-runtime', + ownerDomain: 'model-execution', + purpose: + 'Renew the lease each in-flight request holds on a model runtime on the operator machine, for as long as the request runs.', + invariant: + 'A runtime on the operator machine serves no more requests at once than its connection admits, and a holder that died frees its place.', + criticality: 'correctness', + trigger: 'active-resource', + scope: 'per_instance', + substrate: 'redis-lease', + baseCadenceMs: 10_000, + maxBatch: 1, + maxCycleMs: 5_000, + idleOperationBudgetPerMinute: 0, + hotPathProducerBudget: { + maxAdditionalNetworkRoundTrips: 0, + description: 'Scoped to one request holding a lease; no timer exists at idle.', + }, + disablePolicy: 'never', + recovery: 'An unrenewed lease lapses after its TTL and the next waiter takes the place.', + sites: [{ path: 'packages/redis/src/localRuntimeAdmission.ts', discovery: ['setInterval'] }], + }, { id: 'executor.compute.session_reaper', service: 'executor-compute', diff --git a/packages/schemas/src/credentials/provider.ts b/packages/schemas/src/credentials/provider.ts index 0ec89074..9675b059 100644 --- a/packages/schemas/src/credentials/provider.ts +++ b/packages/schemas/src/credentials/provider.ts @@ -1,4 +1,5 @@ import { z } from 'zod'; +import { EditionIdSchema, type EditionId } from '../edition/descriptor.js'; // --------------------------------------------------------------------------- // Provider taxonomy @@ -63,6 +64,7 @@ export const ProviderIdSchema = z.enum([ 'openrouter', 'fireworks', 'xai', + 'ollama', 'zai', 'runware', 'typesafe', @@ -147,9 +149,21 @@ export const ProviderDefinitionSchema = z.object({ iconName: z.string(), docsUrl: z.string().url().optional(), fields: z.array(CredentialFieldSchema), + /** + * The editions that offer this provider; absent means every edition. A + * runtime on the operator's own machine is reachable only where the platform + * runs on that machine — in the hosted edition the same loopback URL names + * the platform's own host. + */ + editions: z.array(EditionIdSchema).optional(), }); export type ProviderDefinition = z.infer; +/** Whether `edition` offers `provider`. */ +export function isProviderInEdition(provider: ProviderDefinition, edition: EditionId): boolean { + return provider.editions === undefined || provider.editions.includes(edition); +} + // --------------------------------------------------------------------------- // Credential metadata (API response — values NEVER included) // --------------------------------------------------------------------------- diff --git a/packages/schemas/src/credentials/registry.test.ts b/packages/schemas/src/credentials/registry.test.ts index 7b6caaf3..95916434 100644 --- a/packages/schemas/src/credentials/registry.test.ts +++ b/packages/schemas/src/credentials/registry.test.ts @@ -8,6 +8,9 @@ import { getSecretFieldIds, getConfigFieldIds, getRequiredFieldIds, + getProvidersForEdition, + getRequiredSecretFieldIds, + localRuntimeId, } from './registry.js'; import { ProviderDefinitionSchema, ProviderIdSchema } from './provider.js'; @@ -53,11 +56,12 @@ describe('Provider Registry', () => { it('getProvidersByCategory filters correctly', () => { const llmProviders = getProvidersByCategory('llm'); - expect(llmProviders).toHaveLength(6); + expect(llmProviders).toHaveLength(7); expect(llmProviders.map((p) => p.providerId).sort()).toEqual([ 'anthropic', 'fireworks', 'google', + 'ollama', 'openai', 'openrouter', 'xai', @@ -120,4 +124,29 @@ describe('Provider Registry', () => { ); } }); + + it('offers a runtime on the operator machine only in the local edition', () => { + const local = getProvidersForEdition('community-local').map((p) => p.providerId); + const hosted = getProvidersForEdition('enterprise').map((p) => p.providerId); + expect(local).toContain('ollama'); + expect(hosted).not.toContain('ollama'); + expect(hosted).toContain('openai'); + }); + + it('reads the secrets a credential must carry from the registry', () => { + expect(getRequiredSecretFieldIds('openai')).toEqual(['api_key']); + expect(getRequiredSecretFieldIds('ollama')).toEqual([]); + }); + + it('names one runtime however its URL spells this machine', () => { + const names = [ + undefined, + 'http://127.0.0.1:11434', + 'http://localhost:11434/', + 'http://[::1]:11434', + 'http://host.docker.internal:11434', + ]; + expect(new Set(names.map(localRuntimeId))).toEqual(new Set(['this-machine:11434'])); + expect(localRuntimeId('http://gpu-box:11434')).toBe('gpu-box:11434'); + }); }); diff --git a/packages/schemas/src/credentials/registry.ts b/packages/schemas/src/credentials/registry.ts index e9be9fd8..5d49169c 100644 --- a/packages/schemas/src/credentials/registry.ts +++ b/packages/schemas/src/credentials/registry.ts @@ -1,4 +1,23 @@ -import type { ProviderDefinition, ProviderId } from './provider.js'; +import type { EditionId } from '../edition/descriptor.js'; +import { isProviderInEdition, type ProviderDefinition, type ProviderId } from './provider.js'; + +export const OLLAMA_DEFAULT_BASE_URL = 'http://127.0.0.1:11434'; +export const OLLAMA_DEFAULT_CONTEXT_TOKENS = 32_768; + +const THIS_MACHINE_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]', 'host.docker.internal']); + +/** + * The runtime a base URL reaches, as one id however it is spelled (Plan 323 + * D7). The host's loopback goes by several names — and a container reaches + * it as `host.docker.internal` — so two connections that differ only in the + * name still share one runtime's capacity. + */ +export function localRuntimeId(baseUrl: string | undefined): string { + const url = new URL(baseUrl || OLLAMA_DEFAULT_BASE_URL); + const port = url.port || (url.protocol === 'https:' ? '443' : '80'); + const host = THIS_MACHINE_HOSTS.has(url.hostname) ? 'this-machine' : url.hostname; + return `${host}:${port}`; +} // --------------------------------------------------------------------------- // Registry entries @@ -123,6 +142,69 @@ export const PROVIDER_REGISTRY: readonly ProviderDefinition[] = [ }, ], }, + { + providerId: 'ollama', + category: 'llm', + displayName: 'Ollama', + description: + 'Models running on this computer through Ollama. Nothing leaves the machine for inference, and nothing is billed.', + iconName: 'Desktop', + docsUrl: 'https://docs.ollama.com', + editions: ['community-local'], + fields: [ + { + fieldId: 'base_url', + label: 'Ollama URL', + type: 'url', + required: false, + defaultValue: OLLAMA_DEFAULT_BASE_URL, + // Loopback under every name it goes by, and the host as a container + // reaches it. A runtime on another machine is an operator's decision + // (CREDENTIAL_EXTRA_BASE_URL_ORIGINS), never a tenant's. + allowedOrigins: [ + 'http://127.0.0.1:11434', + 'http://localhost:11434', + 'http://[::1]:11434', + 'http://host.docker.internal:11434', + ], + helpText: 'Where Ollama listens. The default is Ollama on this computer.', + }, + { + fieldId: 'token', + label: 'Token', + type: 'secret', + required: false, + helpText: 'Only for an Ollama behind a proxy that asks for a bearer token.', + }, + { + fieldId: 'context_tokens', + label: 'Context window (tokens)', + type: 'number', + required: false, + defaultValue: String(OLLAMA_DEFAULT_CONTEXT_TOKENS), + helpText: + 'The largest prompt and answer a request may use. A larger window holds longer conversations and costs memory.', + }, + { + fieldId: 'max_parallel', + label: 'Requests at once', + type: 'number', + required: false, + defaultValue: '1', + helpText: + 'How many requests this computer serves at the same time. Match OLLAMA_NUM_PARALLEL; each one needs its own context memory.', + }, + { + fieldId: 'keep_alive', + label: 'Keep loaded for', + type: 'text', + required: false, + defaultValue: '30m', + helpText: + 'How long a model stays in memory after its last request, as Ollama reads it (30m, 2h, -1 for always).', + }, + ], + }, { providerId: 'zai', category: 'coding', @@ -360,6 +442,21 @@ export function getAllProviders(): readonly ProviderDefinition[] { return PROVIDER_REGISTRY; } +/** The providers an edition offers. */ +export function getProvidersForEdition(edition: EditionId): ProviderDefinition[] { + return PROVIDER_REGISTRY.filter((p) => isProviderInEdition(p, edition)); +} + +/** + * Secret fields a credential must carry to be usable. A provider whose only + * secret is optional (a local runtime's proxy token) is usable with none. + */ +export function getRequiredSecretFieldIds(providerId: string): string[] { + const provider = registryMap.get(providerId); + if (!provider) return []; + return provider.fields.filter((f) => f.type === 'secret' && f.required).map((f) => f.fieldId); +} + /** Get provider definitions filtered by category. */ export function getProvidersByCategory(category: string): ProviderDefinition[] { return PROVIDER_REGISTRY.filter((p) => p.category === category); diff --git a/packages/server-runtime/src/routes/catalog.ts b/packages/server-runtime/src/routes/catalog.ts index 537f25aa..b1631b81 100644 --- a/packages/server-runtime/src/routes/catalog.ts +++ b/packages/server-runtime/src/routes/catalog.ts @@ -11,7 +11,11 @@ import { retiredRefsFor, type ModelDefinition, } from '@aflow/ai-client'; -import { isRecommendedAgentModelRef } from '@aflow/schemas'; +import { + getProviderDefinition, + isProviderInEdition, + isRecommendedAgentModelRef, +} from '@aflow/schemas'; import { getOperationCatalog, getAllOperations, @@ -492,8 +496,14 @@ export const catalogRoutes: FastifyPluginAsync = async (fastify) => { }) as z.infer; }; - // Get models, optionally filtered by provider - let catalogModels = catalog.listModels(provider); + // Get models, optionally filtered by provider. A model whose provider + // this edition does not offer is not listed: a model on the operator's + // own machine cannot be reached from the hosted service. + const edition = fastify.edition.edition; + let catalogModels = catalog.listModels(provider).filter((m) => { + const definition = getProviderDefinition(m.provider); + return definition === undefined || isProviderInEdition(definition, edition); + }); // Filter by capability if specified if (capability) { diff --git a/packages/server-runtime/src/routes/catalogModelsEdition.test.ts b/packages/server-runtime/src/routes/catalogModelsEdition.test.ts new file mode 100644 index 00000000..9cbfbce0 --- /dev/null +++ b/packages/server-runtime/src/routes/catalogModelsEdition.test.ts @@ -0,0 +1,37 @@ +/** + * A model on the operator's own machine is listed only by the edition that + * runs on that machine: from the hosted service the same loopback address + * names the platform's own host, so offering it there offers nothing. + */ +import { describe, expect, it } from 'vitest'; +import Fastify from 'fastify'; +import { serializerCompiler, validatorCompiler } from 'fastify-type-provider-zod'; +import type { EditionId } from '@aflow/schemas'; +import { catalogRoutes } from './catalog.js'; + +async function listedModels(edition: EditionId): Promise { + const app = Fastify({ logger: false }); + app.setValidatorCompiler(validatorCompiler); + app.setSerializerCompiler(serializerCompiler); + app.decorate('edition', { edition } as never); + app.decorate('authenticate', async () => undefined); + await app.register(catalogRoutes); + const res = await app.inject({ method: 'GET', url: '/models?capability=chat' }); + await app.close(); + expect(res.statusCode).toBe(200); + return (JSON.parse(res.body) as { models: Array<{ modelId: string }> }).models.map( + (m) => m.modelId, + ); +} + +describe('GET /catalog/models by edition', () => { + it('lists a model on this computer in the local edition', async () => { + expect(await listedModels('community-local')).toContain('ollama-gemma4-26b'); + }); + + it('does not list it in the hosted edition, and still lists the cloud models', async () => { + const hosted = await listedModels('enterprise'); + expect(hosted).not.toContain('ollama-gemma4-26b'); + expect(hosted).toContain('accounts/fireworks/models/glm-5p3'); + }); +}); diff --git a/packages/server-runtime/src/routes/credentials.ts b/packages/server-runtime/src/routes/credentials.ts index bc45408e..ccaf2f42 100644 --- a/packages/server-runtime/src/routes/credentials.ts +++ b/packages/server-runtime/src/routes/credentials.ts @@ -22,8 +22,9 @@ import { CredentialMetaSchema, CredentialStatusSchema, ProviderDefinitionSchema, - getAllProviders, + getProvidersForEdition, getProviderDefinition, + isProviderInEdition, isAllowedFieldOrigin, getSecretFieldIds, getConfigFieldIds, @@ -186,7 +187,7 @@ export const credentialRoutes: FastifyPluginAsync = async (fastify) => { }, async (request, reply) => { await request.requireTenant(); - reply.send({ providers: [...getAllProviders()] }); + reply.send({ providers: getProvidersForEdition(fastify.edition.edition) }); }, ); @@ -210,7 +211,7 @@ export const credentialRoutes: FastifyPluginAsync = async (fastify) => { await request.requireTenant(); const { providerId } = request.params; const def = getProviderDefinition(providerId); - if (!def) { + if (!def || !isProviderInEdition(def, fastify.edition.edition)) { reply.status(404).send({ error: `Provider not found: ${providerId}` }); return; } @@ -369,7 +370,7 @@ export const credentialRoutes: FastifyPluginAsync = async (fastify) => { // Validate provider exists const def = getProviderDefinition(providerId); - if (!def) { + if (!def || !isProviderInEdition(def, fastify.edition.edition)) { reply.status(400).send({ error: `Unknown provider: ${providerId}` }); return; } @@ -510,7 +511,7 @@ export const credentialRoutes: FastifyPluginAsync = async (fastify) => { // Validate provider exists const def = getProviderDefinition(providerId); - if (!def) { + if (!def || !isProviderInEdition(def, fastify.edition.edition)) { reply.status(400).send({ error: `Unknown provider: ${providerId}` }); return; } diff --git a/packages/server-runtime/src/routes/spaceLlmReadiness.ts b/packages/server-runtime/src/routes/spaceLlmReadiness.ts index 8be3f7e5..fdfc722d 100644 --- a/packages/server-runtime/src/routes/spaceLlmReadiness.ts +++ b/packages/server-runtime/src/routes/spaceLlmReadiness.ts @@ -189,7 +189,7 @@ export const spaceLlmReadinessRoutes: FastifyPluginAsync = async (fastify) => { : resolveRoleModel(defaults, role); const model = catalog.getModel(ref); const inferred = model ? model.provider : inferProviderForModelRef(ref); - const providerId: string | null = inferred === 'local' ? null : inferred; + const providerId: string | null = inferred; // A ref the catalog does not carry names no model this space can run: // assigning a role is gated on a live catalog entry, so the only way to // hold one is for the model to have left the lineup since. Inferring a diff --git a/packages/web-product/src/ui/components/credentials/ConnectProviderDialog.tsx b/packages/web-product/src/ui/components/credentials/ConnectProviderDialog.tsx index a942dcb6..23ffdefb 100644 --- a/packages/web-product/src/ui/components/credentials/ConnectProviderDialog.tsx +++ b/packages/web-product/src/ui/components/credentials/ConnectProviderDialog.tsx @@ -22,9 +22,12 @@ interface ProviderDef { providerId: string; displayName: string; docsUrl?: string; + description?: string; fields: Array<{ fieldId: string; type: string; + required?: boolean; + defaultValue?: string; placeholder?: string; helpText?: string; }>; @@ -66,6 +69,14 @@ export function ConnectProviderDialog({ }); const def = providersQuery.data?.providers.find((p) => p.providerId === providerId); const keyField = def?.fields.find((f) => f.fieldId === 'api_key'); + // A provider that needs no secret — a runtime on this computer — connects + // with its registry defaults, so the operator has nothing to type. + const needsKey = def?.fields.some((f) => f.type === 'secret' && f.required === true) ?? true; + const defaults = Object.fromEntries( + (def?.fields ?? []) + .filter((f) => f.type !== 'secret' && f.defaultValue !== undefined) + .map((f) => [f.fieldId, f.defaultValue!]), + ); const close = () => { setKey(''); @@ -76,7 +87,7 @@ export function ConnectProviderDialog({ }; const save = async () => { - if (!providerId || !key.trim()) return; + if (!providerId || (needsKey && !key.trim())) return; setSaving(true); setError(null); const scope = spaceOnly ? 'space' : 'user'; @@ -85,7 +96,11 @@ export function ConnectProviderDialog({ const putRes = await authFetch(`/api/credentials/${providerId}`, { method: 'PUT', headers: scopedHeaders, - body: JSON.stringify({ scope, secrets: { api_key: key.trim() } }), + body: JSON.stringify( + needsKey + ? { scope, secrets: { api_key: key.trim() } } + : { scope, secrets: {}, config: defaults }, + ), }); if (!putRes.ok) { const body = (await putRes.json().catch(() => null)) as { error?: string } | null; @@ -135,22 +150,29 @@ export function ConnectProviderDialog({ variant="primary" size="sm" loading={saving} - disabled={!key.trim()} + disabled={needsKey && !key.trim()} onClick={() => { void save(); }} > - Save & test + {needsKey ? 'Save & test' : 'Connect & test'} } > - - {spaceOnly - ? 'Workspace key — only used by this workspace' - : 'Personal key — used across all your workspaces'} - + {needsKey ? ( + + {spaceOnly + ? 'Workspace key — only used by this workspace' + : 'Personal key — used across all your workspaces'} + + ) : ( + + {def?.description} + {defaults['base_url'] ? ` It is reached at ${defaults['base_url']}.` : ''} + + )} {def?.docsUrl && ( )} - - - { - setKey((e.target as HTMLInputElement).value); - }} - /> - {keyField?.helpText && ( - - {keyField.helpText} - - )} - + {needsKey && ( + + + { + setKey((e.target as HTMLInputElement).value); + }} + /> + {keyField?.helpText && ( + + {keyField.helpText} + + )} + + )} {error && ( {error} @@ -194,7 +218,9 @@ export function ConnectProviderDialog({ Store for this workspace only - Stored encrypted. Model usage is billed to your provider account. + {needsKey + ? 'Stored encrypted. Model usage is billed to your provider account.' + : 'Change the address, context window or parallelism under Settings → Credentials.'} diff --git a/packages/web-product/src/ui/components/tenant-settings/modelGroups.ts b/packages/web-product/src/ui/components/tenant-settings/modelGroups.ts index c82e2cf4..bc11ef31 100644 --- a/packages/web-product/src/ui/components/tenant-settings/modelGroups.ts +++ b/packages/web-product/src/ui/components/tenant-settings/modelGroups.ts @@ -26,11 +26,19 @@ export const PROVIDER_LABELS: Record = { xai: 'xAI', fireworks: 'Fireworks', openrouter: 'OpenRouter', - local: 'Local', + ollama: 'This computer (Ollama)', }; /** Providers in the order the groups read; anything unlisted follows, alphabetically. */ -const PROVIDER_ORDER = ['anthropic', 'openai', 'xai', 'google', 'fireworks', 'openrouter', 'local']; +const PROVIDER_ORDER = [ + 'anthropic', + 'openai', + 'xai', + 'google', + 'fireworks', + 'openrouter', + 'ollama', +]; function providerRank(provider: string): number { const at = PROVIDER_ORDER.indexOf(provider); diff --git a/scripts/local-model-gates.ts b/scripts/local-model-gates.ts new file mode 100644 index 00000000..01719dda --- /dev/null +++ b/scripts/local-model-gates.ts @@ -0,0 +1,689 @@ +/** + * Plan 323 P0 — feasibility gates for models served by a local Ollama. + * + * Each gate answers one question a P1 decision rests on, against the runtime + * actually installed, and prints JSON so the answer can be reproduced and + * compared across machines and runtime versions. Nothing here touches the + * stack: it talks to the runtime and nothing else. + * + * yarn models:local-gates [--model ]... [--out ] + * + * Gates: + * overflow G1 — what a prompt over `num_ctx` does, and how exact `prompt_eval_count` is + * locality G2 — whether the runtime says where it sends inference + * lifecycle G3 — pull joins, cancellation, resumption, refusal + * embeddings G4 — dimensions, truncation, and retrieval at reduced dimensions (its built-in + * retrieval set is too easy to separate dimensions; it shows support, not quality) + * machine G5 — footprint by context, load and switch time, throughput + * + * `lifecycle` and `embeddings` pull small models (about 1–3 GB) and delete + * what they pulled. `machine` loads every model it is given, one at a time. + */ +import { writeFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import os from 'node:os'; + +const BASE = process.env['OLLAMA_URL'] ?? 'http://127.0.0.1:11434'; + +interface ChatResult { + promptEvalCount: number | null; + evalCount: number | null; + doneReason: string | null; + content: string; + toolCalls: unknown; + loadS: number; + promptS: number; + evalS: number; + wallS: number; + error?: string; +} + +interface ChatMessage { + role: 'system' | 'user' | 'assistant' | 'tool'; + content: string; + thinking?: string; + images?: string[]; +} + +const describe = (e: unknown): string => + e instanceof Error ? `${e.name}: ${e.message}` : JSON.stringify(e); + +const seconds = (ns: unknown): number => (typeof ns === 'number' ? Math.round(ns / 1e7) / 100 : 0); + +async function api(path: string, body?: unknown, signal?: AbortSignal): Promise { + return fetch(`${BASE}${path}`, { + method: body === undefined ? 'GET' : 'POST', + headers: { 'content-type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + ...(signal ? { signal } : {}), + }); +} + +async function json(path: string, body?: unknown): Promise { + const res = await api(path, body); + if (!res.ok) throw new Error(`${path} → ${res.status} ${await res.text()}`); + return (await res.json()) as T; +} + +async function chat( + model: string, + messages: ChatMessage[], + options: { numCtx: number; numPredict?: number; tools?: unknown[]; think?: boolean }, +): Promise { + const started = Date.now(); + const res = await api('/api/chat', { + model, + messages, + stream: false, + think: options.think ?? false, + ...(options.tools ? { tools: options.tools } : {}), + options: { num_ctx: options.numCtx, num_predict: options.numPredict ?? 40, temperature: 0 }, + }); + const wallS = Math.round((Date.now() - started) / 10) / 100; + if (!res.ok) { + return { + promptEvalCount: null, + evalCount: null, + doneReason: null, + content: '', + toolCalls: null, + loadS: 0, + promptS: 0, + evalS: 0, + wallS, + error: `${res.status} ${(await res.text()).slice(0, 400)}`, + }; + } + const d = (await res.json()) as Record; + const message = (d['message'] ?? {}) as Record; + return { + promptEvalCount: typeof d['prompt_eval_count'] === 'number' ? d['prompt_eval_count'] : null, + evalCount: typeof d['eval_count'] === 'number' ? d['eval_count'] : null, + doneReason: typeof d['done_reason'] === 'string' ? d['done_reason'] : null, + content: typeof message['content'] === 'string' ? message['content'] : '', + toolCalls: message['tool_calls'] ?? null, + loadS: seconds(d['load_duration']), + promptS: seconds(d['prompt_eval_duration']), + evalS: seconds(d['eval_duration']), + wallS, + }; +} + +async function unloadAll(): Promise { + const ps = await json<{ models: Array<{ name: string }> }>('/api/ps'); + for (const m of ps.models) await api('/api/generate', { model: m.name, keep_alive: 0 }); +} + +async function show(model: string): Promise> { + return json('/api/show', { model }); +} + +async function pull( + model: string, + signal?: AbortSignal, + onProgress?: (completed: number, total: number) => void, +): Promise<{ events: number; lastStatus: string; firstCompleted: number | null }> { + const res = await api('/api/pull', { model, stream: true }, signal); + if (!res.ok || !res.body) throw new Error(`pull ${model} → ${res.status} ${await res.text()}`); + let events = 0; + let lastStatus = ''; + let firstCompleted: number | null = null; + const decoder = new TextDecoder(); + let buffer = ''; + const reader = res.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + let nl; + while ((nl = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, nl).trim(); + buffer = buffer.slice(nl + 1); + if (!line) continue; + const ev = JSON.parse(line) as Record; + events++; + if (typeof ev['error'] === 'string') throw new Error(`pull ${model}: ${ev['error']}`); + if (typeof ev['status'] === 'string') lastStatus = ev['status']; + if ( + firstCompleted === null && + typeof ev['completed'] === 'number' && + typeof ev['total'] === 'number' && + ev['total'] > 1e6 + ) { + firstCompleted = ev['completed']; + } + if (onProgress && typeof ev['completed'] === 'number' && typeof ev['total'] === 'number') { + onProgress(ev['completed'], ev['total']); + } + } + } + return { events, lastStatus, firstCompleted }; +} + +async function remove(model: string): Promise { + const res = await fetch(`${BASE}/api/delete`, { + method: 'DELETE', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model }), + }); + if (!res.ok) throw new Error(`delete ${model} → ${res.status} ${await res.text()}`); +} + +async function installedTags(): Promise> { + const tags = await json<{ models: Array<{ name: string }> }>('/api/tags'); + return new Set(tags.models.map((m) => m.name)); +} + +const SECRET_SYSTEM: ChatMessage = { + role: 'system', + content: + 'SECRET CODE: PELICAN-42. When asked for the code, reply with the code and nothing else.', +}; +const filler = (items: number): string => + Array.from({ length: items }, (_, i) => `Item ${i} is a blue marble.`).join(' '); + +const SAMPLES: Record = { + english: + 'The orchestrator schedules each tool call as its own step, records its result, and hands the result back to the next agent turn. '.repeat( + 40, + ), + german: + 'Der Orchestrator plant jeden Werkzeugaufruf als eigenen Schritt, speichert das Ergebnis und gibt es an den nächsten Agentenschritt weiter. '.repeat( + 40, + ), + chinese: '编排器将每次工具调用安排为独立的步骤,记录其结果,并将结果交给下一轮智能体。'.repeat( + 40, + ), + code: 'export function admit(limiter: Limiter, signal: AbortSignal): Promise {\n if (limiter.tryAcquire()) return Promise.resolve(false);\n return limiter.acquire(signal).then(() => true);\n}\n'.repeat( + 25, + ), + json: JSON.stringify( + Array.from({ length: 60 }, (_, i) => ({ + id: `step-${i}`, + status: 'SUCCEEDED', + output: { rows: i, ok: true }, + })), + ), +}; + +const PROBE_TOOLS = Array.from({ length: 8 }, (_, i) => ({ + type: 'function', + function: { + name: `tool_${i}`, + description: `Reads records of kind ${i} from the workspace and returns them as a list.`, + parameters: { + type: 'object', + properties: { + query: { type: 'string', description: 'What to look for.' }, + limit: { type: 'integer', minimum: 1, maximum: 100 }, + }, + required: ['query'], + }, + }, +})); + +// 8×8 red PNG. +const TINY_PNG = + 'iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEklEQVR4nGP4z8CAFWEXHbQSACj/P8Fu7N9hAAAAAElFTkSuQmCC'; + +async function gateOverflow(models: string[]): Promise { + const results: Record = {}; + for (const model of models) { + await unloadAll(); + const info = await show(model); + const details = info['details'] as Record; + const ask: ChatMessage = { role: 'user', content: `${filler(650)} What is the secret code?` }; + const small = await chat( + model, + [SECRET_SYSTEM, { role: 'user', content: 'What is the secret code?' }], + { numCtx: 4096 }, + ); + const over = await chat(model, [SECRET_SYSTEM, ask], { numCtx: 4096 }); + const fits = await chat(model, [SECRET_SYSTEM, ask], { numCtx: 16384 }); + const repeat = await chat(model, [SECRET_SYSTEM, { ...ask, content: `${ask.content} ` }], { + numCtx: 16384, + }); + + const ratios: Record< + string, + { chars: number; tokens: number | null; charsPerToken: number | null } + > = {}; + const base = + (await chat(model, [{ role: 'user', content: 'x' }], { numCtx: 16384, numPredict: 1 })) + .promptEvalCount ?? 0; + for (const [kind, text] of Object.entries(SAMPLES)) { + const r = await chat(model, [{ role: 'user', content: text }], { + numCtx: 16384, + numPredict: 1, + }); + const tokens = r.promptEvalCount === null ? null : r.promptEvalCount - base; + ratios[kind] = { + chars: text.length, + tokens, + charsPerToken: tokens ? Math.round((text.length / tokens) * 100) / 100 : null, + }; + } + const plain = await chat(model, [{ role: 'user', content: 'Say ok.' }], { + numCtx: 16384, + numPredict: 1, + }); + const withTools = await chat(model, [{ role: 'user', content: 'Say ok.' }], { + numCtx: 16384, + numPredict: 1, + tools: PROBE_TOOLS, + }); + const withImage = await chat( + model, + [{ role: 'user', content: 'Say ok.', images: [TINY_PNG] }], + { numCtx: 16384, numPredict: 1 }, + ); + const thinkingText = 'Let me reason about the marbles carefully before answering. '.repeat(30); + const noHistory = await chat( + model, + [ + { role: 'user', content: 'Hi' }, + { role: 'assistant', content: 'Hello.' }, + { role: 'user', content: 'Say ok.' }, + ], + { numCtx: 16384, numPredict: 1 }, + ); + const withThinking = await chat( + model, + [ + { role: 'user', content: 'Hi' }, + { role: 'assistant', content: 'Hello.', thinking: thinkingText }, + { role: 'user', content: 'Say ok.' }, + ], + { numCtx: 16384, numPredict: 1 }, + ); + + results[model] = { + runner: details['runner'], + format: details['format'], + quantization: details['quantization_level'], + digest: ( + await json<{ models: Array<{ name: string; digest: string }> }>('/api/tags') + ).models.find((m) => m.name === model)?.digest, + overflow: { + small, + overWindow: { ...over, recalledSecret: over.content.includes('PELICAN-42') }, + fitsWindow: { ...fits, recalledSecret: fits.content.includes('PELICAN-42') }, + repeatedPrefix: repeat, + truncatedSilently: + over.error === undefined && (over.promptEvalCount ?? 0) < (fits.promptEvalCount ?? 0), + }, + charsPerToken: ratios, + overheadTokens: { + eightTools: (withTools.promptEvalCount ?? 0) - (plain.promptEvalCount ?? 0), + oneTinyImage: withImage.error + ? withImage.error + : (withImage.promptEvalCount ?? 0) - (plain.promptEvalCount ?? 0), + replayedThinking: (withThinking.promptEvalCount ?? 0) - (noHistory.promptEvalCount ?? 0), + replayedThinkingChars: thinkingText.length, + }, + }; + } + return results; +} + +async function gateLocality(): Promise { + const status = await api('/api/status'); + const tags = await json<{ models: Array> }>('/api/tags'); + return { + status: status.ok ? await status.json() : `${status.status}`, + remoteModels: tags.models + .filter((m) => typeof m['remote_host'] === 'string') + .map((m) => ({ + name: m['name'], + remoteHost: m['remote_host'], + remoteModel: m['remote_model'], + })), + localModels: tags.models + .filter((m) => typeof m['remote_host'] !== 'string') + .map((m) => m['name']), + }; +} + +async function gateLifecycle(): Promise { + const tag = 'qwen3.5:0.8b-q8_0'; + const preinstalled = (await installedTags()).has(tag); + if (preinstalled) await remove(tag); + + // Cancelled on the first partial progress of the large layer, not on a + // timer: bytes already on disk from an earlier run finish a timed pull first. + const ctl = new AbortController(); + let cancelledAtBytes: number | null = null; + const afterCancel = await pull(tag, ctl.signal, (completed, total) => { + if (cancelledAtBytes === null && total > 1e8 && completed > total / 10 && completed < total) { + cancelledAtBytes = completed; + ctl.abort(); + } + }).then( + (r) => ({ finished: true, ...r }), + (e: unknown) => ({ finished: false, error: describe(e).slice(0, 200) }), + ); + const installedAfterCancel = (await installedTags()).has(tag); + + const resumeStarted = Date.now(); + const [first, second] = await Promise.all([pull(tag), pull(tag)]); + const concurrentS = Math.round((Date.now() - resumeStarted) / 100) / 10; + + const missing = await pull('aflow-gate/does-not-exist:latest').then( + () => 'succeeded', + (e: unknown) => describe(e).slice(0, 300), + ); + + if (!preinstalled) await remove(tag); + return { + tag, + cancelMidPull: + cancelledAtBytes === null + ? 'not exercised: the layers were already on disk, so the pull finished without partial progress' + : { ...afterCancel, cancelledAtBytes, installedAfterCancel }, + resumedAndConcurrent: { + wallS: concurrentS, + first, + second, + resumedFromPartial: + cancelledAtBytes === null ? null : (first.firstCompleted ?? 0) >= cancelledAtBytes, + }, + missingTag: missing, + diskSpaceRefusal: 'not exercised: needs a full disk; P3 checks free space before pulling', + }; +} + +const RETRIEVAL_PAIRS: Array<[string, string]> = [ + [ + 'How do I reset my password?', + 'To change a forgotten password, open Settings, choose Security and follow the reset link we email you.', + ], + [ + 'Which port does Postgres listen on locally?', + 'The local database is published on port 5433 rather than the default 5432.', + ], + [ + 'What happens when a tool call fails five times in a row?', + 'After five consecutive tool failures the agent loop stops and the run is marked as failed.', + ], + [ + 'Can I run models without a cloud key?', + 'Models served by a runtime on your own machine need no provider key.', + ], + [ + 'How are large payloads stored?', + 'Anything large goes to the payload store and only a reference travels through the stream.', + ], + [ + 'Where do approvals show up?', + 'Requests that need your consent appear as cards in the Action Center.', + ], + [ + 'How do I pair my laptop?', + 'Run the pairing command on the machine, which writes its connection file under your home directory.', + ], + [ + 'What does the judge do?', + 'The judge scores a finished run against the rubric of its evaluation suite.', + ], + [ + 'Why is my local model slow?', + 'A model that does not fit in GPU memory runs partly on the CPU and decodes much more slowly.', + ], + [ + 'Is browsing allowed on any site?', + 'Each browser profile has a posture and per-site rules that allow, ask or deny.', + ], + ['How often are titles generated?', 'A session gets its title once, after the first exchange.'], + [ + 'What is a skill?', + 'A skill bundles a workflow with its manifest, evaluation suite and activation rules.', + ], + [ + 'How do I invite a teammate?', + 'Send an invitation from the members page; it expires after seven days.', + ], + [ + 'What limits concurrent harness runs?', + 'The host executor runs only a bounded number of coding harness runs at once.', + ], + ['Does the platform keep my data in Europe?', 'The hosted service runs in a European region.'], + [ + 'How do I delete a model?', + 'Removing a downloaded model frees its disk space; it can be pulled again later.', + ], + [ + 'What is the Helmsman?', + 'The Helmsman is the conversational agent that plans work and commissions other agents.', + ], + [ + 'How are secrets stored?', + 'Credentials are encrypted with an envelope key before they are written.', + ], + [ + 'Can a run be retried?', + 'A failed run can be retried from the session view, which starts it again.', + ], + [ + 'What happens when memory search finds nothing?', + 'An empty search returns no results rather than guessing.', + ], +]; + +function cosine(a: number[], b: number[]): number { + let dot = 0; + let na = 0; + let nb = 0; + for (let i = 0; i < a.length; i++) { + const x = a[i] ?? 0; + const y = b[i] ?? 0; + dot += x * y; + na += x * x; + nb += y * y; + } + return dot / (Math.sqrt(na) * Math.sqrt(nb)); +} + +async function embed( + model: string, + input: string[], + extra: Record = {}, +): Promise<{ vectors: number[][] | null; promptEvalCount: number | null; error?: string }> { + const res = await api('/api/embed', { model, input, ...extra }); + if (!res.ok) + return { + vectors: null, + promptEvalCount: null, + error: `${res.status} ${(await res.text()).slice(0, 300)}`, + }; + const d = (await res.json()) as { embeddings: number[][]; prompt_eval_count?: number }; + return { vectors: d.embeddings, promptEvalCount: d.prompt_eval_count ?? null }; +} + +async function recallAt1(model: string, extra: Record): Promise { + const q = await embed( + model, + RETRIEVAL_PAIRS.map(([query]) => query), + extra, + ); + const p = await embed( + model, + RETRIEVAL_PAIRS.map(([, passage]) => passage), + extra, + ); + if (!q.vectors || !p.vectors) return q.error ?? p.error ?? 'no vectors'; + let hits = 0; + q.vectors.forEach((qv, i) => { + let best = -1; + let bestScore = -Infinity; + p.vectors.forEach((pv, j) => { + const s = cosine(qv, pv); + if (s > bestScore) { + bestScore = s; + best = j; + } + }); + if (best === i) hits++; + }); + return hits / RETRIEVAL_PAIRS.length; +} + +async function gateEmbeddings(models: string[]): Promise { + const results: Record = {}; + const before = await installedTags(); + const isInstalled = (model: string): boolean => + before.has(model) || before.has(`${model}:latest`); + for (const model of models) { + if (!isInstalled(model)) await pull(model); + const native = await embed(model, ['dimension probe']); + const nativeDims = native.vectors?.[0]?.length ?? null; + const at1536 = await embed(model, ['dimension probe'], { dimensions: 1536 }); + const long = 'The orchestrator writes durable state. '.repeat(4000); + const overWindowTruncateFalse = await embed(model, [long], { truncate: false }); + const overWindowDefault = await embed(model, [long]); + results[model] = { + nativeDims, + requested1536: at1536.error ?? at1536.vectors?.[0]?.length ?? null, + overWindow: { + truncateFalse: + overWindowTruncateFalse.error ?? + `accepted (${overWindowTruncateFalse.promptEvalCount} tokens)`, + default: + overWindowDefault.error ?? + `accepted silently (${overWindowDefault.promptEvalCount} tokens of ~${Math.round(long.length / 4)})`, + }, + recallAt1: { + native: await recallAt1(model, {}), + ...(nativeDims && nativeDims > 1536 + ? { at1536: await recallAt1(model, { dimensions: 1536 }) } + : {}), + ...(nativeDims && nativeDims > 768 + ? { at768: await recallAt1(model, { dimensions: 768 }) } + : {}), + }, + retrievalSet: `${RETRIEVAL_PAIRS.length} pairs — indicative only`, + }; + if (!isInstalled(model)) await remove(model); + } + return results; +} + +function gpuWorkingSetBytes(): number | string { + if (process.platform !== 'darwin') return 'not macOS'; + try { + const out = execFileSync( + 'swift', + ['-e', 'import Metal; print(MTLCreateSystemDefaultDevice()!.recommendedMaxWorkingSetSize)'], + { encoding: 'utf8', timeout: 180_000 }, + ); + return Number(out.trim()); + } catch (e) { + return `unavailable: ${describe(e).slice(0, 120)}`; + } +} + +async function gateMachine(models: string[]): Promise { + const results: Record = {}; + let previous: string | null = null; + for (const model of models) { + await unloadAll(); + const coldStarted = Date.now(); + const cold = await chat(model, [{ role: 'user', content: 'Say ok.' }], { + numCtx: 4096, + numPredict: 1, + }); + const coldWallS = Math.round((Date.now() - coldStarted) / 100) / 10; + const footprint: Record = {}; + for (const numCtx of [4096, 32768]) { + await chat(model, [{ role: 'user', content: 'Say ok.' }], { numCtx, numPredict: 1 }); + const ps = await json<{ models: Array> }>('/api/ps'); + const m = ps.models.find((x) => x['name'] === model); + footprint[`ctx${numCtx}`] = { + size: m?.['size'], + sizeVram: m?.['size_vram'], + contextLength: m?.['context_length'], + }; + } + const prefill = await chat(model, [{ role: 'user', content: `${filler(1200)} Say ok.` }], { + numCtx: 32768, + numPredict: 1, + }); + const decode = await chat( + model, + [{ role: 'user', content: 'Count from 1 to 200, separated by commas.' }], + { numCtx: 32768, numPredict: 256 }, + ); + let switchFromPrevious: unknown = null; + if (previous) { + await chat(previous, [{ role: 'user', content: 'Say ok.' }], { numCtx: 4096, numPredict: 1 }); + const started = Date.now(); + await chat(model, [{ role: 'user', content: 'Say ok.' }], { numCtx: 4096, numPredict: 1 }); + switchFromPrevious = { from: previous, wallS: Math.round((Date.now() - started) / 100) / 10 }; + } + results[model] = { + coldLoad: { wallS: coldWallS, loadS: cold.loadS, firstPromptS: cold.promptS }, + footprint, + prefillTokensPerS: + prefill.promptEvalCount && prefill.promptS + ? Math.round(prefill.promptEvalCount / prefill.promptS) + : null, + prefillTokens: prefill.promptEvalCount, + decodeTokensPerS: + decode.evalCount && decode.evalS ? Math.round(decode.evalCount / decode.evalS) : null, + switchFromPrevious, + }; + previous = model; + } + return { + host: { + platform: process.platform, + arch: process.arch, + cpu: os.cpus()[0]?.model, + totalMemoryBytes: os.totalmem(), + freeMemoryBytes: os.freemem(), + }, + gpuWorkingSetBytes: gpuWorkingSetBytes(), + serverConditions: + 'KV cache type, flash attention and parallelism are server environment, not visible over the API; record them beside the result', + models: results, + }; +} + +async function main(): Promise { + const [gate, ...rest] = process.argv.slice(2); + const models: string[] = []; + let out: string | null = null; + for (let i = 0; i < rest.length; i++) { + if (rest[i] === '--model' && rest[i + 1]) models.push(rest[++i]); + else if (rest[i] === '--out' && rest[i + 1]) out = rest[++i]!; + } + const version = await json<{ version: string }>('/api/version'); + const run: Record Promise> = { + overflow: () => gateOverflow(models), + locality: () => gateLocality(), + lifecycle: () => gateLifecycle(), + embeddings: () => + gateEmbeddings(models.length ? models : ['embeddinggemma', 'qwen3-embedding:4b']), + machine: () => gateMachine(models), + }; + const fn = gate ? run[gate] : undefined; + if (!fn) { + console.error( + `usage: local-model-gates.ts <${Object.keys(run).join('|')}> [--model ]... [--out ]`, + ); + process.exit(2); + } + const result = { + gate, + runtime: { url: BASE, version: version.version }, + measuredAt: new Date().toISOString(), + result: await fn(), + }; + const text = JSON.stringify(result, null, 2); + if (out) writeFileSync(out, `${text}\n`); + console.log(text); +} + +main().catch((err: unknown) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/verify-local-models.ts b/scripts/verify-local-models.ts new file mode 100644 index 00000000..39363dc0 --- /dev/null +++ b/scripts/verify-local-models.ts @@ -0,0 +1,245 @@ +/** + * Measure what a model served by the local Ollama can do, and check the + * catalog entry that names it (Plan 323 D9, compatibility tier). + * + * The catalog's flags for a local model are the runtime's own answers for one + * build: a tag pulled again can be another digest, and a runtime upgrade can + * add or remove a feature (the MLX runner refuses `format` outright). So each + * flag here comes from a request, never from a model card, and the result is + * tied to the digest, runner, quantisation and runtime version it was measured + * on. + * + * yarn models:verify-local [...] + * + * With no tags it checks every local entry in the catalog. Exits non-zero when + * a catalog entry disagrees with what was measured, printing the entry the + * measurement supports. Tags not installed are reported as SKIPPED. + */ +import { + createDefaultModelCatalog, + type LocalModelBuild, + type ModelDefinition, +} from '@aflow/ai-client'; +import { OLLAMA_DEFAULT_BASE_URL } from '@aflow/schemas'; + +const BASE = (process.env['OLLAMA_URL'] ?? OLLAMA_DEFAULT_BASE_URL).replace(/\/+$/, ''); +/** Bumped whenever a probe below changes what it measures. */ +const PROBE_VERSION = 1; + +const WEATHER_TOOL = { + type: 'function', + function: { + name: 'get_weather', + description: 'Current weather for one city.', + parameters: { + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + }, + }, +}; + +interface Measured { + build: LocalModelBuild; + capabilities: { + vision: boolean; + functionCalling: boolean; + parallelToolCalls: boolean; + streamedToolCalls: boolean; + structuredOutputs: boolean; + reasoning: boolean; + }; + reasoningSupported: string[]; + contextWindow: number | null; +} + +async function post( + path: string, + body: unknown, +): Promise<{ status: number; json: Record }> { + const res = await fetch(`${BASE}${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + const text = await res.text(); + let json: Record = {}; + try { + json = JSON.parse(text.split('\n').filter(Boolean).at(-1) ?? '{}') as Record; + } catch { + json = { raw: text }; + } + return { status: res.status, json }; +} + +async function chat(tag: string, extra: Record) { + return post('/api/chat', { + model: tag, + stream: false, + think: false, + options: { num_ctx: 4096, num_predict: 256, temperature: 0 }, + ...extra, + }); +} + +function toolCalls( + json: Record, +): Array<{ function?: { name?: string; arguments?: unknown } }> { + const message = json['message'] as + { tool_calls?: Array<{ function?: { name?: string; arguments?: unknown } }> } | undefined; + return message?.tool_calls ?? []; +} + +async function streamedToolCall(tag: string): Promise { + const res = await fetch(`${BASE}/api/chat`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + model: tag, + stream: true, + think: false, + tools: [WEATHER_TOOL], + options: { num_ctx: 4096, num_predict: 256, temperature: 0 }, + messages: [{ role: 'user', content: 'What is the weather in Berlin? Use the tool.' }], + }), + }); + const lines = (await res.text()).split('\n').filter(Boolean); + return lines.some((line) => { + const chunk = JSON.parse(line) as Record; + return toolCalls(chunk).some((c) => c.function?.name === 'get_weather'); + }); +} + +async function measure(tag: string): Promise { + const tags = (await (await fetch(`${BASE}/api/tags`)).json()) as { + models: Array<{ name: string; digest: string; remote_host?: string }>; + }; + const installed = tags.models.find((m) => m.name === tag || m.name === `${tag}:latest`); + if (!installed) return null; + if (installed.remote_host) + throw new Error(`${tag} is served by ${installed.remote_host}, not this computer`); + const version = (await (await fetch(`${BASE}/api/version`)).json()) as { version: string }; + const show = (await post('/api/show', { model: tag })).json; + const details = (show['details'] ?? {}) as Record; + const caps = new Set((show['capabilities'] as string[] | undefined) ?? []); + const modelInfo = (show['model_info'] ?? {}) as Record; + const contextKey = Object.keys(modelInfo).find((k) => k.endsWith('.context_length')); + + const single = await chat(tag, { + tools: [WEATHER_TOOL], + messages: [{ role: 'user', content: 'What is the weather in Berlin? Use the tool.' }], + }); + const parallel = await chat(tag, { + tools: [WEATHER_TOOL], + messages: [ + { + role: 'user', + content: 'What is the weather in Berlin and in Paris? Call the tool once per city.', + }, + ], + }); + const format = await chat(tag, { + format: { type: 'object', properties: { answer: { type: 'integer' } }, required: ['answer'] }, + messages: [{ role: 'user', content: '17 * 23 = ? Reply as JSON.' }], + }); + const thinking = show['thinking'] as { values?: unknown[] } | undefined; + const levels = (thinking?.values ?? []).filter((v): v is string => typeof v === 'string'); + const reasoningSupported = + thinking?.values === undefined + ? [] + : levels.length > 0 + ? ['off', ...levels.filter((l) => ['low', 'medium', 'high'].includes(l))] + : ['off', 'high']; + + return { + build: { + runtimeTag: tag, + digest: installed.digest, + runner: details['runner'] ?? 'unknown', + quantization: (details['quantization_level'] ?? 'unknown').toLowerCase(), + runtimeVersion: version.version, + probeVersion: PROBE_VERSION, + measuredAt: new Date().toISOString().slice(0, 10), + }, + capabilities: { + vision: caps.has('vision'), + functionCalling: toolCalls(single.json).some((c) => c.function?.name === 'get_weather'), + parallelToolCalls: toolCalls(parallel.json).length >= 2, + streamedToolCalls: await streamedToolCall(tag), + structuredOutputs: format.status === 200, + reasoning: caps.has('thinking'), + }, + reasoningSupported, + contextWindow: + typeof modelInfo[contextKey ?? ''] === 'number' + ? (modelInfo[contextKey ?? ''] as number) + : null, + }; +} + +function drift(entry: ModelDefinition, m: Measured): string[] { + const found: string[] = []; + const expect = (label: string, catalog: unknown, measured: unknown) => { + if (JSON.stringify(catalog) !== JSON.stringify(measured)) { + found.push( + `${label}: catalog ${JSON.stringify(catalog)}, measured ${JSON.stringify(measured)}`, + ); + } + }; + expect('capabilities.vision', entry.capabilities.vision, m.capabilities.vision); + expect( + 'capabilities.functionCalling', + entry.capabilities.functionCalling, + m.capabilities.functionCalling, + ); + expect( + 'capabilities.structuredOutputs', + entry.capabilities.structuredOutputs === true, + m.capabilities.structuredOutputs, + ); + expect('capabilities.reasoning', entry.capabilities.reasoning === true, m.capabilities.reasoning); + expect('reasoning.supported', [...(entry.reasoning?.supported ?? [])], m.reasoningSupported); + if (m.contextWindow !== null) expect('contextWindow', entry.contextWindow, m.contextWindow); + expect('local.digest', entry.local?.digest, m.build.digest); + expect('local.runner', entry.local?.runner, m.build.runner); + expect('local.quantization', entry.local?.quantization, m.build.quantization); + expect('local.runtimeVersion', entry.local?.runtimeVersion, m.build.runtimeVersion); + expect('local.probeVersion', entry.local?.probeVersion, m.build.probeVersion); + return found; +} + +async function main(): Promise { + const catalog = createDefaultModelCatalog(); + const entries = catalog.listModels('ollama'); + const requested = process.argv.slice(2); + const tags = requested.length > 0 ? requested : entries.map((e) => e.providerModelId ?? e.id); + let failed = false; + for (const tag of tags) { + const measured = await measure(tag); + if (!measured) { + console.log(`SKIPPED ${tag}: not installed (ollama pull ${tag})`); + continue; + } + const entry = entries.find((e) => (e.providerModelId ?? e.id) === tag); + console.log(`\n${tag}`); + console.log(JSON.stringify(measured, null, 2)); + if (!entry) { + console.log(`NO ENTRY ${tag}: measured above; add a catalog entry from it.`); + continue; + } + const differences = drift(entry, measured); + if (differences.length === 0) { + console.log(`OK ${entry.id}`); + } else { + failed = true; + console.log(`DRIFT ${entry.id}`); + for (const d of differences) console.log(` ${d}`); + } + } + process.exit(failed ? 1 : 0); +} + +main().catch((err: unknown) => { + console.error(err); + process.exit(1); +});