diff --git a/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurnModel.ts b/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurnModel.ts index 474ae23c..8b26b066 100644 --- a/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurnModel.ts +++ b/apps/aflow-executor-ai/src/handlers/ai/handlers/agentTurnModel.ts @@ -10,6 +10,7 @@ import type { AgentTurnInput } from '../schema.js'; import type { HandlerDeps } from './types.js'; import { chatMessageToAiMessage } from './agentMessageConversion.js'; import { toolImageResolver } from './mediaSourceRef.js'; +import { reportRuntimePhases } from './runtimePhaseStatus.js'; import { buildFunctionDeclarations, mapToolCallsToDecision, @@ -438,6 +439,7 @@ async function streamAndCollect( attempt: ctx.job.attempt, signal: ctx.signal, onStreamProgress: ctx.reportProgress, + onRuntimePhase: reportRuntimePhases(ctx, client.getModel(p.model)?.displayName ?? p.model), maxRetries: 0, // Agent turns are interactive — let orchestrator handle retries with user feedback }); @@ -741,6 +743,7 @@ async function runGenerateJsonPath( attempt: ctx.job.attempt, signal: ctx.signal, onStreamProgress: ctx.reportProgress, + onRuntimePhase: reportRuntimePhases(ctx, modelDef?.displayName ?? model), maxRetries: 0, }); diff --git a/apps/aflow-executor-ai/src/handlers/ai/handlers/runtimePhaseStatus.test.ts b/apps/aflow-executor-ai/src/handlers/ai/handlers/runtimePhaseStatus.test.ts new file mode 100644 index 00000000..342382d8 --- /dev/null +++ b/apps/aflow-executor-ai/src/handlers/ai/handlers/runtimePhaseStatus.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { ExecutorContext } from '@aflow/executor-runtime'; +import { reportRuntimePhases } from './runtimePhaseStatus.js'; + +function context() { + const emitLiveDelta = vi.fn(async () => undefined); + const reportProgress = vi.fn(); + const ctx = { emitLiveDelta, reportProgress, log: { warn: vi.fn() } }; + return { ctx: ctx as unknown as ExecutorContext, emitLiveDelta, reportProgress }; +} + +describe('a local model’s silent stretches', () => { + it('are one line each on the status channel, naming the model that loads', () => { + const { ctx, emitLiveDelta } = context(); + const report = reportRuntimePhases(ctx, 'Gemma 4 26B (this computer)'); + + report({ phase: 'loading_model' }); + report({ phase: 'reading_prompt' }); + + expect(emitLiveDelta.mock.calls).toEqual([ + ['status', 'Loading Gemma 4 26B (this computer) into memory…\n'], + ['status', 'Reading the conversation…\n'], + ]); + }); + + it('count as progress, so a long load is not cut as a stall', () => { + const { ctx, reportProgress } = context(); + + reportRuntimePhases(ctx, 'm')({ phase: 'loading_model' }); + + expect(reportProgress).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/aflow-executor-ai/src/handlers/ai/handlers/runtimePhaseStatus.ts b/apps/aflow-executor-ai/src/handlers/ai/handlers/runtimePhaseStatus.ts new file mode 100644 index 00000000..d6c68e97 --- /dev/null +++ b/apps/aflow-executor-ai/src/handlers/ai/handlers/runtimePhaseStatus.ts @@ -0,0 +1,31 @@ +import type { ExecutorContext } from '@aflow/executor-runtime'; +import type { RuntimePhase } from '@aflow/ai-client'; + +export function runtimePhaseLine(phase: RuntimePhase, modelName: string): string { + switch (phase.phase) { + case 'loading_model': + return `Loading ${modelName} into memory…`; + case 'reading_prompt': + return 'Reading the conversation…'; + } +} + +/** + * Puts a local model's silent stretches on the step's `status` channel, where + * the chat shows them in place of a bare spinner until the first token. Each + * phase also slides the idle window: a load is the runtime working, not a + * stall. + */ +export function reportRuntimePhases( + ctx: ExecutorContext, + modelName: string, +): (phase: RuntimePhase) => void { + return (phase) => { + ctx.reportProgress?.(); + ctx.emitLiveDelta('status', `${runtimePhaseLine(phase, modelName)}\n`).catch((err: unknown) => { + ctx.log.warn('Live status delta dropped', { + error: err instanceof Error ? err.message : String(err), + }); + }); + }; +} diff --git a/apps/aflow-executor-host/src/__tests__/machine.test.ts b/apps/aflow-executor-host/src/__tests__/machine.test.ts index 6588a73c..5ac4b418 100644 --- a/apps/aflow-executor-host/src/__tests__/machine.test.ts +++ b/apps/aflow-executor-host/src/__tests__/machine.test.ts @@ -21,16 +21,17 @@ describe('createMachineReader', () => { totalMemoryBytes: 24 * GIB, osRelease: '25.3.0', run, + freeDisk: async () => undefined, }; const first = createMachineReader(options); - expect(first.current()).toEqual({ totalMemoryBytes: 24 * GIB, unifiedMemory: true }); + expect(await first.current()).toEqual({ totalMemoryBytes: 24 * GIB, unifiedMemory: true }); await first.start(); - expect(first.current().gpuWorkingSetBytes).toBe(19_069_665_280); + expect((await first.current()).gpuWorkingSetBytes).toBe(19_069_665_280); const second = createMachineReader(options); await second.start(); - expect(second.current().gpuWorkingSetBytes).toBe(19_069_665_280); + expect((await second.current()).gpuWorkingSetBytes).toBe(19_069_665_280); expect(run).toHaveBeenCalledTimes(1); }); @@ -64,9 +65,10 @@ describe('createMachineReader', () => { totalMemoryBytes: 64 * GIB, osRelease: '6.8', run: async () => '24576\n49152\n', + freeDisk: async () => undefined, }); await nvidia.start(); - expect(nvidia.current()).toEqual({ + expect(await nvidia.current()).toEqual({ totalMemoryBytes: 64 * GIB, unifiedMemory: false, gpuWorkingSetBytes: 49152 * 1024 ** 2, @@ -81,8 +83,32 @@ describe('createMachineReader', () => { run: async () => { throw new Error('nvidia-smi: command not found'); }, + freeDisk: async () => undefined, }); await none.start(); - expect(none.current().gpuWorkingSetBytes).toBeUndefined(); + expect((await none.current()).gpuWorkingSetBytes).toBeUndefined(); + }); + + it('reads the free disk under the model store at every read, since downloads change it', async () => { + let free = 100 * GIB; + const paths: string[] = []; + const reader = createMachineReader({ + hostDir: await hostDir(), + platform: 'linux', + arch: 'x64', + totalMemoryBytes: 16 * GIB, + osRelease: '6.8', + run: async () => '', + modelStore: '/models', + freeDisk: async (path) => { + paths.push(path); + return free; + }, + }); + + expect((await reader.current()).freeDiskBytes).toBe(100 * GIB); + free = 84 * GIB; + expect((await reader.current()).freeDiskBytes).toBe(84 * GIB); + expect(paths).toEqual(['/models', '/models']); }); }); diff --git a/apps/aflow-executor-host/src/browser/approvalStore.ts b/apps/aflow-executor-host/src/browser/approvalStore.ts index 576fe960..36753aad 100644 --- a/apps/aflow-executor-host/src/browser/approvalStore.ts +++ b/apps/aflow-executor-host/src/browser/approvalStore.ts @@ -3,7 +3,7 @@ import { getWriteApprovalGrant, readBrowserAsk, rememberBrowserAsk, - spendBrowserApproval, + spendWriteApprovalGrant, } from '@aflow/redis'; import type { ApprovalStore } from './actionApproval.js'; @@ -14,7 +14,7 @@ export function redisApprovalStore(redis: Redis): ApprovalStore { grant: async (scope, requestHash) => await getWriteApprovalGrant(redis, scope.tenantId, scope.runId, requestHash), spend: async (scope, grant) => - await spendBrowserApproval(redis, scope.tenantId, scope.runId, grant), + await spendWriteApprovalGrant(redis, scope.tenantId, scope.runId, grant), recall: async (scope, callKey) => await readBrowserAsk(redis, scope.tenantId, scope.runId, callKey), remember: async (scope, callKey, requestHash) => { diff --git a/apps/aflow-executor-host/src/index.ts b/apps/aflow-executor-host/src/index.ts index 2a67493b..13e3a1b6 100644 --- a/apps/aflow-executor-host/src/index.ts +++ b/apps/aflow-executor-host/src/index.ts @@ -339,7 +339,7 @@ async function main(): Promise { const inventory: HostInventory = { hostname, observedAt: new Date().toISOString(), - machine: machine.current(), + machine: await machine.current(), runtimes, harnesses, maxConcurrentHarnessRuns, diff --git a/apps/aflow-executor-host/src/machine.ts b/apps/aflow-executor-host/src/machine.ts index 56796973..05d13458 100644 --- a/apps/aflow-executor-host/src/machine.ts +++ b/apps/aflow-executor-host/src/machine.ts @@ -12,7 +12,7 @@ * says it does not know rather than guessing. */ import { execFile } from 'node:child_process'; -import { readFile, writeFile } from 'node:fs/promises'; +import { readFile, statfs, writeFile } from 'node:fs/promises'; import os from 'node:os'; import { join } from 'node:path'; import type { HostMachine } from '@aflow/redis'; @@ -62,9 +62,30 @@ async function readGpuWorkingSet(platform: string, run: RunCommand): Promise 0 ? Math.max(...mib) * 1024 ** 2 : undefined; } +/** + * Where Ollama keeps its downloads: `OLLAMA_MODELS` when the executor's + * environment names it, the runtime's default otherwise. A runtime started + * by a service manager with another `OLLAMA_MODELS` keeps its downloads on a + * disk this cannot see, which only an operator who moved them has. + */ +function defaultModelStore(): string { + return process.env['OLLAMA_MODELS'] ?? join(os.homedir(), '.ollama', 'models'); +} + +export type ReadFreeDisk = (path: string) => Promise; + +const readFreeDisk: ReadFreeDisk = async (path) => { + // The store may not exist before the first pull; its disk is the home's. + for (const candidate of [path, os.homedir()]) { + const stats = await statfs(candidate).catch(() => undefined); + if (stats) return stats.bavail * stats.bsize; + } + return undefined; +}; + export interface MachineReader { /** The machine as read so far; the GPU figure arrives once the probe has run. */ - current(): HostMachine; + current(): Promise; /** Reads the GPU figure, from the cache when it still describes this machine. */ start(): Promise; } @@ -76,19 +97,26 @@ export function createMachineReader(options: { totalMemoryBytes?: number; osRelease?: string; run?: RunCommand; + modelStore?: string; + freeDisk?: ReadFreeDisk; }): MachineReader { const platform = options.platform ?? process.platform; const totalMemoryBytes = options.totalMemoryBytes ?? os.totalmem(); const osRelease = options.osRelease ?? os.release(); const run = options.run ?? runCommand; const cachePath = join(options.hostDir, CACHE_FILE); + const modelStore = options.modelStore ?? defaultModelStore(); + const freeDisk = options.freeDisk ?? readFreeDisk; const machine: HostMachine = { totalMemoryBytes, unifiedMemory: platform === 'darwin' && (options.arch ?? process.arch) === 'arm64', }; return { - current: () => ({ ...machine }), + async current() { + const freeDiskBytes = await freeDisk(modelStore).catch(() => undefined); + return { ...machine, ...(freeDiskBytes !== undefined ? { freeDiskBytes } : {}) }; + }, async start() { const cached = await readFile(cachePath, 'utf8') .then((text) => JSON.parse(text) as CachedReading) diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/dispatchInlineOp.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/dispatchInlineOp.ts index 42455e8a..4f4fcd50 100644 --- a/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/dispatchInlineOp.ts +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/dispatchInlineOp.ts @@ -32,6 +32,7 @@ import { handleWebhookCrudInline, handleWorkflowCrudInline, handlePlanNodeInline, + handleModelRuntimeInline, handleCoachCrudInline, handleProposalCrudInline, handleCatalogAgentInline, @@ -497,6 +498,20 @@ export async function dispatchInlineOp( ...(parentStepExecutionId ? { parentStepExecutionId } : {}), ...workflowExecutionFields, }); + } else if (stepDef.operation.startsWith('model.')) { + await handleModelRuntimeInline({ + redis, + payloadStore, + context, + stepDef, + stepExecutionId, + idempotencyKey, + resolvedInputRef, + attempt, + scheduledAtMs: now, + ...(parentStepExecutionId ? { parentStepExecutionId } : {}), + ...workflowExecutionFields, + }); } else if (stepDef.operation.startsWith('catalog.agent.')) { await handleCatalogAgentInline({ redis, diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/index.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/index.ts index 763817dc..5523d7ab 100644 --- a/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/index.ts +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/index.ts @@ -14,6 +14,7 @@ export { handleScheduleSnoozeInline } from './scheduleSnooze.js'; export { handleWebhookCrudInline } from './webhookCrud.js'; export { handleWorkflowCrudInline } from './workflowCrud/index.js'; export { handlePlanNodeInline } from './plan/index.js'; +export { handleModelRuntimeInline } from './model/index.js'; export { handleCoachCrudInline } from './coachCrud.js'; export { handleProposalCrudInline } from './proposalCrud.js'; export { handleCatalogAgentInline } from './catalogAgent.js'; diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/model/__tests__/modelRuntime.test.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/model/__tests__/modelRuntime.test.ts new file mode 100644 index 00000000..24ddf030 --- /dev/null +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/model/__tests__/modelRuntime.test.ts @@ -0,0 +1,132 @@ +/** + * A change Helmsman proposes to the model runtime parks on the operator's + * approval, runs once for an approval of exactly the request it would make + * now, and asks again — with the spent decision named — rather than ride an + * approval it already used. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const m = vi.hoisted(() => ({ + grant: null as null | { + requestHash: string; + decision: 'approved' | 'denied'; + decidedAt?: string; + }, + spendable: true, + applied: [] as unknown[], + paused: [] as Array>, + succeeded: [] as unknown[], + failed: [] as Array<{ code: string; classification: string }>, +})); + +vi.mock('@aflow/database', () => ({ getDatabase: () => ({}) })); +vi.mock('@aflow/redis', () => ({ + countRuntimeRequests: async () => 0, + getWriteApprovalGrant: async () => m.grant, + spendWriteApprovalGrant: async () => { + const first = m.spendable; + m.spendable = false; + return first; + }, +})); +vi.mock('@aflow/cybernetic-runtime', async () => { + const actual = await vi.importActual( + '@aflow/cybernetic-runtime', + ); + return { + localRuntimeChangeRequest: actual.localRuntimeChangeRequest, + localRoleModels: () => ({ helmsman: 'ollama-gemma4-26b' }), + readSpaceModelDefaults: async () => undefined, + resolveSpaceLocalRuntime: async () => ({ runtimeId: 'this-machine:11434' }), + diagnoseSpaceLocalRuntime: async () => ({ + runtime: { baseUrl: 'http://127.0.0.1:11434', reachable: true, version: '0.40.5' }, + machine: null, + connection: { contextTokens: null, keepAlive: null, parallel: 1 }, + models: [], + issues: [], + }), + applyLocalModelAction: async (_ctx: unknown, change: unknown) => { + m.applied.push(change); + return { ok: true, message: 'Unloaded gemma4:26b.' }; + }, + }; +}); +vi.mock('../../sessionUser.js', () => ({ sessionUser: async () => 'user-1' })); +vi.mock('../../helpers.js', () => ({ + readInlineOpInput: async () => ({ change: { action: 'unload', model: 'ollama-gemma4-26b' } }), + emitStepPaused: async (_args: unknown, data: Record) => { + m.paused.push(data); + }, + emitStepSuccess: async (_args: unknown, data: unknown) => { + m.succeeded.push(data); + }, + emitStepError: async ( + _args: unknown, + code: string, + _msg: string, + _t: number, + classification: string, + ) => { + m.failed.push({ code, classification }); + }, +})); + +const { handleModelRuntimeInline } = await import('../index.js'); + +const args = { + redis: {}, + context: { tenantId: 't', runId: 'run-1', spaceId: 'space-1' }, + stepDef: { operation: 'model.runtime.change' }, +} as never; + +beforeEach(() => { + m.grant = null; + m.spendable = true; + m.applied = []; + m.paused = []; + m.succeeded = []; + m.failed = []; +}); + +describe('model.runtime.change', () => { + it('parks on the operator’s approval and changes nothing', async () => { + await handleModelRuntimeInline(args); + expect(m.applied).toEqual([]); + expect(m.paused[0]).toMatchObject({ + kind: 'write_approval', + target: 'local_model', + change: { action: 'unload' }, + impact: { ownConversationReloads: true }, + }); + }); + + it('runs once on an approval of the request it would make now', async () => { + await handleModelRuntimeInline(args); + const asked = m.paused[0] as { requestHash: string }; + m.grant = { + requestHash: asked.requestHash, + decision: 'approved', + decidedAt: '2026-10-16T21:00:00Z', + }; + m.paused = []; + + await handleModelRuntimeInline(args); + expect(m.applied).toEqual([{ action: 'unload', model: 'ollama-gemma4-26b' }]); + expect(m.succeeded).toEqual([{ message: 'Unloaded gemma4:26b.' }]); + + await handleModelRuntimeInline(args); + expect(m.applied).toHaveLength(1); + expect(m.paused[0]).toMatchObject({ decidedBefore: '2026-10-16T21:00:00Z' }); + }); + + it('fails as final on a denial', async () => { + await handleModelRuntimeInline(args); + const asked = m.paused[0] as { requestHash: string }; + m.grant = { requestHash: asked.requestHash, decision: 'denied' }; + await handleModelRuntimeInline(args); + expect(m.failed).toEqual([ + { code: 'MODEL_RUNTIME_CHANGE_DENIED', classification: 'permission' }, + ]); + expect(m.applied).toEqual([]); + }); +}); diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/model/index.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/model/index.ts new file mode 100644 index 00000000..acbfa139 --- /dev/null +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/model/index.ts @@ -0,0 +1,174 @@ +/** + * `model.runtime.*` — Helmsman reads the model runtime on this computer and + * asks before changing it (Plan 323 D10). The change parks on the + * write-approval pause, as a browser action does; once the operator approves, + * the step comes back here with its input, rebuilds the request from a fresh + * diagnosis, and runs only when the approval it finds is for that same request + * — a change whose impact moved since asks again with the new one. + */ +import { getDatabase } from '@aflow/database'; +import { + applyLocalModelAction, + diagnoseSpaceLocalRuntime, + localRoleModels, + localRuntimeChangeRequest, + readSpaceModelDefaults, + resolveSpaceLocalRuntime, + type SpaceLocalRuntime, +} from '@aflow/cybernetic-runtime'; +import { countRuntimeRequests, getWriteApprovalGrant, spendWriteApprovalGrant } from '@aflow/redis'; +import { + MODEL_RUNTIME_CHANGE_OPERATION_ID, + MODEL_RUNTIME_STATUS_OPERATION_ID, + ModelRuntimeChangeInputSchema, + type LocalModelWriteApprovalRequestPayload, +} from '@aflow/schemas'; +import { getOrchestratorLogger } from '../../../../../lib/orchestratorLogger.js'; +import type { InlineHandlerArgs } from '../types.js'; +import { emitStepError, emitStepPaused, emitStepSuccess, readInlineOpInput } from '../helpers.js'; +import { requireSpaceId } from '../spaceScope.js'; +import { sessionUser } from '../sessionUser.js'; + +interface Scope { + runtime: SpaceLocalRuntime | null; + roles: ReturnType; +} + +async function resolveScope(args: InlineHandlerArgs): Promise { + const db = getDatabase(); + const tenantId = args.context.tenantId as string; + const spaceId = requireSpaceId(args.context); + const userId = await sessionUser(args); + const runtime = userId + ? await resolveSpaceLocalRuntime({ db, redis: args.redis, tenantId, userId, spaceId }) + : null; + const roles = localRoleModels(await readSpaceModelDefaults(db, tenantId, spaceId)); + return { runtime, roles }; +} + +const NO_CONNECTION = + 'This space has no Ollama connection. The operator adds one under Settings → Credentials.'; + +async function handleStatus(args: InlineHandlerArgs, startTime: number): Promise { + const { runtime, roles } = await resolveScope(args); + if (!runtime) { + await emitStepSuccess(args, { diagnostics: null, note: NO_CONNECTION }, startTime); + return; + } + const diagnostics = await diagnoseSpaceLocalRuntime({ runtime, redis: args.redis, roles }); + await emitStepSuccess(args, { diagnostics }, startTime); +} + +async function handleChange(args: InlineHandlerArgs, startTime: number): Promise { + const parsed = ModelRuntimeChangeInputSchema.safeParse((await readInlineOpInput(args)) ?? {}); + if (!parsed.success) { + const issues = parsed.error.issues + .map((i) => (i.path.length > 0 ? `${i.path.join('.')}: ${i.message}` : i.message)) + .join('; '); + await emitStepError( + args, + 'MODEL_RUNTIME_INVALID_INPUT', + `Invalid input for ${MODEL_RUNTIME_CHANGE_OPERATION_ID}: ${issues}. Pass an issue's remedy action as \`change\`; a remedy that assigns a role's model is the operator's, under Settings → Agents.`, + startTime, + 'validation', + ); + return; + } + const { change } = parsed.data; + const { runtime, roles } = await resolveScope(args); + if (!runtime) { + await emitStepError( + args, + 'MODEL_RUNTIME_NO_CONNECTION', + NO_CONNECTION, + startTime, + 'validation', + ); + return; + } + const diagnostics = await diagnoseSpaceLocalRuntime({ runtime, redis: args.redis, roles }); + const request = localRuntimeChangeRequest({ + operationId: MODEL_RUNTIME_CHANGE_OPERATION_ID, + change, + diagnostics, + inFlightRequests: await countRuntimeRequests(args.redis, runtime.runtimeId), + askingModel: roles.helmsman, + }); + + const { tenantId, runId } = args.context; + const grant = await getWriteApprovalGrant(args.redis, tenantId, runId, request.requestHash); + if (grant?.decision === 'denied') { + await emitStepError( + args, + 'MODEL_RUNTIME_CHANGE_DENIED', + `The operator declined this change${grant.reason ? `: ${grant.reason}` : '.'} Do not ask for it again unless they do.`, + startTime, + 'permission', + ); + return; + } + const spent = + grant?.decision === 'approved' && + (await spendWriteApprovalGrant(args.redis, tenantId, runId, grant)); + if (!spent) { + const ask: LocalModelWriteApprovalRequestPayload = { + ...request, + // An approval already used for this exact request is not one for asking again. + ...(grant?.decidedAt ? { decidedBefore: grant.decidedAt } : {}), + }; + await emitStepPaused(args, ask, startTime); + return; + } + + const outcome = await applyLocalModelAction( + { db: getDatabase(), redis: args.redis, tenantId: tenantId as string, runtime }, + change, + ); + if (outcome.ok) { + await emitStepSuccess(args, { message: outcome.message }, startTime); + return; + } + await emitStepError( + args, + `MODEL_RUNTIME_${outcome.reason.toUpperCase()}`, + outcome.message, + startTime, + outcome.reason === 'runtime' ? 'provider' : 'validation', + ); +} + +export async function handleModelRuntimeInline(args: InlineHandlerArgs): Promise { + const operationId = args.stepDef.operation; + const startTime = Date.now(); + try { + switch (operationId) { + case MODEL_RUNTIME_STATUS_OPERATION_ID: + await handleStatus(args, startTime); + break; + case MODEL_RUNTIME_CHANGE_OPERATION_ID: + await handleChange(args, startTime); + break; + default: + await emitStepError( + args, + 'UNKNOWN_OPERATION', + `Unknown model operation: ${operationId}`, + startTime, + 'validation', + ); + } + } catch (err) { + getOrchestratorLogger().error( + `model: ${operationId} failed: ${err instanceof Error ? err.message : String(err)}`, + err instanceof Error ? err : undefined, + { tenantId: args.context.tenantId, sessionId: args.context.runId }, + ); + await emitStepError( + args, + 'MODEL_RUNTIME_OPERATION_FAILED', + `System error: ${operationId} could not be completed due to an internal failure. Do not retry — report this to the operator.`, + startTime, + 'internal', + ); + } +} diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/plan/write.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/plan/write.ts index dcc51920..3a29abf2 100644 --- a/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/plan/write.ts +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/plan/write.ts @@ -12,23 +12,11 @@ import { updatePlanNode, type PlanWriteContext, } from '@aflow/cybernetic-runtime'; -import { getDatabase } from '@aflow/database'; -import { getSessionStateSafe } from '@aflow/redis'; -import { readDurableSessionCreatedBy } from '../../../../cybernetic/harness/helpers.js'; +import { sessionUser } from '../sessionUser.js'; import type { InlineHandlerArgs } from '../types.js'; import { emitStepSuccess } from '../helpers.js'; import { emitPlanError, parsePlanInput, planReadContext } from './shared.js'; -/** Hot state can be gone after a terminal flush, so the durable session row answers then. */ -async function sessionUser(args: InlineHandlerArgs): Promise { - const { tenantId, runId } = args.context; - const state = await getSessionStateSafe(args.redis, tenantId, runId); - return ( - (state.ok ? state.state.createdBy : undefined) ?? - (await readDurableSessionCreatedBy(getDatabase(), tenantId, runId)) - ); -} - function planWriteContext(args: InlineHandlerArgs): PlanWriteContext { return { ...planReadContext(args), diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/sessionUser.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/sessionUser.ts new file mode 100644 index 00000000..1e0fcbb8 --- /dev/null +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/handlers/inlineOps/sessionUser.ts @@ -0,0 +1,14 @@ +import { getDatabase } from '@aflow/database'; +import { getSessionStateSafe } from '@aflow/redis'; +import { readDurableSessionCreatedBy } from '../../../cybernetic/harness/helpers.js'; +import type { InlineHandlerArgs } from './types.js'; + +/** Who started the session. Hot state can be gone after a terminal flush, so the durable session row answers then. */ +export async function sessionUser(args: InlineHandlerArgs): Promise { + const { tenantId, runId } = args.context; + const state = await getSessionStateSafe(args.redis, tenantId, runId); + return ( + (state.ok ? state.state.createdBy : undefined) ?? + (await readDurableSessionCreatedBy(getDatabase(), tenantId, runId)) + ); +} diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/helpers/inlineOperations.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/helpers/inlineOperations.ts index 98fa4780..00a147fb 100644 --- a/apps/aflow-orchestrator/src/services/SessionOrchestrator/helpers/inlineOperations.ts +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/helpers/inlineOperations.ts @@ -16,6 +16,7 @@ export function isInlineOperation(operationId: string): boolean { operationId.startsWith('integration.simulation.') || operationId.startsWith('store.listing.') || operationId.startsWith('plan.') || + operationId.startsWith('model.') || operationId === 'eval.dataset.get' || operationId === 'eval.dataset.list' || operationId === 'eval.case.promote' || diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/scheduling/__tests__/writeApprovalResume.test.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/scheduling/__tests__/writeApprovalResume.test.ts index 0e36cd7e..2b3557d8 100644 --- a/apps/aflow-orchestrator/src/services/SessionOrchestrator/scheduling/__tests__/writeApprovalResume.test.ts +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/scheduling/__tests__/writeApprovalResume.test.ts @@ -291,3 +291,49 @@ describe('the blocked-on cause of an approval pause', () => { }); }); }); + +const LOCAL_MODEL_REQUEST = { + kind: 'write_approval', + target: 'local_model', + change: { action: 'unload', model: 'ollama-gemma4-26b' }, + modelName: 'Gemma 4 26B (this computer)', + runtimeUrl: 'http://127.0.0.1:11434', + impact: { inFlightRequests: 1, ownConversationReloads: true, notes: [] }, + requestHash: 'local-hash-1', +}; + +describe('a change to the model runtime on this computer', () => { + it('parks on the change it asks for', async () => { + expect( + await resolveWriteApprovalBlockedOn(payloadStore, ref(LOCAL_MODEL_REQUEST), 'step-1'), + ).toEqual({ + kind: 'needs_write_approval', + target: 'local_model', + stepExecutionId: 'step-1', + action: 'unload', + modelName: 'Gemma 4 26B (this computer)', + requestHash: 'local-hash-1', + }); + }); + + it('is answered only by a decision made after it was asked again, as a browser action is', async () => { + const spent = grants([ + { requestHash: 'local-hash-1', decision: 'approved', decidedAt: SPENT_AT }, + ]); + expect(await decide({ ...LOCAL_MODEL_REQUEST, decidedBefore: SPENT_AT }, spent)).toEqual({ + decision: 'undecided', + }); + expect(await decide(LOCAL_MODEL_REQUEST, spent)).toEqual({ decision: 'approved' }); + }); + + it('fails on a denial with the operator’s reason, as final', async () => { + const resumed = await decide( + LOCAL_MODEL_REQUEST, + grants([{ requestHash: 'local-hash-1', decision: 'denied', reason: 'A run is going.' }]), + ); + if (resumed?.decision !== 'denied') throw new Error('expected a denial'); + expect(resumed.error.message).toContain('unload Gemma 4 26B (this computer)'); + expect(resumed.error.message).toContain('A run is going.'); + expect(toAgentToolError(resumed.error).retry).toBe(false); + }); +}); diff --git a/apps/aflow-orchestrator/src/services/SessionOrchestrator/scheduling/writeApprovalPause.ts b/apps/aflow-orchestrator/src/services/SessionOrchestrator/scheduling/writeApprovalPause.ts index c5127cfe..57d5d9c3 100644 --- a/apps/aflow-orchestrator/src/services/SessionOrchestrator/scheduling/writeApprovalPause.ts +++ b/apps/aflow-orchestrator/src/services/SessionOrchestrator/scheduling/writeApprovalPause.ts @@ -3,6 +3,7 @@ import { getWriteApprovalGrant } from '@aflow/redis'; import { type AflowError, grantAnswersAsk, + localModelChangePhrase, WriteApprovalRequestPayloadSchema, type WriteApprovalRequestPayload, type SessionBlockedOn, @@ -13,7 +14,8 @@ import type { PayloadStore } from '@aflow/payload-store'; * Best-effort read of an executor-supplied `write_approval` request payload from * a paused step's `requestedInputRef`. Returns `null` for any other paused-step * contract (the common path), so this only fires for an approval pause — an - * API write (Plan 253) or a browser action (Plan 320 D7). + * API write (Plan 253), a browser action (Plan 320 D7) or a change to the + * model runtime on this computer (Plan 323 D10). */ export async function readWriteApprovalRequest( payloadStore: PayloadStore, @@ -57,6 +59,16 @@ export async function resolveWriteApprovalBlockedOn( requestHash: request.requestHash, }; } + if (request.target === 'local_model') { + return { + kind: 'needs_write_approval', + target: 'local_model', + stepExecutionId, + action: request.change.action, + ...(request.modelName !== undefined ? { modelName: request.modelName } : {}), + requestHash: request.requestHash, + }; + } return { kind: 'needs_write_approval', target: 'api', @@ -105,7 +117,7 @@ export async function decideWriteApprovalResume( params.runId, request.requestHash, ); - const decidedBefore = request.target === 'browser' ? request.decidedBefore : undefined; + const decidedBefore = request.target === 'api' ? undefined : request.decidedBefore; if (!grant || !grantAnswersAsk(grant, decidedBefore)) return { decision: 'undecided' }; if (grant.decision === 'approved') return { decision: 'approved' }; return { decision: 'denied', error: writeApprovalDenial(request, grant.reason) }; @@ -124,18 +136,22 @@ export function writeApprovalDenial( ): AflowError { const because = reason ? `. Operator's reason: "${reason}"` : ''; const message = - request.target === 'browser' - ? `The operator denied this ${request.action} on ${request.pageOrigin} in the Action ` + - `Center${because}. It is not a system error, and the same action on the same element ` + - 'with the same value is refused with this reason rather than asked again. Use the ' + - 'reason to decide what to do: propose a different action, or tell the user it was ' + - 'declined and ask how to proceed.' - : 'This write was denied by a human operator in the Action Center' + - because + - '. It is not a system error and will NOT succeed on retry — do not call this ' + - 'endpoint again with the same request. Use the operator’s reason to decide what to ' + - 'do: adjust and propose a different action, or tell the user it was declined and ask ' + - 'how to proceed.'; + request.target === 'local_model' + ? `The operator declined to ${localModelChangePhrase(request)} in the Action Center${because}. ` + + 'It is not a system error, and asking for the same change again is refused with this ' + + 'reason. Tell the user it was declined, and what the issue it would have fixed still costs.' + : request.target === 'browser' + ? `The operator denied this ${request.action} on ${request.pageOrigin} in the Action ` + + `Center${because}. It is not a system error, and the same action on the same element ` + + 'with the same value is refused with this reason rather than asked again. Use the ' + + 'reason to decide what to do: propose a different action, or tell the user it was ' + + 'declined and ask how to proceed.' + : 'This write was denied by a human operator in the Action Center' + + because + + '. It is not a system error and will NOT succeed on retry — do not call this ' + + 'endpoint again with the same request. Use the operator’s reason to decide what to ' + + 'do: adjust and propose a different action, or tell the user it was declined and ask ' + + 'how to proceed.'; return { code: 'write_approval_denied', classification: 'permission', diff --git a/docs/plans/323-models-that-run-on-this-computer.md b/docs/plans/323-models-that-run-on-this-computer.md index f6f3dcf2..d8ebb960 100644 --- a/docs/plans/323-models-that-run-on-this-computer.md +++ b/docs/plans/323-models-that-run-on-this-computer.md @@ -186,6 +186,15 @@ Which local models are _recommended_ for which role follows from the usefulness - **Coverage:** an issue that cannot be stated with a cause and a remedy is not shipped as an issue. A vague warning teaches nothing, to a person or a model. +**D17 — A local model's silent stretches are said in the chat.** Before its first token, a request on a local model can be silent for minutes: the model loading (25–56 s on the measured machine, more for a larger one) and then reading the prompt, which Ollama streams nothing during. A spinner reading _Working…_ through that looks like a stalled chat. + +- **What is known:** only the transitions. Whether a request will load the model is read from `/api/ps` before it is sent: the model is not loaded, or the llama.cpp runner holds it at another window, which reloads it. The MLX runner ignores the window and never reloads for one. Prefill progress is not reported by the runtime, so no percentage or time estimate is shown. +- **The adapter loads first.** A load otherwise happens inside the request's own prefill, where nothing tells the two apart. So when the request would load, the adapter loads the model explicitly, at the window the request uses, and then sends the request. That makes the end of the load observable at no extra cost, since the load would have happened anyway. A load that fails is left for the request to report in its own words. +- **The signal:** `AIRequestOptions.onRuntimePhase` reports `loading_model`, then `reading_prompt`. The Ollama adapter calls it; hosted adapters never do. The status read is made only for a caller that listens. +- **The channel:** the agent turn writes each phase as one line on the step's `status` live-delta channel (_Loading Gemma 4 26B (this computer) into memory…_, _Reading the conversation…_), and each phase slides the step's idle window. The chat's activity row shows the last line in place of _Working…_ until the first `thinking` or `text` delta replaces it with _Thinking…_ or _Writing…_. Nothing is written to the transcript. _Rejected:_ reusing the `activity` channel. Its lines fold into a harness feed, and the chat pins a harness card for any step with an unsettled feed. + +**D18 — In the local edition, the catalog's models on this computer are assignable under the platform recommendation.** A tenant that names no allowlist follows the platform recommendation, which carries only hosted models, so a fresh local install with no cloud key had nothing it could assign. `allowedModelIds` takes the edition: in the local edition, a tenant following the recommendation may also assign every local catalog entry. Nothing is billed for them and the machine is the operator's own. A tenant that named its own set is held to it, as everywhere. They are assignable, not recommended: which local model is recommended for which role is still P2's usefulness tier. + ## 4. The initial lineup The figures below come from model cards and the Ollama library, read when this plan was written. None of them are measured: P0 replaces each of them with a number from the probe, and an entry ships only after its probe passes. "Fits" assumes Q4 weights and leaves room for the stack, the operating system and a browser. @@ -351,6 +360,7 @@ _Built and tested:_ all of the above. The probe measures Gemma 4 26B at 16.15 GB - the one-model-for-every-role preset as the default for a local space, and the cost of mixing models; - an onboarding path for "use a model on this computer", for an operator with no cloud key; - cost shown as local; +- the model loading and the prompt being read, said in the chat while a turn waits on them (D17); - machine memory and free disk in the host inventory; - edition gating; - _optional:_ the approved one-time install on a paired Mac (D2). @@ -365,6 +375,30 @@ _Proof:_ - a pull larger than the free disk is refused before it starts; - reading diagnostics on an unpaired machine loads nothing. +_Built:_ + +- **The record:** `LocalRuntimeDiagnostics` (`@aflow/schemas`) and `diagnoseLocalRuntime` (`@aflow/ai-client`), pure over one observation of `/api/version`, `/api/tags` and `/api/ps`. Its issue codes are `runtime_unreachable`, `model_not_installed`, `window_does_not_fit`, `window_tight`, `model_spilling`, `model_partly_on_cpu`, `build_unverified` and `mixed_local_models`. +- **Remedies:** each is an action from `LocalModelActionSchema` (`pull`, `delete`, `warm`, `unload`, `set_context`, `use_for_every_role`) or an instruction. +- **Shared core:** the runtime, the diagnosis and the actions live in `@aflow/cybernetic-runtime` `localModels/`, so the server's routes and Helmsman's operations reach the same code. +- **Pulls:** a pull is one Redis record per runtime and tag. It is both the claim, which a second request joins, and the progress. Progress renews its lifetime, so a server that dies leaves nothing behind. A cancel is read on the same round trip, and a stopped or failed pull stays readable for ten minutes. The disk check reads the host inventory's `freeDiskBytes`, measured under `OLLAMA_MODELS` or `~/.ollama/models`, and holds 2 GiB of headroom. +- **Readiness:** derives `localFit` and `localIssues` from the record. An `error` issue counts against `ready`, and the banner points at This Computer rather than the model picker. +- **The page:** This Computer gains "Models on this computer": + - the runtime and the machine; + - each issue as a card whose button runs its remedy; + - per model: download with progress and stop, load, unload, use for every role, and delete with a confirmation naming the roles it leaves without a model. +- **Edition gating:** the page renders none of it in the hosted edition, and the routes answer 404 there. +- **"Use for every role":** writes the space's directives through the space update the agent settings already use. It sets the default and clears every role override, and puts the Clerk on `space_default`. +- **Onboarding (D18):** offers the models on this computer beside the hosted ones. Choosing one puts every role on it, adds an Ollama connection unless one already resolves, and skips the key step. +- **Cost:** a run whose every model call ran here reads _Cost: local_, and so does each such step. + +_Not built:_ + +- the trial load for an unpaired machine, beyond **Load now**, which loads at the request window and whose loaded size `/api/ps` then reports; +- recent tokens per second in the record; +- the approved one-time install. + +The "use for every role" remedy is the operator's only, because a role's model is written through the space update, which no executor reaches. Helmsman proposes it as an instruction. + ### P3b — Helmsman reads and asks (D10, D15) **Build:** @@ -385,6 +419,24 @@ _Proof:_ - a card for unloading the model the asking session runs on names that session among the runs it would end; - an approved unload whose affected runs changed after approval parks again with the new impact. +_Built:_ + +- **Operations:** `model.runtime.status` (read) and `model.runtime.change` (write), in a `model` step type whose lane is composed only in the local edition (`isStepTypeComposed`). A hosted Helmsman's catalog does not carry them, in the same way it lacks `host.*` and `browser.*`. +- **Where they run:** inline in the orchestrator, reaching the runtime through the same `@aflow/cybernetic-runtime` `localModels` code as the operator's routes. +- **Capabilities:** the group is `model.runtime`, its bundle is `local_models` (on demand), and migration 220 grants it to the system profiles (read and write to every authoring profile, read to Read Only). +- **Helmsman's surface:** both operations are discoverable, and the status read is pinned on the local edition. +- **Runners and skills:** `isLocalModelOperation` is in `isRunnerExcludedOperation`, and a skill referencing one is invalid (`skill_references_local_model_op`). +- **The change's input** is `LocalRuntimeChangeSchema`: `pull`, `delete`, `warm`, `unload`, `set_context`. A role's model is not among them, because it is the space's setting written through the space update. Helmsman says so rather than proposing it. +- **The approval:** a third approval target, `local_model`, carried through the payload, `blockedOn`, the Action Center extension and card, and the chat's waiting notice. +- **The request** is built from a fresh diagnosis each time the step runs (`localRuntimeChangeRequest`). Its hash covers the change and whether the asking conversation's model is the one changed, so an approved change whose reach moved since its card was shown parks again. +- **What stays out of the hash** is on the card: the requests running on the runtime now (`countRuntimeRequests`, the admission leases held, on any model, since a lease does not record one), and the change's figures (download size against free disk, memory at the window, what an unload frees). They move by the second, and in the hash an approved change would rarely find the request it approved. +- **Who approves:** the card answers to a space admin (`resolverPolicy: ['admin']`), the role the operator's own routes for these changes require. + +- **Spending the approval:** an approval lets one change through. It is spent with `spendWriteApprovalGrant`, now the one spend for every target. Its key family, `aflow:approval-spent:`, sits outside the grant family the host executor may only read, and the host's ACL may write it. The same request asked again names the spent decision (`decidedBefore`), and the orchestrator honours that for every target but the API one. +- **On the asking conversation:** a turn does not hold the runtime while its tool runs, so unloading the model Helmsman runs on ends no turn. Its card says the conversation's next turn loads the model again, and names any other requests the change ends. + +_Not built:_ issue codes and remedies on local-model tool errors. A local model failing an agent's own turn fails that turn, not a tool call it could read, so this waits for an operation that calls a local model as a tool. + ### P4 — The appliance (D11) **Build:** diff --git a/packages/ai-client/src/catalogModels.ts b/packages/ai-client/src/catalogModels.ts index fc0082b6..86fa9f9e 100644 --- a/packages/ai-client/src/catalogModels.ts +++ b/packages/ai-client/src/catalogModels.ts @@ -95,6 +95,7 @@ export const builtInModels: ModelDefinition[] = [ measuredAt: '2026-10-16', // Measured at OLLAMA_KV_CACHE_TYPE=q8_0 (§9 G6). footprint: { weightsBytes: 16_180_670_828, kvBytesPerToken: 120_000 }, + downloadBytes: 16_146_867_483, }, }, diff --git a/packages/ai-client/src/index.ts b/packages/ai-client/src/index.ts index 3a25f071..db817109 100644 --- a/packages/ai-client/src/index.ts +++ b/packages/ai-client/src/index.ts @@ -25,6 +25,7 @@ export type { ReasoningConfig, ToolDefinition, AIRequestOptions, + RuntimePhase, GenerateTextRequest, GenerateJsonRequest, GenerateEmbeddingRequest, @@ -194,8 +195,13 @@ export { type LocalModelFit, type LocalModelFitStatus, } from './localModelFit.js'; +export { diagnoseLocalRuntime, type LocalRuntimeObservation } from './localRuntimeDiagnostics.js'; export { localRuntimeClient, + sameLocalTag, + servesWithoutLoading, + type InstalledLocalModel, + type LocalPullProgress, type LocalRuntimeClient, type LoadedLocalModel, } from './providers/ollamaRuntime.js'; diff --git a/packages/ai-client/src/localRuntimeDiagnostics.test.ts b/packages/ai-client/src/localRuntimeDiagnostics.test.ts new file mode 100644 index 00000000..5d7db2b2 --- /dev/null +++ b/packages/ai-client/src/localRuntimeDiagnostics.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from 'vitest'; +import { createDefaultModelCatalog } from './catalog.js'; +import { diagnoseLocalRuntime, type LocalRuntimeObservation } from './localRuntimeDiagnostics.js'; +import type { LoadedLocalModel, InstalledLocalModel } from './providers/ollamaRuntime.js'; +import type { ModelDefinition, ProviderConfig } from './types.js'; + +const GB = 1e9; +const gemma = createDefaultModelCatalog().getModel('ollama-gemma4-26b')!; +const M3_24GB = { + totalMemoryBytes: 24 * 1024 ** 3, + gpuWorkingSetBytes: 19_069_665_280, + unifiedMemory: true, +}; +const M4_64GB = { + totalMemoryBytes: 64 * 1024 ** 3, + gpuWorkingSetBytes: 48 * 1024 ** 3, + unifiedMemory: true, +}; + +const installed: InstalledLocalModel = { + tag: 'gemma4:26b', + sizeBytes: 16_146_867_483, + digest: gemma.local!.digest, + remoteHost: null, +}; + +function observe(over: Partial = {}): LocalRuntimeObservation { + return { + baseUrl: 'http://127.0.0.1:11434', + version: '0.40.5', + installed: [installed], + loaded: [], + ...over, + }; +} + +function diagnose( + connection: ProviderConfig, + over: Partial = {}, + models: ModelDefinition[] = [gemma], + roles: Record = { default: gemma.id, helmsman: gemma.id }, +) { + return diagnoseLocalRuntime({ observation: observe(over), connection, models, roles }); +} + +const codes = (d: ReturnType) => d.issues.map((i) => i.code); + +describe('diagnoseLocalRuntime', () => { + it('says only that the runtime is down when it does not answer, with how to start it', () => { + const d = diagnose({ machine: M3_24GB }, { version: null, installed: null, loaded: null }); + expect(d.runtime.reachable).toBe(false); + expect(d.issues).toHaveLength(1); + expect(d.issues[0]).toMatchObject({ code: 'runtime_unreachable', severity: 'error' }); + expect(d.issues[0]!.remedy).toMatchObject({ kind: 'instruction' }); + }); + + it('offers the download, with its size, for a role whose model is not downloaded', () => { + const d = diagnose({ machine: M4_64GB }, { installed: [] }); + expect(d.issues).toEqual([ + expect.objectContaining({ + code: 'model_not_installed', + severity: 'error', + remedy: { + kind: 'action', + action: { action: 'pull', model: gemma.id }, + label: 'Download Gemma 4 26B (this computer) (16.1 GB)', + }, + }), + ]); + }); + + it('reports nothing wrong on a machine that fits the model at its full window', () => { + const d = diagnose({ machine: M4_64GB }); + expect(d.issues).toEqual([]); + expect(d.models[0]).toMatchObject({ + installed: true, + build: 'verified', + roles: ['default', 'helmsman'], + }); + expect(d.models[0]!.fit.status).toBe('fits'); + }); + + it('says the 24 GB machine spills at the agents’ window, with nothing in Aflow to fix it', () => { + const d = diagnose({ machine: M3_24GB }); + expect(codes(d)).toEqual(['window_does_not_fit']); + expect(d.issues[0]!.remedy.kind).toBe('instruction'); + }); + + it('hands a configured window larger than the machine fits back to the machine', () => { + const fits90k = { + totalMemoryBytes: 36 * 1024 ** 3, + gpuWorkingSetBytes: 28 * GB, + unifiedMemory: true, + }; + const d = diagnose({ machine: fits90k, contextTokens: 262_144 }); + expect(d.issues[0]).toMatchObject({ + code: 'window_does_not_fit', + remedy: { kind: 'action', action: { action: 'set_context', contextTokens: null } }, + }); + }); + + it('replaces the estimate with what is loaded: a model past the GPU is spilling, and unloading frees it', () => { + const loaded: LoadedLocalModel = { + tag: 'gemma4:26b', + sizeBytes: 22.7 * GB, + gpuBytes: 22.7 * GB, + contextLength: 32_768, + expiresAt: null, + runner: 'mlx', + }; + const d = diagnose({ machine: M3_24GB }, { loaded: [loaded] }); + expect(codes(d)).toEqual(['model_spilling']); + expect(d.issues[0]!.remedy).toMatchObject({ action: { action: 'unload', model: gemma.id } }); + }); + + it('names the part of a loaded model on the CPU', () => { + const loaded: LoadedLocalModel = { + tag: 'gemma4:26b', + sizeBytes: 30 * GB, + gpuBytes: 20 * GB, + contextLength: 131_072, + expiresAt: null, + runner: 'llamacpp', + }; + const d = diagnose({ machine: M4_64GB, contextTokens: 131_072 }, { loaded: [loaded] }); + expect(d.issues[0]).toMatchObject({ code: 'model_partly_on_cpu' }); + expect(d.issues[0]!.cause).toContain('10.0 GB of'); + expect(d.issues[0]!.cause).toContain('128K window'); + }); + + it('marks a downloaded build other than the measured one as unverified', () => { + const d = diagnose({ machine: M4_64GB }, { installed: [{ ...installed, digest: 'other' }] }); + expect(d.models[0]!.build).toBe('unverified'); + expect(codes(d)).toEqual(['build_unverified']); + }); + + it('offers one model for every role when roles use two, naming what each switch costs', () => { + const other: ModelDefinition = { + ...gemma, + id: 'ollama-other', + providerModelId: 'other:7b', + displayName: 'Other', + local: { ...gemma.local!, runtimeTag: 'other:7b' }, + }; + const d = diagnose( + { machine: M4_64GB }, + { installed: [installed, { ...installed, tag: 'other:7b' }] }, + [gemma, other], + { default: gemma.id, helmsman: gemma.id, judge: other.id }, + ); + const mixed = d.issues.find((i) => i.code === 'mixed_local_models'); + expect(mixed?.remedy).toMatchObject({ + action: { action: 'use_for_every_role', model: gemma.id }, + }); + expect(mixed?.cause).toContain('each switch between them loads a model'); + }); +}); diff --git a/packages/ai-client/src/localRuntimeDiagnostics.ts b/packages/ai-client/src/localRuntimeDiagnostics.ts new file mode 100644 index 00000000..3fde1f68 --- /dev/null +++ b/packages/ai-client/src/localRuntimeDiagnostics.ts @@ -0,0 +1,287 @@ +/** + * One diagnosis of a space's runtime on this computer (Plan 323 D15), from + * what the runtime reports, the connection, the catalog and the machine. + * Pure: the observation is taken by the caller, so readiness, the models + * surface and Helmsman can each read the runtime once and describe it the + * same way. + */ +import type { + LocalModelDiagnosis, + LocalModelPull, + LocalModelRole, + LocalRuntimeDiagnostics, + LocalRuntimeIssue, +} from '@aflow/schemas'; +import { contextWindowForModel } from './client.js'; +import { footprintBytes, localModelFit, MIN_USEFUL_WINDOW_TOKENS } from './localModelFit.js'; +import { + sameLocalTag, + type InstalledLocalModel, + type LoadedLocalModel, +} from './providers/ollamaRuntime.js'; +import type { ModelDefinition, ProviderConfig } from './types.js'; + +export interface LocalRuntimeObservation { + baseUrl: string; + /** Null when the runtime does not answer. */ + version: string | null; + installed: InstalledLocalModel[] | null; + loaded: LoadedLocalModel[] | null; +} + +const gb = (bytes: number): string => `${(bytes / 1e9).toFixed(1)} GB`; +const k = (tokens: number): string => `${Math.round(tokens / 1024)}K`; + +const START_OLLAMA = + 'Start Ollama: open the Ollama app, or run `ollama serve` (`brew services start ollama` where Homebrew installed it). ' + + 'If it is not installed, get it from https://ollama.com/download.'; + +function tagOf(model: ModelDefinition): string { + return model.providerModelId ?? model.id; +} + +export function diagnoseLocalRuntime(input: { + observation: LocalRuntimeObservation; + connection: ProviderConfig; + /** The local models the catalog carries, and any model a role is assigned. */ + models: readonly ModelDefinition[]; + roles: Partial>; + freeDiskBytes?: number | undefined; + pulls?: Readonly> | undefined; +}): LocalRuntimeDiagnostics { + const { observation, connection } = input; + const reachable = observation.version !== null; + const machine = connection.machine; + const parallel = connection.parallel ?? 1; + + const models: LocalModelDiagnosis[] = input.models.map((model) => { + const tag = tagOf(model); + const installed = observation.installed?.find((m) => sameLocalTag(m.tag, tag)); + const held = observation.loaded?.find((m) => sameLocalTag(m.tag, tag)); + const fit = localModelFit({ + footprint: model.local?.footprint, + machine, + catalogWindow: model.contextWindow, + parallel, + window: contextWindowForModel(model, connection), + }); + return { + model: model.id, + ref: model.aliases?.[0] ?? model.id, + tag, + displayName: model.displayName, + installed: installed !== undefined, + downloadBytes: installed?.sizeBytes ?? model.local?.downloadBytes ?? null, + roles: (Object.entries(input.roles) as Array<[LocalModelRole, string]>) + .filter(([, assigned]) => assigned === model.id) + .map(([role]) => role), + fit: { + status: fit.status, + window: fit.window, + largestFittingWindow: fit.largestFittingWindow ?? null, + bytesAtWindow: fit.bytesAtWindow ?? null, + }, + loaded: held + ? { + sizeBytes: held.sizeBytes, + gpuBytes: held.gpuBytes, + contextLength: held.contextLength, + expiresAt: held.expiresAt, + } + : null, + build: !installed + ? 'unmeasured' + : installed.digest === model.local?.digest + ? 'verified' + : 'unverified', + pull: input.pulls?.[model.id] ?? null, + }; + }); + + return { + runtime: { baseUrl: observation.baseUrl, reachable, version: observation.version }, + machine: machine + ? { + totalMemoryBytes: machine.totalMemoryBytes, + gpuWorkingSetBytes: machine.gpuWorkingSetBytes ?? null, + freeDiskBytes: input.freeDiskBytes ?? null, + } + : null, + connection: { + contextTokens: connection.contextTokens ?? null, + keepAlive: connection.keepAlive ?? null, + parallel, + }, + models, + issues: reachable + ? issuesFor(models, input.models, connection) + : [ + { + code: 'runtime_unreachable', + severity: 'error', + cause: `Ollama is not answering at ${observation.baseUrl}.`, + remedy: { kind: 'instruction', text: START_OLLAMA }, + }, + ], + }; +} + +function issuesFor( + models: readonly LocalModelDiagnosis[], + definitions: readonly ModelDefinition[], + connection: ProviderConfig, +): LocalRuntimeIssue[] { + const issues: LocalRuntimeIssue[] = []; + const gpu = connection.machine?.gpuWorkingSetBytes; + const used = models.filter((m) => m.roles.length > 0); + // A window the operator set that is larger than the machine fits is the one + // remedy Aflow can apply itself: hand the choice back to the machine. + const automaticWindow = (m: LocalModelDiagnosis): LocalRuntimeIssue['remedy'] | null => + connection.contextTokens !== undefined && + m.fit.largestFittingWindow !== null && + m.fit.largestFittingWindow >= MIN_USEFUL_WINDOW_TOKENS && + m.fit.largestFittingWindow < m.fit.window + ? { + kind: 'action', + action: { action: 'set_context', contextTokens: null }, + label: `Use the largest window that fits (${k(m.fit.largestFittingWindow)})`, + } + : null; + + for (const m of used) { + if (!m.installed) { + issues.push({ + code: 'model_not_installed', + severity: 'error', + model: m.model, + cause: + m.pull?.status === 'downloading' + ? `${m.displayName} is still downloading${ + m.pull.totalBytes + ? ` (${Math.floor((100 * m.pull.completedBytes) / m.pull.totalBytes)}%)` + : '' + }; ${m.roles.join(', ')} cannot run until it is done.` + : `${m.roles.join(', ')} ${m.roles.length === 1 ? 'is' : 'are'} assigned ${m.displayName}, which this computer has not downloaded.`, + remedy: { + kind: 'action', + action: { action: 'pull', model: m.model }, + label: `Download ${m.displayName}${m.downloadBytes ? ` (${gb(m.downloadBytes)})` : ''}`, + }, + }); + continue; + } + + if (m.build === 'unverified') { + issues.push({ + code: 'build_unverified', + severity: 'warning', + model: m.model, + cause: `The downloaded ${m.tag} is a different build from the one Aflow measured, so its tool calling and memory figures are unverified.`, + remedy: { + kind: 'instruction', + text: `Measure it with \`yarn models:verify-local ${m.tag}\`, or download the measured build again.`, + }, + }); + } + + if (m.loaded) { + const onCpu = m.loaded.gpuBytes !== null ? m.loaded.sizeBytes - m.loaded.gpuBytes : 0; + if (onCpu > 0) { + issues.push({ + code: 'model_partly_on_cpu', + severity: 'warning', + model: m.model, + cause: `${gb(onCpu)} of ${m.displayName}'s ${gb(m.loaded.sizeBytes)} runs on the CPU${ + m.loaded.contextLength ? ` at a ${k(m.loaded.contextLength)} window` : '' + }, which is what makes it slow.`, + remedy: automaticWindow(m) ?? { + kind: 'action', + action: { action: 'unload', model: m.model }, + label: `Unload ${m.displayName}`, + }, + }); + continue; + } + if (gpu !== undefined && m.loaded.sizeBytes > gpu) { + issues.push({ + code: 'model_spilling', + severity: 'warning', + model: m.model, + cause: `${m.displayName} is loaded at ${gb(m.loaded.sizeBytes)}, past the ${gb(gpu)} the GPU allows: it holds the cache of every conversation it has read, and is spilling.`, + remedy: { + kind: 'action', + action: { action: 'unload', model: m.model }, + label: `Unload ${m.displayName} to free it`, + }, + }); + continue; + } + } + + if (m.fit.status === 'does_not_fit') { + const smaller = models.find( + (other) => + other.model !== m.model && + other.fit.status === 'fits' && + other.fit.window >= MIN_USEFUL_WINDOW_TOKENS, + ); + issues.push({ + code: 'window_does_not_fit', + severity: 'warning', + model: m.model, + cause: + m.fit.largestFittingWindow !== null && + m.fit.largestFittingWindow >= MIN_USEFUL_WINDOW_TOKENS + ? `At a ${k(m.fit.window)} window ${m.displayName} takes ${gb(m.fit.bytesAtWindow ?? 0)}, past the ${gb(gpu ?? 0)} the GPU allows; it fits ${k(m.fit.largestFittingWindow)}.` + : `${m.displayName} takes ${gb(m.fit.bytesAtWindow ?? 0)} at the ${k(m.fit.window)} window the agents need, past the ${gb(gpu ?? 0)} this computer's GPU allows, so it spills and runs slowly.`, + remedy: + automaticWindow(m) ?? + (smaller + ? { + kind: 'action', + action: { action: 'use_for_every_role', model: smaller.model }, + label: `Use ${smaller.displayName}, which fits`, + } + : { + kind: 'instruction', + text: 'A smaller model, or a machine with more memory, runs it without spilling.', + }), + }); + } else if (m.fit.status === 'tight') { + issues.push({ + code: 'window_tight', + severity: 'info', + model: m.model, + cause: `${m.displayName} fits the GPU at ${gb(m.fit.bytesAtWindow ?? 0)}, but with the system and other apps it outgrows this computer's ${gb(connection.machine?.totalMemoryBytes ?? 0)}, so it slows down whenever they need memory.`, + remedy: { kind: 'instruction', text: 'Close other large apps while it runs.' }, + }); + } + } + + if (used.length > 1) { + const primary = [...used].sort((a, b) => b.roles.length - a.roles.length)[0]!; + const combined = used.reduce((sum, m) => { + const def = definitions.find((d) => d.id === m.model); + return ( + sum + + (def?.local?.footprint + ? footprintBytes(def.local.footprint, m.fit.window, connection.parallel ?? 1) + : 0) + ); + }, 0); + issues.push({ + code: 'mixed_local_models', + severity: 'info', + cause: `Roles use ${used.map((m) => `${m.displayName} (${m.roles.join(', ')})`).join(' and ')}. ${ + combined > 0 ? `Holding them together takes ${gb(combined)}; otherwise ` : '' + }each switch between them loads a model, a pause of up to a minute.`, + remedy: { + kind: 'action', + action: { action: 'use_for_every_role', model: primary.model }, + label: `Use ${primary.displayName} for every role`, + }, + }); + } + + return issues; +} diff --git a/packages/ai-client/src/providers/ollama.test.ts b/packages/ai-client/src/providers/ollama.test.ts index 079a01bd..6fe763f2 100644 --- a/packages/ai-client/src/providers/ollama.test.ts +++ b/packages/ai-client/src/providers/ollama.test.ts @@ -265,6 +265,74 @@ describe('streaming', () => { }); }); +describe('the silent stretches before the first token', () => { + const ps = (models: Array>) => () => json({ models }); + const ask = (onRuntimePhase: (p: { phase: string }) => void) => + createOllamaAdapter({}).generateText({ + ...context, + model: 'gemma4:26b', + messages: [{ role: 'user', content: 'Hi' }], + onRuntimePhase, + }); + + it('loads a model that is not loaded at the window the request uses, and says so first', async () => { + const sent = runtime( + ps([]), + () => json({ done: true }), + done({ content: 'ok' }, { prompt: 12, eval: 1 }), + ); + const phases: string[] = []; + + await ask((p) => phases.push(p.phase)); + + expect(phases).toEqual(['loading_model', 'reading_prompt']); + expect(sent.map((s) => new URL(s.url).pathname)).toEqual([ + '/api/ps', + '/api/generate', + '/api/chat', + ]); + expect((sent[1]?.body['options'] as Record)['num_ctx']).toBe( + (sent[2]?.body['options'] as Record)['num_ctx'], + ); + }); + + it('reports only the reading of the prompt when the model is already loaded', async () => { + runtime( + ps([{ name: 'gemma4:26b', runner: 'mlx', context_length: 32_768 }]), + done({ content: 'ok' }, { prompt: 12, eval: 1 }), + ); + const phases: string[] = []; + + await ask((p) => phases.push(p.phase)); + + expect(phases).toEqual(['reading_prompt']); + }); + + it('leaves a failed load for the request to report in its own words', async () => { + runtime( + ps([]), + () => json({ error: 'model not found' }, 404), + () => json({ error: "model 'gemma4:26b' not found" }, 404), + ); + + const error = await rejection(ask(() => undefined)); + + expect(error.message).toContain('ollama pull'); + }); + + it('asks the runtime nothing extra when no caller listens', async () => { + const sent = runtime(done({ content: 'ok' }, { prompt: 12, eval: 1 })); + + await createOllamaAdapter({}).generateText({ + ...context, + model: 'gemma4:26b', + messages: [{ role: 'user', content: 'Hi' }], + }); + + expect(sent.map((s) => new URL(s.url).pathname)).toEqual(['/api/chat']); + }); +}); + describe('structured output', () => { const schema = z.object({ answer: z.number() }); diff --git a/packages/ai-client/src/providers/ollama.ts b/packages/ai-client/src/providers/ollama.ts index b1f4f850..dfa5b335 100644 --- a/packages/ai-client/src/providers/ollama.ts +++ b/packages/ai-client/src/providers/ollama.ts @@ -29,6 +29,7 @@ import { AIClientError, buildStreamTruncationError } from '../errors.js'; import { moveToolImagesToUserMessages, toolResultText } from './toolResultContent.js'; import { parseJsonResponse } from './jsonResponseParse.js'; import { MIN_ANSWER_TOKENS, sizePrompt, type PromptSizing } from './ollamaPromptSize.js'; +import { localRuntimeClient, servesWithoutLoading } from './ollamaRuntime.js'; interface Fit { sizing: PromptSizing; @@ -262,6 +263,7 @@ export function createOllamaAdapter(config: ProviderConfig): AIProviderAdapter { // §9 G1). Learned from the runtime's own answer rather than written down, so // a runner that gains structured output is used the moment it does. const refusesFormat = new Set(); + const runtime = localRuntimeClient(config); function windowFor(request: AIRequestOptions): number { return request.contextWindow ?? config.contextTokens ?? OLLAMA_DEFAULT_CONTEXT_TOKENS; @@ -329,6 +331,37 @@ export function createOllamaAdapter(config: ProviderConfig): AIProviderAdapter { return chunk; } + /** + * A load otherwise happens inside the request's own prefill, where nothing + * tells it apart from reading the prompt. Loading first, at the window the + * request uses so it does not load again, is what lets a caller say which + * silent stretch it is in. Asked only by a caller that listens, since it + * costs a status read. A load that fails is left for the request itself to + * report, in the words it would have used anyway. + */ + async function reportPhases(request: GenerateTextRequest | GenerateJsonRequest): Promise { + const report = request.onRuntimePhase; + if (!report) return; + const window = windowFor(request); + const loaded = await runtime.loaded(); + if (loaded !== null && !servesWithoutLoading(loaded, request.model, window)) { + report({ phase: 'loading_model' }); + try { + await runtime.warm(request.model, window, request.signal); + } catch (error) { + if (request.signal?.aborted) { + throw normalizeOllamaError(error, { + baseUrl, + model: request.model, + signal: request.signal, + }); + } + } + request.onStreamProgress?.(); + } + report({ phase: 'reading_prompt' }); + } + /** * Plan 323 D5: a request is sent only once it is known to fit. A prompt * whose conservative size fits is sent as is; any other is counted exactly @@ -340,6 +373,7 @@ export function createOllamaAdapter(config: ProviderConfig): AIProviderAdapter { messages: OllamaMessage[], tools: OllamaTool[] | undefined, ): Promise { + await reportPhases(request); const window = windowFor(request); const requested = request.maxTokens ?? MIN_ANSWER_TOKENS; const sizing = sizePrompt(messages, tools); diff --git a/packages/ai-client/src/providers/ollamaRuntime.test.ts b/packages/ai-client/src/providers/ollamaRuntime.test.ts index d6647f27..145ec70e 100644 --- a/packages/ai-client/src/providers/ollamaRuntime.test.ts +++ b/packages/ai-client/src/providers/ollamaRuntime.test.ts @@ -1,5 +1,9 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { localRuntimeClient } from './ollamaRuntime.js'; +import { + localRuntimeClient, + servesWithoutLoading, + type LoadedLocalModel, +} from './ollamaRuntime.js'; afterEach(() => { vi.unstubAllGlobals(); @@ -26,7 +30,14 @@ describe('localRuntimeClient', () => { ), ); expect(await localRuntimeClient({}).loaded()).toEqual([ - { tag: 'gemma4:26b', sizeBytes: 22_677_887_340, contextLength: 32_768, expiresAt: 'x' }, + { + tag: 'gemma4:26b', + sizeBytes: 22_677_887_340, + contextLength: 32_768, + expiresAt: 'x', + runner: null, + gpuBytes: null, + }, ]); }); @@ -58,3 +69,91 @@ describe('localRuntimeClient', () => { ]); }); }); + +describe('downloads', () => { + it('fails a pull whose stream ends without the runtime saying it succeeded', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(`${JSON.stringify({ status: 'pulling 48eb98ec' })}\n`)), + ); + await expect( + localRuntimeClient({}).pull('gemma4:26b', () => undefined, new AbortController().signal), + ).rejects.toThrow('stopped before it finished'); + }); + + it('reports a pull line by line as the runtime streams it, and fails on an error line', async () => { + const lines = [ + { status: 'pulling manifest' }, + { status: 'pulling 48eb98ec', completed: 1_000, total: 16_000 }, + { error: 'max retries exceeded' }, + ]; + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(lines.map((l) => JSON.stringify(l)).join('\n') + '\n')), + ); + const progress: unknown[] = []; + + await expect( + localRuntimeClient({}).pull( + 'gemma4:26b', + (p) => progress.push(p), + new AbortController().signal, + ), + ).rejects.toThrow('max retries exceeded'); + expect(progress).toEqual([ + { status: 'pulling manifest', completedBytes: 0, totalBytes: null }, + { status: 'pulling 48eb98ec', completedBytes: 1_000, totalBytes: 16_000 }, + ]); + }); + + it('lists what is downloaded, marking a model the runtime forwards to its cloud', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response( + JSON.stringify({ + models: [ + { name: 'gemma4:26b', size: 16, digest: 'abc' }, + { name: 'gpt-oss:120b-cloud', size: 1, digest: 'def', remote_host: 'ollama.com' }, + ], + }), + ), + ), + ); + + expect(await localRuntimeClient({}).installed()).toEqual([ + { tag: 'gemma4:26b', sizeBytes: 16, digest: 'abc', remoteHost: null }, + { tag: 'gpt-oss:120b-cloud', sizeBytes: 1, digest: 'def', remoteHost: 'ollama.com' }, + ]); + }); +}); + +describe('servesWithoutLoading', () => { + const held = (runner: string, contextLength: number): LoadedLocalModel => ({ + tag: 'qwen3.5:9b', + sizeBytes: 1, + contextLength, + expiresAt: null, + runner, + gpuBytes: null, + }); + + it('is false when the model is not loaded', () => { + expect(servesWithoutLoading([], 'qwen3.5:9b', 32_768)).toBe(false); + }); + + it('is true for a llama.cpp model only at the window it holds, since another window reloads it', () => { + expect(servesWithoutLoading([held('llamacpp', 32_768)], 'qwen3.5:9b', 32_768)).toBe(true); + expect(servesWithoutLoading([held('llamacpp', 8_192)], 'qwen3.5:9b', 32_768)).toBe(false); + }); + + it('is true for an MLX model at any window, since that runner never reloads for one', () => { + expect(servesWithoutLoading([held('mlx', 8_192)], 'qwen3.5:9b', 32_768)).toBe(true); + }); + + it('reads a tag without a version as :latest', () => { + const latest = { ...held('mlx', 1), tag: 'gemma4:latest' }; + expect(servesWithoutLoading([latest], 'gemma4', 32_768)).toBe(true); + }); +}); diff --git a/packages/ai-client/src/providers/ollamaRuntime.ts b/packages/ai-client/src/providers/ollamaRuntime.ts index 635ea18f..edc2029d 100644 --- a/packages/ai-client/src/providers/ollamaRuntime.ts +++ b/packages/ai-client/src/providers/ollamaRuntime.ts @@ -16,15 +16,50 @@ export interface LoadedLocalModel { sizeBytes: number; contextLength: number | null; expiresAt: string | null; + /** `mlx` or `llamacpp`: the MLX runner ignores the window it was asked for, so it never reloads for a different one. */ + runner: string | null; + /** The part of `sizeBytes` on the GPU; the rest runs on the CPU. */ + gpuBytes: number | null; +} + +export interface InstalledLocalModel { + tag: string; + sizeBytes: number; + digest: string; + /** Set for a model the runtime forwards to its own cloud rather than running here. */ + remoteHost: string | null; +} + +/** One line of a pull's progress, as the runtime streams it. */ +export interface LocalPullProgress { + status: string; + completedBytes: number; + totalBytes: number | null; } export interface LocalRuntimeClient { /** What the runtime holds now, or null when it does not answer. */ loaded(): Promise; /** Load a model at the window its requests will use, so the first one does not wait for it. */ - warm(tag: string, window: number): Promise; + warm(tag: string, window: number, signal?: AbortSignal): Promise; /** Unload a model, freeing its weights and every conversation's cache it holds. */ unload(tag: string): Promise; + /** The runtime's version, or null when it does not answer. */ + version(): Promise; + /** What the runtime has downloaded, or null when it does not answer. */ + installed(): Promise; + /** + * Download a model, reporting progress as the runtime streams it. Aborting + * the signal closes the request, which installs nothing; the next pull of + * the tag resumes from what was already downloaded (§9 G3). + */ + pull( + tag: string, + onProgress: (progress: LocalPullProgress) => void, + signal: AbortSignal, + ): Promise; + /** Delete a downloaded model. */ + remove(tag: string): Promise; } export function localRuntimeClient( @@ -35,43 +70,121 @@ export function localRuntimeClient( 'content-type': 'application/json', ...(config.apiKey ? { authorization: `Bearer ${config.apiKey}` } : {}), }; - const generate = async (body: Record, timeoutMs: number): Promise => { + const generate = async ( + body: Record, + timeoutMs: number, + signal?: AbortSignal, + ): Promise => { + const timeout = AbortSignal.timeout(timeoutMs); const res = await fetch(`${base}/api/generate`, { method: 'POST', headers, body: JSON.stringify(body), - signal: AbortSignal.timeout(timeoutMs), + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }); if (!res.ok) throw new Error(`Ollama ${res.status}: ${await res.text()}`); await res.text(); }; + const read = async (path: string): Promise => { + try { + const res = await fetch(`${base}${path}`, { + headers, + signal: AbortSignal.timeout(STATUS_TIMEOUT_MS), + }); + return res.ok ? ((await res.json()) as T) : null; + } catch { + return null; + } + }; return { - async loaded() { - try { - const res = await fetch(`${base}/api/ps`, { - headers, - signal: AbortSignal.timeout(STATUS_TIMEOUT_MS), - }); - if (!res.ok) return null; - const body = (await res.json()) as { - models?: Array<{ - name?: string; - size?: number; - context_length?: number; - expires_at?: string; - }>; - }; - return (body.models ?? []).map((m) => ({ - tag: m.name ?? '', - sizeBytes: m.size ?? 0, - contextLength: m.context_length ?? null, - expiresAt: m.expires_at ?? null, - })); - } catch { - return null; + async version() { + return (await read<{ version?: string }>('/api/version'))?.version ?? null; + }, + async installed() { + const body = await read<{ + models?: Array<{ name?: string; size?: number; digest?: string; remote_host?: string }>; + }>('/api/tags'); + if (body === null) return null; + return (body.models ?? []).map((m) => ({ + tag: m.name ?? '', + sizeBytes: m.size ?? 0, + digest: m.digest ?? '', + remoteHost: m.remote_host ?? null, + })); + }, + async pull(tag, onProgress, signal) { + const res = await fetch(`${base}/api/pull`, { + method: 'POST', + headers, + body: JSON.stringify({ model: tag, stream: true }), + signal, + }); + if (!res.ok || !res.body) throw new Error(`Ollama ${res.status}: ${await res.text()}`); + const decoder = new TextDecoder(); + let buffer = ''; + let last = ''; + const reader = res.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + let newline; + while ((newline = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (!line) continue; + const chunk = JSON.parse(line) as { + status?: string; + completed?: number; + total?: number; + error?: string; + }; + if (chunk.error) throw new Error(chunk.error); + last = chunk.status ?? last; + onProgress({ + status: chunk.status ?? '', + completedBytes: chunk.completed ?? 0, + totalBytes: chunk.total ?? null, + }); + } } + // A stream the runtime closed early — a restart mid-pull — ends without + // its `success` line, and installed nothing. + if (last !== 'success') { + throw new Error(`the download stopped before it finished (last: ${last || 'nothing'})`); + } + }, + async remove(tag) { + const res = await fetch(`${base}/api/delete`, { + method: 'DELETE', + headers, + body: JSON.stringify({ model: tag }), + signal: AbortSignal.timeout(STATUS_TIMEOUT_MS * 5), + }); + if (!res.ok) throw new Error(`Ollama ${res.status}: ${await res.text()}`); }, - warm: (tag, window) => + async loaded() { + const body = await read<{ + models?: Array<{ + name?: string; + size?: number; + context_length?: number; + expires_at?: string; + runner?: string; + size_vram?: number; + }>; + }>('/api/ps'); + if (body === null) return null; + return (body.models ?? []).map((m) => ({ + tag: m.name ?? '', + sizeBytes: m.size ?? 0, + contextLength: m.context_length ?? null, + expiresAt: m.expires_at ?? null, + runner: m.runner ?? null, + gpuBytes: m.size_vram ?? null, + })); + }, + warm: (tag, window, signal) => generate( { model: tag, @@ -79,7 +192,29 @@ export function localRuntimeClient( options: { num_ctx: window }, }, LOAD_TIMEOUT_MS, + signal, ), unload: (tag) => generate({ model: tag, keep_alive: 0 }, STATUS_TIMEOUT_MS * 5), }; } + +/** The runtime's names for one model: a tag without a version is `:latest`. */ +export function sameLocalTag(a: string, b: string): boolean { + const full = (tag: string): string => (tag.includes(':') ? tag : `${tag}:latest`); + return full(a) === full(b); +} + +/** + * Whether a request at `window` would be served by what is loaded, or would + * load the model first. The llama.cpp runner reloads for a window other than + * the one it holds; the MLX runner ignores the window and never does. + */ +export function servesWithoutLoading( + loaded: readonly LoadedLocalModel[], + tag: string, + window: number, +): boolean { + return loaded.some( + (m) => sameLocalTag(m.tag, tag) && (m.runner === 'mlx' || m.contextLength === window), + ); +} diff --git a/packages/ai-client/src/types.ts b/packages/ai-client/src/types.ts index fcddf16d..bbdd2034 100644 --- a/packages/ai-client/src/types.ts +++ b/packages/ai-client/src/types.ts @@ -286,6 +286,13 @@ export interface ModelReasoningProfile { /** * Common options for all AI requests. */ +/** + * A request's silent stretches on a local runtime, in order. `loading_model` + * is reported only when the request would load the model; `reading_prompt` + * follows it on every request, and the first streamed token ends it. + */ +export type RuntimePhase = { phase: 'loading_model' } | { phase: 'reading_prompt' }; + export interface AIRequestOptions { /** Model identifier (e.g., "gpt-4o", "claude-3-5-sonnet") */ model: string; @@ -316,6 +323,13 @@ export interface AIRequestOptions { */ onStreamProgress?: (() => void) | undefined; + /** + * Called as a request on a model on this computer passes through the silent + * stretches before its first token: the model loading, then the prompt being + * read. Adapters for hosted models never call it. + */ + onRuntimePhase?: ((phase: RuntimePhase) => void) | undefined; + /** Per-request max retries override (SDK-level). Defaults to provider config. */ maxRetries?: number | undefined; @@ -1001,6 +1015,8 @@ export interface LocalModelBuild { probeVersion: number; measuredAt: string; footprint?: LocalModelFootprint; + /** What the runtime downloads for the tag, so a pull too large for the disk is refused before it starts. */ + downloadBytes?: number; } // ============================================================================ diff --git a/packages/cybernetic-runtime/package.json b/packages/cybernetic-runtime/package.json index f923b545..d2ae6f9b 100644 --- a/packages/cybernetic-runtime/package.json +++ b/packages/cybernetic-runtime/package.json @@ -38,6 +38,7 @@ "dependencies": { "@aflow/ai-client": "workspace:*", "@aflow/applet-runtime": "workspace:*", + "@aflow/credential-resolver": "workspace:*", "@aflow/database": "workspace:*", "@aflow/integration-simulator": "workspace:*", "@aflow/lib": "workspace:*", diff --git a/packages/cybernetic-runtime/src/index.ts b/packages/cybernetic-runtime/src/index.ts index fd757d3c..41a9c377 100644 --- a/packages/cybernetic-runtime/src/index.ts +++ b/packages/cybernetic-runtime/src/index.ts @@ -246,3 +246,4 @@ export * from './userLabels.js'; export * from './sessionRehydration.js'; export * from './resumeDispatch.js'; export * from './taskDraftStore.js'; +export * from './localModels/index.js'; diff --git a/packages/cybernetic-runtime/src/localModels/__tests__/actions.test.ts b/packages/cybernetic-runtime/src/localModels/__tests__/actions.test.ts new file mode 100644 index 00000000..2a4de7ad --- /dev/null +++ b/packages/cybernetic-runtime/src/localModels/__tests__/actions.test.ts @@ -0,0 +1,131 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import Redis from 'ioredis-mock'; +import type { Redis as RedisType } from 'ioredis'; +import type { LocalPullProgress, LocalRuntimeClient } from '@aflow/ai-client'; +import { readLocalPulls } from '@aflow/redis'; +import { applyLocalModelAction, cancelLocalModelPull } from '../actions.js'; +import type { SpaceLocalRuntime } from '../runtime.js'; + +const GIB = 1024 ** 3; +const RUNTIME_ID = 'this-machine:11434'; +let redis: RedisType; + +beforeEach(async () => { + redis = new Redis() as unknown as RedisType; + await redis.flushall(); +}); + +afterEach(() => { + redis.disconnect(); +}); + +/** A runtime whose pull waits until the test lets it go, reporting progress each time it is told to. */ +function runtime(freeDiskBytes: number | undefined) { + let report: ((p: LocalPullProgress) => void) | undefined; + let finish: (() => void) | undefined; + let fail: ((e: Error) => void) | undefined; + const pulls: string[] = []; + const client: Partial = { + installed: async () => [], + pull: (tag, onProgress, signal) => + new Promise((resolve, reject) => { + pulls.push(tag); + report = onProgress; + finish = resolve; + fail = reject; + signal.addEventListener('abort', () => reject(new Error('aborted'))); + }), + }; + const local: SpaceLocalRuntime = { + config: {}, + client: client as LocalRuntimeClient, + runtimeId: RUNTIME_ID, + credential: { id: 'c', scope: 'user', scopeId: 'u' }, + hostMachine: + freeDiskBytes === undefined + ? undefined + : { totalMemoryBytes: 24 * GIB, unifiedMemory: true, freeDiskBytes }, + }; + return { + local, + pulls, + report: (p: LocalPullProgress) => report?.(p), + finish: () => finish?.(), + fail: (e: Error) => fail?.(e), + }; +} + +const ctx = (local: SpaceLocalRuntime) => ({ + db: {} as never, + redis, + tenantId: 't', + runtime: local, +}); +const pull = { action: 'pull' as const, model: 'ollama-gemma4-26b' }; + +describe('a download to this computer', () => { + it('is refused before it starts when the disk cannot hold it', async () => { + const r = runtime(10 * GIB); + const outcome = await applyLocalModelAction(ctx(r.local), pull); + expect(outcome).toMatchObject({ ok: false, reason: 'no_disk' }); + expect(r.pulls).toEqual([]); + }); + + it('joins a pull already downloading instead of starting another', async () => { + const r = runtime(100 * GIB); + expect(await applyLocalModelAction(ctx(r.local), pull)).toMatchObject({ ok: true }); + const second = await applyLocalModelAction(ctx(r.local), pull); + expect(second).toEqual({ ok: true, message: 'gemma4:26b is already downloading.' }); + expect(r.pulls).toEqual(['gemma4:26b']); + }); + + it('stops when the operator cancels, and says it stopped rather than failed', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const r = runtime(100 * GIB); + await applyLocalModelAction(ctx(r.local), pull); + expect(await cancelLocalModelPull(redis, r.local, 'ollama-gemma4-26b')).toBe(true); + + vi.setSystemTime(Date.now() + 5_000); + r.report({ status: 'pulling 48eb', completedBytes: 1, totalBytes: 2 }); + await vi.waitFor(async () => { + expect( + (await readLocalPulls(redis, RUNTIME_ID, ['gemma4:26b'])).get('gemma4:26b')?.status, + ).toBe('interrupted'); + }); + vi.useRealTimers(); + }); + + it('records why a pull failed', async () => { + const r = runtime(100 * GIB); + await applyLocalModelAction(ctx(r.local), pull); + r.fail(new Error('max retries exceeded')); + await vi.waitFor(async () => { + expect( + (await readLocalPulls(redis, RUNTIME_ID, ['gemma4:26b'])).get('gemma4:26b'), + ).toMatchObject({ + status: 'failed', + error: 'max retries exceeded', + }); + }); + }); + + it('leaves the model for the runtime to list once it is installed', async () => { + const r = runtime(100 * GIB); + await applyLocalModelAction(ctx(r.local), pull); + r.finish(); + await vi.waitFor(async () => { + expect((await readLocalPulls(redis, RUNTIME_ID, ['gemma4:26b'])).size).toBe(0); + }); + }); +}); + +describe('a role assignment', () => { + it('is left to the space settings, which set every role at once', async () => { + const r = runtime(undefined); + const outcome = await applyLocalModelAction(ctx(r.local), { + action: 'use_for_every_role', + model: 'ollama-gemma4-26b', + }); + expect(outcome).toMatchObject({ ok: false, reason: 'not_here' }); + }); +}); diff --git a/packages/cybernetic-runtime/src/localModels/__tests__/proposal.test.ts b/packages/cybernetic-runtime/src/localModels/__tests__/proposal.test.ts new file mode 100644 index 00000000..e31ed130 --- /dev/null +++ b/packages/cybernetic-runtime/src/localModels/__tests__/proposal.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest'; +import type { LocalRuntimeDiagnostics } from '@aflow/schemas'; +import { localRuntimeChangeRequest } from '../proposal.js'; + +const GEMMA = 'ollama-gemma4-26b'; + +function diagnostics(loadedBytes: number | null): LocalRuntimeDiagnostics { + return { + runtime: { baseUrl: 'http://127.0.0.1:11434', reachable: true, version: '0.40.5' }, + machine: { totalMemoryBytes: 24e9, gpuWorkingSetBytes: 19.07e9, freeDiskBytes: 195e9 }, + connection: { contextTokens: null, keepAlive: null, parallel: 1 }, + models: [ + { + model: GEMMA, + ref: 'gemma-local', + tag: 'gemma4:26b', + displayName: 'Gemma 4 26B (this computer)', + installed: true, + downloadBytes: 16.1e9, + roles: ['default', 'helmsman'], + fit: { + status: 'does_not_fit', + window: 32_768, + largestFittingWindow: 16_384, + bytesAtWindow: 20.1e9, + }, + loaded: + loadedBytes === null + ? null + : { + sizeBytes: loadedBytes, + gpuBytes: loadedBytes, + contextLength: 32_768, + expiresAt: null, + }, + build: 'verified', + pull: null, + }, + ], + issues: [], + }; +} + +const unload = (inFlightRequests: number, loadedBytes: number | null = 22.7e9) => + localRuntimeChangeRequest({ + operationId: 'model.runtime.change', + change: { action: 'unload', model: GEMMA }, + diagnostics: diagnostics(loadedBytes), + inFlightRequests, + askingModel: GEMMA, + }); + +describe('a proposed change to the model runtime', () => { + it('names the requests an unload ends and that this conversation loads the model again', () => { + const request = unload(2); + expect(request.modelName).toBe('Gemma 4 26B (this computer)'); + expect(request.impact).toEqual({ + inFlightRequests: 2, + ownConversationReloads: true, + notes: ['Frees 22.7 GB and every conversation it holds.'], + }); + }); + + it('asks again when what it reaches has moved since the card was shown', () => { + const elsewhere = localRuntimeChangeRequest({ + operationId: 'model.runtime.change', + change: { action: 'unload', model: GEMMA }, + diagnostics: diagnostics(22.7e9), + inFlightRequests: 0, + askingModel: undefined, + }); + expect(elsewhere.requestHash).not.toBe(unload(0).requestHash); + }); + + it('keeps one request across figures that move by the second', () => { + expect(unload(1).requestHash).toBe(unload(0).requestHash); + expect(unload(0, 18e9).requestHash).toBe(unload(0).requestHash); + }); + + it('counts no ended requests for a change that ends none', () => { + const request = localRuntimeChangeRequest({ + operationId: 'model.runtime.change', + change: { action: 'pull', model: GEMMA }, + diagnostics: diagnostics(null), + inFlightRequests: 3, + askingModel: GEMMA, + }); + expect(request.impact.inFlightRequests).toBe(0); + expect(request.impact.ownConversationReloads).toBe(false); + expect(request.impact.notes[0]).toBe('Downloads 16.1 GB of the 195.0 GB free on disk.'); + }); +}); diff --git a/packages/cybernetic-runtime/src/localModels/actions.ts b/packages/cybernetic-runtime/src/localModels/actions.ts new file mode 100644 index 00000000..caf99737 --- /dev/null +++ b/packages/cybernetic-runtime/src/localModels/actions.ts @@ -0,0 +1,219 @@ +/** + * The changes the platform makes to a space's runtime on this computer (Plan + * 323 D10), applied the same way whoever asked: the operator from a card, or + * Helmsman once the operator approved. + */ +import { eq } from 'drizzle-orm'; +import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'; +import { contextWindowForModel, createDefaultModelCatalog, sameLocalTag } from '@aflow/ai-client'; +import { createTenantContext, providerCredentials, withTenantSchema } from '@aflow/database'; +import { + claimLocalPull, + endLocalPull, + recordLocalPullProgress, + requestLocalPullCancel, + type Redis, +} from '@aflow/redis'; +import { THIS_MACHINE_RUNTIME_HOST, type LocalModelAction, type TenantId } from '@aflow/schemas'; +import type { SpaceLocalRuntime } from './runtime.js'; + +/** Room a download leaves on the disk: the runtime writes a manifest and temporary parts beside the layers. */ +export const PULL_DISK_HEADROOM_BYTES = 2 * 1024 ** 3; +/** Progress is written at most this often; the runtime reports many times a second. */ +const PULL_PROGRESS_INTERVAL_MS = 1_000; + +export type LocalModelActionOutcome = + | { ok: true; message: string } + | { + ok: false; + reason: 'not_local' | 'no_disk' | 'runtime' | 'not_here'; + message: string; + }; + +const gb = (bytes: number): string => `${(bytes / 1e9).toFixed(1)} GB`; + +export interface LocalModelActionContext { + db: PostgresJsDatabase; + redis: Redis | undefined; + tenantId: string; + runtime: SpaceLocalRuntime; + log?: { warn(message: string, fields?: Record): void }; +} + +export async function applyLocalModelAction( + ctx: LocalModelActionContext, + action: LocalModelAction, +): Promise { + if (action.action === 'set_context') return setContext(ctx, action.contextTokens); + if (action.action === 'use_for_every_role') { + return { + ok: false, + reason: 'not_here', + message: + "A role's model is the space's setting: assign it under Settings → Agents, where every role is set at once.", + }; + } + const model = createDefaultModelCatalog().getModel(action.model); + if (model?.provider !== 'ollama') { + return { + ok: false, + reason: 'not_local', + message: `${action.model} is not a model on this computer.`, + }; + } + const tag = model.providerModelId ?? model.id; + const { client } = ctx.runtime; + try { + switch (action.action) { + case 'pull': + return await startPull(ctx, tag, model.local?.downloadBytes); + case 'delete': + await client.remove(tag); + return { ok: true, message: `Deleted ${tag}.` }; + case 'warm': { + const window = contextWindowForModel(model, ctx.runtime.config); + await client.warm(tag, window); + return { ok: true, message: `Loaded ${tag} at a ${window}-token window.` }; + } + case 'unload': + await client.unload(tag); + return { ok: true, message: `Unloaded ${tag}.` }; + } + } catch (error) { + return { + ok: false, + reason: 'runtime', + message: `Ollama refused: ${error instanceof Error ? error.message : String(error)}`, + }; + } +} + +export async function cancelLocalModelPull( + redis: Redis, + runtime: SpaceLocalRuntime, + modelId: string, +): Promise { + const model = createDefaultModelCatalog().getModel(modelId); + if (model?.provider !== 'ollama') return false; + return requestLocalPullCancel(redis, runtime.runtimeId, model.providerModelId ?? model.id); +} + +async function startPull( + ctx: LocalModelActionContext, + tag: string, + downloadBytes: number | undefined, +): Promise { + const { redis, runtime } = ctx; + if (!redis) { + return { + ok: false, + reason: 'runtime', + message: 'Downloads need Redis to record their progress.', + }; + } + const installed = await runtime.client.installed(); + if (installed?.some((m) => sameLocalTag(m.tag, tag))) + return { ok: true, message: `${tag} is already downloaded.` }; + const free = runtime.runtimeId.startsWith(`${THIS_MACHINE_RUNTIME_HOST}:`) + ? runtime.hostMachine?.freeDiskBytes + : undefined; + if ( + downloadBytes !== undefined && + free !== undefined && + free < downloadBytes + PULL_DISK_HEADROOM_BYTES + ) { + return { + ok: false, + reason: 'no_disk', + message: `${tag} is ${gb(downloadBytes)} and this computer has ${gb(free)} free; free ${gb( + downloadBytes + PULL_DISK_HEADROOM_BYTES - free, + )} and download it again.`, + }; + } + if (!(await claimLocalPull(redis, runtime.runtimeId, tag))) { + return { ok: true, message: `${tag} is already downloading.` }; + } + // Held by this process for the length of the download, not awaited by the + // request that started it. The claim's lifetime is what the request leaves + // behind: a process that dies stops renewing it, and the runtime keeps the + // layers it already has for the next pull to resume from. + void runPull(redis, runtime, tag, ctx.log); + return { ok: true, message: `Downloading ${tag}.` }; +} + +async function runPull( + redis: Redis, + runtime: SpaceLocalRuntime, + tag: string, + log: LocalModelActionContext['log'], +): Promise { + const controller = new AbortController(); + let cancelled = false; + let lastWrite = 0; + let writing: Promise = Promise.resolve(); + try { + await runtime.client.pull( + tag, + (progress) => { + const now = Date.now(); + if (now - lastWrite < PULL_PROGRESS_INTERVAL_MS) return; + lastWrite = now; + writing = recordLocalPullProgress(redis, runtime.runtimeId, tag, { + completedBytes: progress.completedBytes, + totalBytes: progress.totalBytes, + detail: progress.status, + }) + .then((read) => { + if (read.cancelled) { + cancelled = true; + controller.abort(); + } + }) + .catch(() => undefined); + }, + controller.signal, + ); + await writing; + await endLocalPull(redis, runtime.runtimeId, tag, { status: 'installed' }); + } catch (error) { + await writing; + await endLocalPull( + redis, + runtime.runtimeId, + tag, + cancelled + ? { status: 'interrupted', error: null } + : { status: 'failed', error: error instanceof Error ? error.message : String(error) }, + ).catch(() => undefined); + if (!cancelled) log?.warn('Local model pull failed', { tag, error: String(error) }); + } +} + +async function setContext( + ctx: LocalModelActionContext, + contextTokens: number | null, +): Promise { + const { credential } = ctx.runtime; + await withTenantSchema(ctx.db, createTenantContext(ctx.tenantId as TenantId), async (tx) => { + const db = tx; + const [row] = await db + .select({ configJson: providerCredentials.configJson }) + .from(providerCredentials) + .where(eq(providerCredentials.id, credential.id)) + .limit(1); + const config = { ...((row?.configJson as Record | undefined) ?? {}) }; + if (contextTokens === null) delete config['context_tokens']; + else config['context_tokens'] = String(contextTokens); + await db + .update(providerCredentials) + .set({ configJson: config, updatedAt: new Date() }) + .where(eq(providerCredentials.id, credential.id)); + }); + return { + ok: true, + message: + contextTokens === null + ? 'Requests now get the largest window that fits this computer.' + : `Requests now get a ${contextTokens}-token window.`, + }; +} diff --git a/packages/cybernetic-runtime/src/localModels/diagnose.ts b/packages/cybernetic-runtime/src/localModels/diagnose.ts new file mode 100644 index 00000000..14084d60 --- /dev/null +++ b/packages/cybernetic-runtime/src/localModels/diagnose.ts @@ -0,0 +1,87 @@ +import { + createDefaultModelCatalog, + diagnoseLocalRuntime, + type ModelDefinition, +} from '@aflow/ai-client'; +import { readLocalPulls, type Redis } from '@aflow/redis'; +import { + DEFAULT_CYBERNETIC_MODEL, + LOCAL_MODEL_ROLES, + OLLAMA_DEFAULT_BASE_URL, + resolveRoleModel, + THIS_MACHINE_RUNTIME_HOST, + type DirectiveModelDefaults, + type LocalModelPull, + type LocalModelRole, + type LocalRuntimeDiagnostics, +} from '@aflow/schemas'; +import type { SpaceLocalRuntime } from './runtime.js'; + +/** The local model each role resolves to; a role on a hosted model is left out. */ +export function localRoleModels( + defaults: DirectiveModelDefaults | undefined, +): Partial> { + const catalog = createDefaultModelCatalog(); + const roles: Partial> = {}; + for (const role of LOCAL_MODEL_ROLES) { + const ref = + role === 'default' + ? (defaults?.default ?? DEFAULT_CYBERNETIC_MODEL) + : resolveRoleModel(defaults, role); + const model = catalog.getModel(ref); + if (model?.provider === 'ollama') roles[role] = model.id; + } + return roles; +} + +/** The lineup — every local model the catalog carries — and any other a role names. */ +function lineup(roles: Partial>): ModelDefinition[] { + const catalog = createDefaultModelCatalog(); + const models = catalog.listModels('ollama'); + for (const id of Object.values(roles)) { + const model = catalog.getModel(id); + if (model && !models.some((m) => m.id === model.id)) models.push(model); + } + return models; +} + +/** Reads the runtime once and diagnoses it (Plan 323 D15). */ +export async function diagnoseSpaceLocalRuntime(input: { + runtime: SpaceLocalRuntime; + redis: Redis | undefined; + roles: Partial>; +}): Promise { + const { runtime, redis, roles } = input; + const models = lineup(roles); + const tags = models.map((m) => m.providerModelId ?? m.id); + const [version, installed, loaded, pullsByTag] = await Promise.all([ + runtime.client.version(), + runtime.client.installed(), + runtime.client.loaded(), + redis + ? readLocalPulls(redis, runtime.runtimeId, tags).catch( + () => new Map(), + ) + : Promise.resolve(new Map()), + ]); + const pulls: Record = {}; + for (const model of models) { + const pull = pullsByTag.get(model.providerModelId ?? model.id); + if (pull) pulls[model.id] = pull; + } + // This machine's free disk says nothing about a runtime on another one. + const onThisMachine = runtime.runtimeId.startsWith(`${THIS_MACHINE_RUNTIME_HOST}:`); + return diagnoseLocalRuntime({ + observation: { + baseUrl: runtime.config.baseUrl ?? OLLAMA_DEFAULT_BASE_URL, + version, + installed, + loaded, + }, + connection: runtime.config, + models, + roles, + freeDiskBytes: onThisMachine ? runtime.hostMachine?.freeDiskBytes : undefined, + pulls, + }); +} diff --git a/packages/cybernetic-runtime/src/localModels/index.ts b/packages/cybernetic-runtime/src/localModels/index.ts new file mode 100644 index 00000000..b9bebc9f --- /dev/null +++ b/packages/cybernetic-runtime/src/localModels/index.ts @@ -0,0 +1,14 @@ +export { + resolveSpaceLocalRuntime, + type SpaceLocalRuntime, + type SpaceLocalRuntimeScope, +} from './runtime.js'; +export { diagnoseSpaceLocalRuntime, localRoleModels } from './diagnose.js'; +export { + applyLocalModelAction, + cancelLocalModelPull, + PULL_DISK_HEADROOM_BYTES, + type LocalModelActionContext, + type LocalModelActionOutcome, +} from './actions.js'; +export { localRuntimeChangeRequest, readSpaceModelDefaults } from './proposal.js'; diff --git a/packages/cybernetic-runtime/src/localModels/proposal.ts b/packages/cybernetic-runtime/src/localModels/proposal.ts new file mode 100644 index 00000000..4f851eda --- /dev/null +++ b/packages/cybernetic-runtime/src/localModels/proposal.ts @@ -0,0 +1,129 @@ +/** + * The approval request for a change Helmsman proposes to the runtime (Plan 323 + * D10). Built from one diagnosis, when the change is asked for and again when + * an approved change is about to run: the hash covers the change and whether + * it reloads the asking conversation's model, so an approval acts only on the + * change its card showed, and one whose reach moved since asks again. + */ +import { eq } from 'drizzle-orm'; +import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'; +import { createTenantContext, spaces, withTenantSchema } from '@aflow/database'; +import { + stableHash, + type DirectiveModelDefaults, + type LocalModelWriteApprovalRequestPayload, + type LocalRuntimeChange, + type LocalRuntimeDiagnostics, + type TenantId, +} from '@aflow/schemas'; + +const gb = (bytes: number): string => `${(bytes / 1e9).toFixed(1)} GB`; + +export async function readSpaceModelDefaults( + db: PostgresJsDatabase, + tenantId: string, + spaceId: string, +): Promise { + const [row] = (await withTenantSchema( + db, + createTenantContext(tenantId as TenantId), + async (tx: unknown) => + (tx as PostgresJsDatabase) + .select({ directives: spaces.directives }) + .from(spaces) + .where(eq(spaces.id, spaceId)) + .limit(1), + )) as Array<{ directives: { modelDefaults?: DirectiveModelDefaults } | null }>; + return row?.directives?.modelDefaults; +} + +function notesFor(change: LocalRuntimeChange, diagnostics: LocalRuntimeDiagnostics): string[] { + if (change.action === 'set_context') { + const used = diagnostics.models.filter((m) => m.roles.length > 0); + return [ + change.contextTokens === null + ? 'Each model gets the largest window that fits this computer.' + : `Requests get ${String(change.contextTokens)} tokens of window, whatever the machine fits.`, + ...used.map( + (m) => + `${m.displayName} now fits ${ + m.fit.largestFittingWindow !== null + ? `${String(m.fit.largestFittingWindow)} tokens` + : 'an unknown window' + } on this computer.`, + ), + ]; + } + const model = diagnostics.models.find((m) => m.model === change.model); + if (!model) return []; + const gpu = diagnostics.machine?.gpuWorkingSetBytes ?? null; + switch (change.action) { + case 'pull': { + const free = diagnostics.machine?.freeDiskBytes ?? null; + return [ + model.downloadBytes !== null + ? `Downloads ${gb(model.downloadBytes)}${free !== null ? ` of the ${gb(free)} free on disk` : ''}.` + : 'Its download size is not known before it starts.', + `Fit on this computer: ${model.fit.status.replaceAll('_', ' ')}.`, + ]; + } + case 'delete': + return [ + `Frees ${model.downloadBytes !== null ? gb(model.downloadBytes) : 'its'} disk; downloading it again takes as much.`, + ...(model.roles.length > 0 + ? [`${model.roles.join(', ')} cannot run until it is downloaded again.`] + : []), + ]; + case 'warm': + return [ + model.fit.bytesAtWindow !== null + ? `Takes about ${gb(model.fit.bytesAtWindow)}${gpu !== null ? ` of the ${gb(gpu)} the GPU allows` : ''} at a ${String(model.fit.window)}-token window.` + : `Loads at a ${String(model.fit.window)}-token window.`, + ]; + case 'unload': + return [ + model.loaded + ? `Frees ${gb(model.loaded.sizeBytes)} and every conversation it holds.` + : 'It is not loaded now; nothing is freed.', + ]; + } +} + +/** Changes that end requests running on the runtime. */ +const ENDS_REQUESTS: ReadonlySet = new Set(['unload', 'delete']); + +export function localRuntimeChangeRequest(input: { + operationId: string; + change: LocalRuntimeChange; + diagnostics: LocalRuntimeDiagnostics; + inFlightRequests: number; + /** The model the asking conversation runs on, when it runs on one here. */ + askingModel: string | undefined; +}): Omit { + const { change, diagnostics } = input; + const model = + 'model' in change ? diagnostics.models.find((m) => m.model === change.model) : undefined; + const impact = { + inFlightRequests: ENDS_REQUESTS.has(change.action) ? input.inFlightRequests : 0, + ownConversationReloads: + 'model' in change && ENDS_REQUESTS.has(change.action) && input.askingModel === change.model, + notes: notesFor(change, diagnostics), + }; + return { + kind: 'write_approval', + target: 'local_model', + change, + ...(model ? { modelName: model.displayName } : {}), + runtimeUrl: diagnostics.runtime.baseUrl, + impact, + // What moves by the second — free disk, a loaded model's growing cache, + // the requests in flight on the runtime, which no lease ties to a model — + // informs the card and stays out of the hash, or an approved change would + // rarely find the request it approved. + requestHash: stableHash({ + operationId: input.operationId, + change, + ownConversationReloads: impact.ownConversationReloads, + }), + }; +} diff --git a/packages/cybernetic-runtime/src/localModels/runtime.ts b/packages/cybernetic-runtime/src/localModels/runtime.ts new file mode 100644 index 00000000..93e63e3b --- /dev/null +++ b/packages/cybernetic-runtime/src/localModels/runtime.ts @@ -0,0 +1,58 @@ +/** + * The space's runtime on the operator's machine, resolved as the AI executor + * resolves it — same credential chain, same defaults, same machine — so what + * the operator and Helmsman read and change is the runtime the space's models + * actually run on (Plan 323). + */ +import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'; +import { localRuntimeClient, type LocalRuntimeClient, type ProviderConfig } from '@aflow/ai-client'; +import { + CredentialResolver, + createProviderCredentialDbLoader, + providerConfigFromCredential, +} from '@aflow/credential-resolver'; +import { readLocalMachine, type HostMachine, type Redis } from '@aflow/redis'; +import { localRuntimeId, type CredentialScope } from '@aflow/schemas'; + +export interface SpaceLocalRuntime { + config: ProviderConfig; + client: LocalRuntimeClient; + runtimeId: string; + /** The connection the runtime resolved from, which a change to its window writes to. */ + credential: { id: string; scope: CredentialScope; scopeId: string }; + /** This machine as its host executor last published it; absent where none is paired. */ + hostMachine: HostMachine | undefined; +} + +export interface SpaceLocalRuntimeScope { + db: PostgresJsDatabase; + redis: Redis | undefined; + tenantId: string; + userId: string; + spaceId: string; +} + +export async function resolveSpaceLocalRuntime( + scope: SpaceLocalRuntimeScope, +): Promise { + const resolver = new CredentialResolver({ loader: createProviderCredentialDbLoader(scope.db) }); + const resolved = await resolver + .resolve('ollama', { + tenantId: scope.tenantId, + credentialOwnerId: scope.userId, + spaceId: scope.spaceId, + }) + .catch(() => null); + if (!resolved) return null; + const hostMachine = scope.redis + ? await readLocalMachine(scope.redis).catch(() => undefined) + : undefined; + const config = providerConfigFromCredential('ollama', resolved, { machine: hostMachine }); + return { + config, + client: localRuntimeClient(config), + runtimeId: localRuntimeId(config.baseUrl), + credential: { id: resolved.credentialId, scope: resolved.scope, scopeId: resolved.scopeId }, + hostMachine, + }; +} diff --git a/packages/cybernetic-runtime/src/scheduling/opTaskOnlyValidator.ts b/packages/cybernetic-runtime/src/scheduling/opTaskOnlyValidator.ts index 590b1790..8b132ffc 100644 --- a/packages/cybernetic-runtime/src/scheduling/opTaskOnlyValidator.ts +++ b/packages/cybernetic-runtime/src/scheduling/opTaskOnlyValidator.ts @@ -1,5 +1,10 @@ import type { WorkflowTask } from '@aflow/schemas'; -import { getOperation, isEvalPlaneOperation, isPlanOperation } from '@aflow/schemas'; +import { + getOperation, + isEvalPlaneOperation, + isLocalModelOperation, + isPlanOperation, +} from '@aflow/schemas'; /** Teaching error template — `{toolId}` is replaced at validation time. */ export const OP_TASK_ONLY_AGENT_TOOL_ERROR = @@ -94,3 +99,26 @@ export function validateSkillPlanSeparation(tasks: readonly WorkflowTask[]): str } return null; } + +/** Teaching error template — `{opId}` is replaced at validation time. */ +export const LOCAL_MODEL_TASK_ERROR = + 'Operation {opId} reads or changes the model runtime on this computer, which only Helmsman reaches, and ' + + 'only with the operator’s approval — a run does not reconfigure the machine it runs on. Remove the reference.'; + +/** + * Plan 323 D10 — no skill task, agent tool surface or operation task may + * reference a `model.*` operation. + */ +export function validateSkillLocalModelSeparation(tasks: readonly WorkflowTask[]): string | null { + for (const task of tasks) { + if (typeof task.operation === 'string' && isLocalModelOperation(task.operation)) { + return LOCAL_MODEL_TASK_ERROR.replace('{opId}', task.operation); + } + for (const toolId of collectAgentTaskToolIds(task)) { + if (isLocalModelOperation(toolId)) { + return LOCAL_MODEL_TASK_ERROR.replace('{opId}', toolId); + } + } + } + return null; +} diff --git a/packages/cybernetic-runtime/src/skillValidity/skillValidity.ts b/packages/cybernetic-runtime/src/skillValidity/skillValidity.ts index 50f7f712..d81d8994 100644 --- a/packages/cybernetic-runtime/src/skillValidity/skillValidity.ts +++ b/packages/cybernetic-runtime/src/skillValidity/skillValidity.ts @@ -32,6 +32,7 @@ import { validateAgentOpTaskOnlyTools, validateSkillEvalPlaneSeparation, validateSkillPlanSeparation, + validateSkillLocalModelSeparation, } from '../scheduling/opTaskOnlyValidator.js'; import { CONTAINS_RESERVED_FIELDS } from '../evalRunnerCriterion.js'; @@ -211,6 +212,16 @@ export function materializeAndValidateSkillConfig( }); } + const localModelError = validateSkillLocalModelSeparation(materializedTasks); + if (localModelError) { + diagnostics.push({ + code: 'skill_references_local_model_op', + dimension: 'ref', + severity: 'error', + detail: localModelError, + }); + } + // 3. Bundle-level checks (compose / ratify / install only). if (input.bundle) { appendBundleDiagnostics(materializedTasks, input.bundle, diagnostics); diff --git a/packages/cybernetic-runtime/tsconfig.json b/packages/cybernetic-runtime/tsconfig.json index 43eceaaa..8eb86bab 100644 --- a/packages/cybernetic-runtime/tsconfig.json +++ b/packages/cybernetic-runtime/tsconfig.json @@ -13,6 +13,7 @@ "references": [ { "path": "../ai-client" }, { "path": "../applet-runtime" }, + { "path": "../credential-resolver" }, { "path": "../database" }, { "path": "../integration-simulator" }, { "path": "../lib" }, diff --git a/packages/database/src/tenant/migrations/index.ts b/packages/database/src/tenant/migrations/index.ts index 3ef177f2..565f4144 100644 --- a/packages/database/src/tenant/migrations/index.ts +++ b/packages/database/src/tenant/migrations/index.ts @@ -211,6 +211,7 @@ import { applyMigration216 } from './migration216.js'; import { applyMigration217 } from './migration217.js'; import { applyMigration218 } from './migration218.js'; import { applyMigration219 } from './migration219.js'; +import { applyMigration220 } from './migration220.js'; /** Migrations 1–33 — skipped when schema_migrations already records version 34. */ export const PRE_TAXONOMY_MIGRATIONS: readonly TenantMigrationFn[] = [ @@ -427,6 +428,7 @@ export const POST_TAXONOMY_MIGRATIONS: readonly TenantMigrationFn[] = [ applyMigration217, applyMigration218, applyMigration219, + applyMigration220, ]; export { applyMigration001 } from './migration001.js'; @@ -638,3 +640,4 @@ export { applyMigration216 } from './migration216.js'; export { applyMigration217 } from './migration217.js'; export { applyMigration218 } from './migration218.js'; export { applyMigration219 } from './migration219.js'; +export { applyMigration220 } from './migration220.js'; diff --git a/packages/database/src/tenant/migrations/migration220.ts b/packages/database/src/tenant/migrations/migration220.ts new file mode 100644 index 00000000..302045ec --- /dev/null +++ b/packages/database/src/tenant/migrations/migration220.ts @@ -0,0 +1,36 @@ +import type postgres from 'postgres'; + +/** + * The `model.runtime` capability group (Plan 323 P3b): Helmsman reads the + * model runtime on this computer and proposes changes to it, each one approved + * by the operator on a card before it runs. Every system profile that authors + * takes read and write — the approval, not the profile, is what lets a change + * through — and `Read Only` takes read. + */ +export async function applyMigration220( + sqlClient: postgres.Sql, + schemaName: string, +): Promise { + await sqlClient.unsafe(` + UPDATE "${schemaName}".capability_profiles + SET allowed_capabilities = allowed_capabilities + || '[ + {"capabilityGroupId":"model.runtime","accessMode":"read"}, + {"capabilityGroupId":"model.runtime","accessMode":"write"} + ]'::jsonb + WHERE is_system_profile = true + AND name IN ('Full Access', 'Standard', 'Personal Safe') + AND NOT (allowed_capabilities @> '[{"capabilityGroupId":"model.runtime","accessMode":"write"}]'::jsonb); + + UPDATE "${schemaName}".capability_profiles + SET allowed_capabilities = allowed_capabilities + || '[{"capabilityGroupId":"model.runtime","accessMode":"read"}]'::jsonb + WHERE is_system_profile = true + AND name = 'Read Only' + AND NOT (allowed_capabilities @> '[{"capabilityGroupId":"model.runtime","accessMode":"read"}]'::jsonb); + + INSERT INTO "${schemaName}".schema_migrations (version, description) + VALUES (220, 'Plan 323 P3b — the model.runtime capability group') + ON CONFLICT (version) DO NOTHING; + `); +} diff --git a/packages/executor-runtime/src/types.ts b/packages/executor-runtime/src/types.ts index 25ab4f1d..d409d634 100644 --- a/packages/executor-runtime/src/types.ts +++ b/packages/executor-runtime/src/types.ts @@ -438,6 +438,7 @@ export function getDefaultPendingTimeoutMs(stepType: StepType): number { case 'host': case 'browser': case 'plan': + case 'model': return PENDING_TIMEOUT_MS_DEFAULT; } } diff --git a/packages/platform-artifacts/src/__tests__/helmsmanLocalModelSeparation.test.ts b/packages/platform-artifacts/src/__tests__/helmsmanLocalModelSeparation.test.ts new file mode 100644 index 00000000..5aea73d6 --- /dev/null +++ b/packages/platform-artifacts/src/__tests__/helmsmanLocalModelSeparation.test.ts @@ -0,0 +1,110 @@ +/** + * Plan 323 D10 — the model runtime on this computer is reached by Helmsman + * alone, its read pinned on the local edition, and by no skill: a run does not + * reconfigure the machine it runs on. Probed with a violation, so the test + * fails the moment the rule admits one. + */ +import { describe, expect, it } from 'vitest'; +import type { WorkflowTask } from '@aflow/schemas'; +import { + getOperation, + isLocalModelOperation, + isRunnerExcludedOperation, + MODEL_RUNTIME_CHANGE_OPERATION_ID, + MODEL_RUNTIME_STATUS_OPERATION_ID, +} from '@aflow/schemas'; +import { + LOCAL_MODEL_TASK_ERROR, + materializeAndValidateSkillConfig, +} from '@aflow/cybernetic-runtime'; +import { CYBERNETIC_AGENTS } from '../cyberneticAgents.js'; +import { composeHelmsmanSurface } from '../helmsmanSurface.js'; + +const MODEL_OPS = [MODEL_RUNTIME_STATUS_OPERATION_ID, MODEL_RUNTIME_CHANGE_OPERATION_ID]; + +function catalogOf(flowId: string): { coreOperations: string[]; allowedOperationIds: string[] } { + const agent = CYBERNETIC_AGENTS.find((a) => a.flowId === flowId); + const step = (agent?.steps as Array> | undefined)?.find( + (s) => s['stepId'] === 'agent', + ); + const catalog = (step?.['config'] as Record | undefined)?.['catalog'] as + { coreOperations?: string[]; discovery?: { allowedOperationIds?: string[] } } | undefined; + return { + coreOperations: catalog?.coreOperations ?? [], + allowedOperationIds: catalog?.discovery?.allowedOperationIds ?? [], + }; +} + +const LOCAL = { + edition: 'community-local', + codeLane: 'present', + hostLane: 'present', + browserLane: 'present', +} as const; +const HOSTED = { + edition: 'enterprise', + codeLane: 'present', + hostLane: 'absent', + browserLane: 'absent', +} as const; + +describe('the model runtime is Helmsman’s, never a Runner’s (Plan 323 D10)', () => { + it('lets Helmsman discover both operations, each a callable agent tool', () => { + const helmsman = catalogOf('cybernetic-helmsman'); + expect(helmsman.allowedOperationIds.filter(isLocalModelOperation).sort()).toEqual( + [...MODEL_OPS].sort(), + ); + for (const id of MODEL_OPS) { + expect(getOperation(id)?.agentTool, id).toBe(true); + expect(isRunnerExcludedOperation(id), id).toBe(true); + } + }); + + it('pins the status read on the local edition, and composes neither in the hosted one', () => { + const helmsman = catalogOf('cybernetic-helmsman'); + const authored = { + coreOperations: helmsman.coreOperations, + promotableOperations: helmsman.allowedOperationIds, + }; + const local = composeHelmsmanSurface(authored, LOCAL); + expect(local.coreOperations).toContain(MODEL_RUNTIME_STATUS_OPERATION_ID); + expect(local.promotableOperations).toContain(MODEL_RUNTIME_CHANGE_OPERATION_ID); + const hosted = composeHelmsmanSurface(authored, HOSTED); + expect( + [...hosted.coreOperations, ...hosted.promotableOperations].filter(isLocalModelOperation), + ).toEqual([]); + }); + + it('gives no other platform agent one', () => { + for (const agent of CYBERNETIC_AGENTS) { + if (agent.flowId === 'cybernetic-helmsman') continue; + const { coreOperations, allowedOperationIds } = catalogOf(agent.flowId); + expect( + [...coreOperations, ...allowedOperationIds].filter(isLocalModelOperation), + agent.flowId, + ).toEqual([]); + } + }); + + it('refuses a skill whose Runner references model.runtime.change', () => { + const { validity } = materializeAndValidateSkillConfig({ + tasks: [ + { + taskId: 'runner', + name: 'Runner', + type: 'agent', + goal: 'Free the machine.', + dependsOn: [], + context: { strategy: 'static', tools: ['model.runtime.change'] }, + } as unknown as WorkflowTask, + ], + }); + expect(validity.status).toBe('invalid'); + expect(validity.diagnostics).toContainEqual( + expect.objectContaining({ + code: 'skill_references_local_model_op', + detail: LOCAL_MODEL_TASK_ERROR.replace('{opId}', 'model.runtime.change'), + }), + ); + }); +}); diff --git a/packages/platform-artifacts/src/cyberneticAgents.ts b/packages/platform-artifacts/src/cyberneticAgents.ts index b876d897..962c23a1 100644 --- a/packages/platform-artifacts/src/cyberneticAgents.ts +++ b/packages/platform-artifacts/src/cyberneticAgents.ts @@ -139,6 +139,11 @@ export const HELMSMAN_DISCOVERY_PRESET: readonly string[] = [ 'browser.page.screenshot', 'browser.page.handoff', 'browser.profile.list', + // The model runtime on this computer (local edition only, by its lane). The + // read is pinned there (`helmsmanSurface.ts`); a change is promoted when an + // issue names one, and runs only once the operator approves its card. + 'model.runtime.status', + 'model.runtime.change', 'mcp.server.list', 'mcp.server.get', 'mcp.server.upsert', diff --git a/packages/platform-artifacts/src/helmsmanSurface.test.ts b/packages/platform-artifacts/src/helmsmanSurface.test.ts index fbdefbd7..e5a61aa4 100644 --- a/packages/platform-artifacts/src/helmsmanSurface.test.ts +++ b/packages/platform-artifacts/src/helmsmanSurface.test.ts @@ -35,7 +35,7 @@ const APPLIANCE: ComposedLanes = { }; describe('composeHelmsmanSurface', () => { - it('returns the registry default unchanged for a hosted edition composing every lane', () => { + it('returns the registry default for a hosted edition composing every lane, less the models on this computer', () => { const authored = registryDefaults(); const composed = composeHelmsmanSurface(authored, { ...HOSTED, @@ -44,7 +44,9 @@ describe('composeHelmsmanSurface', () => { }); expect(composed.coreOperations).toEqual([...authored.coreOperations]); - expect(composed.promotableOperations).toEqual([...HELMSMAN_DISCOVERY_PRESET]); + expect(composed.promotableOperations).toEqual( + HELMSMAN_DISCOVERY_PRESET.filter((op) => !op.startsWith('model.')), + ); }); it('drops only the machine operations from the hosted ceiling while no machine is paired', () => { @@ -54,7 +56,7 @@ describe('composeHelmsmanSurface', () => { expect(composed.coreOperations).toEqual([...authored.coreOperations]); expect(composed.promotableOperations).toEqual( authored.promotableOperations.filter( - (op) => !op.startsWith('host.') && !op.startsWith('browser.'), + (op) => !op.startsWith('host.') && !op.startsWith('browser.') && !op.startsWith('model.'), ), ); expect(authored.promotableOperations.some((op) => op.startsWith('host.'))).toBe(true); @@ -72,7 +74,7 @@ describe('composeHelmsmanSurface', () => { expect(authored.promotableOperations.some((op) => op.startsWith('code.'))).toBe(true); }); - it('pins the machine reads, the harness and opening a page on the local edition', () => { + it('pins the machine reads, the harness, opening a page and the model runtime’s status on the local edition', () => { const authored = registryDefaults(); const composed = composeHelmsmanSurface(authored, APPLIANCE); @@ -84,14 +86,15 @@ describe('composeHelmsmanSurface', () => { 'host.file.list', 'host.file.get', 'browser.page.open', + 'model.runtime.status', ]); expect(removed).toEqual([ 'workflow.campaign.list', 'workflow.campaign.get', 'ui.surface.visualize', ]); - // The three reads and the harness are paid for; opening a page is not. - expect(composed.coreOperations.length).toBe(authored.coreOperations.length + 1); + // The three reads and the harness are paid for; opening a page and the runtime's status are not. + expect(composed.coreOperations.length).toBe(authored.coreOperations.length + 2); }); it('keeps every other browser operation promotable on the local edition, as one bundle', () => { @@ -139,7 +142,13 @@ describe('composeHelmsmanSurface', () => { const uncomposed = authored.promotableOperations.filter((op) => op.startsWith('code.')); expect(uncomposed.length).toBeGreaterThan(0); expect(removed.sort()).toEqual( - [...uncomposed, 'host.harness.run', 'host.file.list', 'host.file.get'].sort(), + [ + ...uncomposed, + 'host.harness.run', + 'host.file.list', + 'host.file.get', + 'model.runtime.status', + ].sort(), ); expect(added).toEqual([ 'workflow.campaign.list', diff --git a/packages/platform-artifacts/src/helmsmanSurface.ts b/packages/platform-artifacts/src/helmsmanSurface.ts index f9335c19..34bb0969 100644 --- a/packages/platform-artifacts/src/helmsmanSurface.ts +++ b/packages/platform-artifacts/src/helmsmanSurface.ts @@ -23,20 +23,23 @@ export interface HelmsmanSurface { } /** - * The machine reads, the harness and opening a page, pinned on the local - * edition. + * The machine reads, the harness, opening a page and the model runtime's + * status, pinned on the local edition. * * Assessment is steering work and happens on most turns, and the harness is how * a task is commissioned once the assessment is done. Promoting either one by * hand is a round trip on every task that touches the operator's machine. * Opening a page is the browser's one every-turn cost; the operations that - * act on a page promote together once one is open. + * act on a page promote together once one is open. The runtime's status is + * what answers "why is this slow" on a machine where the model Helmsman runs + * on is the likeliest cause, and it takes no input. */ const LOCAL_PINNED_OPERATIONS: readonly string[] = [ 'host.harness.run', 'host.file.list', 'host.file.get', 'browser.page.open', + 'model.runtime.status', ]; /** diff --git a/packages/redis/src/__tests__/browserApproval.test.ts b/packages/redis/src/__tests__/browserApproval.test.ts index 21a10145..66a15b5e 100644 --- a/packages/redis/src/__tests__/browserApproval.test.ts +++ b/packages/redis/src/__tests__/browserApproval.test.ts @@ -2,14 +2,13 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import Redis from 'ioredis-mock'; import type { Redis as RedisType } from 'ioredis'; +import { browserAskKey, readBrowserAsk, rememberBrowserAsk } from '../browserApproval.js'; import { - browserApprovalSpentKey, - browserAskKey, - readBrowserAsk, - rememberBrowserAsk, - spendBrowserApproval, -} from '../browserApproval.js'; -import { setWriteApprovalGrant, writeApprovalGrantKey } from '../writeApproval.js'; + setWriteApprovalGrant, + spendWriteApprovalGrant, + writeApprovalGrantKey, + writeApprovalSpentKey, +} from '../writeApproval.js'; const TENANT = '00000000-0000-0000-0000-000000000001'; const RUN = 'run-1'; @@ -28,12 +27,12 @@ afterEach(() => { describe('a browser approval', () => { it('is spent once, and a later approval of the same request is a new one to spend', async () => { const first = { requestHash: 'h1', decidedAt: '2026-10-04T12:00:00.000Z' }; - expect(await spendBrowserApproval(redis, TENANT, RUN, first)).toBe(true); - expect(await spendBrowserApproval(redis, TENANT, RUN, first)).toBe(false); + expect(await spendWriteApprovalGrant(redis, TENANT, RUN, first)).toBe(true); + expect(await spendWriteApprovalGrant(redis, TENANT, RUN, first)).toBe(false); const second = { ...first, decidedAt: '2026-10-04T12:05:00.000Z' }; - expect(await spendBrowserApproval(redis, TENANT, RUN, second)).toBe(true); - expect(await spendBrowserApproval(redis, 'other-tenant', RUN, first)).toBe(true); + expect(await spendWriteApprovalGrant(redis, TENANT, RUN, second)).toBe(true); + expect(await spendWriteApprovalGrant(redis, 'other-tenant', RUN, first)).toBe(true); }); it('outlives the grant it marks', async () => { @@ -42,13 +41,13 @@ describe('a browser approval', () => { decision: 'approved', decidedAt: '2026-10-04T12:00:00.000Z', }); - await spendBrowserApproval(redis, TENANT, RUN, { + await spendWriteApprovalGrant(redis, TENANT, RUN, { requestHash: 'h1', decidedAt: '2026-10-04T12:00:00.000Z', }); const grantTtl = await redis.ttl(writeApprovalGrantKey(TENANT, RUN, 'h1')); const spentTtl = await redis.ttl( - browserApprovalSpentKey(TENANT, RUN, { + writeApprovalSpentKey(TENANT, RUN, { requestHash: 'h1', decidedAt: '2026-10-04T12:00:00.000Z', }), @@ -65,10 +64,10 @@ describe('a browser approval', () => { it('keeps its records outside the grant family the host identity may only read', () => { for (const key of [ browserAskKey(TENANT, RUN, 'call-1'), - browserApprovalSpentKey(TENANT, RUN, { requestHash: 'h1' }), + writeApprovalSpentKey(TENANT, RUN, { requestHash: 'h1' }), ]) { - expect(key.startsWith('aflow:browser-ask:')).toBe(true); + expect(key.startsWith('aflow:write-approval:')).toBe(false); } - expect(writeApprovalGrantKey(TENANT, RUN, 'h1').startsWith('aflow:browser-ask:')).toBe(false); + expect(writeApprovalGrantKey(TENANT, RUN, 'h1').startsWith('aflow:write-approval:')).toBe(true); }); }); diff --git a/packages/redis/src/__tests__/localModelPulls.test.ts b/packages/redis/src/__tests__/localModelPulls.test.ts new file mode 100644 index 00000000..c8403ed7 --- /dev/null +++ b/packages/redis/src/__tests__/localModelPulls.test.ts @@ -0,0 +1,73 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import Redis from 'ioredis-mock'; +import type { Redis as RedisType } from 'ioredis'; +import { + claimLocalPull, + endLocalPull, + readLocalPulls, + recordLocalPullProgress, + requestLocalPullCancel, +} from '../localModelPulls.js'; + +const RUNTIME = 'this-machine:11434'; +const TAG = 'gemma4:26b'; +let redis: RedisType; + +beforeEach(async () => { + redis = new Redis() as unknown as RedisType; + await redis.flushall(); +}); + +afterEach(() => { + redis.disconnect(); +}); + +describe('a pull on a local runtime', () => { + it('is claimed once: a second request while it downloads joins it', async () => { + expect(await claimLocalPull(redis, RUNTIME, TAG)).toBe(true); + expect(await claimLocalPull(redis, RUNTIME, TAG)).toBe(false); + }); + + it('reports its progress to whoever reads it', async () => { + await claimLocalPull(redis, RUNTIME, TAG, new Date('2026-10-16T20:00:00Z')); + await recordLocalPullProgress(redis, RUNTIME, TAG, { + completedBytes: 4_000, + totalBytes: 16_000, + detail: 'pulling 48eb98ec', + }); + + expect((await readLocalPulls(redis, RUNTIME, [TAG, 'other:1b'])).get(TAG)).toEqual({ + status: 'downloading', + completedBytes: 4_000, + totalBytes: 16_000, + detail: 'pulling 48eb98ec', + error: null, + startedAt: '2026-10-16T20:00:00.000Z', + }); + }); + + it('hears a cancel on its next progress, wherever it was asked', async () => { + await claimLocalPull(redis, RUNTIME, TAG); + expect(await requestLocalPullCancel(redis, RUNTIME, TAG)).toBe(true); + const { cancelled } = await recordLocalPullProgress(redis, RUNTIME, TAG, { + completedBytes: 1, + totalBytes: null, + detail: '', + }); + expect(cancelled).toBe(true); + }); + + it('leaves no record once installed, and a readable one once stopped, which a new pull replaces', async () => { + await claimLocalPull(redis, RUNTIME, TAG); + await endLocalPull(redis, RUNTIME, TAG, { status: 'interrupted', error: null }); + expect((await readLocalPulls(redis, RUNTIME, [TAG])).get(TAG)?.status).toBe('interrupted'); + + expect(await claimLocalPull(redis, RUNTIME, TAG)).toBe(true); + await endLocalPull(redis, RUNTIME, TAG, { status: 'installed' }); + expect((await readLocalPulls(redis, RUNTIME, [TAG])).size).toBe(0); + }); + + it('cannot be cancelled once it is not downloading', async () => { + expect(await requestLocalPullCancel(redis, RUNTIME, TAG)).toBe(false); + }); +}); diff --git a/packages/redis/src/__tests__/localRuntimeAdmission.test.ts b/packages/redis/src/__tests__/localRuntimeAdmission.test.ts index d8ea8475..a144f6b5 100644 --- a/packages/redis/src/__tests__/localRuntimeAdmission.test.ts +++ b/packages/redis/src/__tests__/localRuntimeAdmission.test.ts @@ -1,7 +1,11 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import Redis from 'ioredis-mock'; import type { Redis as RedisType } from 'ioredis'; -import { acquireRuntimeLease, RuntimeLeaseAbortedError } from '../localRuntimeAdmission.js'; +import { + acquireRuntimeLease, + countRuntimeRequests, + RuntimeLeaseAbortedError, +} from '../localRuntimeAdmission.js'; const RUNTIME = 'this-machine:11434'; let redis: RedisType; @@ -122,3 +126,15 @@ describe('acquireRuntimeLease', () => { await otherRuntime?.release(); }); }); + +describe('countRuntimeRequests', () => { + it('counts the leases held now, and not a waiter', async () => { + const held = await take('a', { limit: 1 }); + const waiting = take('b', { wait: true, limit: 1, signal: AbortSignal.timeout(200) }).catch( + () => undefined, + ); + expect(await countRuntimeRequests(redis, RUNTIME)).toBe(1); + await held.release(); + await waiting; + }); +}); diff --git a/packages/redis/src/__tests__/streamRetention.test.ts b/packages/redis/src/__tests__/streamRetention.test.ts index 40c2f0de..87442283 100644 --- a/packages/redis/src/__tests__/streamRetention.test.ts +++ b/packages/redis/src/__tests__/streamRetention.test.ts @@ -288,6 +288,10 @@ const STACK_REDIS = await stackRedis(TEST_DB); const STREAM = 'aflow:jobs:retention-test'; const GROUP = 'exec_retention_test'; +const HYGIENE_STREAM = `test:stream-retention:hygiene:${process.pid}`; +const HYGIENE_GROUP = 'test-hygiene'; +const HYGIENE_GROUPS = [{ stream: HYGIENE_STREAM, group: HYGIENE_GROUP }]; + describe.skipIf(!STACK_REDIS.available)('retention against real Redis', () => { let redis: RedisType; @@ -357,11 +361,12 @@ describe.skipIf(!STACK_REDIS.available)('retention against real Redis', () => { }); it('never lets consumer hygiene delete a consumer that still holds pending entries', async () => { - // Must be a stream cleanupStaleConsumers actually visits: it walks keys built - // from EXECUTOR_JOB_STEP_TYPES, so a test-only stream name would make the - // zero-removal assertion pass even with the guard deleted. - const enumerated = StreamKeys.jobStream('eval'); - const enumeratedGroup = ConsumerGroups.executor('eval'); + // Hygiene is handed this stream to visit, so the zero-removal assertion + // exercises the pending guard rather than passing because the stream was + // never looked at. A stream of the test's own: the live stack's job streams + // share this Redis, and their idle consumers are not this test's to reap. + const enumerated = HYGIENE_STREAM; + const enumeratedGroup = HYGIENE_GROUP; await redis.del(enumerated); await redis.xgroup('CREATE', enumerated, enumeratedGroup, '0', 'MKSTREAM'); try { @@ -378,7 +383,7 @@ describe.skipIf(!STACK_REDIS.available)('retention against real Redis', () => { ); // maxIdleMs 0 makes every consumer idle enough; only the pending guard is left. - const cleanup = await cleanupStaleConsumers(redis, 0); + const cleanup = await cleanupStaleConsumers(redis, 0, HYGIENE_GROUPS); expect(cleanup.streamsChecked).toBeGreaterThan(0); expect(cleanup.consumersRemoved).toBe(0); @@ -392,8 +397,8 @@ describe.skipIf(!STACK_REDIS.available)('retention against real Redis', () => { }); it('does let hygiene reap a consumer that holds nothing, leaving the stream trimmable', async () => { - const enumerated = StreamKeys.jobStream('eval'); - const enumeratedGroup = ConsumerGroups.executor('eval'); + const enumerated = HYGIENE_STREAM; + const enumeratedGroup = HYGIENE_GROUP; await redis.del(enumerated); await redis.xgroup('CREATE', enumerated, enumeratedGroup, '0', 'MKSTREAM'); try { @@ -414,7 +419,7 @@ describe.skipIf(!STACK_REDIS.available)('retention against real Redis', () => { // the same millisecond reports idle 0 — so a threshold of 0 reaps nothing on // a machine fast enough to get here inside one tick, and the test fails for // being quick. -1 is what "however briefly it has been idle" spells. - const cleanup = await cleanupStaleConsumers(redis, -1); + const cleanup = await cleanupStaleConsumers(redis, -1, HYGIENE_GROUPS); expect(cleanup.consumersRemoved).toBeGreaterThan(0); const result = await trimToAckedFrontier(redis, enumerated); diff --git a/packages/redis/src/browserApproval.ts b/packages/redis/src/browserApproval.ts index aacf1267..3f81815d 100644 --- a/packages/redis/src/browserApproval.ts +++ b/packages/redis/src/browserApproval.ts @@ -1,10 +1,5 @@ import type { Redis } from 'ioredis'; -import { - stableHash, - WRITE_APPROVAL_GRANT_TTL_SECONDS, - type WriteApprovalGrant, -} from '@aflow/schemas'; - +import { WRITE_APPROVAL_GRANT_TTL_SECONDS } from '@aflow/schemas'; /** * What the host executor keeps beside a browser action's approval (Plan 320 * D7). The grant itself is the write-approval grant, minted only at the @@ -17,23 +12,15 @@ import { * fresh dispatch after an approval can find the approval even when the page * has changed underneath it and the request it would make now hashes * differently; - * - the **spend** — set once, by the action an approval lets through, so an - * approval performs one action. Keyed by the grant's own decision time, so - * a later approval of the same request is a new grant to spend. + * - the **spend** (`spendWriteApprovalGrant`) — set once, by the action an + * approval lets through, so an approval performs one action. Keyed by the + * grant's own decision time, so a later approval of the same request is a + * new grant to spend. */ export function browserAskKey(tenantId: string, runId: string, callKey: string): string { return `aflow:browser-ask:call:${tenantId}:${runId}:${callKey}`; } -export function browserApprovalSpentKey( - tenantId: string, - runId: string, - grant: Pick, -): string { - const decision = stableHash({ requestHash: grant.requestHash, decidedAt: grant.decidedAt ?? '' }); - return `aflow:browser-ask:spent:${tenantId}:${runId}:${decision}`; -} - export async function rememberBrowserAsk( redis: Redis, tenantId: string, @@ -57,24 +44,3 @@ export async function readBrowserAsk( ): Promise { return await redis.get(browserAskKey(tenantId, runId, callKey)); } - -/** - * Spends an approval. True for the one caller that spent it; false when it was - * spent already. Outlives the grant it marks, so a spent grant never reads as - * fresh again. - */ -export async function spendBrowserApproval( - redis: Redis, - tenantId: string, - runId: string, - grant: Pick, -): Promise { - const set = await redis.set( - browserApprovalSpentKey(tenantId, runId, grant), - '1', - 'EX', - WRITE_APPROVAL_GRANT_TTL_SECONDS, - 'NX', - ); - return set === 'OK'; -} diff --git a/packages/redis/src/hostInventory.ts b/packages/redis/src/hostInventory.ts index 5383d119..65574d9c 100644 --- a/packages/redis/src/hostInventory.ts +++ b/packages/redis/src/hostInventory.ts @@ -170,6 +170,11 @@ export const HostMachineSchema = z.object({ totalMemoryBytes: z.number().int().positive(), gpuWorkingSetBytes: z.number().int().positive().optional(), unifiedMemory: z.boolean(), + /** + * Free space on the disk that holds the model runtime's downloads, read at + * each publish, so a pull that cannot fit is refused before it starts. + */ + freeDiskBytes: z.number().int().nonnegative().optional(), }); export type HostMachine = z.infer; diff --git a/packages/redis/src/index.ts b/packages/redis/src/index.ts index 8a047d33..70c27066 100644 --- a/packages/redis/src/index.ts +++ b/packages/redis/src/index.ts @@ -310,14 +310,10 @@ export { setWriteApprovalGrant, getWriteApprovalGrant, hostPushRequestHash, + spendWriteApprovalGrant, + writeApprovalSpentKey, } from './writeApproval.js'; -export { - browserAskKey, - browserApprovalSpentKey, - rememberBrowserAsk, - readBrowserAsk, - spendBrowserApproval, -} from './browserApproval.js'; +export { browserAskKey, rememberBrowserAsk, readBrowserAsk } from './browserApproval.js'; // Memory embedding operations export { @@ -532,6 +528,7 @@ export * from './browserHandoff.js'; export * from './sessionResidue.js'; export { acquireRuntimeLease, + countRuntimeRequests, runtimeAdmission, RuntimeLeaseAbortedError, RUNTIME_LEASE_TTL_MS, @@ -540,3 +537,12 @@ export { type RuntimeLease, type AcquireRuntimeLeaseInput, } from './localRuntimeAdmission.js'; +export { + claimLocalPull, + recordLocalPullProgress, + requestLocalPullCancel, + endLocalPull, + readLocalPulls, + LOCAL_PULL_LEASE_MS, + LOCAL_PULL_ENDED_TTL_MS, +} from './localModelPulls.js'; diff --git a/packages/redis/src/localModelPulls.ts b/packages/redis/src/localModelPulls.ts new file mode 100644 index 00000000..39970704 --- /dev/null +++ b/packages/redis/src/localModelPulls.ts @@ -0,0 +1,146 @@ +/** + * Downloads to a model runtime on the operator's machine, one per runtime and + * tag (Plan 323 D10). + * + * The record is the pull's claim as well as its progress. Whoever starts a + * pull holds it with a lifetime its progress renews, so a second request finds + * it and joins rather than starting another, and a server that dies mid-pull + * leaves nothing behind once the lifetime runs out — the runtime itself kept + * what it downloaded, and the next pull resumes from there. A pull that ended + * badly keeps its record a while longer, so the operator reads why. + */ +import type { Redis } from 'ioredis'; +import type { LocalModelPull } from '@aflow/schemas'; + +const KEY_PREFIX = 'aflow:local-pull'; +/** + * Progress arrives several times a second while bytes flow, but the runtime + * says nothing while it checks a large layer's digest, which takes minutes on + * a slow disk. Ten minutes of silence is a pull nobody holds. + */ +export const LOCAL_PULL_LEASE_MS = 600_000; +/** How long a stopped or failed pull stays readable. */ +export const LOCAL_PULL_ENDED_TTL_MS = 600_000; + +function pullKey(runtimeId: string, tag: string): string { + return `${KEY_PREFIX}:{${runtimeId}}:${tag}`; +} + +function parse(fields: Record): LocalModelPull | null { + const status = fields['status']; + if (status !== 'downloading' && status !== 'interrupted' && status !== 'failed') return null; + const total = fields['totalBytes']; + return { + status, + completedBytes: Number(fields['completedBytes'] ?? 0), + totalBytes: total ? Number(total) : null, + detail: fields['detail'] || null, + error: fields['error'] || null, + startedAt: fields['startedAt'] ?? new Date(0).toISOString(), + }; +} + +/** + * KEYS: the pull. ARGV: startedAt, lease ms. + * Claims the pull unless one is downloading; a stopped or failed one is replaced. + */ +const CLAIM = ` +if redis.call('HGET', KEYS[1], 'status') == 'downloading' then return 0 end +redis.call('DEL', KEYS[1]) +redis.call('HSET', KEYS[1], 'status', 'downloading', 'completedBytes', '0', 'startedAt', ARGV[1]) +redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[2])) +return 1 +`; + +/** Starts a pull's record, or answers false when one is already downloading — the caller joins it. */ +export async function claimLocalPull( + redis: Redis, + runtimeId: string, + tag: string, + now: Date = new Date(), +): Promise { + const claimed = await redis.eval( + CLAIM, + 1, + pullKey(runtimeId, tag), + now.toISOString(), + String(LOCAL_PULL_LEASE_MS), + ); + return claimed === 1; +} + +/** + * Records progress and renews the claim in one round trip, answering whether + * the operator asked to stop it — read here so a cancel reaches whichever + * server holds the pull. + */ +export async function recordLocalPullProgress( + redis: Redis, + runtimeId: string, + tag: string, + progress: { completedBytes: number; totalBytes: number | null; detail: string }, +): Promise<{ cancelled: boolean }> { + const key = pullKey(runtimeId, tag); + const results = await redis + .multi() + // The status rides every write, so a record that lapsed under a pull + // still running is rebuilt whole rather than as one nothing can read. + .hset(key, { + status: 'downloading', + completedBytes: String(progress.completedBytes), + totalBytes: progress.totalBytes === null ? '' : String(progress.totalBytes), + detail: progress.detail, + }) + .pexpire(key, LOCAL_PULL_LEASE_MS) + .hget(key, 'cancel') + .exec(); + return { cancelled: results?.[2]?.[1] === '1' }; +} + +export async function requestLocalPullCancel( + redis: Redis, + runtimeId: string, + tag: string, +): Promise { + const key = pullKey(runtimeId, tag); + if ((await redis.hget(key, 'status')) !== 'downloading') return false; + await redis.hset(key, 'cancel', '1'); + return true; +} + +/** A pull that installed its model leaves no record: the runtime lists the model instead. */ +export async function endLocalPull( + redis: Redis, + runtimeId: string, + tag: string, + outcome: { status: 'installed' } | { status: 'interrupted' | 'failed'; error: string | null }, +): Promise { + const key = pullKey(runtimeId, tag); + if (outcome.status === 'installed') { + await redis.del(key); + return; + } + await redis + .multi() + .hset(key, { status: outcome.status, error: outcome.error ?? '', cancel: '' }) + .pexpire(key, LOCAL_PULL_ENDED_TTL_MS) + .exec(); +} + +export async function readLocalPulls( + redis: Redis, + runtimeId: string, + tags: readonly string[], +): Promise> { + const pulls = new Map(); + if (tags.length === 0) return pulls; + const pipeline = redis.pipeline(); + for (const tag of tags) pipeline.hgetall(pullKey(runtimeId, tag)); + const results = (await pipeline.exec()) ?? []; + tags.forEach((tag, i) => { + const fields = results[i]?.[1] as Record | undefined; + const pull = fields ? parse(fields) : null; + if (pull) pulls.set(tag, pull); + }); + return pulls; +} diff --git a/packages/redis/src/localRuntimeAdmission.ts b/packages/redis/src/localRuntimeAdmission.ts index 34ad5cc5..0e6cef22 100644 --- a/packages/redis/src/localRuntimeAdmission.ts +++ b/packages/redis/src/localRuntimeAdmission.ts @@ -154,6 +154,19 @@ async function release(redis: Redis, runtimeId: string, holderId: string): Promi * `wait` is set. Returns null only when `wait` is false and the runtime is * busy. A cancelled wait leaves the queue and throws `RuntimeLeaseAbortedError`. */ +/** + * The requests running on a runtime now: the holders whose lease has not + * lapsed. What a change that ends them — an unload, a delete — would end. + */ +export async function countRuntimeRequests( + redis: Redis, + runtimeId: string, + now: number = Date.now(), +): Promise { + const [holders] = runtimeKeys(runtimeId); + return await redis.zcount(holders, `(${String(now)}`, '+inf'); +} + export async function acquireRuntimeLease( input: AcquireRuntimeLeaseInput, ): Promise { diff --git a/packages/redis/src/streams/hygiene.ts b/packages/redis/src/streams/hygiene.ts index 55ca7155..99428d7e 100644 --- a/packages/redis/src/streams/hygiene.ts +++ b/packages/redis/src/streams/hygiene.ts @@ -76,8 +76,22 @@ export interface StreamCleanupResult { streamsChecked: number; } +export interface ConsumerGroupRef { + stream: string; + group: string; +} + +/** Every executor job stream and the group its executors read it through. */ +export function executorJobGroups(): ConsumerGroupRef[] { + return EXECUTOR_JOB_STEP_TYPES.map((stepType) => ({ + stream: StreamKeys.jobStream(stepType), + group: ConsumerGroups.executor(stepType), + })); +} + /** - * Remove stale consumers from all known consumer groups. + * Remove stale consumers from the given consumer groups — every executor job + * stream's unless told otherwise. * Call on orchestrator/executor startup to clean up consumers left behind by * processes that were killed without graceful shutdown. * @@ -87,19 +101,13 @@ export interface StreamCleanupResult { export async function cleanupStaleConsumers( redis: Redis, maxIdleMs = 60_000, + groups: readonly ConsumerGroupRef[] = executorJobGroups(), ): Promise { let consumersRemoved = 0; let streamsChecked = 0; - // Job stream groups - for (const stepType of EXECUTOR_JOB_STEP_TYPES) { - const n = await cleanupGroupConsumers( - redis, - StreamKeys.jobStream(stepType), - ConsumerGroups.executor(stepType), - maxIdleMs, - ); - consumersRemoved += n; + for (const { stream, group } of groups) { + consumersRemoved += await cleanupGroupConsumers(redis, stream, group, maxIdleMs); streamsChecked++; } diff --git a/packages/redis/src/writeApproval.ts b/packages/redis/src/writeApproval.ts index 8f1e7a25..0e3710eb 100644 --- a/packages/redis/src/writeApproval.ts +++ b/packages/redis/src/writeApproval.ts @@ -75,3 +75,39 @@ export async function getWriteApprovalGrant( return null; } } + +/** + * Outside the grant family (`aflow:write-approval:`), which the host executor + * may only read: spending one cannot approve anything, so the identity that + * performs the action may write it. + */ +export function writeApprovalSpentKey( + tenantId: string, + runId: string, + grant: Pick, +): string { + const decision = stableHash({ requestHash: grant.requestHash, decidedAt: grant.decidedAt ?? '' }); + return `aflow:approval-spent:${tenantId}:${runId}:${decision}`; +} + +/** + * Spends an approval that lets one action through — a browser action, a change + * to the model runtime. True for the one caller that spent it; false when it + * was spent already. Outlives the grant it marks, so a spent grant never reads + * as fresh again. + */ +export async function spendWriteApprovalGrant( + redis: Redis, + tenantId: string, + runId: string, + grant: Pick, +): Promise { + const set = await redis.set( + writeApprovalSpentKey(tenantId, runId, grant), + '1', + 'EX', + WRITE_APPROVAL_GRANT_TTL_SECONDS, + 'NX', + ); + return set === 'OK'; +} diff --git a/packages/run-view/src/index.ts b/packages/run-view/src/index.ts index 2da9c6f3..0faa9fb9 100644 --- a/packages/run-view/src/index.ts +++ b/packages/run-view/src/index.ts @@ -6,6 +6,7 @@ export { type LiveDeltaAction, type HarnessActivityState, unsettledHarnessSteps, + latestStatusLine, } from './reducer.js'; export { diff --git a/packages/run-view/src/reducer-status-line.test.ts b/packages/run-view/src/reducer-status-line.test.ts new file mode 100644 index 00000000..72b3a9ce --- /dev/null +++ b/packages/run-view/src/reducer-status-line.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest'; +import { latestStatusLine, runViewReducer, initialRunViewState } from './index.js'; + +describe('a status frame', () => { + it('leaves the step in its last whole line', () => { + expect(latestStatusLine('Loading Gemma 4 26B into memory…\nReading the conversation…\n')).toBe( + 'Reading the conversation…', + ); + }); + + it('says nothing new when it carries no whole line', () => { + expect(latestStatusLine('Reading the conv')).toBeUndefined(); + expect(latestStatusLine('Loading…\nReading the conv')).toBe('Loading…'); + }); + + it('adds nothing to the conversation', () => { + const state = runViewReducer(initialRunViewState, { + type: 'LIVE_DELTA', + stepExecutionId: 'step-1', + channel: 'status', + offset: 0, + delta: 'Loading Gemma 4 26B into memory…\n', + timestamp: new Date(0).toISOString(), + }); + expect(state.messages).toEqual([]); + expect(state.harnessActivity).toEqual({}); + }); +}); diff --git a/packages/run-view/src/reducer.ts b/packages/run-view/src/reducer.ts index 1adc34ed..c732ef86 100644 --- a/packages/run-view/src/reducer.ts +++ b/packages/run-view/src/reducer.ts @@ -34,3 +34,4 @@ export function runViewReducer(state: RunViewState, action: RunViewAction): RunV export const initialRunViewState = INITIAL_STATE; export { unsettledHarnessSteps } from './reducer/events/harnessActivity.js'; +export { latestStatusLine } from './reducer/events/agentStreaming.js'; diff --git a/packages/run-view/src/reducer/events/agentStreaming.ts b/packages/run-view/src/reducer/events/agentStreaming.ts index da37b75e..4cb68dd3 100644 --- a/packages/run-view/src/reducer/events/agentStreaming.ts +++ b/packages/run-view/src/reducer/events/agentStreaming.ts @@ -14,6 +14,8 @@ import { applyActivityDelta } from './harnessActivity.js'; * Only the channels that carry what a step is *saying* are here. A channel * that carries what it is *doing* — `activity` — belongs under the step, not * in the conversation, so it has no message and folds into its own state. + * `status` has no message either: it is the label of the step's spinner, + * which the client reads from the frame with `latestStatusLine`. */ const CHANNEL_MESSAGE: Partial< Record @@ -58,3 +60,17 @@ export function applyLiveDelta(state: RunViewState, action: LiveDeltaAction): Ru ? { ...state, messages, streamingStepExecutionId: stepExecutionId } : { ...state, messages }; } + +/** + * The status a `status` frame leaves the step in: its last whole line. A frame + * carrying no whole line — a chunk cut mid-line — says nothing new, and the + * status before it stands. + */ +export function latestStatusLine(delta: string): string | undefined { + const lines = delta.split('\n').slice(0, -1); + for (let i = lines.length - 1; i >= 0; i--) { + const line = lines[i]?.trim(); + if (line) return line; + } + return undefined; +} diff --git a/packages/schemas/src/artifact/operationDefinition.ts b/packages/schemas/src/artifact/operationDefinition.ts index 032876e5..8a7c9367 100644 --- a/packages/schemas/src/artifact/operationDefinition.ts +++ b/packages/schemas/src/artifact/operationDefinition.ts @@ -41,6 +41,7 @@ export const StepTypeSchema = z.enum([ 'host', // 298: the operator's own machine, through the paired native executor 'browser', 'plan', + 'model', // 323: the model runtime on this computer, local edition only ]); export type StepType = z.infer; diff --git a/packages/schemas/src/catalog/capabilityBundles.ts b/packages/schemas/src/catalog/capabilityBundles.ts index f743bde5..ffc9b203 100644 --- a/packages/schemas/src/catalog/capabilityBundles.ts +++ b/packages/schemas/src/catalog/capabilityBundles.ts @@ -225,6 +225,13 @@ export const CAPABILITY_BUNDLES: readonly CapabilityBundle[] = [ tier: 'on_demand', capabilityGroupIds: ['browser.page', 'browser.profile'], }, + { + id: 'local_models', + label: 'Models on this computer', + hint: 'Read why the model on this computer is slow or not answering, and propose the fix for the operator to approve: download, delete, load, unload, or the window its requests get.', + tier: 'on_demand', + capabilityGroupIds: ['model.runtime'], + }, { id: 'http', label: 'Direct HTTP', diff --git a/packages/schemas/src/catalog/index.ts b/packages/schemas/src/catalog/index.ts index a7ed4528..8698328d 100644 --- a/packages/schemas/src/catalog/index.ts +++ b/packages/schemas/src/catalog/index.ts @@ -26,6 +26,7 @@ export { isPrivilegedOperation, isEvalPlaneOperation, isPlanOperation, + isLocalModelOperation, isRunnerExcludedOperation, getOperationCapability, deriveCapabilityGroups, diff --git a/packages/schemas/src/catalog/registry.ts b/packages/schemas/src/catalog/registry.ts index e575bd0e..f07d7bdd 100644 --- a/packages/schemas/src/catalog/registry.ts +++ b/packages/schemas/src/catalog/registry.ts @@ -45,6 +45,7 @@ import { AppletOperationRegistrations } from '../applet/operations.js'; import { EvalOperationRegistrations } from '../operations/evalOps.js'; import { EvalBatchOperationRegistrations } from '../operations/evalBatchOps.js'; import { PlanOperationRegistrations } from '../operations/plan/registrations.js'; +import { ModelOperationRegistrations } from '../operations/model/registrations.js'; // ============================================================================ // Aggregate all registrations @@ -84,6 +85,7 @@ const ALL_REGISTRATIONS: OperationRegistration[] = [ ...EvalOperationRegistrations, ...EvalBatchOperationRegistrations, ...PlanOperationRegistrations, + ...ModelOperationRegistrations, ]; // ============================================================================ @@ -377,6 +379,15 @@ export function isPlanOperation(operationId: string): boolean { return operationId === 'plan' || operationId.startsWith('plan.'); } +/** + * Plan 323 D10 — the model runtime on this computer is reconfigured by the + * operator, or by Helmsman with their approval, and never by a skill's run: a + * run does not reconfigure the machine it runs on. + */ +export function isLocalModelOperation(operationId: string): boolean { + return operationId === 'model' || operationId.startsWith('model.'); +} + /** * Operations that never reach a Runner, through any channel: a delegation's * `runner_tools`, a capability grant's direct or promotable tier, or the @@ -385,7 +396,11 @@ export function isPlanOperation(operationId: string): boolean { * of them at once. */ export function isRunnerExcludedOperation(operationId: string): boolean { - return isEvalPlaneOperation(operationId) || isPlanOperation(operationId); + return ( + isEvalPlaneOperation(operationId) || + isPlanOperation(operationId) || + isLocalModelOperation(operationId) + ); } // ============================================================================ diff --git a/packages/schemas/src/credentials/index.ts b/packages/schemas/src/credentials/index.ts index 5829d56f..629e537e 100644 --- a/packages/schemas/src/credentials/index.ts +++ b/packages/schemas/src/credentials/index.ts @@ -1,2 +1,3 @@ export * from './provider.js'; export * from './registry.js'; +export * from './localRuntimeDiagnostics.js'; diff --git a/packages/schemas/src/credentials/localRuntimeDiagnostics.ts b/packages/schemas/src/credentials/localRuntimeDiagnostics.ts new file mode 100644 index 00000000..ea9287d8 --- /dev/null +++ b/packages/schemas/src/credentials/localRuntimeDiagnostics.ts @@ -0,0 +1,185 @@ +/** + * What is wrong with a space's models on this computer and what fixes it + * (Plan 323 D15). One record, computed by `diagnoseLocalRuntime`, read by + * readiness, the models surface and Helmsman alike, so the same fault is + * described in the same words everywhere and its fix is the same action. + */ +import { z } from 'zod'; + +/** The roles a space assigns a model to, as readiness reports them. */ +export const LOCAL_MODEL_ROLES = ['default', 'helmsman', 'runner', 'coach', 'judge'] as const; +export const LocalModelRoleSchema = z.enum(LOCAL_MODEL_ROLES); +export type LocalModelRole = z.infer; + +const model = z.string().min(1).describe('The catalog id of a model on this computer.'); + +const pull = z + .object({ action: z.literal('pull'), model }) + .describe('Download a model to this computer.'); +const remove = z + .object({ action: z.literal('delete'), model }) + .describe('Delete a downloaded model from this computer, freeing its disk.'); +const warm = z + .object({ action: z.literal('warm'), model }) + .describe('Load a model now, at the window its requests use.'); +const unload = z + .object({ action: z.literal('unload'), model }) + .describe( + 'Unload a model, freeing its memory and every conversation it holds. Runs using it reload it on their next turn.', + ); +const setContext = z + .object({ + action: z.literal('set_context'), + contextTokens: z + .number() + .int() + .positive() + .nullable() + .describe('The window every request gets, or null for the largest that fits the machine.'), + }) + .describe("Set the context window on the space's Ollama connection."); +const useForEveryRole = z + .object({ action: z.literal('use_for_every_role'), model }) + .describe('Assign one model to every role, so the runtime never switches models between them.'); + +/** + * The changes to the runtime itself, which Helmsman may propose for approval. + * A role's model is not among them: it is the space's setting, written where + * every role is set at once. + */ +export const LocalRuntimeChangeSchema = z.discriminatedUnion('action', [ + pull, + remove, + warm, + unload, + setContext, +]); +export type LocalRuntimeChange = z.infer; + +/** + * Every change the platform can make to a space's local runtime. An issue's + * remedy names one with its arguments filled in; the operator runs it from the + * issue's card, and Helmsman proposes the runtime changes for approval. + */ +export const LocalModelActionSchema = z.discriminatedUnion('action', [ + pull, + remove, + warm, + unload, + setContext, + useForEveryRole, +]); +export type LocalModelAction = z.infer; + +export const LocalRuntimeRemedySchema = z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('action'), + action: LocalModelActionSchema, + label: z.string().describe('What the action does, as its button says it.'), + }), + z.object({ + kind: z.literal('instruction'), + text: z.string().describe('What the operator does where nothing in Aflow can fix it.'), + }), +]); +export type LocalRuntimeRemedy = z.infer; + +export const LocalRuntimeIssueCodeSchema = z.enum([ + /** The runtime does not answer at the connection's URL. */ + 'runtime_unreachable', + /** A role is assigned a model the runtime has not downloaded. */ + 'model_not_installed', + /** Even the agents' smallest window spills past the GPU. */ + 'window_does_not_fit', + /** The model fits the GPU, but with everything else running the machine will swap. */ + 'window_tight', + /** A loaded model has grown past the GPU, holding every conversation's cache. */ + 'model_spilling', + /** Part of a loaded model runs on the CPU. */ + 'model_partly_on_cpu', + /** The downloaded build is not the one the catalog entry measured. */ + 'build_unverified', + /** Roles use different models, and each switch between them reloads. */ + 'mixed_local_models', +]); +export type LocalRuntimeIssueCode = z.infer; + +export const LocalRuntimeIssueSchema = z.object({ + code: LocalRuntimeIssueCodeSchema, + /** `error` stops a role from running; the others slow it or cost the operator something. */ + severity: z.enum(['error', 'warning', 'info']), + model: model.optional(), + cause: z.string().describe('What is wrong, with the numbers that show it.'), + remedy: LocalRuntimeRemedySchema, +}); +export type LocalRuntimeIssue = z.infer; + +export const LocalModelFitStatusSchema = z.enum(['fits', 'tight', 'does_not_fit', 'unknown']); + +export const LocalModelPullSchema = z.object({ + status: z.enum(['downloading', 'interrupted', 'failed']), + completedBytes: z.number().nonnegative(), + totalBytes: z.number().nonnegative().nullable(), + /** What the runtime last said: a layer's digest, `verifying`, `writing manifest`. */ + detail: z.string().nullable(), + error: z.string().nullable(), + startedAt: z.string(), +}); +export type LocalModelPull = z.infer; + +export const LocalModelDiagnosisSchema = z.object({ + model, + /** What a role assignment stores for this model: its alias where it has one, since an id names one version. */ + ref: z.string(), + tag: z.string(), + displayName: z.string(), + installed: z.boolean(), + /** The download's size: what `/api/tags` reports once installed, the catalog's measure before. */ + downloadBytes: z.number().nullable(), + roles: z.array(LocalModelRoleSchema), + fit: z.object({ + status: LocalModelFitStatusSchema, + window: z.number().int(), + largestFittingWindow: z.number().int().nullable(), + bytesAtWindow: z.number().nullable(), + }), + /** What the runtime holds now, or null when the model is not loaded. */ + loaded: z + .object({ + sizeBytes: z.number(), + /** The part on the GPU; below `sizeBytes` the rest runs on the CPU. */ + gpuBytes: z.number().nullable(), + contextLength: z.number().nullable(), + expiresAt: z.string().nullable(), + }) + .nullable(), + /** `verified` when the installed build is the one measured; `unmeasured` before it is installed. */ + build: z.enum(['verified', 'unverified', 'unmeasured']), + pull: LocalModelPullSchema.nullable(), +}); +export type LocalModelDiagnosis = z.infer; + +export const LocalRuntimeDiagnosticsSchema = z.object({ + runtime: z.object({ + baseUrl: z.string(), + reachable: z.boolean(), + version: z.string().nullable(), + }), + machine: z + .object({ + totalMemoryBytes: z.number(), + gpuWorkingSetBytes: z.number().nullable(), + freeDiskBytes: z.number().nullable(), + }) + .nullable(), + connection: z.object({ + /** The window the operator set, or null for the largest that fits. */ + contextTokens: z.number().int().nullable(), + keepAlive: z.string().nullable(), + parallel: z.number().int(), + }), + /** The lineup's models and any role's model, installed or not. */ + models: z.array(LocalModelDiagnosisSchema), + issues: z.array(LocalRuntimeIssueSchema), +}); +export type LocalRuntimeDiagnostics = z.infer; diff --git a/packages/schemas/src/edition/composedOperations.ts b/packages/schemas/src/edition/composedOperations.ts index aa51d18b..dd8167a0 100644 --- a/packages/schemas/src/edition/composedOperations.ts +++ b/packages/schemas/src/edition/composedOperations.ts @@ -24,6 +24,8 @@ export function isStepTypeComposed(stepType: string, lanes: ComposedLanes): bool if (stepType === 'code') return lanes.codeLane === 'present'; if (stepType === 'host') return lanes.hostLane === 'present'; if (stepType === 'browser') return lanes.browserLane === 'present'; + // A model on this computer runs only where the platform does. + if (stepType === 'model') return lanes.edition === 'community-local'; return true; } @@ -45,6 +47,7 @@ export function isOperationComposed(operationId: string, lanes: ComposedLanes): */ const LANE_PHRASES: Readonly> = { browser: 'the browser lane, a browser on a paired machine', + model: 'models on this computer, which only the local edition runs', }; /** diff --git a/packages/schemas/src/models/actionCenter.ts b/packages/schemas/src/models/actionCenter.ts index 2be88f9c..9bc45b56 100644 --- a/packages/schemas/src/models/actionCenter.ts +++ b/packages/schemas/src/models/actionCenter.ts @@ -7,6 +7,7 @@ import { BrowserApprovalAskedBySchema, BrowserApprovalValueSummarySchema, BrowserWriteApprovalRequestPayloadSchema, + LocalModelWriteApprovalRequestPayloadSchema, } from '../runtime/requestedInput.js'; // ============================================================================ @@ -449,9 +450,23 @@ export const BrowserWriteApprovalExtensionSchema = z.object({ }); export type BrowserWriteApprovalExtension = z.infer; +/** The same card for a change to the model runtime on this computer (Plan 323 D10). */ +export const LocalModelWriteApprovalExtensionSchema = z.object({ + kind: z.literal('write_approval'), + target: z.literal('local_model'), + change: LocalModelWriteApprovalRequestPayloadSchema.shape.change, + modelName: z.string().optional(), + runtimeUrl: z.string(), + impact: LocalModelWriteApprovalRequestPayloadSchema.shape.impact, +}); +export type LocalModelWriteApprovalExtension = z.infer< + typeof LocalModelWriteApprovalExtensionSchema +>; + export const WriteApprovalExtensionSchema = z.discriminatedUnion('target', [ ApiWriteApprovalExtensionSchema, BrowserWriteApprovalExtensionSchema, + LocalModelWriteApprovalExtensionSchema, ]); export type WriteApprovalExtension = z.infer; diff --git a/packages/schemas/src/operations/index.ts b/packages/schemas/src/operations/index.ts index 114893dd..e8529f51 100644 --- a/packages/schemas/src/operations/index.ts +++ b/packages/schemas/src/operations/index.ts @@ -55,3 +55,4 @@ export * from './store.js'; export * from './evalOps.js'; export * from './evalBatchOps.js'; export * from './plan/index.js'; +export * from './model/index.js'; diff --git a/packages/schemas/src/operations/model/index.ts b/packages/schemas/src/operations/model/index.ts new file mode 100644 index 00000000..209097ac --- /dev/null +++ b/packages/schemas/src/operations/model/index.ts @@ -0,0 +1,2 @@ +export * from './schemas.js'; +export * from './registrations.js'; diff --git a/packages/schemas/src/operations/model/registrations.ts b/packages/schemas/src/operations/model/registrations.ts new file mode 100644 index 00000000..bd3fd04f --- /dev/null +++ b/packages/schemas/src/operations/model/registrations.ts @@ -0,0 +1,71 @@ +import type { OperationRegistration } from '../../catalog/operationCatalog.js'; +import { + ModelRuntimeChangeInputSchema, + ModelRuntimeChangeOutputSchema, + ModelRuntimeStatusInputSchema, + ModelRuntimeStatusOutputSchema, +} from './schemas.js'; + +/** + * Plan 323 D10 — Helmsman reads the model runtime on this computer and asks + * before changing it. Never a skill's (`isLocalModelOperation`): a run does not + * reconfigure the machine it runs on. + */ +export const ModelOperationRegistrations: OperationRegistration[] = [ + { + stepType: 'model', + group: 'runtime', + verb: 'status', + name: 'Model Runtime Status', + actionLabel: 'Checking the models on this computer…', + groupDisplayName: 'Models on this computer', + groupDescription: + 'The model runtime on the operator’s machine: what it has downloaded and loaded, how it fits the machine, and what is wrong with it.', + semanticDescription: + 'Read the diagnosis of the space’s models on this computer: whether Ollama answers, each model’s fit to the machine and what is loaded — how much of it is on the GPU — and every issue with the numbers behind it and its remedy. Reading loads nothing.', + tags: ['model', 'local', 'ollama', 'diagnostics', 'slow'], + idempotency: 'idempotent', + mutates: false, + usage: { + oneLine: 'Why the model on this computer is slow or not answering, and what fixes it.', + whenToUse: [ + 'The operator asks why responses are slow, or why the model is not answering', + 'Before proposing a change to the runtime, to propose the remedy an issue states', + ], + whenNotToUse: ['A model a hosted provider serves — this reads only the one on this computer'], + pitfalls: [ + 'Explain from the issue’s cause, which carries the numbers; propose its remedy as written', + ], + minimalExampleInput: {}, + }, + accessMode: 'read', + inputZod: ModelRuntimeStatusInputSchema, + outputZod: ModelRuntimeStatusOutputSchema, + }, + { + stepType: 'model', + group: 'runtime', + verb: 'change', + name: 'Change Model Runtime', + actionLabel: 'Asking to change the models on this computer…', + semanticDescription: + 'Propose a change to the model runtime on this computer — download, delete, load or unload a model, or set the window its requests get. The operator approves it on a card first, which says what it costs and which requests in flight it ends; nothing changes until they do.', + tags: ['model', 'local', 'ollama', 'pull', 'unload', 'context'], + idempotency: 'non_idempotent', + mutates: true, + usage: { + oneLine: 'Propose an issue’s remedy for the operator to approve.', + whenToUse: ['model.runtime.status names an issue whose remedy is an action'], + whenNotToUse: [ + 'A remedy that assigns a role’s model (`use_for_every_role`) — that is the operator’s, under Settings → Agents; say so instead', + ], + pitfalls: [ + 'Unloading or switching the model this conversation runs on makes its next turn load it again, a pause of up to a minute', + ], + minimalExampleInput: { change: { action: 'unload', model: 'ollama-gemma4-26b' } }, + }, + accessMode: 'write', + inputZod: ModelRuntimeChangeInputSchema, + outputZod: ModelRuntimeChangeOutputSchema, + }, +]; diff --git a/packages/schemas/src/operations/model/schemas.ts b/packages/schemas/src/operations/model/schemas.ts new file mode 100644 index 00000000..a8e2c940 --- /dev/null +++ b/packages/schemas/src/operations/model/schemas.ts @@ -0,0 +1,27 @@ +import { z } from 'zod'; +import { buildOperationId } from '../../catalog/operationId.js'; +import { + LocalRuntimeChangeSchema, + LocalRuntimeDiagnosticsSchema, +} from '../../credentials/localRuntimeDiagnostics.js'; + +export const MODEL_RUNTIME_STATUS_OPERATION_ID = buildOperationId('model', 'runtime', 'status'); +export const MODEL_RUNTIME_CHANGE_OPERATION_ID = buildOperationId('model', 'runtime', 'change'); + +export const ModelRuntimeStatusInputSchema = z.object({}).strict(); + +export const ModelRuntimeStatusOutputSchema = z.object({ + diagnostics: LocalRuntimeDiagnosticsSchema.nullable().describe( + 'The runtime, the machine, every model with its fit and what is loaded, and each issue with its cause and remedy. Null when the space has no Ollama connection.', + ), +}); + +export const ModelRuntimeChangeInputSchema = z.object({ + change: LocalRuntimeChangeSchema.describe( + "The change, as an issue's remedy states it — pass its `action` object unchanged. The operator approves it on a card before it runs.", + ), +}); + +export const ModelRuntimeChangeOutputSchema = z.object({ + message: z.string().describe('What the runtime did.'), +}); diff --git a/packages/schemas/src/runtime/harnessActivity.test.ts b/packages/schemas/src/runtime/harnessActivity.test.ts index fd8fde00..499eb927 100644 --- a/packages/schemas/src/runtime/harnessActivity.test.ts +++ b/packages/schemas/src/runtime/harnessActivity.test.ts @@ -7,7 +7,7 @@ describe('the live channels', () => { it('carries what a step is doing on a channel of its own', () => { // Folded into `text` these lines would render as the agent's own message. expect(LiveDeltaChannelSchema.options).toContain('activity'); - expect(LiveDeltaChannelSchema.options).toEqual(['text', 'thinking', 'activity']); + expect(LiveDeltaChannelSchema.options).toEqual(['text', 'thinking', 'activity', 'status']); }); }); diff --git a/packages/schemas/src/runtime/requestedInput.ts b/packages/schemas/src/runtime/requestedInput.ts index aa8bed2c..d52d68ec 100644 --- a/packages/schemas/src/runtime/requestedInput.ts +++ b/packages/schemas/src/runtime/requestedInput.ts @@ -6,6 +6,7 @@ * SessionHotState.requestedInputRef. The UI loads this to render input forms. */ import { z } from 'zod'; +import { LocalRuntimeChangeSchema } from '../credentials/localRuntimeDiagnostics.js'; import { BROWSER_ACTIONS } from '../operations/browser.js'; import { OAuthConsentIntegrationKindSchema, @@ -219,9 +220,62 @@ export function browserApprovalActionPhrase( } } +/** + * A change Helmsman proposes to the model runtime on this computer (Plan 323 + * D10). What the card shows is what the change costs: the requests in flight + * that an unload or delete ends, whether this conversation's own next turn + * loads the model again, and the change's own figures (a download's size and + * the disk it leaves, the memory a window takes). The change and whether it + * reloads the asking conversation are in the hash, so an approval acts only on + * the change it showed; the figures move by the second and stay out of it. + */ +export const LocalModelWriteApprovalRequestPayloadSchema = z.object({ + kind: z.literal('write_approval'), + target: z.literal('local_model'), + change: LocalRuntimeChangeSchema, + /** The model the change names, as the operator knows it. */ + modelName: z.string().optional(), + runtimeUrl: z.string(), + impact: z.object({ + /** Requests running on the runtime now, on any model; an unload or delete ends those on its model. */ + inFlightRequests: z.number().int().nonnegative(), + /** The asking conversation runs on this model, so its next turn loads it again. */ + ownConversationReloads: z.boolean(), + notes: z.array(z.string()), + }), + decidedBefore: z.string().optional(), + requestHash: z.string(), +}); +export type LocalModelWriteApprovalRequestPayload = z.infer< + typeof LocalModelWriteApprovalRequestPayloadSchema +>; + +/** What a proposed runtime change does, as the card and the Action Center item say it. */ +export function localModelChangePhrase( + request: Pick, +): string { + const { change } = request; + const name = request.modelName ?? ('model' in change ? change.model : ''); + switch (change.action) { + case 'pull': + return `download ${name}`; + case 'delete': + return `delete ${name}`; + case 'warm': + return `load ${name}`; + case 'unload': + return `unload ${name}`; + case 'set_context': + return change.contextTokens === null + ? 'give requests the largest window that fits this computer' + : `give requests a ${String(change.contextTokens)}-token window`; + } +} + export const WriteApprovalRequestPayloadSchema = z.discriminatedUnion('target', [ ApiWriteApprovalRequestPayloadSchema, BrowserWriteApprovalRequestPayloadSchema, + LocalModelWriteApprovalRequestPayloadSchema, ]); export type WriteApprovalRequestPayload = z.infer; diff --git a/packages/schemas/src/runtime/sessionBlockedOn.ts b/packages/schemas/src/runtime/sessionBlockedOn.ts index 0c5150ca..2c05ab04 100644 --- a/packages/schemas/src/runtime/sessionBlockedOn.ts +++ b/packages/schemas/src/runtime/sessionBlockedOn.ts @@ -14,7 +14,7 @@ export const OAuthConsentReasonSchema = z.enum(['never_connected', 'expired']); export type OAuthConsentReason = z.infer; /** - * A step parked on an operator's approval. The two variants mirror the + * A step parked on an operator's approval. The variants mirror the * approval payload's (`target`), so a surface says what is waiting without * reading the payload. */ @@ -46,6 +46,16 @@ const NeedsWriteApprovalSchema = z.discriminatedUnion('target', [ /** Binds the pending decision to the exact action. */ requestHash: z.string(), }), + z.object({ + kind: z.literal('needs_write_approval'), + target: z.literal('local_model'), + /** The paused step awaiting a human approval of a change to the model runtime. */ + stepExecutionId: z.string(), + action: z.string(), + modelName: z.string().optional(), + /** Binds the pending decision to the exact change and its impact. */ + requestHash: z.string(), + }), ]); const SessionBlockedOnByKindSchema = z.discriminatedUnion('kind', [ diff --git a/packages/schemas/src/runtime/streamMessages.ts b/packages/schemas/src/runtime/streamMessages.ts index 98d8f075..0b9a3023 100644 --- a/packages/schemas/src/runtime/streamMessages.ts +++ b/packages/schemas/src/runtime/streamMessages.ts @@ -36,8 +36,13 @@ import { SimulatedFulfillmentReportSchema } from '../simulation/fulfillmentRepor * JSON-encoded `HarnessActivityLine` per newline-terminated delta. It is a * channel of its own because a reader that folded it into `text` would render * a harness's tool calls as the agent's own message. + * + * `status` carries what a step is waiting on before it produces anything — a + * model on this computer loading, or reading a long prompt, both silent for + * tens of seconds. One newline-terminated line per delta; the last whole line + * is the step's status, and the first `thinking` or `text` delta supersedes it. */ -export const LiveDeltaChannelSchema = z.enum(['text', 'thinking', 'activity']); +export const LiveDeltaChannelSchema = z.enum(['text', 'thinking', 'activity', 'status']); export type LiveDeltaChannel = z.infer; /** diff --git a/packages/server-runtime/src/bootstrap/redisAcl.test.ts b/packages/server-runtime/src/bootstrap/redisAcl.test.ts index 354c8f49..bce9bf3f 100644 --- a/packages/server-runtime/src/bootstrap/redisAcl.test.ts +++ b/packages/server-runtime/src/bootstrap/redisAcl.test.ts @@ -10,7 +10,7 @@ import { describe, expect, it } from 'vitest'; import { - browserApprovalSpentKey, + writeApprovalSpentKey, browserAskKey, browserHandoffKey, browserHandoffSpaceIndexKey, @@ -49,14 +49,16 @@ describe('redis acl', () => { }); it('writes the records beside a browser approval, which sit outside the grant family', () => { - const askKeys = [ - browserAskKey('t', 'run-1', 'call'), - browserApprovalSpentKey('t', 'run-1', { requestHash: 'h', decidedAt: 'now' }), - ]; const rules = hostLine.split(' '); expect(rules.filter((rule) => rule.includes('browser-ask'))).toEqual(['~aflow:browser-ask:*']); - for (const key of askKeys) { - expect(key.startsWith('aflow:browser-ask:')).toBe(true); + expect(rules.filter((rule) => rule.includes('approval-spent'))).toEqual([ + '~aflow:approval-spent:*', + ]); + const ask = browserAskKey('t', 'run-1', 'call'); + const spent = writeApprovalSpentKey('t', 'run-1', { requestHash: 'h', decidedAt: 'now' }); + expect(ask.startsWith('aflow:browser-ask:')).toBe(true); + expect(spent.startsWith('aflow:approval-spent:')).toBe(true); + for (const key of [ask, spent]) { expect(key.startsWith('aflow:write-approval:')).toBe(false); } }); diff --git a/packages/server-runtime/src/bootstrap/redisAcl.ts b/packages/server-runtime/src/bootstrap/redisAcl.ts index 5b87a2a5..25c4fd89 100644 --- a/packages/server-runtime/src/bootstrap/redisAcl.ts +++ b/packages/server-runtime/src/bootstrap/redisAcl.ts @@ -166,6 +166,7 @@ const HOST_KEY_PATTERNS = [ // grant stays read-only above — so writing one can only make the executor // ask again or refuse. '~aflow:browser-ask:*', + '~aflow:approval-spent:*', ] as const; /** diff --git a/packages/server-runtime/src/lib/agentModelPolicy.test.ts b/packages/server-runtime/src/lib/agentModelPolicy.test.ts index fff81974..2d43fd69 100644 --- a/packages/server-runtime/src/lib/agentModelPolicy.test.ts +++ b/packages/server-runtime/src/lib/agentModelPolicy.test.ts @@ -20,36 +20,40 @@ describe('canonical comparison', () => { it('admits a directive alias against an allowlist stored as ids', () => { // The gate's original bug: default directives hold `glm-pro`, the Settings // page saves catalog ids, and comparing the strings refused the same model. - const allowed = allowedModelIds([GLM_ID]); + const allowed = allowedModelIds([GLM_ID], 'enterprise'); expect(isModelIdAllowed('glm-pro', allowed)).toBe(true); expect(isModelIdAllowed(GLM_ID, allowed)).toBe(true); }); it('admits an id against an allowlist stored as an alias', () => { - const allowed = allowedModelIds(['luna']); + const allowed = allowedModelIds(['luna'], 'enterprise'); expect(isModelIdAllowed('gpt-6-luna', allowed)).toBe(true); }); it('still refuses a model the tenant did not enable', () => { - expect(isModelIdAllowed('claude-sonnet-5-5', allowedModelIds(['luna']))).toBe(false); + expect(isModelIdAllowed('claude-sonnet-5-5', allowedModelIds(['luna'], 'enterprise'))).toBe( + false, + ); }); it('refuses a ref that names no model', () => { - expect(isModelIdAllowed('not-a-model', allowedModelIds(['luna']))).toBe(false); + expect(isModelIdAllowed('not-a-model', allowedModelIds(['luna'], 'enterprise'))).toBe(false); }); it('falls back to the platform recommendations when the tenant never chose', () => { - expect(isModelIdAllowed('claude-sonnet-5-5', allowedModelIds(null))).toBe(true); + expect(isModelIdAllowed('claude-sonnet-5-5', allowedModelIds(null, 'enterprise'))).toBe(true); }); }); describe('tenantDefaultModelId', () => { it('keeps the platform default when the tenant allows it', () => { - expect(tenantDefaultModelId(allowedModelIds([GLM_ID, 'gpt-6-luna']))).toBe(GLM_ID); + expect(tenantDefaultModelId(allowedModelIds([GLM_ID, 'gpt-6-luna'], 'enterprise'))).toBe( + GLM_ID, + ); }); it('picks one the tenant does allow when the platform default is excluded', () => { - expect(tenantDefaultModelId(allowedModelIds(['gpt-6-luna']))).toBe('gpt-6-luna'); + expect(tenantDefaultModelId(allowedModelIds(['gpt-6-luna'], 'enterprise'))).toBe('gpt-6-luna'); }); }); @@ -60,7 +64,7 @@ describe('withTenantDefaultModels', () => { // Space creation carries `glm-pro` from a Zod default even when the caller // never named a model; without this every create would 400 for a tenant // that excludes GLM. - const allowed = allowedModelIds(['gemini-3.8-flash']); + const allowed = allowedModelIds(['gemini-3.8-flash'], 'enterprise'); const out = withTenantDefaultModels({ modelDefaults: { default: 'glm-pro' } }, allowed, NONE); expect(out.modelDefaults?.['default']).toBe('gemini-3.8-flash'); }); @@ -68,7 +72,7 @@ describe('withTenantDefaultModels', () => { it('leaves an explicitly requested model alone, even the platform default', () => { // Substituting here would hand the caller a space running a model they did // not choose; the gate refuses the request instead. - const allowed = allowedModelIds(['gemini-3.8-flash']); + const allowed = allowedModelIds(['gemini-3.8-flash'], 'enterprise'); const out = withTenantDefaultModels( { modelDefaults: { default: 'glm-pro' } }, allowed, @@ -78,7 +82,7 @@ describe('withTenantDefaultModels', () => { }); it('leaves a deliberately chosen other model alone for the gate to refuse', () => { - const allowed = allowedModelIds(['gemini-3.8-flash']); + const allowed = allowedModelIds(['gemini-3.8-flash'], 'enterprise'); const out = withTenantDefaultModels( { modelDefaults: { default: 'claude-sonnet-5-5' } }, allowed, @@ -88,13 +92,13 @@ describe('withTenantDefaultModels', () => { }); it('leaves an allowed default untouched', () => { - const allowed = allowedModelIds([GLM_ID]); + const allowed = allowedModelIds([GLM_ID], 'enterprise'); const out = withTenantDefaultModels({ modelDefaults: { default: 'glm-pro' } }, allowed, NONE); expect(out.modelDefaults?.['default']).toBe('glm-pro'); }); it('rehomes only the roles the caller left unnamed', () => { - const allowed = allowedModelIds(['gpt-6-luna']); + const allowed = allowedModelIds(['gpt-6-luna'], 'enterprise'); const out = withTenantDefaultModels( { modelDefaults: { default: 'glm-pro', runner: 'glm-pro', coach: 'gpt-6-luna' } }, allowed, @@ -109,7 +113,7 @@ describe('withTenantDefaultModels', () => { }); describe('the Clerk against the same gate', () => { - const recommended = { ids: allowedModelIds(null), explicit: false }; + const recommended = { ids: allowedModelIds(null, 'enterprise'), explicit: false }; it('lets every value its picker can produce be saved', () => { // Each of these was refused before the gate learned the role: the two @@ -122,7 +126,7 @@ describe('the Clerk against the same gate', () => { }); it('still holds the Clerk to a tenant that named its own set', () => { - const strict = { ids: allowedModelIds(['sonnet']), explicit: true }; + const strict = { ids: allowedModelIds(['sonnet'], 'enterprise'), explicit: true }; expect(offListAgentModelRefs({ modelDefaults: { clerk: 'glm-flash' } }, strict)).toEqual([ 'glm-flash', ]); @@ -141,7 +145,7 @@ describe('the Clerk against the same gate', () => { it('leaves a Clerk mode alone when filling tenant defaults', () => { // The filler runs before the gate. Reading `auto` as an unassignable ref // replaces the operator's "pick one for me" with a pinned foreground model. - const allowed = allowedModelIds([GLM_ID]); + const allowed = allowedModelIds([GLM_ID], 'enterprise'); const filled = withTenantDefaultModels( { modelDefaults: { clerk: 'auto', helmsman: 'some-retired-model' } }, allowed, @@ -153,3 +157,16 @@ describe('the Clerk against the same gate', () => { }); }); }); + +describe('models on this computer', () => { + it('may be assigned in the local edition by a tenant following the recommendation', () => { + expect(isModelIdAllowed('gemma-local', allowedModelIds(null, 'community-local'))).toBe(true); + }); + + it('are held to a tenant that named its own set, and absent from the hosted edition', () => { + expect(isModelIdAllowed('gemma-local', allowedModelIds(['sonnet'], 'community-local'))).toBe( + false, + ); + expect(isModelIdAllowed('gemma-local', allowedModelIds(null, 'enterprise'))).toBe(false); + }); +}); diff --git a/packages/server-runtime/src/lib/agentModelPolicy.ts b/packages/server-runtime/src/lib/agentModelPolicy.ts index 886e179f..9f9d04b8 100644 --- a/packages/server-runtime/src/lib/agentModelPolicy.ts +++ b/packages/server-runtime/src/lib/agentModelPolicy.ts @@ -15,6 +15,7 @@ import { CLERK_SPACE_DEFAULT, DEFAULT_CYBERNETIC_MODEL, effectiveAgentModelRefs, + type EditionId, } from '@aflow/schemas'; /** The catalog id behind a ref, or undefined when it names no model. */ @@ -22,9 +23,23 @@ export function canonicalModelId(ref: string): string | undefined { return createDefaultModelCatalog().getModel(ref)?.id; } -/** Catalog ids a space may assign, given whatever the tenant stored. */ -export function allowedModelIds(stored: readonly string[] | null | undefined): Set { +/** + * Catalog ids a space may assign, given whatever the tenant stored. + * + * In the local edition a tenant following the platform recommendation may also + * assign the catalog's models on this computer (Plan 323 D18): nothing is + * billed for them and the machine is the operator's own, so holding them back + * until a tenant names its own set would leave an operator with no cloud key + * nothing to run. A tenant that named its set is held to it, as everywhere. + */ +export function allowedModelIds( + stored: readonly string[] | null | undefined, + edition: EditionId, +): Set { const ids = new Set(); + if (!(stored && stored.length > 0) && edition === 'community-local') { + for (const model of createDefaultModelCatalog().listModels('ollama')) ids.add(model.id); + } for (const ref of effectiveAgentModelRefs(stored)) { const id = canonicalModelId(ref); // An unresolvable stored ref is dropped rather than compared literally: it diff --git a/packages/server-runtime/src/lib/localRuntime.ts b/packages/server-runtime/src/lib/localRuntime.ts deleted file mode 100644 index abb8af82..00000000 --- a/packages/server-runtime/src/lib/localRuntime.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** - * The space's runtime on the operator's machine, resolved as the AI executor - * resolves it — same credential chain, same defaults, same machine — so what - * readiness reports and what the operator's actions reach is the runtime the - * space's models actually run on (Plan 323). - */ -import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'; -import { localRuntimeClient, type LocalRuntimeClient, type ProviderConfig } from '@aflow/ai-client'; -import { - CredentialResolver, - createProviderCredentialDbLoader, - providerConfigFromCredential, -} from '@aflow/credential-resolver'; -import { readLocalMachine, type Redis } from '@aflow/redis'; - -export interface ResolvedLocalRuntime { - config: ProviderConfig; - client: LocalRuntimeClient; -} - -export async function resolveSpaceLocalRuntime(input: { - db: PostgresJsDatabase; - redis: Redis | undefined; - tenantId: string; - userId: string; - spaceId: string; -}): Promise { - const resolver = new CredentialResolver({ loader: createProviderCredentialDbLoader(input.db) }); - const resolved = await resolver - .resolve('ollama', { - tenantId: input.tenantId, - credentialOwnerId: input.userId, - spaceId: input.spaceId, - }) - .catch(() => null); - if (!resolved) return null; - const machine = input.redis - ? await readLocalMachine(input.redis).catch(() => undefined) - : undefined; - const config = providerConfigFromCredential('ollama', resolved, { machine }); - return { config, client: localRuntimeClient(config) }; -} diff --git a/packages/server-runtime/src/routes/localModels.test.ts b/packages/server-runtime/src/routes/localModels.test.ts index 68264336..c0cb2aa0 100644 --- a/packages/server-runtime/src/routes/localModels.test.ts +++ b/packages/server-runtime/src/routes/localModels.test.ts @@ -1,16 +1,36 @@ /** - * Loading and unloading a model on this computer is the local edition's, and - * only for a model the catalog says runs there. + * The models on this computer are the local edition's, and the actions on them + * reach only a model the catalog says runs there. */ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import Fastify from 'fastify'; import { serializerCompiler, validatorCompiler } from 'fastify-type-provider-zod'; import type { EditionId } from '@aflow/schemas'; -import { localModelRoutes } from './localModels.js'; + +const connected = vi.hoisted(() => ({ value: true })); + +vi.mock('@aflow/cybernetic-runtime', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveSpaceLocalRuntime: async () => + connected.value + ? { + config: {}, + client: {}, + runtimeId: 'this-machine:11434', + credential: { id: 'c', scope: 'user', scopeId: 'u' }, + hostMachine: undefined, + } + : null, + }; +}); + +const { localModelRoutes } = await import('./localModels.js'); const SPACE = '00000000-0000-0000-0000-0000000000a1'; -async function post(edition: EditionId, model: string) { +async function post(edition: EditionId, payload: Record) { const app = Fastify({ logger: false }); app.setValidatorCompiler(validatorCompiler); app.setSerializerCompiler(serializerCompiler); @@ -21,22 +41,36 @@ async function post(edition: EditionId, model: string) { await app.register(localModelRoutes); const res = await app.inject({ method: 'POST', - url: `/${SPACE}/local-models/warm`, - payload: { model }, + url: `/${SPACE}/local-models/actions`, + payload, }); await app.close(); return res; } -describe('POST /spaces/:spaceId/local-models/:action', () => { +describe('POST /spaces/:spaceId/local-models/actions', () => { it('is refused in the hosted edition', async () => { - const res = await post('enterprise', 'gemma-local'); + const res = await post('enterprise', { action: 'warm', model: 'gemma-local' }); expect(res.statusCode).toBe(404); expect(res.json().message).toContain('local edition'); }); it('refuses a model that does not run on this computer', async () => { - const res = await post('community-local', 'glm-pro'); + connected.value = true; + const res = await post('community-local', { action: 'warm', model: 'glm-pro' }); + expect(res.statusCode).toBe(400); + }); + + it('says where to add a connection when the space has none', async () => { + connected.value = false; + const res = await post('community-local', { action: 'warm', model: 'gemma-local' }); + expect(res.statusCode).toBe(409); + expect(res.json().message).toContain('Ollama connection'); + }); + + it('refuses an action the schema does not carry', async () => { + connected.value = true; + const res = await post('community-local', { action: 'reboot', model: 'gemma-local' }); expect(res.statusCode).toBe(400); }); }); diff --git a/packages/server-runtime/src/routes/localModels.ts b/packages/server-runtime/src/routes/localModels.ts index 229c2f6a..6561b3f7 100644 --- a/packages/server-runtime/src/routes/localModels.ts +++ b/packages/server-runtime/src/routes/localModels.ts @@ -1,48 +1,134 @@ /** - * The operator's actions on the space's models on this computer: load one - * before it is first needed, or unload one to give the machine its memory - * back (Plan 323 D10). Operator-only REST, never an operation: no agent loads - * gigabytes or evicts the model a run is on. + * The space's models on this computer (Plan 323 D10, D15): the diagnosis every + * surface reads, and the operator's actions on the runtime — download, delete, + * load, unload, and the window its requests get. Operator REST: Helmsman + * reaches the same actions only through an approved card. * * Registered in every edition and refused outside the local one, where no * model runs on the operator's machine. */ -import type { FastifyPluginAsync } from 'fastify'; +import type { FastifyPluginAsync, FastifyReply, FastifyRequest } from 'fastify'; import { z } from 'zod'; import type { ZodTypeProvider } from 'fastify-type-provider-zod'; import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'; -import { contextWindowForModel, createDefaultModelCatalog } from '@aflow/ai-client'; import type { Redis } from '@aflow/redis'; -import { resolveSpaceLocalRuntime } from '../lib/localRuntime.js'; +import { + applyLocalModelAction, + cancelLocalModelPull, + diagnoseSpaceLocalRuntime, + localRoleModels, + readSpaceModelDefaults, + resolveSpaceLocalRuntime, + type LocalModelActionOutcome, + type SpaceLocalRuntime, +} from '@aflow/cybernetic-runtime'; +import { LocalModelActionSchema, LocalRuntimeDiagnosticsSchema } from '@aflow/schemas'; const ErrorSchema = z.object({ error: z.string(), message: z.string() }); +const OUTCOME_STATUS: Record< + Extract['reason'], + 400 | 409 | 502 +> = { + not_local: 400, + not_here: 400, + no_disk: 409, + runtime: 502, +}; + +const spaceAuthz = (action: 'read' | 'admin') => ({ + authz: { + resource: 'space' as const, + action, + spaceIdFrom: 'param' as const, + resourceIdFrom: 'param' as const, + resourceIdParam: 'spaceId', + }, +}); + export const localModelRoutes: FastifyPluginAsync = async (fastify) => { const app = fastify.withTypeProvider(); + const db = (): PostgresJsDatabase => fastify.appContext.db as PostgresJsDatabase; + const redis = (): Redis | undefined => fastify.appContext.redis as Redis | undefined; - app.post( - '/:spaceId/local-models/:action', + /** Null once it has answered the request itself: outside the local edition, or with no connection. */ + async function runtimeFor( + request: Pick, + spaceId: string, + reply: FastifyReply, + { requireConnection }: { requireConnection: boolean }, + ): Promise<{ runtime: SpaceLocalRuntime | null; tenantId: string } | null> { + if (fastify.edition.edition !== 'community-local') { + await reply.status(404).send({ + error: 'NotFound', + message: + 'Models on this computer belong to the local edition, which runs on the machine they run on.', + }); + return null; + } + const tenant = await request.requireTenant(); + await request.requireSpace(); + const runtime = await resolveSpaceLocalRuntime({ + db: db(), + redis: redis(), + tenantId: tenant.tenantId as string, + userId: request.authUser?.userId as string, + spaceId, + }); + if (!runtime && requireConnection) { + await reply.status(409).send({ + error: 'NoConnection', + message: 'This space has no Ollama connection. Add one under Settings → Credentials.', + }); + return null; + } + return { runtime, tenantId: tenant.tenantId as string }; + } + + app.get( + '/:spaceId/local-models', { - config: { - authz: { - resource: 'space', - action: 'admin', - spaceIdFrom: 'param', - resourceIdFrom: 'param', - resourceIdParam: 'spaceId', + config: spaceAuthz('read'), + schema: { + tags: ['Spaces'], + summary: "Diagnose the space's models on this computer", + params: z.object({ spaceId: z.string().uuid() }), + response: { + 200: z.object({ diagnostics: LocalRuntimeDiagnosticsSchema.nullable() }), + 404: ErrorSchema, }, }, + }, + async (request, reply) => { + const resolved = await runtimeFor(request, request.params.spaceId, reply, { + requireConnection: false, + }); + if (!resolved) return reply; + if (!resolved.runtime) return reply.send({ diagnostics: null }); + const roles = localRoleModels( + await readSpaceModelDefaults(db(), resolved.tenantId, request.params.spaceId), + ); + return reply.send({ + diagnostics: await diagnoseSpaceLocalRuntime({ + runtime: resolved.runtime, + redis: redis(), + roles, + }), + }); + }, + ); + + app.post( + '/:spaceId/local-models/actions', + { + config: spaceAuthz('admin'), schema: { tags: ['Spaces'], - summary: 'Load or unload a model on this computer', - params: z.object({ spaceId: z.string().uuid(), action: z.enum(['warm', 'unload']) }), - body: z.object({ model: z.string().min(1) }), + summary: "Change the space's models on this computer", + params: z.object({ spaceId: z.string().uuid() }), + body: LocalModelActionSchema, response: { - 200: z.object({ - model: z.string(), - action: z.enum(['warm', 'unload']), - window: z.number().nullable(), - }), + 200: z.object({ message: z.string() }), 400: ErrorSchema, 404: ErrorSchema, 409: ErrorSchema, @@ -51,50 +137,52 @@ export const localModelRoutes: FastifyPluginAsync = async (fastify) => { }, }, async (request, reply) => { - if (fastify.edition.edition !== 'community-local') { - return reply.status(404).send({ - error: 'NotFound', - message: - 'Models on this computer belong to the local edition, which runs on the machine they run on.', - }); - } - const tenant = await request.requireTenant(); - await request.requireSpace(); - const { spaceId, action } = request.params; - const model = createDefaultModelCatalog().getModel(request.body.model); - if (model?.provider !== 'ollama') { - return reply.status(400).send({ - error: 'NotLocal', - message: `${request.body.model} is not a model on this computer.`, - }); - } - const runtime = await resolveSpaceLocalRuntime({ - db: fastify.appContext.db as PostgresJsDatabase, - redis: fastify.appContext.redis as Redis | undefined, - tenantId: tenant.tenantId as string, - userId: request.authUser?.userId as string, - spaceId, + const resolved = await runtimeFor(request, request.params.spaceId, reply, { + requireConnection: true, + }); + if (!resolved?.runtime) return reply; + const outcome = await applyLocalModelAction( + { + db: db(), + redis: redis(), + tenantId: resolved.tenantId, + runtime: resolved.runtime, + log: request.log, + }, + request.body, + ); + if (outcome.ok) return reply.send({ message: outcome.message }); + return reply + .status(OUTCOME_STATUS[outcome.reason]) + .send({ error: outcome.reason, message: outcome.message }); + }, + ); + + app.delete( + '/:spaceId/local-models/pulls/:model', + { + config: spaceAuthz('admin'), + schema: { + tags: ['Spaces'], + summary: 'Stop a download to this computer', + params: z.object({ spaceId: z.string().uuid(), model: z.string().min(1) }), + response: { + 200: z.object({ stopped: z.boolean() }), + 404: ErrorSchema, + 409: ErrorSchema, + }, + }, + }, + async (request, reply) => { + const resolved = await runtimeFor(request, request.params.spaceId, reply, { + requireConnection: true, }); - if (!runtime) { - return reply.status(409).send({ - error: 'NoConnection', - message: 'This space has no Ollama connection. Add one under Settings → Credentials.', - }); - } - const tag = model.providerModelId ?? model.id; - const window = action === 'warm' ? contextWindowForModel(model, runtime.config) : null; - try { - if (window !== null) await runtime.client.warm(tag, window); - else await runtime.client.unload(tag); - } catch (error) { - return reply.status(502).send({ - error: 'RuntimeError', - message: `Ollama did not ${action === 'warm' ? 'load' : 'unload'} ${tag}: ${ - error instanceof Error ? error.message : String(error) - }`, - }); - } - return reply.send({ model: model.id, action, window }); + if (!resolved?.runtime) return reply; + const store = redis(); + const stopped = store + ? await cancelLocalModelPull(store, resolved.runtime, request.params.model) + : false; + return reply.send({ stopped }); }, ); }; diff --git a/packages/server-runtime/src/routes/spaceConnections.test.ts b/packages/server-runtime/src/routes/spaceConnections.test.ts index 2dbad6a8..f701c82d 100644 --- a/packages/server-runtime/src/routes/spaceConnections.test.ts +++ b/packages/server-runtime/src/routes/spaceConnections.test.ts @@ -104,6 +104,7 @@ async function buildApp(db: unknown): Promise { app.setValidatorCompiler(validatorCompiler); app.setSerializerCompiler(serializerCompiler); (app as unknown as { appContext: unknown }).appContext = { db }; + app.decorate('edition', { edition: 'enterprise' } as never); app.decorate('authenticate', async (request: FastifyRequest): Promise => { (request as unknown as { authUser: unknown }).authUser = { userId: USER_ID }; }); diff --git a/packages/server-runtime/src/routes/spaceCrudRoutes.ts b/packages/server-runtime/src/routes/spaceCrudRoutes.ts index 65014a87..6da9eadd 100644 --- a/packages/server-runtime/src/routes/spaceCrudRoutes.ts +++ b/packages/server-runtime/src/routes/spaceCrudRoutes.ts @@ -125,7 +125,11 @@ export const spaceCrudRoutes: FastifyPluginAsync = async (fastify) => { let resolvedDirectives: z.infer = directives ?? defaultSpaceDirectives(); - const allowedModels = await allowedAgentModelRefs(db, tenantCtx.tenantId); + const allowedModels = await allowedAgentModelRefs( + db, + tenantCtx.tenantId, + fastify.edition.edition, + ); if (resolvedDirectives) { resolvedDirectives = withTenantDefaultModels( resolvedDirectives, @@ -855,7 +859,11 @@ export const spaceCrudRoutes: FastifyPluginAsync = async (fastify) => { } if (request.body.directives) { - const allowedModels = await allowedAgentModelRefs(db, tenantCtx.tenantId); + const allowedModels = await allowedAgentModelRefs( + db, + tenantCtx.tenantId, + fastify.edition.edition, + ); const offListModels = offListAgentModelRefs(request.body.directives, allowedModels); if (offListModels.length > 0) { // Editors send the full modelDefaults object, so re-sending an diff --git a/packages/server-runtime/src/routes/spaceLlmReadiness.ts b/packages/server-runtime/src/routes/spaceLlmReadiness.ts index 20aad82d..04a99b25 100644 --- a/packages/server-runtime/src/routes/spaceLlmReadiness.ts +++ b/packages/server-runtime/src/routes/spaceLlmReadiness.ts @@ -16,22 +16,25 @@ import { providerCredentials, } from '@aflow/database'; import { - contextWindowForModel, createDefaultModelCatalog, inferProviderForModelRef, isVerifiableProvider, - localModelFit, - type ModelDefinition, } from '@aflow/ai-client'; import type { Redis } from '@aflow/redis'; -import { resolveSpaceLocalRuntime, type ResolvedLocalRuntime } from '../lib/localRuntime.js'; -import type { LoadedLocalModel } from '@aflow/ai-client'; import { DEFAULT_CYBERNETIC_MODEL, + LocalRuntimeIssueSchema, resolveRoleModel, type DirectiveModelDefaults, + type LocalRuntimeDiagnostics, } from '@aflow/schemas'; -import { resolveClerkModel, type ClerkModelResolution } from '@aflow/cybernetic-runtime'; +import { + diagnoseSpaceLocalRuntime, + localRoleModels, + resolveClerkModel, + resolveSpaceLocalRuntime, + type ClerkModelResolution, +} from '@aflow/cybernetic-runtime'; import { tenants } from '@aflow/database'; import { previewCredentialResolution } from '../lib/credentialResolutionPreview.js'; @@ -71,29 +74,16 @@ const RoleReadinessSchema = z.object({ }); function localFitFor( - model: ModelDefinition | undefined, - runtime: ResolvedLocalRuntime | null, - loaded: LoadedLocalModel[] | null, + modelId: string | null, + diagnostics: LocalRuntimeDiagnostics | null, ): z.infer['localFit'] { - if (!model?.local || !runtime) return null; - const parallel = runtime.config.parallel ?? 1; - const fit = localModelFit({ - footprint: model.local.footprint, - machine: runtime.config.machine, - catalogWindow: model.contextWindow, - parallel, - window: contextWindowForModel(model, runtime.config), - }); - const tag = model.providerModelId ?? model.id; - const held = loaded?.find((m) => m.tag === tag || m.tag === `${tag}:latest`); + const model = diagnostics?.models.find((m) => m.model === modelId); + if (!diagnostics || !model) return null; return { - status: fit.status, - window: fit.window, - largestFittingWindow: fit.largestFittingWindow ?? null, - bytesAtWindow: fit.bytesAtWindow ?? null, - gpuWorkingSetBytes: runtime.config.machine?.gpuWorkingSetBytes ?? null, - runtimeReachable: loaded !== null, - loadedBytes: held?.sizeBytes ?? null, + ...model.fit, + gpuWorkingSetBytes: diagnostics.machine?.gpuWorkingSetBytes ?? null, + runtimeReachable: diagnostics.runtime.reachable, + loadedBytes: model.loaded?.sizeBytes ?? null, }; } @@ -173,6 +163,13 @@ const LlmReadinessSchema = z.object({ * banner, which names the provider and offers the one-click fix. */ hasConfiguredProvider: z.boolean(), + /** + * What is wrong with the space's models on this computer, from the same + * diagnosis the models surface and Helmsman read (Plan 323 D15). An `error` + * counts against `ready`: a role whose runtime is down, or whose model is + * not downloaded, cannot answer however valid its connection. + */ + localIssues: z.array(LocalRuntimeIssueSchema), }); const READINESS_ROLES = ['default', 'helmsman', 'runner', 'coach', 'judge'] as const; @@ -272,16 +269,26 @@ export const spaceLlmReadinessRoutes: FastifyPluginAsync = async (fastify) => { const errored = new Map(); let ready = true; - const runtime = roleRefs.some((r) => r.providerId === 'ollama') - ? await resolveSpaceLocalRuntime({ - db, + const localRoles = localRoleModels(defaults); + const runtime = + Object.keys(localRoles).length > 0 + ? await resolveSpaceLocalRuntime({ + db, + redis: fastify.appContext.redis as Redis | undefined, + tenantId: tenant.tenantId as string, + userId: authUser.userId as string, + spaceId, + }) + : null; + const diagnostics = runtime + ? await diagnoseSpaceLocalRuntime({ + runtime, redis: fastify.appContext.redis as Redis | undefined, - tenantId: tenant.tenantId as string, - userId: authUser.userId as string, - spaceId, + roles: localRoles, }) : null; - const loaded = runtime ? await runtime.client.loaded() : null; + const localIssues = diagnostics?.issues ?? []; + if (localIssues.some((issue) => issue.severity === 'error')) ready = false; for (const { role, ref, modelId, providerId, unknownModel } of roleRefs) { const preview = providerId ? previews.get(providerId) : undefined; @@ -299,7 +306,7 @@ export const spaceLlmReadinessRoutes: FastifyPluginAsync = async (fastify) => { providerId, resolved, resolvedScope: preview?.resolvedScope ?? null, - localFit: localFitFor(modelId ? catalog.getModel(modelId) : undefined, runtime, loaded), + localFit: localFitFor(modelId, diagnostics), status, verified, }; @@ -353,6 +360,7 @@ export const spaceLlmReadinessRoutes: FastifyPluginAsync = async (fastify) => { reply.send({ ready, hasConfiguredProvider: configuredRows.length > 0, + localIssues, roles, clerk: { mode: clerk.mode, diff --git a/packages/server-runtime/src/routes/spacesShared.ts b/packages/server-runtime/src/routes/spacesShared.ts index 4357e000..02c6199d 100644 --- a/packages/server-runtime/src/routes/spacesShared.ts +++ b/packages/server-runtime/src/routes/spacesShared.ts @@ -19,6 +19,7 @@ import { DirectiveConnectionRefSchema, EntityDirectivesSchema, CLERK_MODEL_CANDIDATES, + type EditionId, } from '@aflow/schemas'; import { allowedModelIds, @@ -88,6 +89,7 @@ function roleAllowed(role: string, policy: AgentModelPolicy): ReadonlySet { const rows = await db .select({ allowlist: tenants.agentModelAllowlist }) @@ -96,7 +98,7 @@ export async function allowedAgentModelRefs( .limit(1); const stored = rows[0]?.allowlist; const explicit = Array.isArray(stored) && stored.length > 0; - return { ids: allowedModelIds(explicit ? (stored as string[]) : null), explicit }; + return { ids: allowedModelIds(explicit ? (stored as string[]) : null, edition), explicit }; } /** diff --git a/packages/server-runtime/src/routes/tenant-settings.ts b/packages/server-runtime/src/routes/tenant-settings.ts index 3140572e..46d5f22f 100644 --- a/packages/server-runtime/src/routes/tenant-settings.ts +++ b/packages/server-runtime/src/routes/tenant-settings.ts @@ -81,7 +81,7 @@ export const tenantSettingsRoutes: FastifyPluginAsync = async (fastify) => { const raw = rows[0]?.allowlist; const stored = Array.isArray(raw) ? (raw as string[]) : null; reply.send({ - modelIds: [...allowedModelIds(stored)], + modelIds: [...allowedModelIds(stored, fastify.edition.edition)], source: stored ? ('tenant' as const) : ('platform' as const), }); }, @@ -151,7 +151,7 @@ export const tenantSettingsRoutes: FastifyPluginAsync = async (fastify) => { ); return reply.send({ - modelIds: [...allowedModelIds(canonical)], + modelIds: [...allowedModelIds(canonical, fastify.edition.edition)], source: canonical ? ('tenant' as const) : ('platform' as const), }); }, diff --git a/packages/server-runtime/src/services/actionCenter/__tests__/pausedStepWriteApproval.test.ts b/packages/server-runtime/src/services/actionCenter/__tests__/pausedStepWriteApproval.test.ts index 1b834012..03bef3a9 100644 --- a/packages/server-runtime/src/services/actionCenter/__tests__/pausedStepWriteApproval.test.ts +++ b/packages/server-runtime/src/services/actionCenter/__tests__/pausedStepWriteApproval.test.ts @@ -243,3 +243,26 @@ describe('pausedStepSource — who the resolve that resumes the run is', () => { ]); }); }); + +describe('pausedStepSource — a change to the model runtime on this computer', () => { + const LOCAL_MODEL_REQUEST = { + kind: 'write_approval', + target: 'local_model', + change: { action: 'unload', model: 'ollama-gemma4-26b' }, + modelName: 'Gemma 4 26B (this computer)', + runtimeUrl: 'http://127.0.0.1:11434', + impact: { inFlightRequests: 0, ownConversationReloads: true, notes: ['Frees 22.7 GB.'] }, + requestHash: 'local-hash-1', + }; + + it('answers to a space admin, the role the operator’s own routes for it require', async () => { + const [item] = await createPausedStepSource(makeDeps(LOCAL_MODEL_REQUEST)).listOpen(ctx()); + expect(item).toMatchObject({ + kind: 'write_approval', + title: 'Approve on this computer: unload Gemma 4 26B (this computer)', + resolverPolicy: { candidateResolvers: ['admin'] }, + extension: { target: 'local_model', change: { action: 'unload' } }, + }); + expect(item!.summary).toContain('its next turn loads it again'); + }); +}); diff --git a/packages/server-runtime/src/services/actionCenter/sources/pausedStepSource.ts b/packages/server-runtime/src/services/actionCenter/sources/pausedStepSource.ts index b627059c..aeb7d61a 100644 --- a/packages/server-runtime/src/services/actionCenter/sources/pausedStepSource.ts +++ b/packages/server-runtime/src/services/actionCenter/sources/pausedStepSource.ts @@ -14,6 +14,10 @@ import { type BrowserWriteApprovalRequestPayload, type WriteApprovalExtension, BROWSER_PAGE_ACT_OPERATION_ID, + MODEL_RUNTIME_CHANGE_OPERATION_ID, + localModelChangePhrase, + type LocalModelWriteApprovalRequestPayload, + type ResolverPolicy, browserApprovalActionPhrase, derivePauseToken, WriteApprovalRequestPayloadSchema, @@ -580,7 +584,12 @@ function buildWriteApprovalItem( const parsed = WriteApprovalRequestPayloadSchema.safeParse(payload); if (!parsed.success) return null; const req = parsed.data; - const shown = req.target === 'browser' ? browserApprovalShown(req) : apiWriteApprovalShown(req); + const shown = + req.target === 'browser' + ? browserApprovalShown(req) + : req.target === 'local_model' + ? localModelApprovalShown(req) + : apiWriteApprovalShown(req); const origin: ActionCenterItemOrigin = { type: 'step', @@ -606,6 +615,7 @@ function buildWriteApprovalItem( requestedBy: { kind: 'agent' as const, label: 'Agent', sessionId: row.sessionId }, priority: shown.priority, relatesTo: shown.relatesTo, + ...(shown.resolverPolicy ? { resolverPolicy: shown.resolverPolicy } : {}), resolverAuthority: { kind: 'space' }, status: row.status === 'PAUSED' ? 'open' : 'resolved', }; @@ -618,6 +628,7 @@ interface WriteApprovalShown { readonly operationId: string; readonly priority: 'normal' | 'high'; readonly relatesTo: RelatesToEntry[]; + readonly resolverPolicy?: ResolverPolicy; } function apiWriteApprovalShown(req: ApiWriteApprovalRequestPayload): WriteApprovalShown { @@ -685,6 +696,37 @@ function browserApprovalShown(req: BrowserWriteApprovalRequestPayload): WriteApp }; } +function localModelApprovalShown(req: LocalModelWriteApprovalRequestPayload): WriteApprovalShown { + const doing = localModelChangePhrase(req); + const ends = + req.impact.inFlightRequests > 0 + ? ` ${String(req.impact.inFlightRequests)} model request${req.impact.inFlightRequests === 1 ? ' is' : 's are'} running on this computer now; those on this model end.` + : ''; + const own = req.impact.ownConversationReloads + ? ' This conversation runs on it, so its next turn loads it again.' + : ''; + return { + title: `Approve on this computer: ${doing}`, + summary: + `Helmsman wants to ${doing} on ${req.runtimeUrl}. ${req.impact.notes.join(' ')}${ends}${own} ` + + 'Approve to let it do this once, or deny to stop it.', + extension: { + kind: 'write_approval', + target: 'local_model', + change: req.change, + ...(req.modelName !== undefined ? { modelName: req.modelName } : {}), + runtimeUrl: req.runtimeUrl, + impact: req.impact, + }, + operationId: MODEL_RUNTIME_CHANGE_OPERATION_ID, + priority: req.impact.inFlightRequests > 0 ? 'high' : 'normal', + relatesTo: [], + // The operator's own routes for these changes are space-admin only; an + // approval is the same change, so it answers to the same role. + resolverPolicy: { minResolvers: 1, requireAll: false, candidateResolvers: ['admin'] }, + }; +} + function parseOwnerScope(raw: unknown): OAuthConsentExtension['ownerScope'] { if (raw === 'user' || raw === 'space') return raw; return 'user'; diff --git a/packages/web-product/src/ui/components/actionCenter/WriteApprovalCard.tsx b/packages/web-product/src/ui/components/actionCenter/WriteApprovalCard.tsx index 23c27d43..94118dc1 100644 --- a/packages/web-product/src/ui/components/actionCenter/WriteApprovalCard.tsx +++ b/packages/web-product/src/ui/components/actionCenter/WriteApprovalCard.tsx @@ -6,8 +6,10 @@ import type { ActionCenterItem, ApiWriteApprovalExtension, BrowserWriteApprovalExtension, + LocalModelWriteApprovalExtension, WriteApprovalExtension, } from '../../hooks/use-action-center-types.js'; +import { localModelChangePhrase } from '@aflow/schemas'; import type { ActionCenterResolution } from '../../hooks/use-action-center.js'; import { useApiQuery } from '../../hooks/useApiQuery.js'; import { decodeInlinePayload, isInlinePayloadRef } from '../../lib/fetch-payload.js'; @@ -76,14 +78,18 @@ export function WriteApprovalCard({ {TIER_BADGE[extension.writeRiskTier].label} ) : ( - browser + + {extension.target === 'browser' ? 'browser' : 'this computer'} + )} {extension.target === 'api' ? ( - ) : ( + ) : extension.target === 'browser' ? ( + ) : ( + )} {errorMessage && ( @@ -210,6 +216,37 @@ function BrowserActionDetail({ ); } +function LocalModelChangeDetail({ + extension, +}: { + extension: LocalModelWriteApprovalExtension; +}): ReactElement { + const { impact } = extension; + return ( + <> + + {localModelChangePhrase(extension)} on {extension.runtimeUrl} + + {impact.notes.map((note) => ( + + {note} + + ))} + {impact.inFlightRequests > 0 && ( + + {impact.inFlightRequests} model request{impact.inFlightRequests === 1 ? ' is' : 's are'}{' '} + running on this computer now; those on this model end. + + )} + {impact.ownConversationReloads && ( + + This conversation runs on it, so its next turn loads it again — up to a minute. + + )} + + ); +} + function PageScreenshot({ payloadRef }: { payloadRef: string }): ReactElement | null { const inline = isInlinePayloadRef(payloadRef); const { data } = useApiQuery({ diff --git a/packages/web-product/src/ui/components/chat/AwaitingApprovalNotice.tsx b/packages/web-product/src/ui/components/chat/AwaitingApprovalNotice.tsx index a19d06bc..32185474 100644 --- a/packages/web-product/src/ui/components/chat/AwaitingApprovalNotice.tsx +++ b/packages/web-product/src/ui/components/chat/AwaitingApprovalNotice.tsx @@ -5,8 +5,20 @@ import type { SessionBlockedOn } from '@aflow/schemas'; type WriteApprovalBlock = Extract; +const LOCAL_MODEL_VERB: Record = { + pull: 'download', + delete: 'delete', + warm: 'load', + unload: 'unload', + set_context: 'change the window of', +}; + function waitingOn(block: WriteApprovalBlock): string { if (block.target === 'api') return `${block.method} ${block.endpointId} on ${block.urlHost}`; + if (block.target === 'local_model') { + const verb = LOCAL_MODEL_VERB[block.action] ?? block.action; + return `${verb} ${block.modelName ?? 'the models'} on this computer`; + } const element = block.elementName !== undefined ? `${block.elementRole} “${block.elementName}”` diff --git a/packages/web-product/src/ui/components/credentials/LlmReadinessBanner.tsx b/packages/web-product/src/ui/components/credentials/LlmReadinessBanner.tsx index 7244709c..c6802384 100644 --- a/packages/web-product/src/ui/components/credentials/LlmReadinessBanner.tsx +++ b/packages/web-product/src/ui/components/credentials/LlmReadinessBanner.tsx @@ -1,8 +1,11 @@ 'use client'; import { useEffect, useRef } from 'react'; +import Link from 'next/link'; import { Button, Icon, Row, Text } from '@aflow/design-system'; -import { useSpaceLlmReadiness } from '../../hooks/useSpaceLlmReadiness.js'; +import { useSpaceLlmReadiness, type SpaceLlmReadiness } from '../../hooks/useSpaceLlmReadiness.js'; +import { useSpaceFromRoute } from '../providers.js'; +import { spaceRoute } from '../../lib/space-routes.js'; const PROVIDER_LABELS: Record = { anthropic: 'Anthropic', @@ -23,6 +26,11 @@ function joinProviderNames(providerIds: string[]): string { return `${names.slice(0, -1).join(', ')}, and ${names.at(-1)}`; } +/** Something no key or model choice fixes: the runtime on this computer, which its own page manages. */ +function localError(readiness: SpaceLlmReadiness) { + return readiness.localIssues.find((issue) => issue.severity === 'error') ?? null; +} + /** * Standing guidance above the composer: missing or rejected provider keys * render as a one-click fix, recomputed from live readiness — never a @@ -36,10 +44,13 @@ export function LlmReadinessBanner({ onReviewModels: () => void; }) { const { readiness } = useSpaceLlmReadiness(spaceId); + const routeSpace = useSpaceFromRoute(); const autoOpenedSpace = useRef(null); useEffect(() => { if (!readiness || readiness.ready || autoOpenedSpace.current === spaceId) return; + // The model picker cannot start Ollama or download a model. + if (localError(readiness) && readiness.missingProviders.length === 0) return; autoOpenedSpace.current = spaceId; onReviewModels(); }, [onReviewModels, readiness, spaceId]); @@ -49,7 +60,10 @@ export function LlmReadinessBanner({ const missing = readiness.missingProviders; const errored = readiness.erroredProviders; const unknownModels = readiness.unknownModelRoles; - if (missing.length === 0 && errored.length === 0 && unknownModels.length === 0) return null; + const local = localError(readiness); + if (missing.length === 0 && errored.length === 0 && unknownModels.length === 0 && !local) { + return null; + } // Stated first: a model the catalog no longer carries cannot be fixed by any // key, so naming a credential alongside it would offer a remedy that does @@ -70,6 +84,12 @@ export function LlmReadinessBanner({ const modelAlternative = oneModel ? 'a model' : 'models'; const providerAlternative = oneProvider ? 'another provider' : 'other providers'; + const localOnly = + local !== null && missing.length === 0 && errored.length === 0 && !unknownMessage; + const localMessage = local + ? `${local.cause} ${local.remedy.kind === 'instruction' ? local.remedy.text : `${local.remedy.label} on This Computer.`}` + : null; + return ( - {unknownMessage ?? + {(localOnly ? localMessage : null) ?? + unknownMessage ?? (isMissing ? `To run agents, ${subject} requires ${providerKey}. Add ${oneProvider ? 'the key' : 'them'}, or choose ${modelAlternative} from ${providerAlternative}.` : `${oneProvider ? `The ${names} key used by ${subject} was rejected` : `The keys for ${names} used by ${subject} were rejected`}. Update ${oneProvider ? 'the key' : 'them'}, or choose ${modelAlternative} from ${providerAlternative}.`)} - + {localOnly ? ( + + + + ) : ( + + )} ); } diff --git a/packages/web-product/src/ui/components/cybernetic/LocalModelStatus.tsx b/packages/web-product/src/ui/components/cybernetic/LocalModelStatus.tsx index 056269c3..f068c317 100644 --- a/packages/web-product/src/ui/components/cybernetic/LocalModelStatus.tsx +++ b/packages/web-product/src/ui/components/cybernetic/LocalModelStatus.tsx @@ -3,6 +3,7 @@ import { Button, Row, Text } from '@aflow/design-system'; import { useApiMutation } from '../../hooks/useApiQuery.js'; import { llmReadinessQueryKey, type LocalFit } from '../../hooks/useSpaceLlmReadiness.js'; +import { localModelsQueryKey } from '../../hooks/useLocalModels.js'; import { describeLocalFit } from './ModelDefaultsEditor.js'; const gb = (bytes: number): string => `${(bytes / 1e9).toFixed(1)} GB`; @@ -24,10 +25,10 @@ export function LocalModelStatus({ fit: LocalFit; }) { const action = useApiMutation<{ action: 'warm' | 'unload' }>({ - path: ({ action: which }) => `/spaces/${spaceId}/local-models/${which}`, - serialize: () => JSON.stringify({ model }), + path: () => `/spaces/${spaceId}/local-models/actions`, + serialize: ({ action: which }) => JSON.stringify({ action: which, model }), spaceId, - invalidate: [llmReadinessQueryKey(spaceId)], + invalidate: [llmReadinessQueryKey(spaceId), localModelsQueryKey(spaceId)], }); const spilling = fit.loadedBytes !== null && diff --git a/packages/web-product/src/ui/components/host/HostLocalModels.tsx b/packages/web-product/src/ui/components/host/HostLocalModels.tsx new file mode 100644 index 00000000..a424e976 --- /dev/null +++ b/packages/web-product/src/ui/components/host/HostLocalModels.tsx @@ -0,0 +1,314 @@ +'use client'; + +import { useState } from 'react'; +import { + Badge, + Button, + Card, + CardBody, + Column, + Dialog, + Heading, + HelperText, + Row, + Text, +} from '@aflow/design-system'; +import { + LOCAL_MODEL_ROLES, + type EntityDirectives, + type LocalModelDiagnosis, + type LocalRuntimeDiagnostics, + type LocalRuntimeIssue, +} from '@aflow/schemas'; +import { useApiMutation } from '../../hooks/useApiQuery.js'; +import { useSpaceDetail } from '../../hooks/use-space-detail.js'; +import { everyRoleOn, localModelsQueryKey, useLocalModels } from '../../hooks/useLocalModels.js'; +import { llmReadinessQueryKey } from '../../hooks/useSpaceLlmReadiness.js'; +import { describeLocalFit } from '../cybernetic/ModelDefaultsEditor.js'; + +const gb = (bytes: number): string => `${(bytes / 1e9).toFixed(1)} GB`; + +const SEVERITY_BADGE: Record = { + error: 'danger', + warning: 'warning', + info: 'neutral', +}; + +/** + * Models on this computer (Plan 323 D10, D15): what the runtime holds, what is + * wrong with it and the fix for each, and every lifecycle action — none of + * which needs a working model, since the model may be what is not working. + * Local edition only, like the page it sits on. + */ +export function HostLocalModels({ spaceId }: { spaceId: string }) { + const { diagnostics, isLoading, act, stopPull } = useLocalModels(spaceId, true); + const { space } = useSpaceDetail(spaceId); + const [deleting, setDeleting] = useState(null); + + const assignAll = useApiMutation({ + path: () => `/spaces/${spaceId}`, + method: 'PATCH', + serialize: (directives) => JSON.stringify({ directives }), + spaceId, + invalidate: [['space', spaceId], localModelsQueryKey(spaceId), llmReadinessQueryKey(spaceId)], + }); + + const useForEveryRole = (modelId: string): void => { + const model = diagnostics?.models.find((m) => m.model === modelId); + const directives = space?.directives as EntityDirectives | null | undefined; + if (model && directives) assignAll.mutate(everyRoleOn(directives, model)); + }; + + const runRemedy = (issue: LocalRuntimeIssue): void => { + if (issue.remedy.kind !== 'action') return; + const { action } = issue.remedy; + if (action.action === 'use_for_every_role') useForEveryRole(action.model); + else act.mutate(action); + }; + + return ( + + Models on this computer + {isLoading ? ( + Asking Ollama… + ) : diagnostics === null ? ( + + No Ollama connection yet. Add one under Settings → Credentials, and models running on this + computer can answer for any role. + + ) : ( + <> + + {diagnostics.issues.map((issue) => ( + + + + + {issue.severity} + {issue.cause} + + {issue.remedy.kind === 'action' ? ( + + + + ) : ( + {issue.remedy.text} + )} + + + + ))} + {diagnostics.runtime.reachable && + diagnostics.models.map((model) => ( + { + act.mutate({ action, model: model.model }); + }} + onStop={() => { + stopPull.mutate(model.model); + }} + onDelete={() => { + setDeleting(model); + }} + onUseEverywhere={() => { + useForEveryRole(model.model); + }} + /> + ))} + + )} + { + setDeleting(null); + }} + title={`Delete ${deleting?.displayName ?? ''}?`} + width="sm" + footer={ + + + + + } + > + + {deleting && deleting.roles.length > 0 + ? `${deleting.roles.join(', ')} use this model and cannot run until it is downloaded again. ` + : ''} + Downloading it again takes{' '} + {deleting?.downloadBytes ? gb(deleting.downloadBytes) : 'its size'} of transfer. + + + + ); +} + +function RuntimeLine({ diagnostics }: { diagnostics: LocalRuntimeDiagnostics }) { + const { runtime, machine } = diagnostics; + if (!runtime.reachable) return null; + const parts = [`Ollama ${runtime.version ?? ''} at ${runtime.baseUrl}`]; + if (machine?.gpuWorkingSetBytes) parts.push(`${gb(machine.gpuWorkingSetBytes)} of GPU memory`); + if (machine?.freeDiskBytes !== null && machine?.freeDiskBytes !== undefined) { + parts.push(`${gb(machine.freeDiskBytes)} free on disk`); + } + return ( + + {parts.join(' · ')} + + ); +} + +function ModelRow({ + model, + diagnostics, + busy, + onAct, + onStop, + onDelete, + onUseEverywhere, +}: { + model: LocalModelDiagnosis; + diagnostics: LocalRuntimeDiagnostics; + busy: boolean; + onAct: (action: 'pull' | 'warm' | 'unload') => void; + onStop: () => void; + onDelete: () => void; + onUseEverywhere: () => void; +}) { + const downloading = model.pull?.status === 'downloading'; + const everyRole = model.roles.length === LOCAL_MODEL_ROLES.length; + return ( + + + + + {model.displayName} + {model.tag} + {model.installed ? ( + + downloaded{model.downloadBytes ? `, ${gb(model.downloadBytes)}` : ''} + + ) : ( + + not downloaded{model.downloadBytes ? `, ${gb(model.downloadBytes)}` : ''} + + )} + {model.build === 'unverified' && unmeasured build} + {model.roles.length > 0 && {model.roles.join(', ')}} + + + {describeLocalFit({ + ...model.fit, + gpuWorkingSetBytes: diagnostics.machine?.gpuWorkingSetBytes ?? null, + runtimeReachable: true, + loadedBytes: model.loaded?.sizeBytes ?? null, + })} + + {model.loaded && ( + + Loaded, {gb(model.loaded.sizeBytes)} + {model.loaded.gpuBytes !== null && model.loaded.gpuBytes < model.loaded.sizeBytes + ? `, ${gb(model.loaded.sizeBytes - model.loaded.gpuBytes)} of it on the CPU` + : ''} + . + + )} + {model.pull && } + + {downloading ? ( + + ) : !model.installed ? ( + + ) : ( + <> + + {!everyRole && ( + + )} + + + )} + + + + + ); +} + +function PullLine({ pull }: { pull: NonNullable }) { + const percent = + pull.totalBytes && pull.totalBytes > 0 + ? Math.floor((100 * pull.completedBytes) / pull.totalBytes) + : null; + const text = + pull.status === 'downloading' + ? `Downloading${percent !== null ? ` — ${percent}% of ${gb(pull.totalBytes ?? 0)}` : '…'}${ + pull.detail ? ` (${pull.detail})` : '' + }` + : pull.status === 'interrupted' + ? 'Download stopped. Downloading again resumes from what is already here.' + : `Download failed: ${pull.error ?? 'the runtime gave no reason'}. Downloading again resumes from what is already here.`; + return ( + + {text} + + ); +} diff --git a/packages/web-product/src/ui/components/onboarding-flow.tsx b/packages/web-product/src/ui/components/onboarding-flow.tsx index 6745f96c..d946ded0 100644 --- a/packages/web-product/src/ui/components/onboarding-flow.tsx +++ b/packages/web-product/src/ui/components/onboarding-flow.tsx @@ -8,6 +8,12 @@ import { type AgentModelProviderId, } from '@aflow/schemas'; import { CreateSpaceForm, type CreateSpaceResult } from './create-space-form.js'; +import { + PROVIDER_FALLBACK_META, + providerDisplayName, + type ProviderFieldMeta, +} from './onboarding-providers.js'; +import { ModelCard } from './onboarding-model-card.js'; import { useApi, useSpace } from './providers.js'; import { useChatModelOptions } from '../hooks/useChatModelOptions.js'; import { useApiMutation } from '../hooks/useApiQuery.js'; @@ -386,84 +392,6 @@ function StepNameWorkspace({ // Step 3 — Model // ============================================================================= -function ModelCard({ - model, - selected, - onSelect, -}: { - model: RecommendedAgentModel; - selected: boolean; - onSelect: () => void; -}) { - return ( -
{ - if (e.key === 'Enter' || e.key === ' ') { - e.preventDefault(); - onSelect(); - } - }} - style={{ - padding: 'var(--space-3) var(--space-4)', - borderRadius: 'var(--radius-md)', - border: '2px solid', - borderColor: selected ? 'var(--color-interactive-default)' : 'var(--color-border-subtle)', - background: selected ? 'var(--color-surface-1)' : 'transparent', - cursor: 'pointer', - transition: 'border-color 150ms, background 150ms', - }} - > -
- {model.label} - - via {providerDisplayName(model.credentialProviderId)} - - {model.recommended && ( - - Recommended - - )} -
-
- {model.tagline} -
-
- ); -} - -function providerDisplayName(providerId: AgentModelProviderId): string { - switch (providerId) { - case 'anthropic': - return 'Anthropic'; - case 'google': - return 'Google'; - case 'openai': - return 'OpenAI'; - case 'fireworks': - return 'Fireworks AI'; - case 'xai': - return 'xAI'; - } -} - export interface ModelSelection { defaultRef: string; roleOverrides: Partial>; @@ -534,6 +462,12 @@ function StepModel({ (m) => allowed.has(m.modelId) || (m.alias !== undefined && allowed.has(m.alias)), ); }, [modelOptions]); + // Offered only where the tenant may assign them, which outside the local + // edition is never: there is no computer of the operator's behind a hosted one. + const onThisComputer = useMemo( + () => modelOptions.filter((o) => o.provider === 'ollama' && o.allowed), + [modelOptions], + ); const defaultEntry = offered.find((m) => m.recommended) ?? offered[0]; // The policy has not loaded on first render, so the selection is seeded once // it arrives rather than by the state initializer, which would freeze a @@ -572,6 +506,28 @@ function StepModel({ const [saving, setSaving] = useState(false); const [error, setError] = useState(null); + /** + * A model on this computer needs a connection and no key: one is added at + * Ollama's own address unless one already resolves for this workspace, which + * may name another and must not be replaced. + */ + const connectThisComputer = async (spaceId: string): Promise => { + const scoped = { ...headers(), 'X-Space-ID': spaceId }; + const readiness = await authFetch(`/api/spaces/${spaceId}/llm-readiness`, { headers: scoped }); + const roles = readiness.ok + ? ((await readiness.json()) as { roles: Record }).roles + : {}; + if (roles['default']?.resolved) return null; + const put = await authFetch('/api/credentials/ollama', { + method: 'PUT', + headers: scoped, + body: JSON.stringify({ scope: 'user', secrets: {} }), + }); + if (put.ok) return null; + const body = (await put.json().catch(() => null)) as { error?: string } | null; + return body?.error ?? `Could not connect to Ollama on this computer (${put.status}).`; + }; + const handleContinue = async () => { // Nothing to save when no model was offered or chosen: the space keeps // whatever creation assigned it, which the server already held to the @@ -588,11 +544,15 @@ function StepModel({ version: 1, responsibility: 'General-purpose workspace.', }; + const local = onThisComputer.some((m) => m.ref === defaultRef); const modelDefaults: Record = { default: defaultRef }; for (const role of AGENT_ROLES) { const override = roleOverrides[role]; if (override && override !== defaultRef) modelDefaults[role] = override; } + // One GPU serves every role, and each switch between models is a load, + // so the Clerk — the one role that does not inherit — follows too. + if (local) modelDefaults['clerk'] = 'space_default'; // No active space exists during onboarding, so headers() carries no // X-Space-ID — the PATCH handler's requireSpace() needs it explicitly. const res = await authFetch(`/api/spaces/${space.id}`, { @@ -606,6 +566,14 @@ function StepModel({ setSaving(false); return; } + if (local) { + const problem = await connectThisComputer(space.id); + if (problem) { + setError(problem); + setSaving(false); + return; + } + } onDone(selection); } catch { setError('Network error — please try again.'); @@ -642,22 +610,39 @@ function StepModel({ Could not load the available models ({modelOptionsError}). Continue — your workspace keeps the model it was created with, and you can change it in settings. - ) : offered.length === 0 ? ( + ) : offered.length === 0 && onThisComputer.length === 0 ? ( Your administrator has chosen a custom set of models. Continue — your workspace keeps the model it was created with, and you can change it in settings. ) : ( - offered.map((model) => ( - { - setDefaultRef(modelRef(model)); - }} - /> - )) + <> + {offered.map((model) => ( + { + setDefaultRef(modelRef(model)); + }} + /> + ))} + {onThisComputer.map((model) => ( + { + setDefaultRef(model.ref); + }} + /> + ))} + )} @@ -749,46 +734,6 @@ function StepModel({ // Step 4 — Connect provider key(s) // ============================================================================= -interface ProviderFieldMeta { - displayName: string; - placeholder: string; - helpText: string; - docsUrl: string | null; -} - -const PROVIDER_FALLBACK_META: Record = { - anthropic: { - displayName: 'Anthropic', - placeholder: 'sk-ant-...', - helpText: 'Your Anthropic API key.', - docsUrl: 'https://console.anthropic.com/settings/keys', - }, - google: { - displayName: 'Google', - placeholder: 'AIza...', - helpText: 'Your Google AI Studio API key.', - docsUrl: 'https://aistudio.google.com/app/apikey', - }, - openai: { - displayName: 'OpenAI', - placeholder: 'sk-...', - helpText: 'Your OpenAI API key.', - docsUrl: 'https://platform.openai.com/api-keys', - }, - fireworks: { - displayName: 'Fireworks AI', - placeholder: 'fw_...', - helpText: 'Your Fireworks API key.', - docsUrl: 'https://fireworks.ai/account/api-keys', - }, - xai: { - displayName: 'xAI', - placeholder: 'xai-...', - helpText: 'Your xAI API key.', - docsUrl: 'https://console.x.ai', - }, -}; - type ProviderConnectState = | { phase: 'idle' } | { phase: 'saving' } @@ -1442,6 +1387,12 @@ export function OnboardingFlow({ resumeFor }: { resumeFor?: CreateSpaceResult }) initial={selection} onDone={(sel) => { setSelection(sel); + // A model on this computer has no key to connect. + if (sel.defaultRef !== '' && providersForSelection(sel).length === 0) { + setKeysConnected(true); + setStep('ready'); + return; + } setStep('connect'); }} /> diff --git a/packages/web-product/src/ui/components/onboarding-model-card.tsx b/packages/web-product/src/ui/components/onboarding-model-card.tsx new file mode 100644 index 00000000..1670b168 --- /dev/null +++ b/packages/web-product/src/ui/components/onboarding-model-card.tsx @@ -0,0 +1,73 @@ +'use client'; + +import { Badge } from '@aflow/design-system'; + +/** One model offered at onboarding, selectable as a radio. */ +export function ModelCard({ + label, + via, + tagline, + recommended, + selected, + onSelect, +}: { + label: string; + via: string; + tagline: string; + recommended?: boolean | undefined; + selected: boolean; + onSelect: () => void; +}) { + return ( +
{ + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + onSelect(); + } + }} + style={{ + padding: 'var(--space-3) var(--space-4)', + borderRadius: 'var(--radius-md)', + border: '2px solid', + borderColor: selected ? 'var(--color-interactive-default)' : 'var(--color-border-subtle)', + background: selected ? 'var(--color-surface-1)' : 'transparent', + cursor: 'pointer', + transition: 'border-color 150ms, background 150ms', + }} + > +
+ {label} + + via {via} + + {recommended && ( + + Recommended + + )} +
+
+ {tagline} +
+
+ ); +} diff --git a/packages/web-product/src/ui/components/onboarding-providers.ts b/packages/web-product/src/ui/components/onboarding-providers.ts new file mode 100644 index 00000000..b3fd1273 --- /dev/null +++ b/packages/web-product/src/ui/components/onboarding-providers.ts @@ -0,0 +1,56 @@ +import type { AgentModelProviderId } from '@aflow/schemas'; + +export function providerDisplayName(providerId: AgentModelProviderId): string { + switch (providerId) { + case 'anthropic': + return 'Anthropic'; + case 'google': + return 'Google'; + case 'openai': + return 'OpenAI'; + case 'fireworks': + return 'Fireworks AI'; + case 'xai': + return 'xAI'; + } +} + +export interface ProviderFieldMeta { + displayName: string; + placeholder: string; + helpText: string; + docsUrl: string | null; +} + +export const PROVIDER_FALLBACK_META: Record = { + anthropic: { + displayName: 'Anthropic', + placeholder: 'sk-ant-...', + helpText: 'Your Anthropic API key.', + docsUrl: 'https://console.anthropic.com/settings/keys', + }, + google: { + displayName: 'Google', + placeholder: 'AIza...', + helpText: 'Your Google AI Studio API key.', + docsUrl: 'https://aistudio.google.com/app/apikey', + }, + openai: { + displayName: 'OpenAI', + placeholder: 'sk-...', + helpText: 'Your OpenAI API key.', + docsUrl: 'https://platform.openai.com/api-keys', + }, + fireworks: { + displayName: 'Fireworks AI', + placeholder: 'fw_...', + helpText: 'Your Fireworks API key.', + docsUrl: 'https://fireworks.ai/account/api-keys', + }, + xai: { + displayName: 'xAI', + placeholder: 'xai-...', + helpText: 'Your xAI API key.', + docsUrl: 'https://console.x.ai', + }, +}; diff --git a/packages/web-product/src/ui/components/run-timeline/RunSummaryBar.tsx b/packages/web-product/src/ui/components/run-timeline/RunSummaryBar.tsx index 5f74dd3e..677a4612 100644 --- a/packages/web-product/src/ui/components/run-timeline/RunSummaryBar.tsx +++ b/packages/web-product/src/ui/components/run-timeline/RunSummaryBar.tsx @@ -13,6 +13,11 @@ export interface RunSummary { totalCostUsd: number; models: Set; totalCacheReadTokens: number; + /** + * Every model call ran on this computer (Plan 323 D13). Nothing was billed, + * which is a fact about where it ran rather than a price of zero. + */ + ranLocally: boolean; } function formatDuration(ms: number): string { @@ -89,7 +94,11 @@ export function RunSummaryBar({ summary }: { summary: RunSummary }) { detail={`${formatNumber(summary.totalPromptTokens)} in / ${formatNumber(summary.totalCompletionTokens)} out`} /> ) : null} - {hasCost ? : null} + {hasCost ? ( + + ) : summary.ranLocally && hasTokens ? ( + + ) : null} {summary.totalCacheReadTokens > 0 && summary.totalPromptTokens > 0 ? ( )} - {group.costInfo.totalCostUsd != null && group.costInfo.totalCostUsd > 0 && ( + {group.costInfo.totalCostUsd != null && group.costInfo.totalCostUsd > 0 ? ( {formatCostUsd(group.costInfo.totalCostUsd)} - )} + ) : group.costInfo.provider === 'ollama' && (group.costInfo.totalTokens ?? 0) > 0 ? ( + + local + + ) : null} {(group.costInfo.cacheReadTokens ?? 0) > 0 && (group.costInfo.promptTokens ?? 0) > 0 && ( { expect(parked?.stepType).toBe('host'); }); }); + +describe('a run on a model on this computer', () => { + const step = (provider: string, totalCostUsd: number): TimelineEntry => + ({ + kind: 'step', + group: { status: 'succeeded', costInfo: { provider, totalTokens: 100, totalCostUsd } }, + }) as unknown as TimelineEntry; + + it('ran locally when every model call did', () => { + expect(computeRunSummary([step('ollama', 0), step('ollama', 0)]).ranLocally).toBe(true); + }); + + it('did not when any call went to a hosted model', () => { + expect(computeRunSummary([step('ollama', 0), step('anthropic', 0.02)]).ranLocally).toBe(false); + expect(computeRunSummary([]).ranLocally).toBe(false); + }); +}); diff --git a/packages/web-product/src/ui/components/run-timeline/buildTimelineEntries.ts b/packages/web-product/src/ui/components/run-timeline/buildTimelineEntries.ts index 3decf996..c39ce9d8 100644 --- a/packages/web-product/src/ui/components/run-timeline/buildTimelineEntries.ts +++ b/packages/web-product/src/ui/components/run-timeline/buildTimelineEntries.ts @@ -5,8 +5,18 @@ import { ev } from './eventAccessor'; import type { RunSummary } from './RunSummaryBar'; import type { StepGroup, TimelineEntry } from './types'; -/** Aggregated run-level summary computed from all step groups */ +/** The provider a model on this computer is served by. */ +const LOCAL_PROVIDER = 'ollama'; +/** The steps that called a model, and every one of them called one on this computer. */ +function ranLocally(entries: TimelineEntry[]): boolean { + const providers = entries.flatMap((e) => + e.kind === 'step' && e.group.costInfo?.provider ? [e.group.costInfo.provider] : [], + ); + return providers.length > 0 && providers.every((p) => p === LOCAL_PROVIDER); +} + +/** Aggregated run-level summary computed from all step groups */ export function computeRunSummary(entries: TimelineEntry[]): RunSummary { const summary: RunSummary = { totalSteps: 0, @@ -19,6 +29,7 @@ export function computeRunSummary(entries: TimelineEntry[]): RunSummary { totalCostUsd: 0, models: new Set(), totalCacheReadTokens: 0, + ranLocally: ranLocally(entries), }; // Check for server-side usageSummary from a terminal event diff --git a/packages/web-product/src/ui/hooks/use-action-center-types.ts b/packages/web-product/src/ui/hooks/use-action-center-types.ts index 20a56698..e129dcd1 100644 --- a/packages/web-product/src/ui/hooks/use-action-center-types.ts +++ b/packages/web-product/src/ui/hooks/use-action-center-types.ts @@ -171,6 +171,7 @@ export interface OAuthConsentExtension { export type { ApiWriteApprovalExtension, BrowserWriteApprovalExtension, + LocalModelWriteApprovalExtension, WriteApprovalExtension, } from '@aflow/schemas'; export type ActionCenterItemExtension = diff --git a/packages/web-product/src/ui/hooks/use-activity-bubble.ts b/packages/web-product/src/ui/hooks/use-activity-bubble.ts index d4358a01..bb00a681 100644 --- a/packages/web-product/src/ui/hooks/use-activity-bubble.ts +++ b/packages/web-product/src/ui/hooks/use-activity-bubble.ts @@ -315,7 +315,7 @@ export function useActivityBubble( } // The step in flight is producing output — "Thinking…" while it reasons, - // "Writing…" once it writes. The producer of this signal clears it the + // "Writing…" once it writes — or, before either, says what it waits on. The producer of this signal clears it the // moment a durable event supersedes it (the step ends, the run rests, or a // new step takes over), so its mere presence means it is current — no // wall-clock comparison, which would be at the mercy of client/server @@ -331,7 +331,7 @@ export function useActivityBubble( operationId: isSameStep ? ctx.operationId : 'ai.agent.turn', stepType: isSameStep ? ctx.stepType : 'ai', ...(isSameStep && ctx.stepName ? { stepName: ctx.stepName } : {}), - label: liveDelta.channel === 'thinking' ? 'Thinking…' : 'Writing…', + label: liveDelta.status ?? (liveDelta.channel === 'thinking' ? 'Thinking…' : 'Writing…'), startedAtMs: liveDelta.atMs, ttlMs: ACTIVE_STEP_MAX_AGE_MS, }; diff --git a/packages/web-product/src/ui/hooks/use-run-reducer.ts b/packages/web-product/src/ui/hooks/use-run-reducer.ts index 0094a3bc..147ad659 100644 --- a/packages/web-product/src/ui/hooks/use-run-reducer.ts +++ b/packages/web-product/src/ui/hooks/use-run-reducer.ts @@ -11,6 +11,7 @@ import type { LiveDeltaFrame } from './session-events-broker.js'; import { runViewReducer, initialRunViewState, + latestStatusLine, type Message, type RunViewState, type RunViewAction, @@ -28,6 +29,8 @@ export interface LiveStreamActivity { stepExecutionId: string; channel: LiveDeltaFrame['channel']; atMs: number; + /** What a `status` frame says the step is waiting on — a model loading, a prompt being read. */ + status?: string; } interface UseRunReducerReturn { @@ -465,6 +468,17 @@ export function useRunReducer(runId: string | null, flowName?: string): UseRunRe timestamp: new Date().toISOString(), ...(flowNameRef.current ? { senderName: flowNameRef.current } : {}), }); + if (frame.channel === 'status') { + const status = latestStatusLine(frame.delta); + if (status === undefined) return; + setLiveActivity({ + stepExecutionId: frame.stepExecutionId, + channel: frame.channel, + atMs: Date.now(), + status, + }); + return; + } setLiveActivity({ stepExecutionId: frame.stepExecutionId, channel: frame.channel, diff --git a/packages/web-product/src/ui/hooks/useLocalModels.ts b/packages/web-product/src/ui/hooks/useLocalModels.ts new file mode 100644 index 00000000..3c454c7b --- /dev/null +++ b/packages/web-product/src/ui/hooks/useLocalModels.ts @@ -0,0 +1,73 @@ +'use client'; + +import type { + EntityDirectives, + LocalModelAction, + LocalModelDiagnosis, + LocalRuntimeDiagnostics, +} from '@aflow/schemas'; +import { useEffect, useState } from 'react'; +import { useApiMutation, useApiQuery } from './useApiQuery.js'; +import { llmReadinessQueryKey } from './useSpaceLlmReadiness.js'; + +/** While bytes are flowing the page reads the record this often; otherwise only when something changes it. */ +const DOWNLOAD_POLL_MS = 2_000; + +export const localModelsQueryKey = (spaceId: string) => ['space', spaceId, 'localModels'] as const; + +/** + * The space's models on this computer, from the one diagnosis readiness and + * Helmsman read too (Plan 323 D15). + */ +export function useLocalModels(spaceId: string, enabled: boolean) { + // Polled only while a download runs; read from the last answer, since the + // interval has to be chosen before this render's answer exists. + const [downloading, setDownloading] = useState(false); + const query = useApiQuery<{ diagnostics: LocalRuntimeDiagnostics | null }>({ + key: localModelsQueryKey(spaceId), + path: `/spaces/${spaceId}/local-models`, + enabled: enabled && spaceId.length > 0, + ...(downloading ? { refetchInterval: DOWNLOAD_POLL_MS } : {}), + spaceId, + }); + const nowDownloading = + query.data?.diagnostics?.models.some((m) => m.pull?.status === 'downloading') ?? false; + useEffect(() => { + setDownloading(nowDownloading); + }, [nowDownloading]); + const invalidate = [localModelsQueryKey(spaceId), llmReadinessQueryKey(spaceId)]; + const act = useApiMutation({ + path: () => `/spaces/${spaceId}/local-models/actions`, + serialize: (action) => JSON.stringify(action), + spaceId, + invalidate, + }); + const stopPull = useApiMutation({ + path: (model) => `/spaces/${spaceId}/local-models/pulls/${encodeURIComponent(model)}`, + method: 'DELETE', + spaceId, + invalidate, + }); + return { + diagnostics: query.data?.diagnostics ?? null, + isLoading: query.isLoading, + act, + stopPull, + }; +} + +/** + * The directives with every role on one model: the default set to it, each + * role's override cleared so it inherits, and the Clerk on the space default — + * the one role that does not inherit unless told to. One GPU serves every + * role, and a role on another model is a load each time the work switches. + */ +export function everyRoleOn( + directives: EntityDirectives, + model: LocalModelDiagnosis, +): EntityDirectives { + return { + ...directives, + modelDefaults: { default: model.ref, clerk: 'space_default' }, + }; +} diff --git a/packages/web-product/src/ui/hooks/useSpaceLlmReadiness.ts b/packages/web-product/src/ui/hooks/useSpaceLlmReadiness.ts index 3b0b69be..3c3bbb47 100644 --- a/packages/web-product/src/ui/hooks/useSpaceLlmReadiness.ts +++ b/packages/web-product/src/ui/hooks/useSpaceLlmReadiness.ts @@ -1,5 +1,6 @@ 'use client'; +import type { LocalRuntimeIssue } from '@aflow/schemas'; import { useApiQuery } from './useApiQuery.js'; export interface RoleReadiness { @@ -59,6 +60,8 @@ export interface SpaceLlmReadiness { * is whether setup has ever happened, not whether the workspace can run now. */ hasConfiguredProvider: boolean; + /** What is wrong with the space's models on this computer; an `error` counts against `ready`. */ + localIssues: LocalRuntimeIssue[]; } /** Invalidate with this key after credential or model-defaults mutations. */ diff --git a/packages/web-product/src/ui/screens/space-host-bindings.tsx b/packages/web-product/src/ui/screens/space-host-bindings.tsx index c88270de..d6cba0af 100644 --- a/packages/web-product/src/ui/screens/space-host-bindings.tsx +++ b/packages/web-product/src/ui/screens/space-host-bindings.tsx @@ -18,6 +18,7 @@ import { useSpace, useSpaceFromRoute } from '../components/providers.js'; import { useEdition } from '../hooks/useEdition.js'; import { useApiQuery, useApiMutation } from '../hooks/useApiQuery.js'; import { HostBrowserProfiles } from '../components/host/HostBrowserProfiles.js'; +import { HostLocalModels } from '../components/host/HostLocalModels.js'; interface HostBinding { hostBindingId: string; @@ -123,8 +124,8 @@ export function SpaceHostBindingsPage() { This Computer - Folders on your machine that this workspace can reach, and the browser its agents use - there. + Folders on your machine that this workspace can reach, the browser its agents use there, + and the models that run on it. @@ -248,6 +249,8 @@ export function SpaceHostBindingsPage() { + + {spaceId.length > 0 && } ); diff --git a/yarn.lock b/yarn.lock index c37fd403..94c3e894 100644 --- a/yarn.lock +++ b/yarn.lock @@ -333,6 +333,7 @@ __metadata: dependencies: "@aflow/ai-client": "workspace:*" "@aflow/applet-runtime": "workspace:*" + "@aflow/credential-resolver": "workspace:*" "@aflow/database": "workspace:*" "@aflow/integration-simulator": "workspace:*" "@aflow/lib": "workspace:*"