feat(323): P3a — a local model's window is sized from the machine it runs on

apps/aflow-executor-ai/src/handlers/ai/aiClient.ts

@@ -7,6 +7,7 @@ import {
 } from '@aflow/ai-client';
 import type { ExecutorContext } from '@aflow/executor-runtime';
 import { localRuntimeId, type AflowError } from '@aflow/schemas';
+import { readLocalMachine, type HostMachine, type Redis } from '@aflow/redis';
 import {
   CredentialResolver,
   credentialMissingMessage,
@@ -132,6 +133,25 @@ let credentialResolver: CredentialResolver | null = null;
 /**
  * Initialize the credential resolver. Call once at executor startup.
  */
+// The machine a local runtime runs on, from the paired machine's inventory.
+// Read at most this often: it changes when the machine does.
+const LOCAL_MACHINE_TTL_MS = 60_000;
+let machineSource: Redis | null = null;
+let machineReading: { value: HostMachine | undefined; expiresAt: number } | null = null;
+
+/** Where the machine a local runtime runs on is read from (Plan 323 D8). */
+export function initLocalMachineSource(redis: Redis): void {
+  machineSource = redis;
+}
+
+async function localMachine(): Promise<HostMachine | undefined> {
+  if (!machineSource) return undefined;
+  if (machineReading && machineReading.expiresAt > Date.now()) return machineReading.value;
+  const value = await readLocalMachine(machineSource).catch(() => undefined);
+  machineReading = { value, expiresAt: Date.now() + LOCAL_MACHINE_TTL_MS };
+  return value;
+}
+
 export function initCredentialResolver(loader: CredentialLoader): void {
   credentialResolver = new CredentialResolver({ loader });
 }
@@ -301,15 +321,21 @@ export async function getAIClientForContext(
 
   // 4. Check client cache (keyed by credentialId + updatedAt for rotation safety)
   // When a credential is rotated in-place, updatedAt changes → cache miss → new client
-  const cacheKey = `${resolved.credentialId}:${resolved.updatedAt}`;
+
+  // 5. Build single-provider AIClient
+  // The machine is part of the key: a GPU figure that arrives after the first
+  // client was built changes the window that client would size.
+  const machine = aiProvider === 'ollama' ? await localMachine() : undefined;
+  const cacheKey = `${resolved.credentialId}:${resolved.updatedAt}:${machine?.gpuWorkingSetBytes ?? '-'}`;
   const cached = CLIENT_CACHE.get(cacheKey);
   if (cached && cached.expiresAt > Date.now()) {
     return cached.client;
   }
 
-  // 5. Build single-provider AIClient
   const client = createAIClient({
-    providers: { [aiProvider]: providerConfigFromCredential(credentialProviderId, resolved) },
+    providers: {
+      [aiProvider]: providerConfigFromCredential(credentialProviderId, resolved, { machine }),
+    },
     defaultProvider: aiProvider,
   });
 

apps/aflow-executor-ai/src/index.ts

@@ -40,6 +40,7 @@ import { createServiceLogger } from '@aflow/executor-runtime';
 import { AiHandler } from './handlers/ai/index.js';
 import {
   initCredentialResolver,
+  initLocalMachineSource,
   getCredentialResolver,
   invalidateAllCredentialCaches,
 } from './handlers/ai/aiClient.js';
@@ -140,6 +141,7 @@ async function main(): Promise<void> {
   if (shouldRunStepExecutor) {
     const db = database.db;
 
+    initLocalMachineSource(redis);
     initCredentialResolver(async (tenantId, providerId) => {
       const tenantContext = createTenantContext(tenantId as TenantId);
       try {

apps/aflow-executor-host/src/__tests__/harnessBoundary.test.ts

@@ -198,14 +198,17 @@ describe('one spawn path', () => {
     // pair, which Node has no call for; it runs nothing else. keepAwake.ts
     // starts the `caffeinate` or `systemd-inhibit` that holds the machine awake
     // while a step runs, a system call by another name that reads and runs no
-    // workload.
+    // workload. machine.ts asks Metal (through `swift`) or `nvidia-smi` how much
+    // GPU memory a model runtime on this machine has, once, and runs nothing it
+    // was handed.
     // Named rather than matched, so a new unconfined spawn is argued for here.
     expect(importers).toEqual([
       'browser/harnessBrowser.ts',
       'credentialFetch.ts',
       'folderPicker.ts',
       'harnessDiscovery.ts',
       'keepAwake.ts',
+      'machine.ts',
       'orphans.ts',
       'runtimes.ts',
       'sandboxedRun.ts',

apps/aflow-executor-host/src/__tests__/machine.test.ts

@@ -0,0 +1,88 @@
+import { mkdtemp, readFile, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { describe, expect, it, vi } from 'vitest';
+import { createMachineReader } from '../machine.js';
+
+const GIB = 1024 ** 3;
+
+async function hostDir(): Promise<string> {
+  return mkdtemp(join(tmpdir(), 'aflow-machine-'));
+}
+
+describe('createMachineReader', () => {
+  it('reads Metal’s working set on Apple silicon once and keeps it for the next start', async () => {
+    const dir = await hostDir();
+    const run = vi.fn(async () => '19069665280\n');
+    const options = {
+      hostDir: dir,
+      platform: 'darwin' as const,
+      arch: 'arm64',
+      totalMemoryBytes: 24 * GIB,
+      osRelease: '25.3.0',
+      run,
+    };
+
+    const first = createMachineReader(options);
+    expect(first.current()).toEqual({ totalMemoryBytes: 24 * GIB, unifiedMemory: true });
+    await first.start();
+    expect(first.current().gpuWorkingSetBytes).toBe(19_069_665_280);
+
+    const second = createMachineReader(options);
+    await second.start();
+    expect(second.current().gpuWorkingSetBytes).toBe(19_069_665_280);
+    expect(run).toHaveBeenCalledTimes(1);
+  });
+
+  it('reads it again once the machine or its OS has changed', async () => {
+    const dir = await hostDir();
+    await writeFile(
+      join(dir, 'machine.json'),
+      JSON.stringify({ totalMemoryBytes: 24 * GIB, osRelease: '25.2.0', gpuWorkingSetBytes: 1 }),
+    );
+    const run = vi.fn(async () => '19069665280');
+    const reader = createMachineReader({
+      hostDir: dir,
+      platform: 'darwin',
+      arch: 'arm64',
+      totalMemoryBytes: 24 * GIB,
+      osRelease: '25.3.0',
+      run,
+    });
+    await reader.start();
+    expect(run).toHaveBeenCalledTimes(1);
+    expect(JSON.parse(await readFile(join(dir, 'machine.json'), 'utf8'))).toMatchObject({
+      osRelease: '25.3.0',
+    });
+  });
+
+  it('takes the largest NVIDIA device, and leaves the figure out where no probe answers', async () => {
+    const nvidia = createMachineReader({
+      hostDir: await hostDir(),
+      platform: 'linux',
+      arch: 'x64',
+      totalMemoryBytes: 64 * GIB,
+      osRelease: '6.8',
+      run: async () => '24576\n49152\n',
+    });
+    await nvidia.start();
+    expect(nvidia.current()).toEqual({
+      totalMemoryBytes: 64 * GIB,
+      unifiedMemory: false,
+      gpuWorkingSetBytes: 49152 * 1024 ** 2,
+    });
+
+    const none = createMachineReader({
+      hostDir: await hostDir(),
+      platform: 'linux',
+      arch: 'x64',
+      totalMemoryBytes: 16 * GIB,
+      osRelease: '6.8',
+      run: async () => {
+        throw new Error('nvidia-smi: command not found');
+      },
+    });
+    await none.start();
+    expect(none.current().gpuWorkingSetBytes).toBeUndefined();
+  });
+});

apps/aflow-executor-host/src/index.ts

@@ -81,6 +81,7 @@ import { loadPairedEnv } from './pairedEnv.js';
 import { followPolicy, watchPolicy } from './policyWatch.js';
 import { killAllProcesses, killProcessesForBinding, reapWithdrawn } from './sandboxedRun.js';
 import { observeRuntimes } from './runtimes.js';
+import { createMachineReader } from './machine.js';
 import { publishingFolders } from './pushApproval.js';
 import { startUnderSignals } from './startUnderSignals.js';
 import { createStartupTimings } from './startupTimings.js';
@@ -288,6 +289,10 @@ async function main(): Promise<void> {
   let lastFolders: HostInventoryFolders = [];
   let lastMaxConcurrentHarnessRuns = HOST_HARNESS_CONCURRENCY_DEFAULT;
   let lastBrowsers: HostInventoryBrowsers = [];
+  const machine = createMachineReader({ hostDir: dirname(policyPath) });
+  // Read in the background: the probe compiles a Swift program, and the
+  // inventory goes out without the GPU figure until it is in.
+  void machine.start();
 
   // Published with a lifetime rather than stored: an inventory that outlives the
   // executor describes a machine nobody is listening on, and inviting a run
@@ -340,6 +345,7 @@ async function main(): Promise<void> {
     const inventory: HostInventory = {
       hostname,
       observedAt: new Date().toISOString(),
+      machine: machine.current(),
       runtimes,
       harnesses,
       maxConcurrentHarnessRuns,

apps/aflow-executor-host/src/machine.ts

@@ -0,0 +1,111 @@
+/**
+ * What this machine offers a model runtime on it (Plan 323 D8), for the
+ * inventory.
+ *
+ * The figure that decides fit is the GPU working set, and nothing cheap
+ * reports it: on Apple silicon it is Metal's `recommendedMaxWorkingSetSize`,
+ * which no `sysctl` carries and Ollama logs but does not serve. Reading it
+ * through Metal compiles a one-line Swift program, which takes seconds to
+ * minutes, so it is read once in the background and cached beside the policy
+ * file, keyed by the machine's memory and OS release so an upgrade reads it
+ * again. Until it is read — or where it cannot be — it is absent, and fit
+ * says it does not know rather than guessing.
+ */
+import { execFile } from 'node:child_process';
+import { readFile, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import { join } from 'node:path';
+import type { HostMachine } from '@aflow/redis';
+
+const CACHE_FILE = 'machine.json';
+/** A first `swift` run compiles its toolchain's caches; minutes, not seconds. */
+const PROBE_TIMEOUT_MS = 300_000;
+const METAL_PROBE =
+  'import Metal; print(MTLCreateSystemDefaultDevice()?.recommendedMaxWorkingSetSize ?? 0)';
+
+interface CachedReading {
+  totalMemoryBytes: number;
+  osRelease: string;
+  gpuWorkingSetBytes: number;
+}
+
+export type RunCommand = (file: string, args: string[], timeoutMs: number) => Promise<string>;
+
+const runCommand: RunCommand = (file, args, timeoutMs) =>
+  new Promise((resolve, reject) => {
+    execFile(file, args, { timeout: timeoutMs, encoding: 'utf8' }, (error, stdout) => {
+      if (error) reject(new Error(`${file} failed: ${error.message}`, { cause: error }));
+      else resolve(stdout);
+    });
+  });
+
+function positive(value: string): number | undefined {
+  const parsed = Number(value.trim());
+  return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : undefined;
+}
+
+async function readGpuWorkingSet(platform: string, run: RunCommand): Promise<number | undefined> {
+  if (platform === 'darwin') {
+    return positive(await run('swift', ['-e', METAL_PROBE], PROBE_TIMEOUT_MS));
+  }
+  // A discrete GPU's memory is the device's own; the largest one is what a
+  // model that does not split across devices can use.
+  const out = await run(
+    'nvidia-smi',
+    ['--query-gpu=memory.total', '--format=csv,noheader,nounits'],
+    PROBE_TIMEOUT_MS,
+  );
+  const mib = out
+    .split('\n')
+    .map(positive)
+    .filter((v): v is number => v !== undefined);
+  return mib.length > 0 ? Math.max(...mib) * 1024 ** 2 : undefined;
+}
+
+export interface MachineReader {
+  /** The machine as read so far; the GPU figure arrives once the probe has run. */
+  current(): HostMachine;
+  /** Reads the GPU figure, from the cache when it still describes this machine. */
+  start(): Promise<void>;
+}
+
+export function createMachineReader(options: {
+  hostDir: string;
+  platform?: NodeJS.Platform;
+  arch?: string;
+  totalMemoryBytes?: number;
+  osRelease?: string;
+  run?: RunCommand;
+}): MachineReader {
+  const platform = options.platform ?? process.platform;
+  const totalMemoryBytes = options.totalMemoryBytes ?? os.totalmem();
+  const osRelease = options.osRelease ?? os.release();
+  const run = options.run ?? runCommand;
+  const cachePath = join(options.hostDir, CACHE_FILE);
+  const machine: HostMachine = {
+    totalMemoryBytes,
+    unifiedMemory: platform === 'darwin' && (options.arch ?? process.arch) === 'arm64',
+  };
+
+  return {
+    current: () => ({ ...machine }),
+    async start() {
+      const cached = await readFile(cachePath, 'utf8')
+        .then((text) => JSON.parse(text) as CachedReading)
+        .catch(() => undefined);
+      if (
+        cached?.totalMemoryBytes === totalMemoryBytes &&
+        cached.osRelease === osRelease &&
+        cached.gpuWorkingSetBytes > 0
+      ) {
+        machine.gpuWorkingSetBytes = cached.gpuWorkingSetBytes;
+        return;
+      }
+      const gpu = await readGpuWorkingSet(platform, run).catch(() => undefined);
+      if (gpu === undefined) return;
+      machine.gpuWorkingSetBytes = gpu;
+      const reading: CachedReading = { totalMemoryBytes, osRelease, gpuWorkingSetBytes: gpu };
+      await writeFile(cachePath, `${JSON.stringify(reading)}\n`).catch(() => undefined);
+    },
+  };
+}

apps/aflow-orchestrator/src/services/cybernetic/__tests__/postRunHooks.test.ts

@@ -66,6 +66,7 @@ vi.mock('@aflow/database', () => ({
 const mockAppendEntityEvent = vi.fn();
 vi.mock('@aflow/redis', () => ({
   runtimeAdmission: () => async () => ({ release: async () => undefined }),
+  readLocalMachine: async () => undefined,
   appendEntityEvent: (...args: unknown[]) => mockAppendEntityEvent(...args),
 }));
 

apps/aflow-orchestrator/src/services/cybernetic/postRunHooks.ts

@@ -41,7 +41,7 @@ import { createTenantContext, withTenantSchema, spaces } from '@aflow/database';
 import { createByokAiClientFactory } from '@aflow/credential-resolver';
 import { eq } from 'drizzle-orm';
 import { EntityDirectivesSchema } from '@aflow/schemas';
-import { appendEntityEvent, runtimeAdmission } from '@aflow/redis';
+import { appendEntityEvent, readLocalMachine, runtimeAdmission } from '@aflow/redis';
 import type { PayloadStore } from '@aflow/payload-store';
 import { randomUUID } from 'node:crypto';
 
@@ -149,7 +149,10 @@ export async function fireCyberneticPostRunHooksStandalone(
 
     // Step 1: Run eval — unconditional when a suite exists; the outcome is
     // recorded on the run as a typed evaluation envelope either way.
-    const byokFactory = createByokAiClientFactory(db, runtimeAdmission(redis, { wait: true }));
+    const byokFactory = createByokAiClientFactory(db, {
+      admission: runtimeAdmission(redis, { wait: true }),
+      machine: () => readLocalMachine(redis),
+    });
     let evalOutcome: EvalRunOutcome;
     try {
       evalOutcome = await runEvaluation({

apps/aflow-orchestrator/src/services/sessionMetadataTask.test.ts

@@ -17,6 +17,7 @@ const mocks = vi.hoisted(() => ({
 
 vi.mock('@aflow/redis', () => ({
   runtimeAdmission: () => async () => ({ release: async () => undefined }),
+  readLocalMachine: async () => undefined,
   claimSessionMetadataCandidates: async () => [
     {
       tenantId: '00000000-0000-0000-0000-000000000001',

apps/aflow-orchestrator/src/services/sessionMetadataTask.ts

@@ -20,6 +20,7 @@ import {
   settleSessionMetadata,
   sessionMetadataLeaseHeld,
   appendEntityEvent,
+  readLocalMachine,
   runtimeAdmission,
   type SessionMetadataCandidate,
 } from '@aflow/redis';
@@ -100,10 +101,10 @@ export function createSessionMetadataTask(deps: SessionMetadataTaskDeps): Backgr
 
       // A title is not worth queueing behind an agent turn on this computer's
       // runtime; a busy one fails this conversation's cycle, and it is due again.
-      byokFactory ??= createByokAiClientFactory(
-        deps.db,
-        runtimeAdmission(deps.redis, { wait: false }),
-      );
+      byokFactory ??= createByokAiClientFactory(deps.db, {
+        admission: runtimeAdmission(deps.redis, { wait: false }),
+        machine: () => readLocalMachine(deps.redis),
+      });
 
       let processed = 0;
       let failed = 0;