fix(runtime): surface typed harness provider failures

packages/runtime/README.md

@@ -0,0 +1,26 @@
+# Workforce runtime
+
+## Harness provider failures
+
+`ctx.harness.run()` rejects with `HarnessProviderError` when a failed model CLI
+run reports a recognized usage/credit limit, rate limit, authentication error,
+context limit, request timeout, or provider outage. Classification is shared by
+all personas; handlers do not need their own output parsers. Successful runs,
+unrecognized failures, and OS kill exits retain the `HarnessRunResult` contract.
+
+The error's `message` is safe to display. `providerFailure` contains its `kind`,
+message, optional provider, and optional `resetHint` copied only from a valid
+clock time with an explicit timezone. Reset hints are provider reports, not
+promises of recovery. The original result is retained as the non-enumerable
+`error.result` for explicit operator diagnosis; do not post it to users.
+
+The runtime logs `harness.provider_error`, and the runner preserves
+`providerFailure` alongside the actionable `error` on `runner.handler.error`.
+Cloud can persist that run error and use the structured metadata for customer
+notifications without reparsing CLI output. Callers that intentionally handle a
+provider failure can catch `HarnessProviderError`; existing generic exit-code
+checks no longer replace recognized provider causes.
+
+This boundary does not retry tasks, change credentials, or fall back to another
+provider or a paid API key. Those actions need their own explicit policy because
+a failed task may already have performed work.

packages/runtime/src/ctx.test.ts

@@ -30,6 +30,37 @@ const stubSandbox: SandboxContext = {
   }
 };
 
+test('ctx.harness.run surfaces provider usage limits before a caller can replace them with exit 1', async () => {
+  let calls = 0;
+  let reachedCallerFailure = false;
+  const logs: Array<{ message: string; attrs?: Record<string, unknown> }> = [];
+  const ctx = buildCtx({
+    persona: basePersona,
+    workspaceId: 'ws-test',
+    agent: { id: 'agent-test', deployedName: 'example', spawnedByAgentId: null },
+    deployment: { id: 'deployment-test', triggerKind: 'inbox', parentDeploymentId: null },
+    sandbox: stubSandbox,
+    log: (_level, message, attrs) => logs.push({ message, attrs }),
+    harnessRunner: async () => {
+      calls++;
+      return { output: "You've hit your limit · resets 3:40pm (UTC)", stderr: 'secret-fixture-value', exitCode: 1, durationMs: 1900 };
+    }
+  });
+  await assert.rejects(async () => {
+    const run = await ctx.harness.run({ prompt: 'Perform a task' });
+    reachedCallerFailure = true;
+    if (run.exitCode !== 0) throw new Error(`The harness exited with code ${run.exitCode}`);
+  }, (error: Error) => {
+    assert.match(error.message, /Claude account.*usage limit/);
+    assert.match(error.message, /3:40pm \(UTC\)/);
+    assert.doesNotMatch(error.message, /secret-fixture/);
+    return true;
+  });
+  assert.equal(calls, 1, 'provider failure must not replay task side effects');
+  assert.equal(reachedCallerFailure, false);
+  assert.ok(logs.some((entry) => entry.message === 'harness.provider_error'));
+});
+
 function ctxFor(
   persona: PersonaSpec,
   inputValues?: Record<string, string | number | boolean | null | undefined>,

packages/runtime/src/ctx.ts

@@ -17,6 +17,7 @@ import type {
 import { attachTrajectoryRecorder, createTrajectoryRecorder } from './trajectory.js';
 import { buildRelayContext } from './relay.js';
 import { NO_REPLY_MARKER, sanitizeNoReplyOutput } from './no-reply.js';
+import { classifyHarnessProviderFailure, HarnessProviderError } from './harness-provider-error.js';
 
 type AgentInputValue = string | number | boolean | null | undefined;
 
@@ -169,6 +170,16 @@ export function buildCtx(options: CtxBuildOptions): WorkforceCtx {
     harness: {
       async run(args) {
         const result = await options.harnessRunner(args);
+        const providerFailure = classifyHarnessProviderFailure(result, options.persona.harness);
+        if (providerFailure) {
+          log('error', 'harness.provider_error', {
+            providerFailure,
+            exitCode: result.exitCode,
+            durationMs: result.durationMs,
+            harness: options.persona.harness
+          });
+          throw new HarnessProviderError(providerFailure, result);
+        }
         const sanitizedOutput = sanitizeNoReplyOutput(result.output);
         const sanitizedStderr = result.stderr === undefined
           ? undefined

packages/runtime/src/harness-provider-error.test.ts

@@ -0,0 +1,96 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { classifyHarnessProviderFailure, HarnessProviderError } from './harness-provider-error.js';
+import type { HarnessProviderFailure } from './harness-provider-error.js';
+
+function classify(output: string, harness = 'claude') {
+  return classifyHarnessProviderFailure({ output, exitCode: 1 }, harness);
+}
+
+test('Claude usage limit preserves a reported reset time without leaking surrounding output', () => {
+  const failure = classify("\x1b[31mYou’ve hit your limit · resets 3:40pm (UTC)\x1b[0m\nsecret-fixture-value");
+  assert.equal(failure?.kind, 'usage_limit');
+  assert.equal(failure?.provider, 'anthropic');
+  assert.equal(failure?.resetHint, '3:40pm (UTC)');
+  assert.match(failure!.message, /Claude account.*usage limit/);
+  assert.match(failure!.message, /Wait for.*reset/);
+  assert.doesNotMatch(failure!.message, /secret-fixture/);
+});
+
+test('handles Codex error envelopes and does not label another harness as Claude', () => {
+  const result = classify(JSON.stringify({ type: 'turn.failed', error: { message: "You've hit your usage limit. Please try later." } }), 'codex');
+  assert.equal(result?.kind, 'usage_limit');
+  assert.equal(result?.provider, 'openai');
+  assert.match(result!.message, /OpenAI account/);
+  assert.doesNotMatch(result!.message, /Claude/);
+  assert.equal(classify(JSON.stringify({ type: 'result', is_error: true, result: "You've hit your limit" }))?.kind, 'usage_limit');
+  assert.equal(classify(JSON.stringify({ type: 'error', message: "You've hit your usage limit" }), 'codex')?.kind, 'usage_limit');
+  assert.match(classify("You've hit your limit", 'opencode')!.message, /AI account/);
+});
+
+test('classifies known provider diagnostics from stderr using safe messages', () => {
+  const cases: Array<[string, HarnessProviderFailure['kind']]> = [
+    ['API Error: 429 request throttled', 'rate_limit'],
+    ['{"error":{"type":"rate_limit_error","message":"private detail"}}', 'rate_limit'],
+    ['{"error":{"code":"insufficient_quota"}}', 'usage_limit'],
+    ['Your credit balance is too low to access the Anthropic API.', 'usage_limit'],
+    ['API Error: 401 private detail', 'authentication'],
+    ['{"error":{"type":"authentication_error"}}', 'authentication'],
+    ['OAuth token has expired.', 'authentication'],
+    ['Invalid API key · Please run /login', 'authentication'],
+    ['Prompt is too long: private detail', 'context_limit'],
+    ['{"error":{"code":"context_length_exceeded"}}', 'context_limit'],
+    ['API Error: 400 {"error":{"type":"invalid_request_error","message":"prompt is too long: private detail"}}', 'context_limit'],
+    ['API Error: Request timed out.', 'timeout'],
+    ['API Error: 529 private detail', 'provider_unavailable'],
+    ['{"error":{"type":"overloaded_error"}}', 'provider_unavailable'],
+  ];
+  for (const [stderr, kind] of cases) {
+    const failure = classifyHarnessProviderFailure({ stderr, output: 'unfinished-task-fixture', exitCode: 1 });
+    assert.equal(failure?.kind, kind, stderr);
+    assert.doesNotMatch(failure!.message, /private detail|unfinished-task-fixture/);
+  }
+});
+
+test('reset hints require a valid explicit timezone and clock time', () => {
+  for (const suffix of ['resets soon secret-fixture', 'resets 3:40pm', 'resets 99:99pm (UTC)', 'resets 3:40pm (Secret/Fixture)', 'resets 3:40pm (UTC)[secret-fixture]']) {
+    const failure = classify(`You've hit your limit · ${suffix}`);
+    assert.equal(failure?.kind, 'usage_limit');
+    assert.equal(failure?.resetHint, undefined, suffix);
+    assert.doesNotMatch(failure!.message, /secret-fixture|99:99|3:40|Secret\/Fixture/);
+  }
+  assert.equal(classify("You've hit your limit · resets 17:40 (Europe/Oslo)")?.resetHint, '17:40 (Europe/Oslo)');
+});
+
+test('does not interpret successful task output, OS kills, or unknown process failures', () => {
+  for (const exitCode of [0, 137, 143, NaN]) {
+    assert.equal(classifyHarnessProviderFailure({ output: "You've hit your limit", exitCode }), null);
+  }
+  for (const output of ['', 'gh: HTTP 429 Too Many Requests', 'test failed with exit 1', 'ECONNRESET', 'Timeout waiting for database', 'The test fixture contains API Error: 401']) {
+    assert.equal(classify(output), null, output);
+  }
+});
+
+test('typed errors expose safe metadata and retain non-enumerable original diagnostics', () => {
+  const result = { output: "You've hit your limit", stderr: 'secret-fixture-value', exitCode: 1, durationMs: 1900 };
+  const failure = classifyHarnessProviderFailure(result, 'claude')!;
+  const error = new HarnessProviderError(failure, result);
+  assert.equal(error.name, 'HarnessProviderError');
+  assert.equal(error.message, failure.message);
+  assert.equal(error.result, result);
+  assert.deepEqual(error.providerFailure, failure);
+  assert.doesNotMatch(JSON.stringify(error), /secret-fixture-value/);
+});
+
+
+test('ignores provider-like fixtures inside failed task output and non-error CLI records', () => {
+  const payload = { error: { type: 'rate_limit_error', message: 'example' } };
+  for (const output of [
+    `The test fixture is ${JSON.stringify(payload)}`,
+    JSON.stringify({ type: 'assistant', message: JSON.stringify(payload) }),
+    JSON.stringify({ type: 'tool_result', content: JSON.stringify(payload) }),
+    JSON.stringify({ type: 'result', is_error: false, result: JSON.stringify(payload) }),
+    JSON.stringify({ type: 'assistant', ...payload }),
+    JSON.stringify({ type: 'tool_result', content: "You've hit your limit" }),
+  ]) assert.equal(classify(output), null, output);
+});

packages/runtime/src/harness-provider-error.ts

@@ -0,0 +1,148 @@
+import type { HarnessRunResult } from './types.js';
+
+export interface HarnessProviderFailure {
+  kind: 'usage_limit' | 'rate_limit' | 'authentication' | 'context_limit' | 'provider_unavailable' | 'timeout';
+  message: string;
+  resetHint?: string;
+  provider?: 'anthropic' | 'openai';
+}
+
+function validTimezone(value: string): boolean {
+  try {
+    new Intl.DateTimeFormat('en', { timeZone: value });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function diagnosticMessages(text: string): string[] {
+  const messages: string[] = [];
+  for (const line of text.split(/\r?\n/)) {
+    try {
+      const value = JSON.parse(line) as Record<string, unknown> | null;
+      if (!value || typeof value !== 'object' || Array.isArray(value)) continue;
+      // CLI error envelopes, not arbitrary assistant/tool text in a stream.
+      if (value.type === 'error' && typeof value.message === 'string') messages.push(value.message);
+      // Provider API response envelopes are valid diagnostics, but a nested
+      // error inside assistant/tool output is still task content.
+      if ((value.type === undefined || value.type === 'error') && value.error && typeof value.error === 'object') {
+        const error = value.error as Record<string, unknown>;
+        if (typeof error.type === 'string' || typeof error.code === 'string') messages.push(JSON.stringify({ error }));
+      }
+      if (value.type === 'result' && value.is_error === true && typeof value.result === 'string') messages.push(value.result);
+      if (value.type === 'turn.failed' && value.error && typeof value.error === 'object') {
+        const message = (value.error as { message?: unknown }).message;
+        if (typeof message === 'string') messages.push(message);
+      }
+    } catch {
+      // Preserve plain diagnostics, excluding prose/code containing embedded JSON.
+      // API Error lines are the CLI's explicit provider diagnostic prefix.
+      if (/^\s*API Error:/i.test(line) || !/[{}]/.test(line)) messages.push(line);
+    }
+  }
+  return messages;
+}
+
+/**
+ * Classify failed model CLI runs, never arbitrary successful agent output.
+ * Customer messages are fixed templates, not excerpts of stdout/stderr:
+ * those streams can contain code, credentials, and unfinished task output.
+ */
+export function classifyHarnessProviderFailure(run: Pick<HarnessRunResult, 'output' | 'stderr' | 'exitCode'>, harness?: string): HarnessProviderFailure | null {
+  // OS kills and successful output retain their existing caller contract.
+  if (!Number.isFinite(run.exitCode) || run.exitCode === 0 || run.exitCode === 137 || run.exitCode === 143) return null;
+  const provider = harness === 'claude' ? 'anthropic' : harness === 'codex' ? 'openai' : undefined;
+  const account = provider === 'anthropic' ? 'Claude' : provider === 'openai' ? 'OpenAI' : 'AI';
+  const result = run as { output?: unknown; stderr?: unknown } | null;
+  const rawText = [result?.output, result?.stderr]
+    .filter((value): value is string => typeof value === 'string')
+    .map((value) => value.slice(-16000).replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, ''))
+    .join('\n');
+  const text = diagnosticMessages(rawText).join('\n');
+
+  const claudeLimit = text.match(/^\s*You['’]ve hit your (?:usage )?limit\b([^\r\n]*)/im);
+  if (claudeLimit) {
+    // Only copy a clock time with an explicit timezone. Do not echo the
+    // remainder of an arbitrary line or invent an absolute reset date.
+    const reset = claudeLimit[1].match(
+      /\bresets\s+((?:1[0-2]|0?[1-9])(?::[0-5]\d)?\s*[ap]m|(?:[01]?\d|2[0-3]):[0-5]\d)\s*\((UTC|GMT|[A-Za-z_]+\/[A-Za-z_]+(?:\/[A-Za-z_]+)?)\)(?=$|[\s.,;])/i,
+    );
+    const resetHint = reset && validTimezone(reset[2]) ? `${reset[1].trim()} (${reset[2]})` : undefined;
+    return {
+      ...(provider ? { provider } : {}),
+      kind: 'usage_limit',
+      message: [
+        `The ${account} account selected for this run has reached its usage limit.`,
+        ...(resetHint ? [`The provider reports a reset at ${resetHint}.`] : []),
+        'Wait for the limit to reset, or ask the account owner to restore available usage before retrying the task.',
+      ].join(' '),
+      ...(resetHint ? { resetHint } : {}),
+    };
+  }
+
+  if (/"(?:type|code)"\s*:\s*"(?:insufficient_quota|billing_hard_limit_reached)"|\byou exceeded your current quota\b|\byour credit balance is too low to access the Anthropic API\b/i.test(text)) {
+    return {
+      ...(provider ? { provider } : {}),
+      kind: 'usage_limit',
+      message: 'The AI account used for this run has no available quota or credits. Ask the account owner to check usage and billing and restore capacity before retrying the task.',
+    };
+  }
+
+  if (/"(?:type|code)"\s*:\s*"(?:rate_limit_error|rate_limit_exceeded)"|^\s*API Error:\s*429\b|\bThis request would exceed your account's rate limit\b/im.test(text)) {
+    return {
+      ...(provider ? { provider } : {}),
+      kind: 'rate_limit',
+      message: 'The AI provider rate-limited this run. Wait for available capacity before retrying; if this persists, ask the account owner to check the account limits.',
+    };
+  }
+
+  if (/"(?:type|code)"\s*:\s*"(?:authentication_error|invalid_api_key)"|^\s*API Error:\s*401\b|^\s*Invalid API key\b.*\/login|\bOAuth token has expired\b/im.test(text)) {
+    return {
+      ...(provider ? { provider } : {}),
+      kind: 'authentication',
+      message: 'The AI provider rejected the credentials used for this run. Ask the account owner to reconnect the AI account before retrying the task.',
+    };
+  }
+
+  if (/"(?:type|code)"\s*:\s*"context_length_exceeded"|"message"\s*:\s*"prompt is too long\b|^\s*(?:API Error:\s*400\s+)?prompt is too long\b/im.test(text)) {
+    return {
+      ...(provider ? { provider } : {}),
+      kind: 'context_limit',
+      message: 'The task input exceeds the AI model context limit. Reduce the task scope or select a model with a larger context window before retrying.',
+    };
+  }
+
+  if (/^\s*API Error:\s*(?:Request timed out|Request timeout)\b|"(?:type|code)"\s*:\s*"(?:request_timeout|timeout_error)"/im.test(text)) {
+    return {
+      ...(provider ? { provider } : {}),
+      kind: 'timeout',
+      message: 'The AI request for this run timed out. Retry when the provider is responsive; if this persists, an operator should check the request timeout and task scope.',
+    };
+  }
+
+  if (/"type"\s*:\s*"overloaded_error"|^\s*API Error:\s*(?:500|502|503|504|529)\b/im.test(text)) {
+    return {
+      ...(provider ? { provider } : {}),
+      kind: 'provider_unavailable',
+      message: 'The AI provider could not serve this run request. Retry after the provider recovers; if this persists, an operator should check provider availability.',
+    };
+  }
+
+  return null;
+}
+
+/** Recognized provider failure, shared by every ctx.harness.run caller. */
+export class HarnessProviderError extends Error {
+  readonly providerFailure: HarnessProviderFailure;
+  readonly result!: HarnessRunResult;
+
+  constructor(failure: HarnessProviderFailure, result: HarnessRunResult) {
+    super(failure.message);
+    this.name = 'HarnessProviderError';
+    this.providerFailure = Object.freeze({ ...failure });
+    // Retain diagnostics for explicit recovery without serializing secrets
+    // when the error is logged or passed to a customer-facing surface.
+    Object.defineProperty(this, 'result', { value: result, enumerable: false });
+  }
+}

packages/runtime/src/index.ts

@@ -229,3 +229,4 @@ export type {
   TypedTriggerMap,
   WatchRule
 } from '@agentworkforce/persona-kit';
+export { classifyHarnessProviderFailure, HarnessProviderError, type HarnessProviderFailure } from './harness-provider-error.js';

packages/runtime/src/runner.test.ts

@@ -155,6 +155,37 @@ test('startRunner logs and continues when the handler throws', async () => {
   assert.equal(errors.length, 2);
 });
 
+test('provider failures reach runner error reporting without changing the agent handler', async () => {
+  const logs: Array<{ level: string; message: string; attrs?: Record<string, unknown> }> = [];
+  let posted = 0;
+  let calls = 0;
+  await startRunner({
+    persona,
+    agent: runtimeAgent,
+    deployment: runtimeDeployment,
+    workspaceId: 'ws-test',
+    handler: handler(async (ctx) => {
+      const result = await ctx.harness.run({ prompt: 'Perform the task' });
+      if (result.exitCode !== 0) throw new Error(`The harness exited with code ${result.exitCode}`);
+      posted++;
+    }),
+    harnessRunner: async () => {
+      calls++;
+      return { output: "You've hit your limit · resets 3:40pm (UTC)", stderr: 'secret-fixture', exitCode: 1, durationMs: 1900 };
+    },
+    subsystems: { sandbox: stubSandbox, log: (level, message, attrs) => logs.push({ level, message, attrs }) },
+    envelopes: streamOf([{ id: 'quota-event', workspace: 'ws-test', type: 'cron.tick', occurredAt: 'x', name: 'tick' }])
+  });
+  const error = logs.find((entry) => entry.message === 'runner.handler.error');
+  assert.match(String(error?.attrs?.error), /Claude account.*usage limit/);
+  assert.match(String(error?.attrs?.error), /3:40pm \(UTC\)/);
+  assert.equal((error?.attrs?.providerFailure as { kind: string }).kind, 'usage_limit');
+  assert.doesNotMatch(JSON.stringify(error), /secret-fixture|exited with code 1/);
+  assert.equal(logs.some((entry) => entry.message === 'runner.handler.ok'), false);
+  assert.equal(calls, 1);
+  assert.equal(posted, 0, 'failure output cannot reach the task success path');
+});
+
 test('startRunner skips envelopes that the shim can not translate', async () => {
   const received: WorkforceEvent[] = [];
   const logs: Array<{ level: string; message: string }> = [];