Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions packages/runtime/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Fixed

- Classify Grok Build usage-balance exhaustion as a typed, customer-safe
`usage_limit` provider failure so scheduled deliveries stop retrying and tell
operators to restore Grok credits.

## [4.2.0] - 2026-10-02

### Dependencies
Expand Down
38 changes: 38 additions & 0 deletions packages/runtime/src/harness-provider-error.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,44 @@ test('handles Codex error envelopes and does not label another harness as Claude
assert.match(classify("You've hit your limit", 'opencode')!.message, /AI account/);
});

test('classifies Grok Build 402 balance exhaustion with a safe actionable message', () => {
const envelope = {
message: 'API error (status 402 Payment Required): Grok Build usage balance exhausted',
http_status: 402,
private_detail: 'secret-fixture-value',
};
const prettyOutput = [
'Internal error: {',
` "message": "${envelope.message}",`,
' "http_status": 402,',
' "private_detail": "secret-fixture-value"',
'}',
].join('\n');
const compactOutput = `Internal error: ${JSON.stringify(envelope)}`;
for (const stderr of [prettyOutput, compactOutput]) {
const failure = classifyHarnessProviderFailure({
output: 'unfinished task output',
stderr,
exitCode: 1,
}, 'grok');
assert.equal(failure?.kind, 'usage_limit');
assert.equal(failure?.provider, 'xai');
assert.match(failure!.message, /Grok account.*no available usage balance/);
assert.match(failure!.message, /add Grok Build credits/);
assert.doesNotMatch(failure!.message, /secret-fixture|http_status/);
}
});

test('requires the Grok provider envelope instead of matching task-authored text', () => {
const exhausted = 'API error (status 402 Payment Required): Grok Build usage balance exhausted';
assert.equal(classify(exhausted, 'claude'), null);
assert.equal(classify(exhausted, 'grok'), null);
assert.equal(classify(`The task output quoted: ${exhausted}`, 'grok'), null);
assert.equal(classify(`Internal error: ${JSON.stringify({ message: exhausted, http_status: 402 })}`, 'grok'), null);
assert.equal(classify(`Internal error: ${JSON.stringify({ message: exhausted, http_status: 400 })}`, 'grok'), null);
assert.equal(classify(`Internal error: ${JSON.stringify({ message: 'billing profile unavailable', http_status: 402 })}`, 'grok'), null);
});

test('classifies known provider diagnostics from stderr using safe messages', () => {
const cases: Array<[string, HarnessProviderFailure['kind']]> = [
['API Error: 429 request throttled', 'rate_limit'],
Expand Down
80 changes: 77 additions & 3 deletions packages/runtime/src/harness-provider-error.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ export interface HarnessProviderFailure {
kind: 'usage_limit' | 'rate_limit' | 'authentication' | 'context_limit' | 'provider_unavailable' | 'timeout';
message: string;
resetHint?: string;
provider?: 'anthropic' | 'openai';
provider?: 'anthropic' | 'openai' | 'xai';
}

function validTimezone(value: string): boolean {
Expand Down Expand Up @@ -44,6 +44,48 @@ function diagnosticMessages(text: string): string[] {
return messages;
}

function internalErrorEnvelopes(text: string): Array<Record<string, unknown>> {
const envelopes: Array<Record<string, unknown>> = [];
const marker = /^\s*Internal error:\s*/gim;
for (const match of text.matchAll(marker)) {
let cursor = (match.index ?? 0) + match[0].length;
if (text[cursor] !== '{') continue;

const start = cursor;
let depth = 0;
let inString = false;
let escaped = false;
for (; cursor < text.length; cursor += 1) {
const char = text[cursor];
if (inString) {
if (escaped) escaped = false;
else if (char === '\\') escaped = true;
else if (char === '"') inString = false;
continue;
}
if (char === '"') {
inString = true;
continue;
}
if (char === '{') depth += 1;
else if (char === '}') {
depth -= 1;
if (depth !== 0) continue;
try {
const value = JSON.parse(text.slice(start, cursor + 1)) as unknown;
if (value && typeof value === 'object' && !Array.isArray(value)) {
envelopes.push(value as Record<string, unknown>);
}
} catch {
// A malformed CLI envelope is not authoritative provider metadata.
}
break;
}
}
}
return envelopes;
}

/**
* Classify failed model CLI runs, never arbitrary successful agent output.
* Customer messages are fixed templates, not excerpts of stdout/stderr:
Expand All @@ -52,8 +94,20 @@ function diagnosticMessages(text: string): string[] {
export function classifyHarnessProviderFailure(run: Pick<HarnessRunResult, 'output' | 'stderr' | 'exitCode'>, harness?: string): HarnessProviderFailure | null {
// OS kills and successful output retain their existing caller contract.
if (!Number.isFinite(run.exitCode) || run.exitCode === 0 || run.exitCode === 137 || run.exitCode === 143) return null;
const provider = harness === 'claude' ? 'anthropic' : harness === 'codex' ? 'openai' : undefined;
const account = provider === 'anthropic' ? 'Claude' : provider === 'openai' ? 'OpenAI' : 'AI';
const provider = harness === 'claude'
? 'anthropic'
: harness === 'codex'
? 'openai'
: harness === 'grok'
? 'xai'
: undefined;
const account = provider === 'anthropic'
? 'Claude'
: provider === 'openai'
? 'OpenAI'
: provider === 'xai'
? 'Grok'
: 'AI';
const result = run as { output?: unknown; stderr?: unknown } | null;
const rawText = [result?.output, result?.stderr]
.filter((value): value is string => typeof value === 'string')
Expand Down Expand Up @@ -81,6 +135,26 @@ export function classifyHarnessProviderFailure(run: Pick<HarnessRunResult, 'outp
};
}

// Grok Build emits this fixed provider diagnostic on stderr inside its
// multiline "Internal error" envelope. Require that process-owned channel,
// plus the 402 status and exact reason, so task-authored stdout cannot be
// promoted into customer-facing provider metadata.
const trustedGrokDiagnostics = typeof result?.stderr === 'string'
? result.stderr.slice(-16000).replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, '')
: '';
const grokUsageBalanceExhausted = provider === 'xai' && internalErrorEnvelopes(trustedGrokDiagnostics).some(
(envelope) =>
envelope.http_status === 402 &&
envelope.message === 'API error (status 402 Payment Required): Grok Build usage balance exhausted',
);
if (grokUsageBalanceExhausted) {
return {
provider,
kind: 'usage_limit',
message: 'The Grok account selected for this run has no available usage balance. Ask the account owner to add Grok Build credits before retrying the task.',
};
}
Comment thread
cursor[bot] marked this conversation as resolved.

if (/"(?:type|code)"\s*:\s*"(?:insufficient_quota|billing_hard_limit_reached)"|\byou exceeded your current quota\b|\byour credit balance is too low to access the Anthropic API\b/i.test(text)) {
return {
...(provider ? { provider } : {}),
Expand Down
Loading