From b5b0e8d17851a813dea9ef0a2d54988e820254d1 Mon Sep 17 00:00:00 2001 From: Ricky Schema Cascade Date: Mon, 5 Oct 2026 07:04:44 +0200 Subject: [PATCH 1/4] fix(runtime): classify Grok usage balance exhaustion --- packages/runtime/CHANGELOG.md | 6 ++++ .../src/harness-provider-error.test.ts | 22 +++++++++++++ .../runtime/src/harness-provider-error.ts | 33 +++++++++++++++++-- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index bef7b2d9..ceec2473 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Classify Grok Build usage-balance exhaustion as a typed, customer-safe + `usage_limit` provider failure so scheduled deliveries stop retrying and tell + operators to restore Grok credits. + ## [4.2.0] - 2026-10-02 ### Dependencies diff --git a/packages/runtime/src/harness-provider-error.test.ts b/packages/runtime/src/harness-provider-error.test.ts index 1d3a6b14..b1f73ba7 100644 --- a/packages/runtime/src/harness-provider-error.test.ts +++ b/packages/runtime/src/harness-provider-error.test.ts @@ -28,6 +28,28 @@ test('handles Codex error envelopes and does not label another harness as Claude assert.match(classify("You've hit your limit", 'opencode')!.message, /AI account/); }); +test('classifies Grok Build 402 balance exhaustion with a safe actionable message', () => { + const output = [ + 'Internal error: {', + ' "message": "API error (status 402 Payment Required): Grok Build usage balance exhausted",', + ' "http_status": 402,', + ' "private_detail": "secret-fixture-value"', + '}', + ].join('\n'); + const failure = classify(output, 'grok'); + assert.equal(failure?.kind, 'usage_limit'); + assert.equal(failure?.provider, 'xai'); + assert.match(failure!.message, /Grok account.*no available usage balance/); + assert.match(failure!.message, /add Grok Build credits/); + assert.doesNotMatch(failure!.message, /secret-fixture|http_status/); +}); + +test('does not classify Grok payment text for another harness or a different 402', () => { + const exhausted = 'API error (status 402 Payment Required): Grok Build usage balance exhausted'; + assert.equal(classify(exhausted, 'claude'), null); + assert.equal(classify('API error (status 402 Payment Required): billing profile unavailable', 'grok'), null); +}); + test('classifies known provider diagnostics from stderr using safe messages', () => { const cases: Array<[string, HarnessProviderFailure['kind']]> = [ ['API Error: 429 request throttled', 'rate_limit'], diff --git a/packages/runtime/src/harness-provider-error.ts b/packages/runtime/src/harness-provider-error.ts index 9e6186e4..4ff8ef51 100644 --- a/packages/runtime/src/harness-provider-error.ts +++ b/packages/runtime/src/harness-provider-error.ts @@ -4,7 +4,7 @@ export interface HarnessProviderFailure { kind: 'usage_limit' | 'rate_limit' | 'authentication' | 'context_limit' | 'provider_unavailable' | 'timeout'; message: string; resetHint?: string; - provider?: 'anthropic' | 'openai'; + provider?: 'anthropic' | 'openai' | 'xai'; } function validTimezone(value: string): boolean { @@ -52,8 +52,20 @@ function diagnosticMessages(text: string): string[] { export function classifyHarnessProviderFailure(run: Pick, harness?: string): HarnessProviderFailure | null { // OS kills and successful output retain their existing caller contract. if (!Number.isFinite(run.exitCode) || run.exitCode === 0 || run.exitCode === 137 || run.exitCode === 143) return null; - const provider = harness === 'claude' ? 'anthropic' : harness === 'codex' ? 'openai' : undefined; - const account = provider === 'anthropic' ? 'Claude' : provider === 'openai' ? 'OpenAI' : 'AI'; + const provider = harness === 'claude' + ? 'anthropic' + : harness === 'codex' + ? 'openai' + : harness === 'grok' + ? 'xai' + : undefined; + const account = provider === 'anthropic' + ? 'Claude' + : provider === 'openai' + ? 'OpenAI' + : provider === 'xai' + ? 'Grok' + : 'AI'; const result = run as { output?: unknown; stderr?: unknown } | null; const rawText = [result?.output, result?.stderr] .filter((value): value is string => typeof value === 'string') @@ -81,6 +93,21 @@ export function classifyHarnessProviderFailure(run: Pick Date: Mon, 5 Oct 2026 07:11:50 +0200 Subject: [PATCH 2/4] fix(runtime): require Grok provider envelope --- .../src/harness-provider-error.test.ts | 7 ++- .../runtime/src/harness-provider-error.ts | 52 +++++++++++++++++-- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/packages/runtime/src/harness-provider-error.test.ts b/packages/runtime/src/harness-provider-error.test.ts index b1f73ba7..5a1bafda 100644 --- a/packages/runtime/src/harness-provider-error.test.ts +++ b/packages/runtime/src/harness-provider-error.test.ts @@ -44,10 +44,13 @@ test('classifies Grok Build 402 balance exhaustion with a safe actionable messag assert.doesNotMatch(failure!.message, /secret-fixture|http_status/); }); -test('does not classify Grok payment text for another harness or a different 402', () => { +test('requires the Grok provider envelope instead of matching task-authored text', () => { const exhausted = 'API error (status 402 Payment Required): Grok Build usage balance exhausted'; assert.equal(classify(exhausted, 'claude'), null); - assert.equal(classify('API error (status 402 Payment Required): billing profile unavailable', 'grok'), null); + assert.equal(classify(exhausted, 'grok'), null); + assert.equal(classify(`The task output quoted: ${exhausted}`, 'grok'), null); + assert.equal(classify(`Internal error: ${JSON.stringify({ message: exhausted, http_status: 400 })}`, 'grok'), null); + assert.equal(classify(`Internal error: ${JSON.stringify({ message: 'billing profile unavailable', http_status: 402 })}`, 'grok'), null); }); test('classifies known provider diagnostics from stderr using safe messages', () => { diff --git a/packages/runtime/src/harness-provider-error.ts b/packages/runtime/src/harness-provider-error.ts index 4ff8ef51..1330af04 100644 --- a/packages/runtime/src/harness-provider-error.ts +++ b/packages/runtime/src/harness-provider-error.ts @@ -44,6 +44,48 @@ function diagnosticMessages(text: string): string[] { return messages; } +function internalErrorEnvelopes(text: string): Array> { + const envelopes: Array> = []; + const marker = /^\s*Internal error:\s*/gim; + for (const match of text.matchAll(marker)) { + let cursor = (match.index ?? 0) + match[0].length; + if (text[cursor] !== '{') continue; + + const start = cursor; + let depth = 0; + let inString = false; + let escaped = false; + for (; cursor < text.length; cursor += 1) { + const char = text[cursor]; + if (inString) { + if (escaped) escaped = false; + else if (char === '\\') escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') { + inString = true; + continue; + } + if (char === '{') depth += 1; + else if (char === '}') { + depth -= 1; + if (depth !== 0) continue; + try { + const value = JSON.parse(text.slice(start, cursor + 1)) as unknown; + if (value && typeof value === 'object' && !Array.isArray(value)) { + envelopes.push(value as Record); + } + } catch { + // A malformed CLI envelope is not authoritative provider metadata. + } + break; + } + } + } + return envelopes; +} + /** * Classify failed model CLI runs, never arbitrary successful agent output. * Customer messages are fixed templates, not excerpts of stdout/stderr: @@ -97,10 +139,12 @@ export function classifyHarnessProviderFailure(run: Pick + envelope.http_status === 402 && + envelope.message === 'API error (status 402 Payment Required): Grok Build usage balance exhausted', + ); + if (grokUsageBalanceExhausted) { return { provider, kind: 'usage_limit', From 4936d9a674550420c4672fe51dd2571d98ead9cb Mon Sep 17 00:00:00 2001 From: Ricky Schema Cascade Date: Mon, 5 Oct 2026 07:13:07 +0200 Subject: [PATCH 3/4] test(runtime): cover compact Grok error envelope --- .../src/harness-provider-error.test.ts | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/runtime/src/harness-provider-error.test.ts b/packages/runtime/src/harness-provider-error.test.ts index 5a1bafda..fd0e425b 100644 --- a/packages/runtime/src/harness-provider-error.test.ts +++ b/packages/runtime/src/harness-provider-error.test.ts @@ -29,19 +29,27 @@ test('handles Codex error envelopes and does not label another harness as Claude }); test('classifies Grok Build 402 balance exhaustion with a safe actionable message', () => { - const output = [ + const envelope = { + message: 'API error (status 402 Payment Required): Grok Build usage balance exhausted', + http_status: 402, + private_detail: 'secret-fixture-value', + }; + const prettyOutput = [ 'Internal error: {', - ' "message": "API error (status 402 Payment Required): Grok Build usage balance exhausted",', + ` "message": "${envelope.message}",`, ' "http_status": 402,', ' "private_detail": "secret-fixture-value"', '}', ].join('\n'); - const failure = classify(output, 'grok'); - assert.equal(failure?.kind, 'usage_limit'); - assert.equal(failure?.provider, 'xai'); - assert.match(failure!.message, /Grok account.*no available usage balance/); - assert.match(failure!.message, /add Grok Build credits/); - assert.doesNotMatch(failure!.message, /secret-fixture|http_status/); + const compactOutput = `Internal error: ${JSON.stringify(envelope)}`; + for (const output of [prettyOutput, compactOutput]) { + const failure = classify(output, 'grok'); + assert.equal(failure?.kind, 'usage_limit'); + assert.equal(failure?.provider, 'xai'); + assert.match(failure!.message, /Grok account.*no available usage balance/); + assert.match(failure!.message, /add Grok Build credits/); + assert.doesNotMatch(failure!.message, /secret-fixture|http_status/); + } }); test('requires the Grok provider envelope instead of matching task-authored text', () => { From 85e87b9dac4bc71be54a5561e8ddabe598f54d05 Mon Sep 17 00:00:00 2001 From: Ricky Schema Cascade Date: Mon, 5 Oct 2026 07:33:08 +0200 Subject: [PATCH 4/4] fix(runtime): trust Grok diagnostics from stderr --- packages/runtime/src/harness-provider-error.test.ts | 9 +++++++-- packages/runtime/src/harness-provider-error.ts | 13 ++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/runtime/src/harness-provider-error.test.ts b/packages/runtime/src/harness-provider-error.test.ts index fd0e425b..c5d05979 100644 --- a/packages/runtime/src/harness-provider-error.test.ts +++ b/packages/runtime/src/harness-provider-error.test.ts @@ -42,8 +42,12 @@ test('classifies Grok Build 402 balance exhaustion with a safe actionable messag '}', ].join('\n'); const compactOutput = `Internal error: ${JSON.stringify(envelope)}`; - for (const output of [prettyOutput, compactOutput]) { - const failure = classify(output, 'grok'); + for (const stderr of [prettyOutput, compactOutput]) { + const failure = classifyHarnessProviderFailure({ + output: 'unfinished task output', + stderr, + exitCode: 1, + }, 'grok'); assert.equal(failure?.kind, 'usage_limit'); assert.equal(failure?.provider, 'xai'); assert.match(failure!.message, /Grok account.*no available usage balance/); @@ -57,6 +61,7 @@ test('requires the Grok provider envelope instead of matching task-authored text assert.equal(classify(exhausted, 'claude'), null); assert.equal(classify(exhausted, 'grok'), null); assert.equal(classify(`The task output quoted: ${exhausted}`, 'grok'), null); + assert.equal(classify(`Internal error: ${JSON.stringify({ message: exhausted, http_status: 402 })}`, 'grok'), null); assert.equal(classify(`Internal error: ${JSON.stringify({ message: exhausted, http_status: 400 })}`, 'grok'), null); assert.equal(classify(`Internal error: ${JSON.stringify({ message: 'billing profile unavailable', http_status: 402 })}`, 'grok'), null); }); diff --git a/packages/runtime/src/harness-provider-error.ts b/packages/runtime/src/harness-provider-error.ts index 1330af04..f6c283ae 100644 --- a/packages/runtime/src/harness-provider-error.ts +++ b/packages/runtime/src/harness-provider-error.ts @@ -135,11 +135,14 @@ export function classifyHarnessProviderFailure(run: Pick envelope.http_status === 402 && envelope.message === 'API error (status 402 Payment Required): Grok Build usage balance exhausted',