Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 73 additions & 1 deletion packages/sdk/src/authored-budget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,76 @@ interface Charge {
unmetered: boolean;
}

type Total = Omit<Charge, 'day'>;

/** `ms` as the budget header writes durations (`45s`, `2h`, `132.9m`), with the value that text denotes. */
function duration(ms: bigint): {text: string; shown: number} {
const n = Number(ms);
if (n < 60_000) { const s = Number((n / 1000).toFixed(1)); return {text: `${s}s`, shown: s * 1000}; }
if (n % 3_600_000 === 0) return {text: `${n / 3_600_000}h`, shown: n};
const m = Number((n / 60_000).toFixed(1));
return {text: `${m}m`, shown: m * 60_000};
}

const usd = (micro: bigint): string => `$${micro / 1_000_000n}.${String(micro % 1_000_000n).padStart(6, '0').replace(/0{1,4}$/, '')}`;

/**
* Whether `micro` microdollars exceeds the decimal `limit`, compared exactly at
* the limit's own precision (legacy `maxDollars` may carry more than six
* decimals, which the kernel compares as written). Undefined if the limit is
* not a plain decimal.
*/
function dollarsOver(micro: bigint, limit: string): boolean | undefined {
const match = /^(\d+)(?:\.(\d+))?$/.exec(limit);
if (match === null) return undefined;
const scale = Math.max(6, match[2]?.length ?? 0);
const limitScaled = BigInt(match[1]! + (match[2] ?? '').padEnd(scale, '0'));
return micro * 10n ** BigInt(scale - 6) > limitScaled;
}

/**
* Both durations, exact to the millisecond whenever rounding would make the
* overrun invisible: the displayed spend must itself read as over the
* displayed limit, compared as quantities so `120m` against `2h` counts too.
*/
function durations(used: bigint, limit: bigint): [string, string] {
const [u, l] = [duration(used), duration(limit)];
return u.shown > l.shown ? [u.text, l.text] : [`${used}ms`, `${limit}ms`];
}

/**
* The kernel refuses admission with only `budget_exceeded`; name which
* declared limit the carried spend crossed and by how much, using the same
* strict comparisons as `machine/budget.rs`. The kernel stays the authority: a
* refusal this accumulator cannot explain keeps the generic wording.
*/
export function budgetExceededMessage(limit: KernelBudgetSpec, total: Total, spec?: KernelRunSpec): string {
const crossed: string[] = [];
if (limit.max_wallclock_ms !== undefined && total.ms > BigInt(limit.max_wallclock_ms)) {
const [used, declared] = durations(total.ms, BigInt(limit.max_wallclock_ms));
crossed.push(`wallclock ${used} used of ${declared} declared`);
}
if (limit.max_dollars !== undefined && dollarsOver(total.micro, limit.max_dollars) === true) {
const exact = /^(\d+)(?:\.(\d{1,6}))?$/.exec(limit.max_dollars);
const declared = exact === null ? `$${limit.max_dollars}`
: usd(BigInt(exact[1]!) * 1_000_000n + BigInt((exact[2] ?? '').padEnd(6, '0')));
crossed.push(`dollars ${usd(total.micro)}${total.unmetered ? ' metered (some steps unmetered)' : ''} used of ${declared} declared`);
}
if (limit.max_tokens !== undefined && total.input + total.output > BigInt(limit.max_tokens)) {
crossed.push(`tokens ${total.input + total.output} used of ${limit.max_tokens} declared`);
}
if (limit.max_tokens_in !== undefined && total.input > BigInt(limit.max_tokens_in)) {
crossed.push(`input tokens ${total.input} used of ${limit.max_tokens_in} declared`);
}
if (limit.max_tokens_out !== undefined && total.output > BigInt(limit.max_tokens_out)) {
crossed.push(`output tokens ${total.output} used of ${limit.max_tokens_out} declared`);
}
const next = spec?.steps.length === 1 && typeof spec.steps[0]?.id === 'string' ? `step "${spec.steps[0].id}"` : 'the next step';
if (crossed.length === 0) return `Flow budget exceeded before ${next}.`;
const scope = limit.window === 'day' ? "in today's window of the flow's budget header" : "in the flow's budget header";
return `Flow budget exceeded before ${next}: ${crossed.join('; ')} ${scope}.`;
}

/** Serialized admission for the internal authored runner's separate step runs. */
export class AuthoredBudget {
private readonly limit: KernelBudgetSpec | undefined;
Expand Down Expand Up @@ -66,7 +136,9 @@ export class AuthoredBudget {
};
const outcome = await journal.runStart({ ...spec, budget: { ...this.limit, prior_spend: priorSpend } }, undefined, admissionKey);
try {
if (outcome.completion_reason === 'budget_exceeded') throw new AuthoredFlowExecutionError('step_failed', 'Flow budget exceeded before the next step.', 'budget_exceeded', outcome.run_id);
if (outcome.completion_reason === 'budget_exceeded') {
throw new AuthoredFlowExecutionError('step_failed', budgetExceededMessage(this.limit!, total, spec), 'budget_exceeded', outcome.run_id);
}
return await consume(outcome);
} finally {
let seq = 1;
Expand Down
63 changes: 62 additions & 1 deletion packages/sdk/tests/budget-attribution.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { EventEmitter } from 'node:events';
import { describe, expect, it, vi } from 'vitest';
import { decodeProviderResult, decodeWrapperResult, requirePricedUsage } from '../src/worker-usage.js';
import { pricedUsage, MODEL_PRICING } from '../src/model-pricing.js';
import { AuthoredBudget } from '../src/authored-budget.js';
import { AuthoredBudget, budgetExceededMessage } from '../src/authored-budget.js';
import type { JournalClient } from '../src/journal-client.js';
import type { StepDispatchEvent } from '../src/protocol.js';
import { LlmWorker } from '../src/llm-worker.js';
Expand Down Expand Up @@ -68,3 +68,64 @@ describe('budget attribution', () => {
expect(calls[1]?.[0].budget.prior_spend).toMatchObject({tokens_in:500, tokens_out:500, dollars:'0.000000', wallclock_ms:5, dollars_unmetered:true});
});
});

describe('authored budget refusal names the limit it crossed', () => {
const charge = (payload: Record<string, unknown>) => ({seq: 1, entry_type: 'step.completed', at_ms: 0, payload});
// The shape of Cloud run f28314ed: 26 steps summed to 132.9 min of step
// wallclock under `{ wallclock: "2h", dollars: 25 }`, and the refusal said
// only "Flow budget exceeded before the next step." — not which limit.
it('reports wallclock used against the declared header on refusal', async () => {
const budget = new AuthoredBudget({wallclock: '2h', dollars: 25});
let started = 0;
const client = {
runStart: vi.fn(async () => ({run_id: `r${++started}`, status: started === 1 ? 'completed' : 'failed',
completion_reason: started === 1 ? 'success' : 'budget_exceeded', completed_steps: started === 1 ? 1 : 0})),
journalRead: vi.fn(async (run: string, seq: number) => ({entries: run === 'r1' && seq === 1
? [charge({budget: {tokens_in: 878, tokens_out: 257687, dollars: '6.446600'}, spend: {wallclock_ms: 7_974_000}})] : []})),
};
const step = (id: string) => ({version: '0.1.0', steps: [{id}]}) as never;
await budget.execute(client as unknown as JournalClient, step('agent-26'), async () => 'ok');
const refused = budget.execute(client as unknown as JournalClient, step('run-27'), async () => 'unreachable');
await expect(refused).rejects.toMatchObject({
message: 'step_failed: Flow budget exceeded before step "run-27": wallclock 132.9m used of 2h declared in the flow\'s budget header.',
code: 'step_failed', completionReason: 'budget_exceeded',
});
});
it('names every crossed dimension, marks unmetered dollars, and scopes a day window', () => {
const total = {input: 900n, output: 300n, micro: 25_500_000n, ms: 30_000n, unmetered: true};
expect(budgetExceededMessage({max_dollars: '25', max_tokens: 1000, max_wallclock_ms: 60_000, window: 'day'}, total))
.toBe("Flow budget exceeded before the next step: dollars $25.50 metered (some steps unmetered) used of $25.00 declared; tokens 1200 used of 1000 declared in today's window of the flow's budget header.");
expect(budgetExceededMessage({max_tokens_in: 800, max_tokens_out: 300}, total))
.toBe("Flow budget exceeded before the next step: input tokens 900 used of 800 declared in the flow's budget header.");
});
it('prints exact milliseconds when rounding would show a strict overrun as equal (PR #594 review)', () => {
const zero = {input: 0n, output: 0n, micro: 0n, unmetered: false};
expect(budgetExceededMessage({max_wallclock_ms: 120_000}, {...zero, ms: 120_001n}))
.toBe("Flow budget exceeded before the next step: wallclock 120001ms used of 120000ms declared in the flow's budget header.");
expect(budgetExceededMessage({max_wallclock_ms: 60_000}, {...zero, ms: 60_001n}))
.toBe("Flow budget exceeded before the next step: wallclock 60001ms used of 60000ms declared in the flow's budget header.");
// Mixed units: the limit renders in hours, the spend in rounded minutes.
expect(budgetExceededMessage({max_wallclock_ms: 7_200_000}, {...zero, ms: 7_200_001n}))
.toBe("Flow budget exceeded before the next step: wallclock 7200001ms used of 7200000ms declared in the flow's budget header.");
expect(budgetExceededMessage({max_wallclock_ms: 7_200_000}, {...zero, ms: 7_206_000n}))
.toBe("Flow budget exceeded before the next step: wallclock 120.1m used of 2h declared in the flow's budget header.");
expect(budgetExceededMessage({max_wallclock_ms: 10}, {...zero, ms: 20n}))
.toBe("Flow budget exceeded before the next step: wallclock 20ms used of 10ms declared in the flow's budget header.");
});
it('names a legacy sub-microdollar dollar limit, compared at its own precision (PR #594 review)', () => {
const spent = {input: 0n, output: 0n, micro: 1n, ms: 0n, unmetered: false};
expect(budgetExceededMessage({max_dollars: '0.0000005'}, spent))
.toBe("Flow budget exceeded before the next step: dollars $0.000001 used of $0.0000005 declared in the flow's budget header.");
// Redundant trailing zeros are still the same limit, and equal is not over.
expect(budgetExceededMessage({max_dollars: '0.0000010000'}, spent)).toBe('Flow budget exceeded before the next step.');
expect(budgetExceededMessage({max_dollars: '0.0000009999'}, spent))
.toBe("Flow budget exceeded before the next step: dollars $0.000001 used of $0.0000009999 declared in the flow's budget header.");
});
it('keeps the generic wording when the carried totals do not explain the kernel refusal', () => {
// Equal is not over: the kernel compares strictly, so this total alone
// did not trip it and the accumulator must not invent a reason.
expect(budgetExceededMessage({max_wallclock_ms: 60_000, max_dollars: '1'}, {input: 0n, output: 0n, micro: 1_000_000n, ms: 60_000n, unmetered: false}))
.toBe('Flow budget exceeded before the next step.');
});
});

Loading