Skip to content

feat!: Rename LDAIMetrics.usage and LDAIGraphMetrics.usage to .tokens #1366

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jsonbailey merged 1 commit into from
May 7, 2026
Merged

feat!: Rename LDAIMetrics.usage and LDAIGraphMetrics.usage to .tokens #1366

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions packages/ai-providers/server-ai-langchain/__tests__/LangChainAgentRunner.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ it('returns content with no toolCalls when the agent returns a simple response',
expect(result.content).toBe('done');
expect(result.metrics.success).toBe(true);
expect(result.metrics.toolCalls).toBeUndefined();
expect(result.metrics.usage).toEqual({ total: 6, input: 4, output: 2 });
expect(result.metrics.tokens).toEqual({ total: 6, input: 4, output: 2 });
});

it('extracts tool calls and aggregates usage from multi-step agent messages', async () => {
Expand Down Expand Up @@ -56,7 +56,7 @@ it('extracts tool calls and aggregates usage from multi-step agent messages', as

expect(result.content).toBe('Answer is 42.');
expect(result.metrics.toolCalls).toEqual(['lookup']);
expect(result.metrics.usage).toEqual({ total: 28, input: 16, output: 12 });
expect(result.metrics.tokens).toEqual({ total: 28, input: 16, output: 12 });
});

it('returns success=false when the agent throws', async () => {
Expand All @@ -83,5 +83,5 @@ it('handles empty messages array gracefully', async () => {
expect(result.content).toBe('');
expect(result.metrics.success).toBe(true);
expect(result.metrics.toolCalls).toBeUndefined();
expect(result.metrics.usage).toBeUndefined();
expect(result.metrics.tokens).toBeUndefined();
});
2 changes: 1 addition & 1 deletion packages/ai-providers/server-ai-langchain/__tests__/LangChainHelper.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ it('returns success=true with usage from the response', () => {
message.usage_metadata = { total_tokens: 3, input_tokens: 1, output_tokens: 2 };
expect(getAIMetricsFromResponse(message)).toEqual({
success: true,
usage: { total: 3, input: 1, output: 2 },
tokens: { total: 3, input: 1, output: 2 },
});
});

Expand Down
2 changes: 1 addition & 1 deletion packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ describe('LangChainModelRunner', () => {
expect(result.content).toBe('hello');
expect(result.metrics).toEqual({
success: true,
usage: { total: 12, input: 7, output: 5 },
tokens: { total: 12, input: 7, output: 5 },
});
expect(result.raw).toBe(response);
});
Expand Down
2 changes: 1 addition & 1 deletion packages/ai-providers/server-ai-langchain/src/LangChainAgentRunner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ export class LangChainAgentRunner implements Runner {

const metrics: LDAIMetrics = {
success: true,
usage: sumTokenUsageFromMessages(messages),
tokens: sumTokenUsageFromMessages(messages),
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
};

Expand Down
2 changes: 1 addition & 1 deletion packages/ai-providers/server-ai-langchain/src/LangChainHelper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ export function getAIUsageFromResponse(response: AIMessage): LDTokenUsage | unde
export function getAIMetricsFromResponse(response: AIMessage): LDAIMetrics {
return {
success: true,
usage: getAIUsageFromResponse(response),
tokens: getAIUsageFromResponse(response),
};
}

Expand Down
4 changes: 2 additions & 2 deletions packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ export class LangChainModelRunner implements Runner {

const metrics = {
success: true,
usage: { total: 0, input: 0, output: 0 },
tokens: { total: 0, input: 0, output: 0 },
};

return {
Expand All @@ -111,7 +111,7 @@ export class LangChainModelRunner implements Runner {
content: '',
metrics: {
success: false,
usage: { total: 0, input: 0, output: 0 },
tokens: { total: 0, input: 0, output: 0 },
},
};
}
Expand Down
4 changes: 2 additions & 2 deletions packages/ai-providers/server-ai-openai/__tests__/OpenAIAgentRunner.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ describe('OpenAIAgentRunner', () => {
expect(result.content).toBe('Done');
expect(result.metrics.success).toBe(true);
expect(result.metrics.toolCalls).toBeUndefined();
expect(result.metrics.usage).toEqual({ total: 12, input: 8, output: 4 });
expect(result.metrics.tokens).toEqual({ total: 12, input: 8, output: 4 });
});

it('reports tool calls from newItems with LD config name mapping', async () => {
Expand All @@ -55,7 +55,7 @@ describe('OpenAIAgentRunner', () => {

expect(result.content).toBe('The answer is 42.');
expect(result.metrics.toolCalls).toEqual(['lookup']);
expect(result.metrics.usage).toEqual({ total: 28, input: 16, output: 12 });
expect(result.metrics.tokens).toEqual({ total: 28, input: 16, output: 12 });
});

it('returns an unsuccessful RunnerResult when the agent run throws', async () => {
Expand Down
2 changes: 1 addition & 1 deletion packages/ai-providers/server-ai-openai/__tests__/OpenAIHelper.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ it('returns success=true with usage extracted from the response', () => {

expect(metrics).toEqual({
success: true,
usage: { total: 3, input: 1, output: 2 },
tokens: { total: 3, input: 1, output: 2 },
});
});

Expand Down
2 changes: 1 addition & 1 deletion packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ describe('OpenAIModelRunner', () => {
expect(result.content).toBe('Hello there!');
expect(result.metrics).toEqual({
success: true,
usage: { total: 15, input: 10, output: 5 },
tokens: { total: 15, input: 10, output: 5 },
});
expect(result.raw).toBe(mockResponse);
expect(result.parsed).toBeUndefined();
Expand Down
4 changes: 2 additions & 2 deletions packages/ai-providers/server-ai-openai/src/OpenAIAgentRunner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,10 @@ export class OpenAIAgentRunner implements Runner {
[],
);

const usage: LDTokenUsage | undefined = getAIUsageFromAgentResult(result);
const tokens: LDTokenUsage | undefined = getAIUsageFromAgentResult(result);
const metrics: LDAIMetrics = {
success: true,
usage,
tokens,
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
};

Expand Down
2 changes: 1 addition & 1 deletion packages/ai-providers/server-ai-openai/src/OpenAIHelper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ export function getAIUsageFromResponse(response: any): LDTokenUsage | undefined
export function getAIMetricsFromResponse(response: any): LDAIMetrics {
return {
success: true,
usage: getAIUsageFromResponse(response),
tokens: getAIUsageFromResponse(response),
};
}

Expand Down
4 changes: 2 additions & 2 deletions packages/ai-providers/server-ai-vercel/__tests__/VercelHelper.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ describe('getAIMetricsFromResponse', () => {
getAIMetricsFromResponse({
usage: { totalTokens: 5, promptTokens: 2, completionTokens: 3 },
}),
).toEqual({ success: true, usage: { total: 5, input: 2, output: 3 } });
).toEqual({ success: true, tokens: { total: 5, input: 2, output: 3 } });
});

it('marks success=false when finishReason is "error"', () => {
Expand All @@ -83,7 +83,7 @@ describe('getAIMetricsFromStream', () => {
});
expect(result).toEqual({
success: true,
usage: { total: 100, input: 49, output: 51 },
tokens: { total: 100, input: 49, output: 51 },
});
});

Expand Down
4 changes: 2 additions & 2 deletions packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ describe('VercelModelRunner', () => {
expect(out.content).toBe('Hi!');
expect(out.metrics).toEqual({
success: true,
usage: { total: 12, input: 7, output: 5 },
tokens: { total: 12, input: 7, output: 5 },
});
expect(out.raw).toBe(result);
});
Expand Down Expand Up @@ -87,7 +87,7 @@ describe('VercelModelRunner', () => {

const out = await runner.run('hello');

expect(out.metrics.usage).toEqual({ total: 100, input: 40, output: 60 });
expect(out.metrics.tokens).toEqual({ total: 100, input: 40, output: 60 });
});

it('returns success=false when generateText throws', async () => {
Expand Down
18 changes: 9 additions & 9 deletions packages/ai-providers/server-ai-vercel/src/VercelHelper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,16 @@ export function mapUsageDataToLDTokenUsage(usageData: ModelUsageTokens): LDToken
export function getAIMetricsFromResponse(response: TextResponse): LDAIMetrics {
const finishReason = response?.finishReason ?? 'unknown';

let usage: LDTokenUsage | undefined;
let tokens: LDTokenUsage | undefined;
if (response?.totalUsage) {
usage = mapUsageDataToLDTokenUsage(response.totalUsage);
tokens = mapUsageDataToLDTokenUsage(response.totalUsage);
} else if (response?.usage) {
usage = mapUsageDataToLDTokenUsage(response.usage);
tokens = mapUsageDataToLDTokenUsage(response.usage);
}

return {
success: finishReason !== 'error',
usage,
tokens,
};
}

Expand All @@ -66,24 +66,24 @@ export function getAIMetricsFromResponse(response: TextResponse): LDAIMetrics {
export async function getAIMetricsFromStream(stream: StreamResponse): Promise<LDAIMetrics> {
const finishReason = (await stream.finishReason?.catch(() => 'error')) ?? 'unknown';

let usage: LDTokenUsage | undefined;
let tokens: LDTokenUsage | undefined;

if (stream.totalUsage) {
const usageData = await stream.totalUsage.catch(() => undefined);
if (usageData) {
usage = mapUsageDataToLDTokenUsage(usageData);
tokens = mapUsageDataToLDTokenUsage(usageData);
}
}

if (!usage && stream.usage) {
if (!tokens && stream.usage) {
const usageData = await stream.usage.catch(() => undefined);
if (usageData) {
usage = mapUsageDataToLDTokenUsage(usageData);
tokens = mapUsageDataToLDTokenUsage(usageData);
}
}

return {
success: finishReason !== 'error',
usage,
tokens,
};
}
22 changes: 11 additions & 11 deletions packages/sdk/server-ai/__tests__/Judge.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ describe('Judge', () => {
},
metrics: {
success: true,
usage: {
tokens: {
total: 100,
input: 50,
output: 50,
Expand Down Expand Up @@ -254,7 +254,7 @@ describe('Judge', () => {
},
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand Down Expand Up @@ -283,7 +283,7 @@ describe('Judge', () => {
},
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand Down Expand Up @@ -356,7 +356,7 @@ describe('Judge', () => {
},
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand Down Expand Up @@ -391,7 +391,7 @@ describe('Judge', () => {
},
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand Down Expand Up @@ -426,7 +426,7 @@ describe('Judge', () => {
},
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand Down Expand Up @@ -462,7 +462,7 @@ describe('Judge', () => {
},
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand Down Expand Up @@ -512,7 +512,7 @@ describe('Judge', () => {
parsed: undefined,
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand All @@ -538,7 +538,7 @@ describe('Judge', () => {
parsed: {},
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand Down Expand Up @@ -568,7 +568,7 @@ describe('Judge', () => {
},
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand Down Expand Up @@ -642,7 +642,7 @@ describe('Judge', () => {
},
metrics: {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
},
};

Expand Down
2 changes: 1 addition & 1 deletion packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -723,7 +723,7 @@ describe('trackMetricsOf', () => {
const mockResult = { response: 'test' };
const mockMetrics = {
success: true,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
};

const metricsExtractor = jest.fn().mockReturnValue(mockMetrics);
Expand Down
8 changes: 4 additions & 4 deletions packages/sdk/server-ai/__tests__/ManagedAgentGraph.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@ describe('ManagedAgentGraph', () => {
success: true,
path: ['node-a', 'node-b'],
durationMs: 1500,
usage: { total: 100, input: 50, output: 50 },
tokens: { total: 100, input: 50, output: 50 },
nodeMetrics: {
'node-a': { success: true, usage: { total: 40, input: 20, output: 20 } },
'node-b': { success: true, usage: { total: 60, input: 30, output: 30 } },
'node-a': { success: true, tokens: { total: 40, input: 20, output: 20 } },
'node-b': { success: true, tokens: { total: 60, input: 30, output: 30 } },
},
},
};
Expand Down Expand Up @@ -103,7 +103,7 @@ describe('ManagedAgentGraph', () => {
nodeMetrics: {
n1: {
success: true,
usage: { total: 10, input: 5, output: 5 },
tokens: { total: 10, input: 5, output: 5 },
durationMs: 200,
toolCalls: ['tool-a'],
},
Expand Down
8 changes: 4 additions & 4 deletions packages/sdk/server-ai/__tests__/ManagedModel.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ describe('ManagedModel', () => {
it('passes the prompt directly to the runner without prepending config messages', async () => {
const runnerResult: RunnerResult = {
content: 'Response from model',
metrics: { success: true, usage: { total: 10, input: 4, output: 6 } },
metrics: { success: true, tokens: { total: 10, input: 4, output: 6 } },
};

mockTracker.trackMetricsOf.mockImplementation(async (_extractor, func) => func());
Expand All @@ -68,7 +68,7 @@ describe('ManagedModel', () => {
content: 'Hi there',
metrics: {
success: true,
usage: { total: 12, input: 5, output: 7 },
tokens: { total: 12, input: 5, output: 7 },
toolCalls: ['tool-1'],
durationMs: 42,
},
Expand Down Expand Up @@ -99,7 +99,7 @@ describe('ManagedModel', () => {
it('forwards the runner result through tracker.trackMetricsOf', async () => {
const runnerResult: RunnerResult = {
content: 'tracked',
metrics: { success: true, usage: { total: 1, input: 1, output: 0 } },
metrics: { success: true, tokens: { total: 1, input: 1, output: 0 } },
};

mockTracker.trackMetricsOf.mockImplementation(async (_extractor, func) => func());
Expand All @@ -117,7 +117,7 @@ describe('ManagedModel', () => {
it('does not retain conversation state across runs', async () => {
const runnerResult: RunnerResult = {
content: 'ok',
metrics: { success: true, usage: { total: 1, input: 1, output: 0 } },
metrics: { success: true, tokens: { total: 1, input: 1, output: 0 } },
};

mockTracker.trackMetricsOf.mockImplementation(async (_extractor, func) => func());
Expand Down
Loading
Loading