diff --git a/pkg/ai/agent/anthropic/client.go b/pkg/ai/agent/anthropic/client.go index 2827057a20d..4147d122c9c 100644 --- a/pkg/ai/agent/anthropic/client.go +++ b/pkg/ai/agent/anthropic/client.go @@ -321,16 +321,13 @@ func parseAnthropicResponse(response *anthropic.Message) (*types.Response, error } } - // Extract usage information. - if response.Usage.InputTokens > 0 || response.Usage.OutputTokens > 0 { - result.Usage = &types.Usage{ - InputTokens: response.Usage.InputTokens, - OutputTokens: response.Usage.OutputTokens, - TotalTokens: response.Usage.InputTokens + response.Usage.OutputTokens, - CacheReadTokens: response.Usage.CacheReadInputTokens, - CacheCreationTokens: response.Usage.CacheCreationInputTokens, - } - } + // Extract usage information. InputTokens includes cache reads and writes, as for the other providers. + result.Usage = base.UsageFromAnthropic( + response.Usage.InputTokens, + response.Usage.OutputTokens, + response.Usage.CacheReadInputTokens, + response.Usage.CacheCreationInputTokens, + ) return result, nil } diff --git a/pkg/ai/agent/anthropic/client_test.go b/pkg/ai/agent/anthropic/client_test.go index f45b78dc3df..7f9d5d65766 100644 --- a/pkg/ai/agent/anthropic/client_test.go +++ b/pkg/ai/agent/anthropic/client_test.go @@ -1423,6 +1423,8 @@ func TestParseAnthropicResponse_NoUsage(t *testing.T) { assert.Nil(t, result.Usage) } +// TestParseAnthropicResponse_WithCacheTokens checks that cache reads and writes are counted in InputTokens, +// because the Anthropic API reports them outside input_tokens. func TestParseAnthropicResponse_WithCacheTokens(t *testing.T) { response := &anthropic.Message{ StopReason: "end_turn", @@ -1439,9 +1441,27 @@ func TestParseAnthropicResponse_WithCacheTokens(t *testing.T) { require.NotNil(t, result.Usage) assert.Equal(t, int64(30), result.Usage.CacheReadTokens) assert.Equal(t, int64(20), result.Usage.CacheCreationTokens) - assert.Equal(t, int64(100), result.Usage.InputTokens) + assert.Equal(t, int64(150), result.Usage.InputTokens) assert.Equal(t, int64(50), result.Usage.OutputTokens) - assert.Equal(t, int64(150), result.Usage.TotalTokens) + assert.Equal(t, int64(200), result.Usage.TotalTokens) +} + +// TestParseAnthropicResponse_OnlyCacheReadTokens checks that usage is kept when the whole prompt was read +// from the cache and input_tokens is 0. +func TestParseAnthropicResponse_OnlyCacheReadTokens(t *testing.T) { + response := &anthropic.Message{ + StopReason: "end_turn", + Usage: anthropic.Usage{ + CacheReadInputTokens: 2048, + }, + } + + result, err := parseAnthropicResponse(response) + require.NoError(t, err) + require.NotNil(t, result.Usage) + assert.Equal(t, int64(2048), result.Usage.InputTokens) + assert.Equal(t, int64(2048), result.Usage.TotalTokens) + assert.Equal(t, int64(2048), result.Usage.CacheReadTokens) } func TestParseAnthropicResponse_EmptyContent(t *testing.T) { @@ -1812,6 +1832,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens int64 cacheRead int64 cacheCreate int64 + expectedInput int64 expectedTotal int64 expectUsageNil bool }{ @@ -1821,6 +1842,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 50, cacheRead: 0, cacheCreate: 0, + expectedInput: 100, expectedTotal: 150, expectUsageNil: false, }, @@ -1830,6 +1852,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 0, cacheRead: 0, cacheCreate: 0, + expectedInput: 0, expectedTotal: 0, expectUsageNil: true, }, @@ -1839,7 +1862,8 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 200, cacheRead: 300, cacheCreate: 100, - expectedTotal: 700, + expectedInput: 900, // Cache reads and writes are part of the prompt. + expectedTotal: 1100, expectUsageNil: false, }, { @@ -1848,6 +1872,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 0, cacheRead: 0, cacheCreate: 0, + expectedInput: 1000, expectedTotal: 1000, expectUsageNil: false, }, @@ -1857,6 +1882,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 500, cacheRead: 0, cacheCreate: 0, + expectedInput: 0, expectedTotal: 500, expectUsageNil: false, }, @@ -1882,7 +1908,7 @@ func TestUsageCalculations(t *testing.T) { } else { require.NotNil(t, result.Usage) assert.Equal(t, tt.expectedTotal, result.Usage.TotalTokens) - assert.Equal(t, tt.inputTokens, result.Usage.InputTokens) + assert.Equal(t, tt.expectedInput, result.Usage.InputTokens) assert.Equal(t, tt.outputTokens, result.Usage.OutputTokens) assert.Equal(t, tt.cacheRead, result.Usage.CacheReadTokens) assert.Equal(t, tt.cacheCreate, result.Usage.CacheCreationTokens) diff --git a/pkg/ai/agent/base/usage.go b/pkg/ai/agent/base/usage.go new file mode 100644 index 00000000000..09c20b8c5f8 --- /dev/null +++ b/pkg/ai/agent/base/usage.go @@ -0,0 +1,30 @@ +package base + +import ( + "github.com/cloudposse/atmos/pkg/ai/types" +) + +// UsageFromAnthropic converts token counts in the Anthropic Messages API format into types.Usage. +// +// The Anthropic API, and Claude on Amazon Bedrock, report prompt cache reads and writes outside +// input_tokens: the full prompt is input_tokens + cache_read_input_tokens + cache_creation_input_tokens. +// The OpenAI-compatible and Gemini APIs already count cached tokens in their prompt total, so this adds +// the cache counts to InputTokens. InputTokens is then the full prompt size for every provider, and +// CacheReadTokens and CacheCreationTokens are the parts of it read from and written to the cache. +// +// Returns nil when the response reports no tokens. +func UsageFromAnthropic(inputTokens, outputTokens, cacheReadTokens, cacheCreationTokens int64) *types.Usage { + if inputTokens <= 0 && outputTokens <= 0 && cacheReadTokens <= 0 && cacheCreationTokens <= 0 { + return nil + } + + promptTokens := inputTokens + cacheReadTokens + cacheCreationTokens + + return &types.Usage{ + InputTokens: promptTokens, + OutputTokens: outputTokens, + TotalTokens: promptTokens + outputTokens, + CacheReadTokens: cacheReadTokens, + CacheCreationTokens: cacheCreationTokens, + } +} diff --git a/pkg/ai/agent/base/usage_test.go b/pkg/ai/agent/base/usage_test.go new file mode 100644 index 00000000000..512053ce37b --- /dev/null +++ b/pkg/ai/agent/base/usage_test.go @@ -0,0 +1,66 @@ +package base + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/ai/types" +) + +// TestUsageFromAnthropic checks that cache reads and writes are counted in InputTokens and TotalTokens, +// so the prompt count means the same as for the OpenAI-compatible and Gemini providers. +func TestUsageFromAnthropic(t *testing.T) { + tests := []struct { + name string + input, output, cacheRead, cacheCreate int64 + expected *types.Usage + }{ + { + name: "no cache use", + input: 100, output: 50, + expected: &types.Usage{InputTokens: 100, OutputTokens: 50, TotalTokens: 150}, + }, + { + name: "cache read only", + input: 90, output: 12, cacheRead: 5526, + expected: &types.Usage{InputTokens: 5616, OutputTokens: 12, TotalTokens: 5628, CacheReadTokens: 5526}, + }, + { + name: "cache write only (first request)", + input: 90, output: 12, cacheCreate: 5526, + expected: &types.Usage{InputTokens: 5616, OutputTokens: 12, TotalTokens: 5628, CacheCreationTokens: 5526}, + }, + { + name: "cache read and write", + input: 100, output: 50, cacheRead: 30, cacheCreate: 20, + expected: &types.Usage{InputTokens: 150, OutputTokens: 50, TotalTokens: 200, CacheReadTokens: 30, CacheCreationTokens: 20}, + }, + { + name: "whole prompt read from cache", + input: 0, output: 0, cacheRead: 2048, + expected: &types.Usage{InputTokens: 2048, TotalTokens: 2048, CacheReadTokens: 2048}, + }, + { + name: "output only", + input: 0, output: 50, + expected: &types.Usage{OutputTokens: 50, TotalTokens: 50}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + usage := UsageFromAnthropic(tt.input, tt.output, tt.cacheRead, tt.cacheCreate) + + require.NotNil(t, usage) + assert.Equal(t, tt.expected, usage) + assert.LessOrEqual(t, usage.CacheReadTokens+usage.CacheCreationTokens, usage.InputTokens) + }) + } +} + +// TestUsageFromAnthropic_NoTokens checks that a response reporting no tokens yields no usage. +func TestUsageFromAnthropic_NoTokens(t *testing.T) { + assert.Nil(t, UsageFromAnthropic(0, 0, 0, 0)) +} diff --git a/pkg/ai/agent/bedrock/cache_test.go b/pkg/ai/agent/bedrock/cache_test.go index 87b295676e9..4d1f375ce41 100644 --- a/pkg/ai/agent/bedrock/cache_test.go +++ b/pkg/ai/agent/bedrock/cache_test.go @@ -151,7 +151,8 @@ func TestSendMessageWithToolsAndHistory_NoSystemField(t *testing.T) { assert.NotContains(t, *requestBody, "system") } -// TestParseBedrockResponse_CacheUsage checks cache read and creation token counts are read from the response. +// TestParseBedrockResponse_CacheUsage checks cache read and creation token counts are read from the response +// and counted in InputTokens, because Claude on Bedrock reports them outside input_tokens. func TestParseBedrockResponse_CacheUsage(t *testing.T) { responseBody := `{ "content": [{"type": "text", "text": "ok"}], @@ -163,13 +164,31 @@ func TestParseBedrockResponse_CacheUsage(t *testing.T) { require.NoError(t, err) require.NotNil(t, result.Usage) - assert.Equal(t, int64(90), result.Usage.InputTokens) + assert.Equal(t, int64(5916), result.Usage.InputTokens) assert.Equal(t, int64(12), result.Usage.OutputTokens) - assert.Equal(t, int64(102), result.Usage.TotalTokens) + assert.Equal(t, int64(5928), result.Usage.TotalTokens) assert.Equal(t, int64(5526), result.Usage.CacheReadTokens) assert.Equal(t, int64(300), result.Usage.CacheCreationTokens) } +// TestParseBedrockResponse_OnlyCacheReadUsage checks that usage is kept when the whole prompt was read +// from the cache and input_tokens is 0. +func TestParseBedrockResponse_OnlyCacheReadUsage(t *testing.T) { + responseBody := `{ + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 0, "output_tokens": 0, "cache_read_input_tokens": 2048} + }` + + result, err := parseBedrockResponse([]byte(responseBody)) + + require.NoError(t, err) + require.NotNil(t, result.Usage) + assert.Equal(t, int64(2048), result.Usage.InputTokens) + assert.Equal(t, int64(2048), result.Usage.TotalTokens) + assert.Equal(t, int64(2048), result.Usage.CacheReadTokens) +} + // TestExtractCacheConfig checks the cache settings match the Anthropic provider's defaults and overrides. func TestExtractCacheConfig(t *testing.T) { tests := []struct { diff --git a/pkg/ai/agent/bedrock/client.go b/pkg/ai/agent/bedrock/client.go index 60d4fa32fd8..6f1003d315d 100644 --- a/pkg/ai/agent/bedrock/client.go +++ b/pkg/ai/agent/bedrock/client.go @@ -464,17 +464,14 @@ func parseBedrockResponse(responseBody []byte) (*types.Response, error) { processBedrockContentBlock(&apiResponse.Content[i], result) } - // Extract usage information. - if apiResponse.Usage.InputTokens > 0 || apiResponse.Usage.OutputTokens > 0 { - result.Usage = &types.Usage{ - InputTokens: apiResponse.Usage.InputTokens, - OutputTokens: apiResponse.Usage.OutputTokens, - TotalTokens: apiResponse.Usage.InputTokens + apiResponse.Usage.OutputTokens, - // Claude on Bedrock reports cache usage in the same fields as the Anthropic API. - CacheReadTokens: apiResponse.Usage.CacheReadInputTokens, - CacheCreationTokens: apiResponse.Usage.CacheCreationInputTokens, - } - } + // Extract usage information. Claude on Bedrock reports cache usage in the same fields as the Anthropic API, + // and InputTokens includes cache reads and writes, as for the other providers. + result.Usage = base.UsageFromAnthropic( + apiResponse.Usage.InputTokens, + apiResponse.Usage.OutputTokens, + apiResponse.Usage.CacheReadInputTokens, + apiResponse.Usage.CacheCreationInputTokens, + ) return result, nil } diff --git a/pkg/ai/formatter/formatter.go b/pkg/ai/formatter/formatter.go index 79e00a65efb..2cc47f865b9 100644 --- a/pkg/ai/formatter/formatter.go +++ b/pkg/ai/formatter/formatter.go @@ -60,7 +60,7 @@ type ToolCallResult struct { // TokenUsage contains token usage statistics. type TokenUsage struct { - // Prompt tokens used in the request. + // Prompt tokens used in the request, including tokens read from or written to the prompt cache. Prompt int64 `json:"prompt"` // Completion tokens used in the response. @@ -69,10 +69,10 @@ type TokenUsage struct { // Total tokens used (prompt + completion). Total int64 `json:"total"` - // Cached tokens read from prompt cache (if supported). + // Cached tokens read from prompt cache (if supported). They are included in Prompt. Cached int64 `json:"cached,omitempty"` - // CacheCreation tokens used to create cache (if supported). + // CacheCreation tokens written to the prompt cache (if supported). They are included in Prompt. CacheCreation int64 `json:"cache_creation,omitempty"` } diff --git a/pkg/ai/types/response.go b/pkg/ai/types/response.go index b7eba75c9e8..5449fe590f4 100644 --- a/pkg/ai/types/response.go +++ b/pkg/ai/types/response.go @@ -17,7 +17,8 @@ type Response struct { // Usage contains token usage statistics from the AI provider. type Usage struct { - // InputTokens is the number of tokens in the request/prompt. + // InputTokens is the number of tokens in the request/prompt, including tokens read from or written to + // the prompt cache. InputTokens int64 // OutputTokens is the number of tokens in the response/completion. @@ -26,10 +27,12 @@ type Usage struct { // TotalTokens is the sum of input and output tokens. TotalTokens int64 - // CacheReadTokens is the number of tokens read from prompt cache (if supported). + // CacheReadTokens is the number of prompt tokens read from prompt cache (if supported). + // They are included in InputTokens. CacheReadTokens int64 - // CacheCreationTokens is the number of tokens used to create cache (if supported). + // CacheCreationTokens is the number of prompt tokens written to the prompt cache (if supported). + // They are included in InputTokens. CacheCreationTokens int64 } diff --git a/website/docs/cli/commands/ai/exec.mdx b/website/docs/cli/commands/ai/exec.mdx index 1e92a528a27..4cb35c36f60 100644 --- a/website/docs/cli/commands/ai/exec.mdx +++ b/website/docs/cli/commands/ai/exec.mdx @@ -193,6 +193,8 @@ Output: } ``` +`tokens.prompt` counts every input token, including tokens read from or written to the prompt cache, for every provider that reports token usage. `tokens.cached` (tokens read from the cache) and `tokens.cache_creation` (tokens written to the cache) are parts of `tokens.prompt` and are left out when zero. + ### Markdown Format Formatted output with tool execution summaries: