From 3e94fd9e27b4fce716decdc9ebbbf43fd97b57b7 Mon Sep 17 00:00:00 2001 From: Haris Farooq Date: Mon, 9 Nov 2026 21:50:59 +0500 Subject: [PATCH] fix(ai): count prompt cache reads and writes in Anthropic and Bedrock input tokens The Anthropic API and Claude on Bedrock report cache reads and writes outside input_tokens (total input = input_tokens + cache_read_input_tokens + cache_creation_input_tokens). The OpenAI-compatible and Gemini clients already put cached tokens inside InputTokens, so the same field meant different things per provider, and with caching on the Anthropic and Bedrock prompt counts in the chat footer and `atmos ai exec` JSON showed only the uncached tail. Add base.UsageFromAnthropic, used by both clients, so InputTokens and TotalTokens include cache reads and writes, and keep usage when a reply reports only cached input. Document that the cache counts are part of the prompt count. --- pkg/ai/agent/anthropic/client.go | 17 +++---- pkg/ai/agent/anthropic/client_test.go | 34 ++++++++++++-- pkg/ai/agent/base/usage.go | 30 ++++++++++++ pkg/ai/agent/base/usage_test.go | 66 +++++++++++++++++++++++++++ pkg/ai/agent/bedrock/cache_test.go | 25 ++++++++-- pkg/ai/agent/bedrock/client.go | 19 ++++---- pkg/ai/formatter/formatter.go | 6 +-- pkg/ai/types/response.go | 9 ++-- website/docs/cli/commands/ai/exec.mdx | 2 + 9 files changed, 174 insertions(+), 34 deletions(-) create mode 100644 pkg/ai/agent/base/usage.go create mode 100644 pkg/ai/agent/base/usage_test.go diff --git a/pkg/ai/agent/anthropic/client.go b/pkg/ai/agent/anthropic/client.go index 2827057a20d..4147d122c9c 100644 --- a/pkg/ai/agent/anthropic/client.go +++ b/pkg/ai/agent/anthropic/client.go @@ -321,16 +321,13 @@ func parseAnthropicResponse(response *anthropic.Message) (*types.Response, error } } - // Extract usage information. - if response.Usage.InputTokens > 0 || response.Usage.OutputTokens > 0 { - result.Usage = &types.Usage{ - InputTokens: response.Usage.InputTokens, - OutputTokens: response.Usage.OutputTokens, - TotalTokens: response.Usage.InputTokens + response.Usage.OutputTokens, - CacheReadTokens: response.Usage.CacheReadInputTokens, - CacheCreationTokens: response.Usage.CacheCreationInputTokens, - } - } + // Extract usage information. InputTokens includes cache reads and writes, as for the other providers. + result.Usage = base.UsageFromAnthropic( + response.Usage.InputTokens, + response.Usage.OutputTokens, + response.Usage.CacheReadInputTokens, + response.Usage.CacheCreationInputTokens, + ) return result, nil } diff --git a/pkg/ai/agent/anthropic/client_test.go b/pkg/ai/agent/anthropic/client_test.go index f45b78dc3df..7f9d5d65766 100644 --- a/pkg/ai/agent/anthropic/client_test.go +++ b/pkg/ai/agent/anthropic/client_test.go @@ -1423,6 +1423,8 @@ func TestParseAnthropicResponse_NoUsage(t *testing.T) { assert.Nil(t, result.Usage) } +// TestParseAnthropicResponse_WithCacheTokens checks that cache reads and writes are counted in InputTokens, +// because the Anthropic API reports them outside input_tokens. func TestParseAnthropicResponse_WithCacheTokens(t *testing.T) { response := &anthropic.Message{ StopReason: "end_turn", @@ -1439,9 +1441,27 @@ func TestParseAnthropicResponse_WithCacheTokens(t *testing.T) { require.NotNil(t, result.Usage) assert.Equal(t, int64(30), result.Usage.CacheReadTokens) assert.Equal(t, int64(20), result.Usage.CacheCreationTokens) - assert.Equal(t, int64(100), result.Usage.InputTokens) + assert.Equal(t, int64(150), result.Usage.InputTokens) assert.Equal(t, int64(50), result.Usage.OutputTokens) - assert.Equal(t, int64(150), result.Usage.TotalTokens) + assert.Equal(t, int64(200), result.Usage.TotalTokens) +} + +// TestParseAnthropicResponse_OnlyCacheReadTokens checks that usage is kept when the whole prompt was read +// from the cache and input_tokens is 0. +func TestParseAnthropicResponse_OnlyCacheReadTokens(t *testing.T) { + response := &anthropic.Message{ + StopReason: "end_turn", + Usage: anthropic.Usage{ + CacheReadInputTokens: 2048, + }, + } + + result, err := parseAnthropicResponse(response) + require.NoError(t, err) + require.NotNil(t, result.Usage) + assert.Equal(t, int64(2048), result.Usage.InputTokens) + assert.Equal(t, int64(2048), result.Usage.TotalTokens) + assert.Equal(t, int64(2048), result.Usage.CacheReadTokens) } func TestParseAnthropicResponse_EmptyContent(t *testing.T) { @@ -1812,6 +1832,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens int64 cacheRead int64 cacheCreate int64 + expectedInput int64 expectedTotal int64 expectUsageNil bool }{ @@ -1821,6 +1842,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 50, cacheRead: 0, cacheCreate: 0, + expectedInput: 100, expectedTotal: 150, expectUsageNil: false, }, @@ -1830,6 +1852,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 0, cacheRead: 0, cacheCreate: 0, + expectedInput: 0, expectedTotal: 0, expectUsageNil: true, }, @@ -1839,7 +1862,8 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 200, cacheRead: 300, cacheCreate: 100, - expectedTotal: 700, + expectedInput: 900, // Cache reads and writes are part of the prompt. + expectedTotal: 1100, expectUsageNil: false, }, { @@ -1848,6 +1872,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 0, cacheRead: 0, cacheCreate: 0, + expectedInput: 1000, expectedTotal: 1000, expectUsageNil: false, }, @@ -1857,6 +1882,7 @@ func TestUsageCalculations(t *testing.T) { outputTokens: 500, cacheRead: 0, cacheCreate: 0, + expectedInput: 0, expectedTotal: 500, expectUsageNil: false, }, @@ -1882,7 +1908,7 @@ func TestUsageCalculations(t *testing.T) { } else { require.NotNil(t, result.Usage) assert.Equal(t, tt.expectedTotal, result.Usage.TotalTokens) - assert.Equal(t, tt.inputTokens, result.Usage.InputTokens) + assert.Equal(t, tt.expectedInput, result.Usage.InputTokens) assert.Equal(t, tt.outputTokens, result.Usage.OutputTokens) assert.Equal(t, tt.cacheRead, result.Usage.CacheReadTokens) assert.Equal(t, tt.cacheCreate, result.Usage.CacheCreationTokens) diff --git a/pkg/ai/agent/base/usage.go b/pkg/ai/agent/base/usage.go new file mode 100644 index 00000000000..09c20b8c5f8 --- /dev/null +++ b/pkg/ai/agent/base/usage.go @@ -0,0 +1,30 @@ +package base + +import ( + "github.com/cloudposse/atmos/pkg/ai/types" +) + +// UsageFromAnthropic converts token counts in the Anthropic Messages API format into types.Usage. +// +// The Anthropic API, and Claude on Amazon Bedrock, report prompt cache reads and writes outside +// input_tokens: the full prompt is input_tokens + cache_read_input_tokens + cache_creation_input_tokens. +// The OpenAI-compatible and Gemini APIs already count cached tokens in their prompt total, so this adds +// the cache counts to InputTokens. InputTokens is then the full prompt size for every provider, and +// CacheReadTokens and CacheCreationTokens are the parts of it read from and written to the cache. +// +// Returns nil when the response reports no tokens. +func UsageFromAnthropic(inputTokens, outputTokens, cacheReadTokens, cacheCreationTokens int64) *types.Usage { + if inputTokens <= 0 && outputTokens <= 0 && cacheReadTokens <= 0 && cacheCreationTokens <= 0 { + return nil + } + + promptTokens := inputTokens + cacheReadTokens + cacheCreationTokens + + return &types.Usage{ + InputTokens: promptTokens, + OutputTokens: outputTokens, + TotalTokens: promptTokens + outputTokens, + CacheReadTokens: cacheReadTokens, + CacheCreationTokens: cacheCreationTokens, + } +} diff --git a/pkg/ai/agent/base/usage_test.go b/pkg/ai/agent/base/usage_test.go new file mode 100644 index 00000000000..512053ce37b --- /dev/null +++ b/pkg/ai/agent/base/usage_test.go @@ -0,0 +1,66 @@ +package base + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/ai/types" +) + +// TestUsageFromAnthropic checks that cache reads and writes are counted in InputTokens and TotalTokens, +// so the prompt count means the same as for the OpenAI-compatible and Gemini providers. +func TestUsageFromAnthropic(t *testing.T) { + tests := []struct { + name string + input, output, cacheRead, cacheCreate int64 + expected *types.Usage + }{ + { + name: "no cache use", + input: 100, output: 50, + expected: &types.Usage{InputTokens: 100, OutputTokens: 50, TotalTokens: 150}, + }, + { + name: "cache read only", + input: 90, output: 12, cacheRead: 5526, + expected: &types.Usage{InputTokens: 5616, OutputTokens: 12, TotalTokens: 5628, CacheReadTokens: 5526}, + }, + { + name: "cache write only (first request)", + input: 90, output: 12, cacheCreate: 5526, + expected: &types.Usage{InputTokens: 5616, OutputTokens: 12, TotalTokens: 5628, CacheCreationTokens: 5526}, + }, + { + name: "cache read and write", + input: 100, output: 50, cacheRead: 30, cacheCreate: 20, + expected: &types.Usage{InputTokens: 150, OutputTokens: 50, TotalTokens: 200, CacheReadTokens: 30, CacheCreationTokens: 20}, + }, + { + name: "whole prompt read from cache", + input: 0, output: 0, cacheRead: 2048, + expected: &types.Usage{InputTokens: 2048, TotalTokens: 2048, CacheReadTokens: 2048}, + }, + { + name: "output only", + input: 0, output: 50, + expected: &types.Usage{OutputTokens: 50, TotalTokens: 50}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + usage := UsageFromAnthropic(tt.input, tt.output, tt.cacheRead, tt.cacheCreate) + + require.NotNil(t, usage) + assert.Equal(t, tt.expected, usage) + assert.LessOrEqual(t, usage.CacheReadTokens+usage.CacheCreationTokens, usage.InputTokens) + }) + } +} + +// TestUsageFromAnthropic_NoTokens checks that a response reporting no tokens yields no usage. +func TestUsageFromAnthropic_NoTokens(t *testing.T) { + assert.Nil(t, UsageFromAnthropic(0, 0, 0, 0)) +} diff --git a/pkg/ai/agent/bedrock/cache_test.go b/pkg/ai/agent/bedrock/cache_test.go index 87b295676e9..4d1f375ce41 100644 --- a/pkg/ai/agent/bedrock/cache_test.go +++ b/pkg/ai/agent/bedrock/cache_test.go @@ -151,7 +151,8 @@ func TestSendMessageWithToolsAndHistory_NoSystemField(t *testing.T) { assert.NotContains(t, *requestBody, "system") } -// TestParseBedrockResponse_CacheUsage checks cache read and creation token counts are read from the response. +// TestParseBedrockResponse_CacheUsage checks cache read and creation token counts are read from the response +// and counted in InputTokens, because Claude on Bedrock reports them outside input_tokens. func TestParseBedrockResponse_CacheUsage(t *testing.T) { responseBody := `{ "content": [{"type": "text", "text": "ok"}], @@ -163,13 +164,31 @@ func TestParseBedrockResponse_CacheUsage(t *testing.T) { require.NoError(t, err) require.NotNil(t, result.Usage) - assert.Equal(t, int64(90), result.Usage.InputTokens) + assert.Equal(t, int64(5916), result.Usage.InputTokens) assert.Equal(t, int64(12), result.Usage.OutputTokens) - assert.Equal(t, int64(102), result.Usage.TotalTokens) + assert.Equal(t, int64(5928), result.Usage.TotalTokens) assert.Equal(t, int64(5526), result.Usage.CacheReadTokens) assert.Equal(t, int64(300), result.Usage.CacheCreationTokens) } +// TestParseBedrockResponse_OnlyCacheReadUsage checks that usage is kept when the whole prompt was read +// from the cache and input_tokens is 0. +func TestParseBedrockResponse_OnlyCacheReadUsage(t *testing.T) { + responseBody := `{ + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 0, "output_tokens": 0, "cache_read_input_tokens": 2048} + }` + + result, err := parseBedrockResponse([]byte(responseBody)) + + require.NoError(t, err) + require.NotNil(t, result.Usage) + assert.Equal(t, int64(2048), result.Usage.InputTokens) + assert.Equal(t, int64(2048), result.Usage.TotalTokens) + assert.Equal(t, int64(2048), result.Usage.CacheReadTokens) +} + // TestExtractCacheConfig checks the cache settings match the Anthropic provider's defaults and overrides. func TestExtractCacheConfig(t *testing.T) { tests := []struct { diff --git a/pkg/ai/agent/bedrock/client.go b/pkg/ai/agent/bedrock/client.go index 60d4fa32fd8..6f1003d315d 100644 --- a/pkg/ai/agent/bedrock/client.go +++ b/pkg/ai/agent/bedrock/client.go @@ -464,17 +464,14 @@ func parseBedrockResponse(responseBody []byte) (*types.Response, error) { processBedrockContentBlock(&apiResponse.Content[i], result) } - // Extract usage information. - if apiResponse.Usage.InputTokens > 0 || apiResponse.Usage.OutputTokens > 0 { - result.Usage = &types.Usage{ - InputTokens: apiResponse.Usage.InputTokens, - OutputTokens: apiResponse.Usage.OutputTokens, - TotalTokens: apiResponse.Usage.InputTokens + apiResponse.Usage.OutputTokens, - // Claude on Bedrock reports cache usage in the same fields as the Anthropic API. - CacheReadTokens: apiResponse.Usage.CacheReadInputTokens, - CacheCreationTokens: apiResponse.Usage.CacheCreationInputTokens, - } - } + // Extract usage information. Claude on Bedrock reports cache usage in the same fields as the Anthropic API, + // and InputTokens includes cache reads and writes, as for the other providers. + result.Usage = base.UsageFromAnthropic( + apiResponse.Usage.InputTokens, + apiResponse.Usage.OutputTokens, + apiResponse.Usage.CacheReadInputTokens, + apiResponse.Usage.CacheCreationInputTokens, + ) return result, nil } diff --git a/pkg/ai/formatter/formatter.go b/pkg/ai/formatter/formatter.go index 79e00a65efb..2cc47f865b9 100644 --- a/pkg/ai/formatter/formatter.go +++ b/pkg/ai/formatter/formatter.go @@ -60,7 +60,7 @@ type ToolCallResult struct { // TokenUsage contains token usage statistics. type TokenUsage struct { - // Prompt tokens used in the request. + // Prompt tokens used in the request, including tokens read from or written to the prompt cache. Prompt int64 `json:"prompt"` // Completion tokens used in the response. @@ -69,10 +69,10 @@ type TokenUsage struct { // Total tokens used (prompt + completion). Total int64 `json:"total"` - // Cached tokens read from prompt cache (if supported). + // Cached tokens read from prompt cache (if supported). They are included in Prompt. Cached int64 `json:"cached,omitempty"` - // CacheCreation tokens used to create cache (if supported). + // CacheCreation tokens written to the prompt cache (if supported). They are included in Prompt. CacheCreation int64 `json:"cache_creation,omitempty"` } diff --git a/pkg/ai/types/response.go b/pkg/ai/types/response.go index b7eba75c9e8..5449fe590f4 100644 --- a/pkg/ai/types/response.go +++ b/pkg/ai/types/response.go @@ -17,7 +17,8 @@ type Response struct { // Usage contains token usage statistics from the AI provider. type Usage struct { - // InputTokens is the number of tokens in the request/prompt. + // InputTokens is the number of tokens in the request/prompt, including tokens read from or written to + // the prompt cache. InputTokens int64 // OutputTokens is the number of tokens in the response/completion. @@ -26,10 +27,12 @@ type Usage struct { // TotalTokens is the sum of input and output tokens. TotalTokens int64 - // CacheReadTokens is the number of tokens read from prompt cache (if supported). + // CacheReadTokens is the number of prompt tokens read from prompt cache (if supported). + // They are included in InputTokens. CacheReadTokens int64 - // CacheCreationTokens is the number of tokens used to create cache (if supported). + // CacheCreationTokens is the number of prompt tokens written to the prompt cache (if supported). + // They are included in InputTokens. CacheCreationTokens int64 } diff --git a/website/docs/cli/commands/ai/exec.mdx b/website/docs/cli/commands/ai/exec.mdx index 1e92a528a27..4cb35c36f60 100644 --- a/website/docs/cli/commands/ai/exec.mdx +++ b/website/docs/cli/commands/ai/exec.mdx @@ -193,6 +193,8 @@ Output: } ``` +`tokens.prompt` counts every input token, including tokens read from or written to the prompt cache, for every provider that reports token usage. `tokens.cached` (tokens read from the cache) and `tokens.cache_creation` (tokens written to the cache) are parts of `tokens.prompt` and are left out when zero. + ### Markdown Format Formatted output with tool execution summaries: