Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 7 additions & 10 deletions pkg/ai/agent/anthropic/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,16 +321,13 @@ func parseAnthropicResponse(response *anthropic.Message) (*types.Response, error
}
}

// Extract usage information.
if response.Usage.InputTokens > 0 || response.Usage.OutputTokens > 0 {
result.Usage = &types.Usage{
InputTokens: response.Usage.InputTokens,
OutputTokens: response.Usage.OutputTokens,
TotalTokens: response.Usage.InputTokens + response.Usage.OutputTokens,
CacheReadTokens: response.Usage.CacheReadInputTokens,
CacheCreationTokens: response.Usage.CacheCreationInputTokens,
}
}
// Extract usage information. InputTokens includes cache reads and writes, as for the other providers.
result.Usage = base.UsageFromAnthropic(
response.Usage.InputTokens,
response.Usage.OutputTokens,
response.Usage.CacheReadInputTokens,
response.Usage.CacheCreationInputTokens,
)

return result, nil
}
Expand Down
34 changes: 30 additions & 4 deletions pkg/ai/agent/anthropic/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1423,6 +1423,8 @@ func TestParseAnthropicResponse_NoUsage(t *testing.T) {
assert.Nil(t, result.Usage)
}

// TestParseAnthropicResponse_WithCacheTokens checks that cache reads and writes are counted in InputTokens,
// because the Anthropic API reports them outside input_tokens.
func TestParseAnthropicResponse_WithCacheTokens(t *testing.T) {
response := &anthropic.Message{
StopReason: "end_turn",
Expand All @@ -1439,9 +1441,27 @@ func TestParseAnthropicResponse_WithCacheTokens(t *testing.T) {
require.NotNil(t, result.Usage)
assert.Equal(t, int64(30), result.Usage.CacheReadTokens)
assert.Equal(t, int64(20), result.Usage.CacheCreationTokens)
assert.Equal(t, int64(100), result.Usage.InputTokens)
assert.Equal(t, int64(150), result.Usage.InputTokens)
assert.Equal(t, int64(50), result.Usage.OutputTokens)
assert.Equal(t, int64(150), result.Usage.TotalTokens)
assert.Equal(t, int64(200), result.Usage.TotalTokens)
}

// TestParseAnthropicResponse_OnlyCacheReadTokens checks that usage is kept when the whole prompt was read
// from the cache and input_tokens is 0.
func TestParseAnthropicResponse_OnlyCacheReadTokens(t *testing.T) {
response := &anthropic.Message{
StopReason: "end_turn",
Usage: anthropic.Usage{
CacheReadInputTokens: 2048,
},
}

result, err := parseAnthropicResponse(response)
require.NoError(t, err)
require.NotNil(t, result.Usage)
assert.Equal(t, int64(2048), result.Usage.InputTokens)
assert.Equal(t, int64(2048), result.Usage.TotalTokens)
assert.Equal(t, int64(2048), result.Usage.CacheReadTokens)
}

func TestParseAnthropicResponse_EmptyContent(t *testing.T) {
Expand Down Expand Up @@ -1812,6 +1832,7 @@ func TestUsageCalculations(t *testing.T) {
outputTokens int64
cacheRead int64
cacheCreate int64
expectedInput int64
expectedTotal int64
expectUsageNil bool
}{
Expand All @@ -1821,6 +1842,7 @@ func TestUsageCalculations(t *testing.T) {
outputTokens: 50,
cacheRead: 0,
cacheCreate: 0,
expectedInput: 100,
expectedTotal: 150,
expectUsageNil: false,
},
Expand All @@ -1830,6 +1852,7 @@ func TestUsageCalculations(t *testing.T) {
outputTokens: 0,
cacheRead: 0,
cacheCreate: 0,
expectedInput: 0,
expectedTotal: 0,
expectUsageNil: true,
},
Expand All @@ -1839,7 +1862,8 @@ func TestUsageCalculations(t *testing.T) {
outputTokens: 200,
cacheRead: 300,
cacheCreate: 100,
expectedTotal: 700,
expectedInput: 900, // Cache reads and writes are part of the prompt.
expectedTotal: 1100,
expectUsageNil: false,
},
{
Expand All @@ -1848,6 +1872,7 @@ func TestUsageCalculations(t *testing.T) {
outputTokens: 0,
cacheRead: 0,
cacheCreate: 0,
expectedInput: 1000,
expectedTotal: 1000,
expectUsageNil: false,
},
Expand All @@ -1857,6 +1882,7 @@ func TestUsageCalculations(t *testing.T) {
outputTokens: 500,
cacheRead: 0,
cacheCreate: 0,
expectedInput: 0,
expectedTotal: 500,
expectUsageNil: false,
},
Expand All @@ -1882,7 +1908,7 @@ func TestUsageCalculations(t *testing.T) {
} else {
require.NotNil(t, result.Usage)
assert.Equal(t, tt.expectedTotal, result.Usage.TotalTokens)
assert.Equal(t, tt.inputTokens, result.Usage.InputTokens)
assert.Equal(t, tt.expectedInput, result.Usage.InputTokens)
assert.Equal(t, tt.outputTokens, result.Usage.OutputTokens)
assert.Equal(t, tt.cacheRead, result.Usage.CacheReadTokens)
assert.Equal(t, tt.cacheCreate, result.Usage.CacheCreationTokens)
Expand Down
30 changes: 30 additions & 0 deletions pkg/ai/agent/base/usage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package base

import (
"github.com/cloudposse/atmos/pkg/ai/types"
)

// UsageFromAnthropic converts token counts in the Anthropic Messages API format into types.Usage.
//
// The Anthropic API, and Claude on Amazon Bedrock, report prompt cache reads and writes outside
// input_tokens: the full prompt is input_tokens + cache_read_input_tokens + cache_creation_input_tokens.
// The OpenAI-compatible and Gemini APIs already count cached tokens in their prompt total, so this adds
// the cache counts to InputTokens. InputTokens is then the full prompt size for every provider, and
// CacheReadTokens and CacheCreationTokens are the parts of it read from and written to the cache.
//
// Returns nil when the response reports no tokens.
func UsageFromAnthropic(inputTokens, outputTokens, cacheReadTokens, cacheCreationTokens int64) *types.Usage {
if inputTokens <= 0 && outputTokens <= 0 && cacheReadTokens <= 0 && cacheCreationTokens <= 0 {
return nil
}

promptTokens := inputTokens + cacheReadTokens + cacheCreationTokens

return &types.Usage{
InputTokens: promptTokens,
OutputTokens: outputTokens,
TotalTokens: promptTokens + outputTokens,
CacheReadTokens: cacheReadTokens,
CacheCreationTokens: cacheCreationTokens,
}
}
66 changes: 66 additions & 0 deletions pkg/ai/agent/base/usage_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package base

import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/cloudposse/atmos/pkg/ai/types"
)

// TestUsageFromAnthropic checks that cache reads and writes are counted in InputTokens and TotalTokens,
// so the prompt count means the same as for the OpenAI-compatible and Gemini providers.
func TestUsageFromAnthropic(t *testing.T) {
tests := []struct {
name string
input, output, cacheRead, cacheCreate int64
expected *types.Usage
}{
{
name: "no cache use",
input: 100, output: 50,
expected: &types.Usage{InputTokens: 100, OutputTokens: 50, TotalTokens: 150},
},
{
name: "cache read only",
input: 90, output: 12, cacheRead: 5526,
expected: &types.Usage{InputTokens: 5616, OutputTokens: 12, TotalTokens: 5628, CacheReadTokens: 5526},
},
{
name: "cache write only (first request)",
input: 90, output: 12, cacheCreate: 5526,
expected: &types.Usage{InputTokens: 5616, OutputTokens: 12, TotalTokens: 5628, CacheCreationTokens: 5526},
},
{
name: "cache read and write",
input: 100, output: 50, cacheRead: 30, cacheCreate: 20,
expected: &types.Usage{InputTokens: 150, OutputTokens: 50, TotalTokens: 200, CacheReadTokens: 30, CacheCreationTokens: 20},
},
{
name: "whole prompt read from cache",
input: 0, output: 0, cacheRead: 2048,
expected: &types.Usage{InputTokens: 2048, TotalTokens: 2048, CacheReadTokens: 2048},
},
{
name: "output only",
input: 0, output: 50,
expected: &types.Usage{OutputTokens: 50, TotalTokens: 50},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
usage := UsageFromAnthropic(tt.input, tt.output, tt.cacheRead, tt.cacheCreate)

require.NotNil(t, usage)
assert.Equal(t, tt.expected, usage)
assert.LessOrEqual(t, usage.CacheReadTokens+usage.CacheCreationTokens, usage.InputTokens)
})
}
}

// TestUsageFromAnthropic_NoTokens checks that a response reporting no tokens yields no usage.
func TestUsageFromAnthropic_NoTokens(t *testing.T) {
assert.Nil(t, UsageFromAnthropic(0, 0, 0, 0))
}
25 changes: 22 additions & 3 deletions pkg/ai/agent/bedrock/cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,8 @@ func TestSendMessageWithToolsAndHistory_NoSystemField(t *testing.T) {
assert.NotContains(t, *requestBody, "system")
}

// TestParseBedrockResponse_CacheUsage checks cache read and creation token counts are read from the response.
// TestParseBedrockResponse_CacheUsage checks cache read and creation token counts are read from the response
// and counted in InputTokens, because Claude on Bedrock reports them outside input_tokens.
func TestParseBedrockResponse_CacheUsage(t *testing.T) {
responseBody := `{
"content": [{"type": "text", "text": "ok"}],
Expand All @@ -163,13 +164,31 @@ func TestParseBedrockResponse_CacheUsage(t *testing.T) {

require.NoError(t, err)
require.NotNil(t, result.Usage)
assert.Equal(t, int64(90), result.Usage.InputTokens)
assert.Equal(t, int64(5916), result.Usage.InputTokens)
assert.Equal(t, int64(12), result.Usage.OutputTokens)
assert.Equal(t, int64(102), result.Usage.TotalTokens)
assert.Equal(t, int64(5928), result.Usage.TotalTokens)
assert.Equal(t, int64(5526), result.Usage.CacheReadTokens)
assert.Equal(t, int64(300), result.Usage.CacheCreationTokens)
}

// TestParseBedrockResponse_OnlyCacheReadUsage checks that usage is kept when the whole prompt was read
// from the cache and input_tokens is 0.
func TestParseBedrockResponse_OnlyCacheReadUsage(t *testing.T) {
responseBody := `{
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 0, "output_tokens": 0, "cache_read_input_tokens": 2048}
}`

result, err := parseBedrockResponse([]byte(responseBody))

require.NoError(t, err)
require.NotNil(t, result.Usage)
assert.Equal(t, int64(2048), result.Usage.InputTokens)
assert.Equal(t, int64(2048), result.Usage.TotalTokens)
assert.Equal(t, int64(2048), result.Usage.CacheReadTokens)
}

// TestExtractCacheConfig checks the cache settings match the Anthropic provider's defaults and overrides.
func TestExtractCacheConfig(t *testing.T) {
tests := []struct {
Expand Down
19 changes: 8 additions & 11 deletions pkg/ai/agent/bedrock/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -464,17 +464,14 @@ func parseBedrockResponse(responseBody []byte) (*types.Response, error) {
processBedrockContentBlock(&apiResponse.Content[i], result)
}

// Extract usage information.
if apiResponse.Usage.InputTokens > 0 || apiResponse.Usage.OutputTokens > 0 {
result.Usage = &types.Usage{
InputTokens: apiResponse.Usage.InputTokens,
OutputTokens: apiResponse.Usage.OutputTokens,
TotalTokens: apiResponse.Usage.InputTokens + apiResponse.Usage.OutputTokens,
// Claude on Bedrock reports cache usage in the same fields as the Anthropic API.
CacheReadTokens: apiResponse.Usage.CacheReadInputTokens,
CacheCreationTokens: apiResponse.Usage.CacheCreationInputTokens,
}
}
// Extract usage information. Claude on Bedrock reports cache usage in the same fields as the Anthropic API,
// and InputTokens includes cache reads and writes, as for the other providers.
result.Usage = base.UsageFromAnthropic(
apiResponse.Usage.InputTokens,
apiResponse.Usage.OutputTokens,
apiResponse.Usage.CacheReadInputTokens,
apiResponse.Usage.CacheCreationInputTokens,
)

return result, nil
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/ai/formatter/formatter.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ type ToolCallResult struct {

// TokenUsage contains token usage statistics.
type TokenUsage struct {
// Prompt tokens used in the request.
// Prompt tokens used in the request, including tokens read from or written to the prompt cache.
Prompt int64 `json:"prompt"`

// Completion tokens used in the response.
Expand All @@ -69,10 +69,10 @@ type TokenUsage struct {
// Total tokens used (prompt + completion).
Total int64 `json:"total"`

// Cached tokens read from prompt cache (if supported).
// Cached tokens read from prompt cache (if supported). They are included in Prompt.
Cached int64 `json:"cached,omitempty"`

// CacheCreation tokens used to create cache (if supported).
// CacheCreation tokens written to the prompt cache (if supported). They are included in Prompt.
CacheCreation int64 `json:"cache_creation,omitempty"`
}

Expand Down
9 changes: 6 additions & 3 deletions pkg/ai/types/response.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ type Response struct {

// Usage contains token usage statistics from the AI provider.
type Usage struct {
// InputTokens is the number of tokens in the request/prompt.
// InputTokens is the number of tokens in the request/prompt, including tokens read from or written to
// the prompt cache.
InputTokens int64

// OutputTokens is the number of tokens in the response/completion.
Expand All @@ -26,10 +27,12 @@ type Usage struct {
// TotalTokens is the sum of input and output tokens.
TotalTokens int64

// CacheReadTokens is the number of tokens read from prompt cache (if supported).
// CacheReadTokens is the number of prompt tokens read from prompt cache (if supported).
// They are included in InputTokens.
CacheReadTokens int64

// CacheCreationTokens is the number of tokens used to create cache (if supported).
// CacheCreationTokens is the number of prompt tokens written to the prompt cache (if supported).
// They are included in InputTokens.
CacheCreationTokens int64
}

Expand Down
2 changes: 2 additions & 0 deletions website/docs/cli/commands/ai/exec.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@ Output:
}
```

`tokens.prompt` counts every input token, including tokens read from or written to the prompt cache, for every provider that reports token usage. `tokens.cached` (tokens read from the cache) and `tokens.cache_creation` (tokens written to the cache) are parts of `tokens.prompt` and are left out when zero.

### Markdown Format

Formatted output with tool execution summaries:
Expand Down
Loading