From 47f551c5175fb55970363efdf4e0dcc77ab79738 Mon Sep 17 00:00:00 2001 From: Haris Farooq Date: Tue, 20 Oct 2026 18:53:08 +0500 Subject: [PATCH] 0.4.0: read cache hit rates and dollars lost from Langfuse, so teams without Bedrock invocation logging can check their cache from the traces they already have New `cachecanary langfuse` command (also in the GitHub Action). It reads only token counts, model names and timestamps for the project's generations through the Langfuse public API (v2 on Langfuse v4, v1 on older servers), and reports hit rate and input cost per model like `logs` does. Langfuse stores the counts the way each integration sends them, and some don't add up: Strands before 1.54 makes the input look like 0 (so the cache looks perfect), Strands 1.53 sends its cache writes under a name Langfuse doesn't recognise, and LiteLLM's langfuse callback counts cache writes twice. When a call carries a total that was really sent, the uncached input is rebuilt from it and the output says how many calls were fixed. LiteLLM routing prefixes are stripped so one model stays one group and keeps its global endpoint price. Tested against records that Strands 1.61 and 1.53, LiteLLM (both Langfuse integrations) and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where the numbers matched a separate recount for every model. --- .github/workflows/ci.yml | 25 +- README.md | 45 +- action.yml | 6 +- site/index.html | 2 +- src/cachecanary/__init__.py | 2 +- src/cachecanary/cli.py | 122 ++- src/cachecanary/langfuse.py | 403 ++++++++++ .../langfuse_observations_2026-10.json | 487 ++++++++++++ tests/test_action.py | 15 + tests/test_langfuse.py | 721 ++++++++++++++++++ 10 files changed, 1809 insertions(+), 19 deletions(-) create mode 100644 src/cachecanary/langfuse.py create mode 100644 tests/fixtures/langfuse_observations_2026-10.json create mode 100644 tests/test_langfuse.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ef043a..01ae670 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,28 @@ jobs: command: diff args: examples/request_bad.json examples/request_next.json + - name: langfuse on the saved real records (must fail the 80% target) + id: langfuse_file + continue-on-error: true + uses: ./ + with: + command: langfuse + args: --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8 + + # Proves Langfuse settings given as step env reach the action: the host below is unreachable on purpose, + # so the step must fail with "could not reach Langfuse at http://127.0.0.1:9". + - name: langfuse reads keys and host from step env (must fail, unreachable host) + id: langfuse_env + continue-on-error: true + uses: ./ + env: + LANGFUSE_PUBLIC_KEY: pk-lf-ci + LANGFUSE_SECRET_KEY: sk-lf-ci + LANGFUSE_HOST: http://127.0.0.1:9 + with: + command: langfuse + args: --since 1h + - name: unknown command (must fail) id: bad_command continue-on-error: true @@ -63,7 +85,8 @@ jobs: shell: bash run: | set -euo pipefail - for o in "${{ steps.lint_short.outcome }}" "${{ steps.diff_changed.outcome }}" "${{ steps.bad_command.outcome }}"; do + for o in "${{ steps.lint_short.outcome }}" "${{ steps.diff_changed.outcome }}" "${{ steps.bad_command.outcome }}" \ + "${{ steps.langfuse_file.outcome }}" "${{ steps.langfuse_env.outcome }}"; do if [ "$o" != "failure" ]; then echo "::error::expected failure, got '$o'"; exit 1; fi done echo "All expected failures happened." diff --git a/README.md b/README.md index db7e43c..1140874 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Website: [cachecanary.com](https://cachecanary.com). It has guides that show exa What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region). -CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production. +CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs or your Langfuse traces to show hit rates in production. ## Install @@ -57,6 +57,7 @@ The last one is easy to miss. The same prompt caches fine on Sonnet 4.6 (minimum | `diff previous.json next.json` | Explains why the second request missed the first one's cache | the cached part changed | | `probe request.json --model ID [--region R] [--stream]` | Sends the request twice and checks the second one read from cache | nothing was read from cache | | `logs files-or-folders... [--by model\|principal] [--min-hit 0.5] [--price 3]` | Hit rate and input cost per model or IAM principal from Bedrock invocation logs, plus how much the misses cost | a group is under the threshold | +| `langfuse [--since 7d] [--environment E] [--by model\|name\|environment] [--min-hit 0.5] [--price 3]` | The same, from the generations your Langfuse project already records (no invocation logging needed) | a group is under the threshold | Exit code 2 means it couldn't run at all (bad file, no model access, no AWS credentials). `--json` and `--github` go before the command. @@ -93,6 +94,16 @@ Problems show up as annotations on the pull request, plus a short table in the j with: command: probe args: tests/fixtures/agent_request.json --model us.anthropic.claude-sonnet-4-6 --region us-west-2 + +# Reads last week's hit rate from Langfuse and fails under 80%. Good as a scheduled workflow. +- uses: Haarris/cachecanary@v0 + env: + LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} + LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} + LANGFUSE_HOST: https://us.cloud.langfuse.com + with: + command: langfuse + args: --since 7d --environment production --min-hit 0.8 ``` `args` takes the same arguments as the CLI, separated by spaces. Paths with spaces aren't supported. @@ -124,20 +135,48 @@ How the dollars work: It reads S3 deliveries, CloudWatch exports and `aws logs filter-log-events` output, gzipped or not, including streamed responses. Bedrock logs InvokeModel calls under an inference profile ARN and Converse calls under the short model ID, so CacheCanary merges the two. +## Hit rates from Langfuse + +If you trace your agents with [Langfuse](https://langfuse.com), you don't need invocation logging. Langfuse already stores the cache counts for every model call, and `langfuse` reads them through the Langfuse API: + +```bash +export LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-... +export LANGFUSE_HOST=https://us.cloud.langfuse.com # or https://cloud.langfuse.com (EU), or your own server +cachecanary langfuse --since 7d --environment production --min-hit 0.8 +``` + +It asks Langfuse only for token counts, model names and timestamps, never prompts or outputs. It works with Langfuse Cloud and self-hosted v3 and v4 (on v4 it uses the faster v2 observations API). + +Langfuse stores the counts the way each integration sends them, and some don't add up. Strands before 1.54 sent Bedrock's input count after the cache was already taken out, and Langfuse takes the cache out again, so the input often shows as 0 and the cache looks perfect. LiteLLM's `langfuse` callback leaves cache writes inside the input count as well as reporting them on their own. When a call has a total that was really sent, CacheCanary checks the parts against it, rebuilds the uncached input from the total (Bedrock's total includes the cached tokens), and tells you how many calls it fixed. + +This is the output on real Langfuse records in this repo. They came from Strands 1.61 and 1.53, LiteLLM (both Langfuse integrations) and LangChain, each making one cache write and one cache read: + +```text +$ cachecanary langfuse --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8 +us.anthropic.claude-sonnet-4-6: hit 50% over 10 calls (read 29240, write 29240, uncached 100, no usage 0) <-- below threshold + input cost $0.13 at list price; about $0.07 of it lost to cache misses (target: 80% hit rate) +List prices: Amazon Bedrock on-demand, Oct 2026 (global. IDs at list, others +10%). Use --price for your own rate. +Read 10 generations. +2 generation(s): Langfuse's input count did not add up with the total that was sent, so the uncached input was rebuilt from the total (seen with Strands before 1.54 and LiteLLM's langfuse callback). +``` + +Only model calls (generations) are counted. Agent and chain spans can carry the summed usage of the calls under them, so counting them would count tokens twice. `--file` reads a saved observations API response instead of calling Langfuse. + ## How it's tested -The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 added blocks hit, 22 miss). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). +The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 added blocks hit, 22 miss). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). The Langfuse reader is tested against records that Strands, LiteLLM and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where its numbers matched a separate recount for every model. To run the live checks yourself, use an AWS Region without production traffic: `python scripts/live_test.py --region `. ## Privacy -Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials. +Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials, and `langfuse` only reads token counts from your own Langfuse project with your own keys. ## Limits - Token counts in `lint` are estimates. On English text Bedrock counted about 11% more than CacheCanary did, so near a model's minimum it warns you and suggests running `probe`. - Bedrock keeps cache counts inside the logged response body, and only bodies up to 100 KB are stored inline. Bigger ones show up as `unparsed` and aren't in the dollar figures. +- `langfuse` can only check a call's counts when the integration sent a total. Calls with no cache fields at all are counted as uncached and listed in the output, because Langfuse can't tell "caching was off" from "the integration doesn't report cache tokens". - Dollar figures use list prices, which change. Batch, provisioned throughput and priority tiers aren't covered; use `--price` for those. - Bedrock only for now. diff --git a/action.yml b/action.yml index 5ed8043..3e18a1a 100644 --- a/action.yml +++ b/action.yml @@ -7,7 +7,7 @@ branding: inputs: command: - description: "One of: lint, diff, probe, logs" + description: "One of: lint, diff, probe, logs, langfuse" required: true args: description: > @@ -38,8 +38,8 @@ runs: CC_ARGS: ${{ inputs.args }} run: | case "$CC_COMMAND" in - lint|diff|probe|logs) ;; - *) echo "::error title=CacheCanary::unknown command '$CC_COMMAND' (use lint, diff, probe or logs)"; exit 2 ;; + lint|diff|probe|logs|langfuse) ;; + *) echo "::error title=CacheCanary::unknown command '$CC_COMMAND' (use lint, diff, probe, logs or langfuse)"; exit 2 ;; esac read -r -a CC_ARGV <<< "$CC_ARGS" cachecanary --github "$CC_COMMAND" "${CC_ARGV[@]}" diff --git a/site/index.html b/site/index.html index dde2afe..7714e4a 100644 --- a/site/index.html +++ b/site/index.html @@ -31,7 +31,7 @@ "url": "https://cachecanary.com/", "applicationCategory": "DeveloperApplication", "operatingSystem": "macOS, Linux, Windows", - "softwareVersion": "0.3.2", + "softwareVersion": "0.4.0", "downloadUrl": "https://pypi.org/project/cachecanary/", "license": "https://www.apache.org/licenses/LICENSE-2.0", "offers": { diff --git a/src/cachecanary/__init__.py b/src/cachecanary/__init__.py index 6c1a163..1934f55 100644 --- a/src/cachecanary/__init__.py +++ b/src/cachecanary/__init__.py @@ -1,3 +1,3 @@ """CacheCanary: know the moment your Claude prompt cache breaks on Amazon Bedrock.""" -__version__ = "0.3.2" +__version__ = "0.4.0" diff --git a/src/cachecanary/cli.py b/src/cachecanary/cli.py index 66a229f..95f3dab 100644 --- a/src/cachecanary/cli.py +++ b/src/cachecanary/cli.py @@ -4,6 +4,7 @@ cachecanary diff previous.json next.json [--model ID] cachecanary probe request.json --model ID [--region R] [--stream] cachecanary logs FILE_OR_FOLDER ... [--by model|principal|model+principal] [--min-hit 0.5] [--price 3] + cachecanary langfuse [--since 7d | --from DATE --to DATE] [--environment E] [--by model|name|environment] Exit codes: 0 ok, 1 caching problem found (use to gate CI), 2 could not run (bad input, AWS error). Add --github (before the command) for GitHub Actions annotations and a job summary. @@ -11,11 +12,13 @@ import argparse import json +import os import sys from dataclasses import asdict +from datetime import datetime, timezone from pathlib import Path -from cachecanary import __version__, diff, lint, logs, pricing, probe +from cachecanary import __version__, diff, langfuse, lint, logs, pricing, probe from cachecanary import github as gh from cachecanary.request import RequestError, normalize @@ -163,6 +166,12 @@ def cmd_logs(args) -> int: except (OSError, UnicodeDecodeError, EOFError, json.JSONDecodeError) as exc: raise InputError(f"could not read {path}: {exc}") from exc groups = logs.aggregate(records, by=args.by) + notes = [f"Skipped {stats.bad_lines} unreadable line(s)."] if stats.bad_lines else [] + return _report(groups, args, "logs", "No invocation log records found.", notes) + + +def _report(groups, args, source: str, empty: str, notes: list[str]) -> int: + """Hit rate and dollars per group, shared by `logs` and `langfuse`.""" target = args.min_hit if args.min_hit is not None else DEFAULT_TARGET_HIT data, lines, failing = {}, [], False total_cost = total_lost = 0.0 @@ -177,7 +186,7 @@ def cmd_logs(args) -> int: flag, failing = " <-- below threshold", True lines.append(f"{key}: hit {ratio:.0%} over {g.calls} calls " f"(read {g.usage.cache_read}, write {g.usage.cache_write}, uncached {g.usage.uncached_input}, " - f"unparsed {g.unparsed}){flag}") + f"{'no usage' if source == 'langfuse' else 'unparsed'} {g.unparsed}){flag}") if g.usage.total_input == 0: continue if cost is None: @@ -202,9 +211,11 @@ def cmd_logs(args) -> int: if any_priced and not args.price: lines.append("List prices: Amazon Bedrock on-demand, Oct 2026 (global. IDs at list, others +10%). " "Use --price for your own rate.") - if stats.bad_lines: - lines.append(f"Skipped {stats.bad_lines} unreadable line(s).") - _emit(data, args.json, lines or ["No invocation log records found."]) + lines.extend(notes) + if args.json: + print(json.dumps({"groups": data, "notes": notes} if source == "langfuse" else data, indent=2)) + else: + print("\n".join(lines if data else [empty, *notes])) if args.github: rows = [] for key, d in data.items(): @@ -216,13 +227,59 @@ def cmd_logs(args) -> int: rows.append([key, d["calls"], f"{d['hit_ratio']:.0%}", d["cache_read"], d["cache_write"], d["uncached_input"], d["unparsed"], _money(d["input_cost_usd"]) if priced else "", _money(d["lost_usd"]) if priced else "", "❌" if below else ""]) - gh.append_summary("### CacheCanary logs\n\n" + (gh.table( - ["group", "calls", "hit rate", "read", "write", "uncached", "unparsed", "input cost", - f"lost vs {target:.0%} hit", "below threshold"], rows) - if rows else "No invocation log records found.")) + summary = gh.table(["group", "calls", "hit rate", "read", "write", "uncached", + "no usage" if source == "langfuse" else "unparsed", "input cost", + f"lost vs {target:.0%} hit", "below threshold"], rows) if rows else empty + gh.append_summary(f"### CacheCanary {source}\n\n" + summary + "".join(f"\n\n{n}" for n in notes)) return EXIT_PROBLEM if failing else EXIT_OK +def cmd_langfuse(args) -> int: + if args.file: + observations, api, capped = [], "file", False + for path in args.file: + observations.extend(langfuse.iter_file(Path(path))) + window = "" + else: + now = datetime.now(timezone.utc) + to_time = args.to or now + from_time = args.from_ or to_time - args.since + if from_time >= to_time: + raise InputError("--from must be earlier than --to") + client = langfuse.Client(args.host or os.environ.get("LANGFUSE_HOST") or os.environ.get("LANGFUSE_BASE_URL"), + os.environ.get("LANGFUSE_PUBLIC_KEY", ""), os.environ.get("LANGFUSE_SECRET_KEY", ""), + progress=langfuse.stderr_progress) + try: + fetched = client.fetch_generations(langfuse.iso(from_time), langfuse.iso(to_time) if args.to else None, + args.environment, args.max_calls) + finally: + if sys.stderr.isatty(): + print(file=sys.stderr) + observations, api, capped = fetched.observations, fetched.api, fetched.capped + window = f" from {langfuse.iso(from_time)} to {langfuse.iso(to_time)}" + groups, q = langfuse.aggregate(observations, by=args.by) + notes = [] + if q.generations: + notes.append(f"Read {q.generations:,} generations{window}" + (f" (Langfuse API {api})" if api != "file" else "") + ".") + if capped: + notes.append(f"Stopped at --max-calls {args.max_calls:,}; older generations in the window were not read.") + if q.corrected: + notes.append(f"{q.corrected:,} generation(s): Langfuse's input count did not add up with the total that was sent, " + "so the uncached input was rebuilt from the total (seen with Strands before 1.54 and LiteLLM's " + "langfuse callback).") + if q.total_mismatch: + notes.append(f"{q.total_mismatch:,} generation(s) carry a total smaller than their parts; their counts were used as sent.") + if q.no_cache_fields: + notes.append(f"{q.no_cache_fields:,} generation(s) have no cache fields at all: caching was not used for them, " + "the integration does not report cache tokens, or Langfuse estimated the tokens itself.") + if q.no_usage: + notes.append(f"{q.no_usage:,} generation(s) have no token counts.") + if q.skipped_other_types: + notes.append(f"Ignored {q.skipped_other_types:,} non-generation observation(s) (agent and chain spans can repeat " + "the usage of the calls under them).") + return _report(groups, args, "langfuse", "No generations found in Langfuse for this window.", notes) + + def _unit(value: str) -> float: f = float(value) if not 0 <= f <= 1: @@ -240,6 +297,30 @@ def _price(value: str) -> float: return f +def _since(value: str): + try: + return langfuse.parse_since(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(str(exc)) from None + + +def _when(value: str): + try: + return langfuse.parse_time(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(str(exc)) from None + + +def _positive(value: str) -> int: + try: + n = int(value) + except ValueError: + raise argparse.ArgumentTypeError("must be a whole number") from None + if n < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return n + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="cachecanary", description="CacheCanary: catch silent prompt-cache breakage for Claude on Amazon Bedrock.") parser.add_argument("--version", action="version", version=f"cachecanary {__version__}") @@ -273,6 +354,27 @@ def build_parser() -> argparse.ArgumentParser: "also the target for the dollars lost figure (default 0.9)") p.add_argument("--price", type=_price, help="your input price in USD per million tokens, instead of list prices") p.set_defaults(func=cmd_logs) + + p = sub.add_parser("langfuse", help="cache hit rates from the generations in a Langfuse project", + description="Reads token counts (no prompts or outputs) for the project's generations through " + "the Langfuse public API. Keys come from LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY; " + "the host from --host, LANGFUSE_HOST or LANGFUSE_BASE_URL (default " + f"{langfuse.DEFAULT_HOST}).") + p.add_argument("--host", help="Langfuse URL, e.g. https://us.cloud.langfuse.com or your self-hosted URL") + window = p.add_mutually_exclusive_group() + window.add_argument("--since", type=_since, default=langfuse.parse_since("7d"), + help="how far back to read, e.g. 24h, 7d, 2w (default 7d)") + window.add_argument("--from", dest="from_", type=_when, help="start of the window (date or ISO time, UTC)") + p.add_argument("--to", type=_when, help="end of the window (default now)") + p.add_argument("--environment", action="append", help="only this Langfuse environment (repeatable)") + p.add_argument("--by", choices=["model", "name", "environment"], default="model") + p.add_argument("--min-hit", type=_unit, help="fail if any group's hit ratio is below this (0-1); " + "also the target for the dollars lost figure (default 0.9)") + p.add_argument("--price", type=_price, help="your input price in USD per million tokens, instead of list prices") + p.add_argument("--max-calls", type=_positive, default=20000, + help="stop after this many generations (default 20000)") + p.add_argument("--file", action="append", help="read a saved observations API response instead of calling Langfuse") + p.set_defaults(func=cmd_langfuse) return parser @@ -280,7 +382,7 @@ def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: return args.func(args) - except (InputError, RequestError, probe.ProbeError) as exc: + except (InputError, RequestError, probe.ProbeError, langfuse.LangfuseError) as exc: print(f"cachecanary: error: {exc}", file=sys.stderr) if getattr(args, "github", False): print(gh.annotation("error", str(exc), title="CacheCanary could not run")) diff --git a/src/cachecanary/langfuse.py b/src/cachecanary/langfuse.py new file mode 100644 index 0000000..1cffd93 --- /dev/null +++ b/src/cachecanary/langfuse.py @@ -0,0 +1,403 @@ +"""Cache hit rates from the generations Langfuse already records, for teams without Bedrock +invocation logging. + +Langfuse keeps each generation's token counts in `usageDetails`, a free-form map whose key names +depend on how the data was sent (checked against Langfuse, langfuse-python, Strands, LiteLLM and +the Vercel AI SDK source, Oct 2026): + +- OpenTelemetry (Strands, LiteLLM OTel, OpenInference, Pydantic AI): `input` (Langfuse subtracts + the cache counts from the reported input), `input_cached_tokens`, `input_cache_creation`, `total`. +- Vercel AI SDK: `input_cached_tokens` or `input_cache_read`, and `input_cache_creation` / + `input_cache_write`, with `input_cache_creation_5m` / `_1h` split out of the aggregate. +- LangChain: `input_cache_read` / `input_cache_creation`, or Anthropic's `cache_read_input_tokens` / + `cache_creation_input_tokens` plus `cache_creation_ephemeral_5m_input_tokens` / `_1h` inside it. +- LiteLLM's `langfuse` callback and hand-written SDK calls: `cache_read_input_tokens` / + `cache_creation_input_tokens`. +- Strands also sends `cache_write_input_tokens`, which Langfuse stores unchanged (before Strands + 1.54.0 it is the only cache write count). + +Two known sources send numbers that don't add up, and Langfuse stores them as sent: +Strands before 1.54.0 reported Bedrock's already-uncached input, which Langfuse then reduced by the +cache counts again (often to 0, so the cache looks perfect), and LiteLLM's `langfuse` callback +leaves cache writes inside `input` as well as reporting them separately. When a generation carries +a total that was really sent (not one Langfuse added up itself), CacheCanary checks the parts +against it and rebuilds the uncached input as total - output - cache read - cache write; Bedrock +totals include the cache counts. +""" + +import base64 +import json +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from cachecanary import __version__ +from cachecanary.logs import Group +from cachecanary.models import canonical_model_id +from cachecanary.usage import Usage + +DEFAULT_HOST = "https://cloud.langfuse.com" +V2_PATH = "/api/public/v2/observations" +V1_PATH = "/api/public/observations" +V2_PAGE_SIZE = 1000 # API maximum +V1_PAGE_SIZE = 100 # API maximum +# Only what the report needs: no prompts, outputs or metadata are requested. +V2_FIELDS = "core,basic,model,usage" + +READ_KEYS = ("input_cache_read", "input_cached_tokens", "cache_read_input_tokens") +# Vercel AI SDK: input_cache_creation is what is left after the TTL split, so the parts add up. +SPLIT_WRITE_KEYS = ("input_cache_creation", "input_cache_creation_5m", "input_cache_creation_1h") +# LangChain + Anthropic: the aggregate already contains the TTL parts. +ANTHROPIC_WRITE_KEY = "cache_creation_input_tokens" +# Names Langfuse does not recognise and stores unchanged. Strands sends its deprecated +# gen_ai.usage.cache_write_input_tokens attribute as this key; before 1.54 it is the only write count. +PASSTHROUGH_WRITE_KEYS = ("cache_write_input_tokens",) +ANTHROPIC_TTL_KEYS = ("cache_creation_ephemeral_5m_input_tokens", "cache_creation_ephemeral_1h_input_tokens") +WRITE_1H_KEYS = ("input_cache_creation_1h", "cache_creation_ephemeral_1h_input_tokens") +CACHE_KEYS = frozenset(READ_KEYS + SPLIT_WRITE_KEYS + (ANTHROPIC_WRITE_KEY, "input_cache_write") + + ANTHROPIC_TTL_KEYS + PASSTHROUGH_WRITE_KEYS) + + +class LangfuseError(Exception): + pass + + +@dataclass +class Normalized: + usage: Usage | None + status: str # "ok" | "corrected" | "total-mismatch" | "no-usage" + has_cache_fields: bool = False + + +def _num(value) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)) and value >= 0 and value == value and value != float("inf"): + return int(value) + if isinstance(value, str): + try: + parsed = float(value) + except ValueError: + return None + return _num(parsed) + return None + + +def normalize_usage(details) -> Normalized: + """Turn one generation's usageDetails into uncached / read / write / output counts.""" + if not isinstance(details, dict): + return Normalized(None, "no-usage") + values = {k: n for k, v in details.items() if isinstance(k, str) and (n := _num(v)) is not None} + if not any(v for k, v in values.items() if k != "total"): + return Normalized(None, "no-usage") + + has_cache = any(k in values for k in CACHE_KEYS) + read = max((values.get(k, 0) for k in READ_KEYS), default=0) + split_write = sum(values.get(k, 0) for k in SPLIT_WRITE_KEYS) + anthropic_write = max(values.get(ANTHROPIC_WRITE_KEY, 0), sum(values.get(k, 0) for k in ANTHROPIC_TTL_KEYS)) + # One integration sends one family; the largest guards against the same writes under two names. + write = max(split_write, anthropic_write, values.get("input_cache_write", 0), + *(values.get(k, 0) for k in PASSTHROUGH_WRITE_KEYS)) + write_1h = min(max((values.get(k, 0) for k in WRITE_1H_KEYS), default=0), write) + + output = sum(v for k, v in values.items() if k == "output" or k.startswith("output_")) + # Other input categories (e.g. input_audio) are still uncached input. Priority-tier and + # modality keys repeat tokens already counted elsewhere. + extra_input = sum(v for k, v in values.items() + if k.startswith("input_") and k not in CACHE_KEYS + and "priority" not in k and "modality" not in k) + uncached = values.get("input", 0) + extra_input + + total = values.get("total") + # Langfuse fills in total = sum of every value when none was sent; such a total proves nothing. + sent_total = total is not None and total != sum(v for k, v in values.items() if k != "total") + status = "ok" + if sent_total: + derived = total - output - read - write + if derived < 0: + status = "total-mismatch" + elif derived != uncached: + uncached, status = derived, "corrected" + return Normalized(Usage(uncached, read, write, output, write_1h), status, has_cache) + + +@dataclass +class Quality: + generations: int = 0 + skipped_other_types: int = 0 + no_usage: int = 0 + no_cache_fields: int = 0 + corrected: int = 0 + total_mismatch: int = 0 + + +# LiteLLM's Bedrock routing prefixes (litellm.llms.bedrock.common_utils.strip_bedrock_routing_prefix) and its +# optional Region path ("bedrock/us-west-2/"). LiteLLM's `langfuse` callback logs e.g. +# "converse/us.anthropic.claude-sonnet-4-6"; left in place, the prefix splits one model into two groups and hides +# a "global." endpoint from pricing. +_ROUTE_PREFIXES = ("bedrock/", "chat_completions/", "converse/", "invoke/", "openai/", "mantle/") +_REGION_PATH = re.compile(r"^[a-z]{2}(?:-gov)?-[a-z]+-\d+/") + + +def model_id(raw) -> str | None: + """The Bedrock model ID behind a Langfuse model name: LiteLLM prefixes and ARN forms removed.""" + if not isinstance(raw, str) or not raw.strip(): + return None + name = raw.strip() + changed = True + while changed: + changed = False + for prefix in _ROUTE_PREFIXES: + if name.startswith(prefix) and len(name) > len(prefix): + name, changed = name[len(prefix):], True + name = _REGION_PATH.sub("", name, count=1) or name + return canonical_model_id(name) + + +def group_key(obs: dict, by: str) -> str: + if by == "name": + return obs.get("name") or "(no name)" + if by == "environment": + return obs.get("environment") or "default" + return model_id(obs.get("model")) or "(no model)" + + +def aggregate(observations, by: str = "model") -> tuple[dict[str, Group], Quality]: + groups: dict[str, Group] = defaultdict(Group) + quality = Quality() + for obs in observations: + if not isinstance(obs, dict): + continue + if (obs.get("type") or "GENERATION").upper() != "GENERATION": + # Agent and chain spans can repeat the summed usage of the calls under them. + quality.skipped_other_types += 1 + continue + quality.generations += 1 + g = groups[group_key(obs, by)] + g.calls += 1 + result = normalize_usage(obs.get("usageDetails")) + if result.usage is None: + quality.no_usage += 1 + g.unparsed += 1 + continue + quality.corrected += result.status == "corrected" + quality.total_mismatch += result.status == "total-mismatch" + quality.no_cache_fields += not result.has_cache_fields + model = model_id(obs.get("model")) or "(no model)" + g.usage = g.usage + result.usage + g.by_model[model] = g.by_model.get(model, Usage()) + result.usage + return dict(groups), quality + + +# ---------------------------------------------------------------- reading saved API responses + +def iter_file(path: Path): + """Observations from a saved API response ({"data": [...]}), a list, or JSON Lines of either.""" + try: + text = path.read_text(encoding="utf-8").strip() + except FileNotFoundError as exc: + raise LangfuseError(f"file not found: {path}") from exc + except (OSError, UnicodeDecodeError) as exc: + raise LangfuseError(f"could not read {path}: {exc}") from exc + if not text: + return + try: + docs = [json.loads(text)] + except json.JSONDecodeError: + docs = [] + for n, line in enumerate(text.splitlines(), 1): + if line.strip(): + try: + docs.append(json.loads(line)) + except json.JSONDecodeError as exc: + raise LangfuseError(f"{path} line {n} is not valid JSON: {exc}") from exc + for doc in docs: + if isinstance(doc, dict) and isinstance(doc.get("data"), list): + yield from doc["data"] + elif isinstance(doc, list): + yield from doc + elif isinstance(doc, dict): + yield doc + else: + raise LangfuseError(f"{path} does not hold Langfuse observations") + + +# ---------------------------------------------------------------- the API + +@dataclass +class FetchResult: + observations: list = field(default_factory=list) + api: str = "" # "v2" | "v1" + capped: bool = False + + +class Client: + def __init__(self, host: str, public_key: str, secret_key: str, *, timeout: float = 60, + sleep=None, opener=None, progress=None): + host = (host or DEFAULT_HOST).strip().rstrip("/") + if not re.match(r"^https?://[^/\s]+", host): + raise LangfuseError(f"Langfuse host must start with https:// (got {host!r})") + if not public_key or not secret_key: + raise LangfuseError("set LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY (a project's API keys) " + "to read from Langfuse, or pass --file with a saved API response") + self.host = host + token = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode() + self.headers = {"Authorization": f"Basic {token}", "Accept": "application/json", + "User-Agent": f"cachecanary/{__version__}"} + self.timeout, self.progress = timeout, progress + self.sleep = sleep or time.sleep + self.opener = opener or urllib.request.urlopen + + def get(self, path: str, params: dict) -> tuple[int, dict | None]: + """(status, JSON body). 404 is returned to the caller; other failures raise.""" + query = urllib.parse.urlencode({k: v for k, v in params.items() if v is not None}, doseq=True) + url = f"{self.host}{path}?{query}" + rate_waits = server_retries = 0 + while True: + try: + with self.opener(urllib.request.Request(url, headers=self.headers), timeout=self.timeout) as resp: + raw = resp.read() + break + except urllib.error.HTTPError as exc: + status = exc.code + if status == 404: + return 404, _json_or_none(exc.read()) + if status in (401, 403): + raise LangfuseError( + f"Langfuse rejected the API keys (HTTP {status}) at {self.host}. Check LANGFUSE_PUBLIC_KEY / " + "LANGFUSE_SECRET_KEY and that the host matches the project's region " + "(EU https://cloud.langfuse.com, US https://us.cloud.langfuse.com).") from None + if status == 429 and rate_waits < 6: + rate_waits += 1 + self.sleep(_retry_after(exc.headers)) + continue + if status >= 500 and server_retries < 3: + server_retries += 1 + self.sleep(2 ** server_retries) + continue + raise LangfuseError(f"Langfuse returned HTTP {status} for {path}: {_message(exc.read())}") from None + except (urllib.error.URLError, TimeoutError, ConnectionError) as exc: + if server_retries < 3: + server_retries += 1 + self.sleep(2 ** server_retries) + continue + reason = getattr(exc, "reason", exc) + raise LangfuseError(f"could not reach Langfuse at {self.host}: {reason}") from None + body = _json_or_none(raw) + if not isinstance(body, dict): + raise LangfuseError(f"Langfuse sent a response that is not JSON for {path}; is {self.host} a Langfuse host?") + return 200, body + + def fetch_generations(self, from_time: str, to_time: str | None, environments: list[str] | None, + max_calls: int) -> FetchResult: + result = FetchResult() + common = {"type": "GENERATION", "fromStartTime": from_time, "toStartTime": to_time, + "environment": environments or None} + status, body = self.get(V2_PATH, {**common, "fields": V2_FIELDS, "limit": V2_PAGE_SIZE}) + if status == 200: + result.api = "v2" + seen_cursors = set() + while True: + result.capped = self._take(result, body, max_calls) + cursor = (body.get("meta") or {}).get("cursor") + if result.capped or not cursor or not body.get("data"): + return result + if cursor in seen_cursors: + raise LangfuseError("Langfuse returned the same page cursor twice; stopping to avoid a loop") + seen_cursors.add(cursor) + status, body = self.get(V2_PATH, {**common, "fields": V2_FIELDS, "limit": V2_PAGE_SIZE, + "cursor": cursor}) + if status != 200: + raise LangfuseError(f"Langfuse returned HTTP {status} part-way through the results") + # Langfuse v3, or v4 before its write mode is switched on, has only the v1 list. + v2_message = _message(body) + result.api, page = "v1", 1 + while True: + status, body = self.get(V1_PATH, {**common, "page": page, "limit": V1_PAGE_SIZE}) + if status != 200: + raise LangfuseError( + f"this Langfuse host serves neither observations API (v2: {v2_message}; v1: {_message(body)})") + result.capped = self._take(result, body, max_calls) + total_pages = _num((body.get("meta") or {}).get("totalPages")) or 0 + if result.capped or not body.get("data") or page >= total_pages: + return result + page += 1 + + def _take(self, result: FetchResult, body: dict, max_calls: int) -> bool: + data = body.get("data") + if not isinstance(data, list): + raise LangfuseError("Langfuse response has no 'data' list; unexpected API response") + room = max_calls - len(result.observations) + result.observations.extend(data[:room]) + if self.progress: + self.progress(len(result.observations)) + return len(data) > room or (len(result.observations) >= max_calls and _more(body)) + + +def _more(body: dict) -> bool: + meta = body.get("meta") or {} + return bool(meta.get("cursor")) or (_num(meta.get("totalPages")) or 0) > (_num(meta.get("page")) or 0) + + +def _retry_after(headers) -> float: + value = headers.get("Retry-After") if headers else None + seconds = _num(value) + return float(min(max(seconds if seconds is not None else 5, 1), 60)) + + +def _json_or_none(raw): + try: + return json.loads(raw.decode("utf-8") if isinstance(raw, (bytes, bytearray)) else raw) + except (ValueError, UnicodeDecodeError, AttributeError, TypeError): + return None + + +def _message(body) -> str: + if isinstance(body, (bytes, bytearray)): + body = _json_or_none(body) or body.decode("utf-8", "replace") + if isinstance(body, dict): + body = body.get("message") or body.get("error") or json.dumps(body) + text = str(body or "no details").strip().replace("\n", " ") + return text[:200] + + +# ---------------------------------------------------------------- time window + +_SINCE = re.compile(r"^(\d+)\s*([mhdw])$") + + +def parse_since(value: str) -> timedelta: + match = _SINCE.match(value.strip().lower()) + if not match or int(match.group(1)) == 0: + raise ValueError("use a number and a unit, e.g. 30m, 24h, 7d or 2w") + amount, unit = int(match.group(1)), match.group(2) + return {"m": timedelta(minutes=amount), "h": timedelta(hours=amount), + "d": timedelta(days=amount), "w": timedelta(weeks=amount)}[unit] + + +def parse_time(value: str) -> datetime: + """A date (2026-10-01, midnight UTC) or an ISO 8601 time; times without an offset are UTC.""" + text = value.strip() + if text.endswith(("Z", "z")): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError: + raise ValueError("use a date like 2026-10-01 or a time like 2026-10-01T09:00:00Z") from None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def iso(dt: datetime) -> str: + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def stderr_progress(count: int) -> None: + if sys.stderr.isatty(): + print(f"\rRead {count:,} generations from Langfuse...", end="", file=sys.stderr, flush=True) diff --git a/tests/fixtures/langfuse_observations_2026-10.json b/tests/fixtures/langfuse_observations_2026-10.json new file mode 100644 index 0000000..e05ccde --- /dev/null +++ b/tests/fixtures/langfuse_observations_2026-10.json @@ -0,0 +1,487 @@ +{ + "data": [ + { + "id": "e03a666e33105a6b", + "traceId": "f95c49980df3697bfa3dc1e5d5b835b4", + "startTime": "2026-10-20T13:25:12.156Z", + "endTime": "2026-10-20T13:25:12.159Z", + "projectId": "cc-proj", + "parentObservationId": "11a7e2e3d8570bc7", + "type": "GENERATION", + "name": "chat", + "level": "DEFAULT", + "statusMessage": "", + "version": "1.61.1", + "environment": "new", + "model": "us.anthropic.claude-sonnet-4-6", + "internalModelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelParameters": {}, + "usageDetails": { + "cache_write_input_tokens": 5848, + "input": 10, + "output": 4, + "total": 5862, + "input_cached_tokens": 0, + "input_cache_creation": 5848 + }, + "inputUsage": 11706, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "input": 3e-05, + "output": 6e-05, + "input_cached_tokens": 0, + "input_cache_creation": 0.02193, + "total": 0.02202 + }, + "inputCost": 0.02196, + "outputCost": 6e-05, + "totalCost": 0.02202, + "usagePricingTierId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "usagePricingTierName": "Standard", + "latency": 0.003, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": false, + "bookmarked": false, + "public": false, + "modelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "inputPrice": "0.000003", + "outputPrice": "0.000015", + "totalPrice": null + }, + { + "id": "b71656d4a1eafb78", + "traceId": "e190ec5b63c736af0cdd2b59f6267ac2", + "startTime": "2026-10-20T13:25:12.160Z", + "endTime": "2026-10-20T13:25:12.161Z", + "projectId": "cc-proj", + "parentObservationId": "1a73e933403d121e", + "type": "GENERATION", + "name": "chat", + "level": "DEFAULT", + "statusMessage": "", + "version": "1.61.1", + "environment": "new", + "model": "us.anthropic.claude-sonnet-4-6", + "internalModelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelParameters": {}, + "usageDetails": { + "cache_write_input_tokens": 0, + "input": 10, + "output": 4, + "total": 5862, + "input_cached_tokens": 5848, + "input_cache_creation": 0 + }, + "inputUsage": 5858, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "input": 3e-05, + "output": 6e-05, + "input_cached_tokens": 0.0017544, + "input_cache_creation": 0, + "total": 0.0018444 + }, + "inputCost": 0.0017844, + "outputCost": 6e-05, + "totalCost": 0.0018444, + "usagePricingTierId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "usagePricingTierName": "Standard", + "latency": 0.001, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": false, + "bookmarked": false, + "public": false, + "modelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "inputPrice": "0.000003", + "outputPrice": "0.000015", + "totalPrice": null + }, + { + "id": "d69858da09a7d382", + "traceId": "e9b703ad83f0ad74781ad5621fb679a3", + "startTime": "2026-10-20T13:25:41.926Z", + "endTime": "2026-10-20T13:25:41.928Z", + "projectId": "cc-proj", + "parentObservationId": "d3e1ac7411c6075b", + "type": "GENERATION", + "name": "chat", + "level": "DEFAULT", + "statusMessage": "", + "version": "1.53.0", + "environment": "old", + "model": "us.anthropic.claude-sonnet-4-6", + "internalModelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelParameters": {}, + "usageDetails": { + "cache_write_input_tokens": 5848, + "input": 10, + "output": 4, + "total": 5862, + "input_cached_tokens": 0 + }, + "inputUsage": 5858, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "input": 3e-05, + "output": 6e-05, + "input_cached_tokens": 0, + "total": 9e-05 + }, + "inputCost": 3e-05, + "outputCost": 6e-05, + "totalCost": 9e-05, + "usagePricingTierId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "usagePricingTierName": "Standard", + "latency": 0.002, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": false, + "bookmarked": false, + "public": false, + "modelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "inputPrice": "0.000003", + "outputPrice": "0.000015", + "totalPrice": null + }, + { + "id": "a8356582de1890bd", + "traceId": "600c065255c2c4fa466404ef8f5e3e39", + "startTime": "2026-10-20T13:25:41.929Z", + "endTime": "2026-10-20T13:25:41.930Z", + "projectId": "cc-proj", + "parentObservationId": "a6c9ba5bc573ebd8", + "type": "GENERATION", + "name": "chat", + "level": "DEFAULT", + "statusMessage": "", + "version": "1.53.0", + "environment": "old", + "model": "us.anthropic.claude-sonnet-4-6", + "internalModelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelParameters": {}, + "usageDetails": { + "cache_write_input_tokens": 0, + "input": 0, + "output": 4, + "total": 5862, + "input_cached_tokens": 5848 + }, + "inputUsage": 5848, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "input": 0, + "output": 6e-05, + "input_cached_tokens": 0.0017544, + "total": 0.001814399999 + }, + "inputCost": 0.0017544, + "outputCost": 6e-05, + "totalCost": 0.001814399999, + "usagePricingTierId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "usagePricingTierName": "Standard", + "latency": 0.001, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": false, + "bookmarked": false, + "public": false, + "modelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "inputPrice": "0.000003", + "outputPrice": "0.000015", + "totalPrice": null + }, + { + "id": "8cf6be70bb67110d", + "traceId": "d1ab8f83c59f4d758c38c8230bea9e3c", + "startTime": "2026-10-20T13:26:52.654Z", + "endTime": "2026-10-20T13:26:53.845Z", + "projectId": "cc-proj", + "parentObservationId": null, + "type": "GENERATION", + "name": "litellm-legacy", + "level": "DEFAULT", + "statusMessage": "", + "version": "", + "environment": "default", + "model": "converse/us.anthropic.claude-sonnet-4-6", + "internalModelId": "", + "modelParameters": {}, + "usageDetails": { + "input": 5858, + "output": 4, + "total": 5862, + "cache_creation_input_tokens": 5848, + "cache_read_input_tokens": 0 + }, + "inputUsage": 11706, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "total": 0.024222 + }, + "inputCost": null, + "outputCost": null, + "totalCost": 0.024222, + "usagePricingTierId": null, + "usagePricingTierName": null, + "latency": 1.191, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": true, + "bookmarked": false, + "public": false, + "modelId": null, + "inputPrice": null, + "outputPrice": null, + "totalPrice": null + }, + { + "id": "5840b5e28267154a", + "traceId": "b4ece235720641cfb401704a8569de88", + "startTime": "2026-10-20T13:26:53.849Z", + "endTime": "2026-10-20T13:26:53.851Z", + "projectId": "cc-proj", + "parentObservationId": null, + "type": "GENERATION", + "name": "litellm-legacy", + "level": "DEFAULT", + "statusMessage": "", + "version": "", + "environment": "default", + "model": "converse/us.anthropic.claude-sonnet-4-6", + "internalModelId": "", + "modelParameters": {}, + "usageDetails": { + "input": 10, + "output": 4, + "total": 5862, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 5848 + }, + "inputUsage": 5858, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "total": 0.00202884 + }, + "inputCost": null, + "outputCost": null, + "totalCost": 0.00202884, + "usagePricingTierId": null, + "usagePricingTierName": null, + "latency": 0.002, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": true, + "bookmarked": false, + "public": false, + "modelId": null, + "inputPrice": null, + "outputPrice": null, + "totalPrice": null + }, + { + "id": "6a60a2f50d5ce145", + "traceId": "a62d88e92d867b2475029c27a55ad62c", + "startTime": "2026-10-20T13:27:23.264Z", + "endTime": "2026-10-20T13:27:23.268Z", + "projectId": "cc-proj", + "parentObservationId": null, + "type": "GENERATION", + "name": "ChatBedrockConverse", + "level": "DEFAULT", + "statusMessage": "", + "version": "", + "environment": "langchain", + "model": "us.anthropic.claude-sonnet-4-6", + "internalModelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelParameters": {}, + "usageDetails": { + "input": 10, + "output": 4, + "total": 5862, + "input_cache_creation": 5848, + "input_cache_read": 0 + }, + "inputUsage": 5858, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "input": 3e-05, + "output": 6e-05, + "input_cache_creation": 0.02193, + "input_cache_read": 0, + "total": 0.02202 + }, + "inputCost": 0.02196, + "outputCost": 6e-05, + "totalCost": 0.02202, + "usagePricingTierId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "usagePricingTierName": "Standard", + "latency": 0.004, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": true, + "bookmarked": false, + "public": false, + "modelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "inputPrice": "0.000003", + "outputPrice": "0.000015", + "totalPrice": null + }, + { + "id": "3bbde9fa299dbbf6", + "traceId": "906e3c10ae7f00e561b639cfff81e00e", + "startTime": "2026-10-20T13:27:23.269Z", + "endTime": "2026-10-20T13:27:23.270Z", + "projectId": "cc-proj", + "parentObservationId": null, + "type": "GENERATION", + "name": "ChatBedrockConverse", + "level": "DEFAULT", + "statusMessage": "", + "version": "", + "environment": "langchain", + "model": "us.anthropic.claude-sonnet-4-6", + "internalModelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelParameters": {}, + "usageDetails": { + "input": 10, + "output": 4, + "total": 5862, + "input_cache_creation": 0, + "input_cache_read": 5848 + }, + "inputUsage": 5858, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "input": 3e-05, + "output": 6e-05, + "input_cache_creation": 0, + "input_cache_read": 0.0017544, + "total": 0.0018444 + }, + "inputCost": 0.0017844, + "outputCost": 6e-05, + "totalCost": 0.0018444, + "usagePricingTierId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "usagePricingTierName": "Standard", + "latency": 0.001, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": true, + "bookmarked": false, + "public": false, + "modelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "inputPrice": "0.000003", + "outputPrice": "0.000015", + "totalPrice": null + }, + { + "id": "07d8fc73ded160ba", + "traceId": "3da66dc41a1ce219f2381f79f20eb487", + "startTime": "2026-10-20T13:27:49.426Z", + "endTime": "2026-10-20T13:27:49.735Z", + "projectId": "cc-proj", + "parentObservationId": null, + "type": "GENERATION", + "name": "litellm-otel", + "level": "DEFAULT", + "statusMessage": "", + "version": "", + "environment": "production", + "model": "us.anthropic.claude-sonnet-4-6", + "internalModelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelParameters": {}, + "usageDetails": { + "input": 10, + "output": 4, + "total": 5862, + "input_cache_creation": 5848 + }, + "inputUsage": 5858, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "total": 0.024222 + }, + "inputCost": null, + "outputCost": null, + "totalCost": 0.024222, + "usagePricingTierId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "usagePricingTierName": "Standard", + "latency": 0.309, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": true, + "bookmarked": false, + "public": false, + "modelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "inputPrice": "0.000003", + "outputPrice": "0.000015", + "totalPrice": null + }, + { + "id": "bf0b64713faddc1f", + "traceId": "245996a7201c9c5f25a219eb52d5b0dd", + "startTime": "2026-10-20T13:27:49.738Z", + "endTime": "2026-10-20T13:27:49.740Z", + "projectId": "cc-proj", + "parentObservationId": null, + "type": "GENERATION", + "name": "litellm-otel", + "level": "DEFAULT", + "statusMessage": "", + "version": "", + "environment": "production", + "model": "us.anthropic.claude-sonnet-4-6", + "internalModelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelParameters": {}, + "usageDetails": { + "input": 10, + "output": 4, + "total": 5862, + "input_cached_tokens": 5848 + }, + "inputUsage": 5858, + "outputUsage": 4, + "totalUsage": 5862, + "costDetails": { + "total": 0.00202884 + }, + "inputCost": null, + "outputCost": null, + "totalCost": 0.00202884, + "usagePricingTierId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "usagePricingTierName": "Standard", + "latency": 0.002, + "timeToFirstToken": null, + "userId": "", + "sessionId": "", + "isRootObservation": true, + "bookmarked": false, + "public": false, + "modelId": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "inputPrice": "0.000003", + "outputPrice": "0.000015", + "totalPrice": null + } + ], + "meta": {} +} \ No newline at end of file diff --git a/tests/test_action.py b/tests/test_action.py index 0fd4a6a..ee2d522 100644 --- a/tests/test_action.py +++ b/tests/test_action.py @@ -117,3 +117,18 @@ def test_license_files_and_metadata(): assert "Copyright 2026 Haris Farooq" in (ROOT / "NOTICE").read_text() text = (ROOT / "pyproject.toml").read_text() assert 'license = "Apache-2.0"' in text and 'license-files = ["LICENSE", "NOTICE"]' in text + + +def test_action_runs_langfuse(tmp_path): + res = run_action("langfuse", "--file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8", tmp_path) + assert res.returncode == 1, res.stderr + assert "::error title=CacheCanary%3A low cache hit rate::us.anthropic.claude-sonnet-4-6" in res.stdout + assert "### CacheCanary langfuse" in (tmp_path / "s.md").read_text() + + +def test_action_langfuse_reads_env_settings(tmp_path, monkeypatch): + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk") + monkeypatch.setenv("LANGFUSE_HOST", "http://127.0.0.1:9") + res = run_action("langfuse", "--since 1h", tmp_path) + assert res.returncode == 2 and "could not reach Langfuse at http://127.0.0.1:9" in res.stdout + res.stderr diff --git a/tests/test_langfuse.py b/tests/test_langfuse.py new file mode 100644 index 0000000..11cade3 --- /dev/null +++ b/tests/test_langfuse.py @@ -0,0 +1,721 @@ +"""`cachecanary langfuse`: usageDetails shapes as each integration stores them, the API client, and the CLI. + +Every usage shape below is derived from the source of the integration that produces it (Oct 2026): +Langfuse's OTel ingestion (OtelIngestionProcessor.extractGenericGenAiUsageDetails and the AI SDK branch), +langfuse-python's LangChain CallbackHandler, LiteLLM's `langfuse` callback, Strands' tracer before and +after 1.54.0, and Langfuse filling in `total` when none is sent (IngestionService.getUsageUnits). +The token numbers are the live Bedrock capture from strands-agents/sdk-python#3546: +inputTokens 10, outputTokens 4, cache 5848, totalTokens 5862. +""" + +import base64 +import io +import json +import threading +import urllib.error +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import parse_qs, urlparse + +import pytest + +from cachecanary import cli, langfuse +from cachecanary.langfuse import normalize_usage + +SONNET = "us.anthropic.claude-sonnet-4-6" + + +def with_langfuse_total(details): + """Langfuse adds total = sum of every value when the integration sent none.""" + return {**details, "total": sum(details.values())} + + +# ---------------------------------------------------------------- one usageDetails shape per integration + +def test_strands_current_otel_shape_is_taken_as_is(): + # Strands >= 1.54 sends input_tokens = 10 + 5848 (whole prompt); Langfuse subtracts the cache counts. + n = normalize_usage({"input": 10, "output": 4, "total": 5862, "input_cached_tokens": 5848, "input_cache_creation": 0}) + assert n.status == "ok" and n.has_cache_fields + assert (n.usage.uncached_input, n.usage.cache_read, n.usage.cache_write, n.usage.output) == (10, 5848, 0, 4) + + +def test_strands_before_1_54_hit_is_rebuilt_from_total(): + # Old Strands sent input_tokens = 10 (Bedrock's uncached count); Langfuse took 5848 off again -> 0. + n = normalize_usage({"input": 0, "output": 4, "total": 5862, "input_cached_tokens": 5848, "input_cache_creation": 0}) + assert n.status == "corrected" + assert (n.usage.uncached_input, n.usage.cache_read) == (10, 5848) + + +def test_strands_before_1_54_write_is_rebuilt_from_total(): + n = normalize_usage({"input": 0, "output": 4, "total": 5862, "input_cached_tokens": 0, "input_cache_creation": 5848}) + assert n.status == "corrected" + assert (n.usage.uncached_input, n.usage.cache_write) == (10, 5848) + + +def test_strands_before_1_54_would_otherwise_look_like_a_perfect_cache(): + n = normalize_usage({"input": 0, "output": 4, "total": 5862, "input_cached_tokens": 5848, "input_cache_creation": 0}) + assert n.usage.hit_ratio == pytest.approx(5848 / 5858) # not 100% + + +def test_litellm_langfuse_callback_write_counted_twice_is_fixed(): + # prompt_tokens = 10 + 0 + 5848; the callback sends input = prompt - cache read, so the write stays inside. + n = normalize_usage({"input": 5858, "output": 4, "total": 5862, + "cache_creation_input_tokens": 5848, "cache_read_input_tokens": 0}) + assert n.status == "corrected" + assert (n.usage.uncached_input, n.usage.cache_write, n.usage.total_input) == (10, 5848, 5858) + + +def test_litellm_langfuse_callback_hit_already_adds_up(): + n = normalize_usage({"input": 10, "output": 4, "total": 5862, + "cache_creation_input_tokens": 0, "cache_read_input_tokens": 5848}) + assert n.status == "ok" and n.usage.uncached_input == 10 + + +def test_langchain_bedrock_converse_shape(): + # usage_metadata input_tokens includes the cache; the handler subtracts input_token_details. + n = normalize_usage({"input": 10, "output": 4, "total": 5862, "input_cache_read": 5848, "input_cache_creation": 0}) + assert n.status == "ok" + assert (n.usage.uncached_input, n.usage.cache_read) == (10, 5848) + + +def test_langchain_anthropic_ttl_parts_are_not_added_to_the_aggregate(): + details = with_langfuse_total({"input": 10, "output": 4, "cache_read_input_tokens": 5000, + "cache_creation_input_tokens": 848, + "cache_creation_ephemeral_5m_input_tokens": 0, + "cache_creation_ephemeral_1h_input_tokens": 848}) + n = normalize_usage(details) + assert n.status == "ok" + assert (n.usage.uncached_input, n.usage.cache_write, n.usage.cache_write_1h) == (10, 848, 848) + + +def test_langchain_anthropic_ttl_parts_without_aggregate(): + n = normalize_usage(with_langfuse_total({"input": 10, "output": 4, + "cache_creation_ephemeral_5m_input_tokens": 300, + "cache_creation_ephemeral_1h_input_tokens": 200})) + assert (n.usage.cache_write, n.usage.cache_write_1h) == (500, 200) + + +def test_ai_sdk_anthropic_ttl_split_adds_up(): + # Langfuse leaves input_cache_creation = aggregate - 5m - 1h. + n = normalize_usage({"input": 7, "output": 50, "total": 18545, "input_cached_tokens": 16399, + "input_cache_creation": 0, "input_cache_creation_5m": 2089}) + assert n.status == "ok" + assert (n.usage.uncached_input, n.usage.cache_read, n.usage.cache_write, n.usage.cache_write_1h) == (7, 16399, 2089, 0) + + +def test_ai_sdk_bedrock_metadata_names(): + n = normalize_usage({"input": 10, "output": 4, "total": 4496, "input_cache_read": 4482, + "input_cache_write": 0, "input_cache_creation": 0}) + assert n.status == "ok" and n.usage.cache_read == 4482 + + +def test_ai_sdk_bedrock_write_under_two_names_counts_once(): + n = normalize_usage(with_langfuse_total({"input": 10, "output": 4, "input_cache_write": 700, + "input_cache_creation": 700})) + assert n.usage.cache_write == 700 + + +def test_hand_written_sdk_call_without_total(): + # e.g. usage_details={"input": ..., "output": ..., "cache_read_input_tokens": ...}; Langfuse sums a total. + n = normalize_usage(with_langfuse_total({"input": 10, "output": 4, "cache_read_input_tokens": 5848})) + assert n.status == "ok" + assert (n.usage.uncached_input, n.usage.cache_read) == (10, 5848) + + +def test_openai_style_subset_shape(): + # prompt_tokens 1000 incl. 900 cached; Langfuse stores input 100 + input_cached_tokens 900. + n = normalize_usage({"input": 100, "input_cached_tokens": 900, "output": 10, "total": 1010}) + assert n.status == "ok" and n.usage.hit_ratio == pytest.approx(0.9) + + +def test_reasoning_output_detail_is_part_of_output(): + # OTel keeps total raw and splits reasoning out of output. + n = normalize_usage({"input": 0, "output": 4, "output_reasoning_tokens": 20, "total": 5882, + "input_cached_tokens": 5848, "input_cache_creation": 0}) + assert n.status == "corrected" + assert (n.usage.uncached_input, n.usage.output) == (10, 24) + + +def test_other_input_categories_count_as_uncached(): + n = normalize_usage(with_langfuse_total({"input": 90, "input_audio": 10, "output": 4})) + assert n.usage.uncached_input == 100 + + +def test_priority_and_modality_keys_are_not_double_counted(): + n = normalize_usage(with_langfuse_total({"input": 100, "input_priority": 100, "input_modality_text": 100, + "output": 4})) + assert n.usage.uncached_input == 100 + + +def test_two_read_names_with_same_value_count_once(): + n = normalize_usage(with_langfuse_total({"input": 10, "output": 4, "input_cached_tokens": 500, + "cache_read_input_tokens": 500})) + assert n.usage.cache_read == 500 + + +def test_total_smaller_than_the_parts_keeps_sent_counts(): + n = normalize_usage({"input": 10, "output": 4, "total": 100, "input_cached_tokens": 5848}) + assert n.status == "total-mismatch" + assert (n.usage.uncached_input, n.usage.cache_read) == (10, 5848) + + +def test_no_cache_fields_is_flagged(): + n = normalize_usage({"input": 100, "output": 5, "total": 105}) + assert n.status == "ok" and not n.has_cache_fields + + +def test_zero_cache_fields_still_count_as_reported(): + n = normalize_usage({"input": 100, "output": 5, "total": 105, "input_cached_tokens": 0, "input_cache_creation": 0}) + assert n.has_cache_fields + + +@pytest.mark.parametrize("details", [None, {}, [], "x", {"total": 0}, {"input": 0, "output": 0, "total": 0}, + {"total": 50}, {"input": -5}, {"input": True}, {"input": "abc"}, + {"input": float("nan")}, {"input": float("inf")}, {"input": None}]) +def test_missing_or_unusable_usage(details): + assert normalize_usage(details).status == "no-usage" + + +def test_numeric_strings_and_floats_are_read(): + n = normalize_usage({"input": "10", "output": 4.0, "input_cached_tokens": "5848", "total": "5862"}) + assert (n.usage.uncached_input, n.usage.cache_read, n.usage.output) == (10, 5848, 4) + + +def test_huge_counts(): + big = 10 ** 12 + n = normalize_usage({"input": big, "output": 1, "input_cached_tokens": big * 9, "total": big * 10 + 1}) + assert n.status == "ok" and n.usage.hit_ratio == pytest.approx(0.9) + + +def test_non_string_keys_are_ignored(): + n = normalize_usage({1: 500, "input": 10, "output": 4}) + assert n.usage.uncached_input == 10 + + +# ---------------------------------------------------------------- aggregation + +def gen(model=SONNET, name="agent", env="production", type_="GENERATION", **usage): + return {"id": "x", "type": type_, "model": model, "name": name, "environment": env, + "usageDetails": usage or {"input": 10, "output": 4, "input_cached_tokens": 5848, "total": 5862}} + + +def test_only_generations_are_counted(): + obs = [gen(), gen(type_="SPAN"), gen(type_="AGENT"), gen(type_="generation")] + groups, q = langfuse.aggregate(obs) + assert groups[SONNET].calls == 2 and q.skipped_other_types == 2 and q.generations == 2 + + +def test_missing_type_counts_as_generation_and_non_dicts_are_skipped(): + obs = [{"model": SONNET, "usageDetails": {"input": 5, "output": 1}}, "junk", None, 3] + groups, q = langfuse.aggregate(obs) + assert groups[SONNET].calls == 1 and q.generations == 1 + + +def test_grouping_and_quality_counts(): + obs = [gen(), gen(input=0, output=4, total=5862, input_cached_tokens=5848), + gen(input=100, output=5), {"type": "GENERATION", "model": SONNET, "usageDetails": {}}, + gen(model="arn:aws:bedrock:us-east-1:1:inference-profile/us.anthropic.claude-sonnet-4-6")] + groups, q = langfuse.aggregate(obs) + g = groups[SONNET] + assert g.calls == 5 and g.unparsed == 1 + assert (q.corrected, q.no_cache_fields, q.no_usage) == (1, 1, 1) + assert set(g.by_model) == {SONNET} + + +def test_group_by_name_and_environment(): + obs = [gen(name="a", env="production"), gen(name="b", env="staging"), gen(name=None, env=None)] + by_name, _ = langfuse.aggregate(obs, by="name") + by_env, _ = langfuse.aggregate(obs, by="environment") + assert set(by_name) == {"a", "b", "(no name)"} + assert set(by_env) == {"production", "staging", "default"} + + +def test_missing_model_is_its_own_group(): + groups, _ = langfuse.aggregate([gen(model=None)]) + assert "(no model)" in groups + + +# ---------------------------------------------------------------- saved API responses + +def test_file_shapes(tmp_path): + one = gen() + (tmp_path / "resp.json").write_text(json.dumps({"data": [one, one], "meta": {}})) + (tmp_path / "list.json").write_text(json.dumps([one])) + (tmp_path / "lines.jsonl").write_text(json.dumps({"data": [one]}) + "\n\n" + json.dumps(one) + "\n") + (tmp_path / "empty.json").write_text(" ") + assert len(list(langfuse.iter_file(tmp_path / "resp.json"))) == 2 + assert len(list(langfuse.iter_file(tmp_path / "list.json"))) == 1 + assert len(list(langfuse.iter_file(tmp_path / "lines.jsonl"))) == 2 + assert list(langfuse.iter_file(tmp_path / "empty.json")) == [] + + +@pytest.mark.parametrize("content,match", [("{bad\n{worse", "line 1 is not valid JSON"), ("42", "does not hold")]) +def test_file_errors(tmp_path, content, match): + p = tmp_path / "x.json" + p.write_text(content) + with pytest.raises(langfuse.LangfuseError, match=match): + list(langfuse.iter_file(p)) + + +def test_file_missing(tmp_path): + with pytest.raises(langfuse.LangfuseError, match="file not found"): + list(langfuse.iter_file(tmp_path / "nope.json")) + + +# ---------------------------------------------------------------- API client with a fake transport + +class FakeResponse(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + +def http_error(url, code, body=b"{}", headers=None): + return urllib.error.HTTPError(url, code, "err", headers or {}, io.BytesIO(body)) + + +class Transport: + """Plays back a list of (status, body[, headers]) per call and records each request.""" + + def __init__(self, replies): + self.replies, self.requests = list(replies), [] + + def __call__(self, request, timeout): + self.requests.append(request) + status, body, *headers = self.replies.pop(0) + raw = body if isinstance(body, bytes) else json.dumps(body).encode() + if status != 200: + raise http_error(request.full_url, status, raw, headers[0] if headers else None) + return FakeResponse(raw) + + def params(self, i): + url = urlparse(self.requests[i].full_url) + return url.path, parse_qs(url.query) + + +def client(transport, sleeps=None): + return langfuse.Client("https://us.cloud.langfuse.com/", "pk-lf-1", "sk-lf-2", opener=transport, + sleep=(sleeps.append if sleeps is not None else lambda s: None)) + + +def page(n, cursor=None): + return {"data": [gen() for _ in range(n)], "meta": {"cursor": cursor} if cursor else {}} + + +def test_v2_request_shape_and_auth(): + t = Transport([(200, page(1))]) + res = client(t).fetch_generations("2026-10-13T00:00:00.000Z", None, ["production", "staging"], 100) + path, q = t.params(0) + assert path == "/api/public/v2/observations" and res.api == "v2" + assert q["type"] == ["GENERATION"] and q["fields"] == ["core,basic,model,usage"] and q["limit"] == ["1000"] + assert q["fromStartTime"] == ["2026-10-13T00:00:00.000Z"] and "toStartTime" not in q + assert q["environment"] == ["production", "staging"] + auth = t.requests[0].get_header("Authorization") + assert base64.b64decode(auth.split()[1]).decode() == "pk-lf-1:sk-lf-2" + assert "io" not in q["fields"][0] and "metadata" not in q["fields"][0] # never asks for prompts or outputs + + +def test_v2_follows_cursors_to_the_end(): + t = Transport([(200, page(2, "c1")), (200, page(2, "c2")), (200, page(1))]) + res = client(t).fetch_generations("t", None, None, 100) + assert len(res.observations) == 5 and not res.capped + assert t.params(1)[1]["cursor"] == ["c1"] and t.params(2)[1]["cursor"] == ["c2"] + + +def test_v2_repeated_cursor_stops(): + t = Transport([(200, page(1, "same")), (200, page(1, "same"))]) + with pytest.raises(langfuse.LangfuseError, match="same page cursor twice"): + client(t).fetch_generations("t", None, None, 100) + + +def test_v2_empty_page_with_cursor_stops(): + t = Transport([(200, {"data": [], "meta": {"cursor": "c"}})]) + assert client(t).fetch_generations("t", None, None, 100).observations == [] + + +def test_falls_back_to_v1_when_v2_is_missing(): + v2_404 = (404, {"message": "The observations v2 API is only available in a Langfuse v4 write mode."}) + t = Transport([v2_404, (200, {"data": [gen()] * 100, "meta": {"page": 1, "totalPages": 2}}), + (200, {"data": [gen()] * 30, "meta": {"page": 2, "totalPages": 2}})]) + res = client(t).fetch_generations("t", "u", None, 1000) + assert res.api == "v1" and len(res.observations) == 130 + path, q = t.params(1) + assert path == "/api/public/observations" and q["page"] == ["1"] and q["limit"] == ["100"] + assert q["toStartTime"] == ["u"] and "fields" not in q + assert t.params(2)[1]["page"] == ["2"] + + +def test_neither_api_available(): + t = Transport([(404, {"message": "v2 off"}), (404, b"not found")]) + with pytest.raises(langfuse.LangfuseError, match="neither observations API.*v2 off"): + client(t).fetch_generations("t", None, None, 10) + + +def test_max_calls_caps_mid_page(): + t = Transport([(200, page(5, "c1"))]) + res = client(t).fetch_generations("t", None, None, 3) + assert len(res.observations) == 3 and res.capped and len(t.requests) == 1 + + +def test_max_calls_exactly_at_last_page_is_not_capped(): + t = Transport([(200, page(3))]) + res = client(t).fetch_generations("t", None, None, 3) + assert len(res.observations) == 3 and not res.capped + + +def test_max_calls_at_page_boundary_with_more_pages_is_capped(): + t = Transport([(200, page(3, "c1"))]) + res = client(t).fetch_generations("t", None, None, 3) + assert res.capped and len(t.requests) == 1 + + +def test_rate_limit_waits_for_retry_after(): + sleeps = [] + t = Transport([(429, {}, {"Retry-After": "7"}), (429, {}, {}), (200, page(1))]) + res = client(t, sleeps).fetch_generations("t", None, None, 10) + assert len(res.observations) == 1 and sleeps == [7.0, 5.0] + + +def test_rate_limit_wait_is_capped_and_gives_up(): + sleeps = [] + t = Transport([(429, {}, {"Retry-After": "9999"})] * 7) + with pytest.raises(langfuse.LangfuseError, match="HTTP 429"): + client(t, sleeps).fetch_generations("t", None, None, 10) + assert sleeps == [60.0] * 6 + + +def test_server_errors_retry_then_succeed(): + sleeps = [] + t = Transport([(502, b"bad gateway"), (500, {}), (200, page(1))]) + assert len(client(t, sleeps).fetch_generations("t", None, None, 10).observations) == 1 + assert sleeps == [2, 4] + + +def test_server_errors_give_up_with_message(): + t = Transport([(500, {"message": "boom"})] * 4) + with pytest.raises(langfuse.LangfuseError, match="HTTP 500.*boom"): + client(t).fetch_generations("t", None, None, 10) + + +@pytest.mark.parametrize("code", [401, 403]) +def test_bad_keys(code): + t = Transport([(code, {"message": "Invalid credentials"})]) + with pytest.raises(langfuse.LangfuseError, match=f"rejected the API keys \\(HTTP {code}\\).*us.cloud.langfuse.com"): + client(t).fetch_generations("t", None, None, 10) + + +def test_other_client_error_is_reported(): + t = Transport([(400, {"message": "Invalid request data"})]) + with pytest.raises(langfuse.LangfuseError, match="HTTP 400.*Invalid request data"): + client(t).fetch_generations("t", None, None, 10) + + +def test_network_errors_retry_then_report(): + calls = [] + + def down(request, timeout): + calls.append(1) + raise urllib.error.URLError("nodename nor servname provided") + + with pytest.raises(langfuse.LangfuseError, match="could not reach Langfuse.*nodename"): + langfuse.Client("https://lf.example.com", "a", "b", opener=down, sleep=lambda s: None).get("/x", {}) + assert len(calls) == 4 + + +def test_non_json_success_body(): + t = Transport([(200, b"login")]) + with pytest.raises(langfuse.LangfuseError, match="not JSON"): + client(t).fetch_generations("t", None, None, 10) + + +def test_response_without_data_list(): + t = Transport([(200, {"items": []})]) + with pytest.raises(langfuse.LangfuseError, match="no 'data' list"): + client(t).fetch_generations("t", None, None, 10) + + +def test_failure_part_way_through_pages(): + t = Transport([(200, page(1, "c1")), (404, {})]) + with pytest.raises(langfuse.LangfuseError, match="part-way"): + client(t).fetch_generations("t", None, None, 10) + + +@pytest.mark.parametrize("host", ["cloud.langfuse.com", "ftp://x", "", "https://"]) +def test_host_validation(host): + if host == "": + assert langfuse.Client(host, "a", "b").host == langfuse.DEFAULT_HOST + return + with pytest.raises(langfuse.LangfuseError, match="must start with https://"): + langfuse.Client(host, "a", "b") + + +@pytest.mark.parametrize("pk,sk", [("", "b"), ("a", ""), ("", "")]) +def test_missing_keys(pk, sk): + with pytest.raises(langfuse.LangfuseError, match="LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY"): + langfuse.Client("https://x.io", pk, sk) + + +# ---------------------------------------------------------------- time window parsing + +@pytest.mark.parametrize("text,hours", [("30m", 0.5), ("24h", 24), ("7d", 168), ("2w", 336), (" 3D ", 72)]) +def test_parse_since(text, hours): + assert langfuse.parse_since(text).total_seconds() == hours * 3600 + + +@pytest.mark.parametrize("text", ["0d", "7", "d", "-1d", "1y", "1.5d", ""]) +def test_parse_since_rejects(text): + with pytest.raises(ValueError): + langfuse.parse_since(text) + + +def test_parse_time_and_iso(): + assert langfuse.iso(langfuse.parse_time("2026-10-01")) == "2026-10-01T00:00:00.000Z" + assert langfuse.iso(langfuse.parse_time("2026-10-01T09:30:00Z")) == "2026-10-01T09:30:00.000Z" + assert langfuse.iso(langfuse.parse_time("2026-10-01T14:30:00+05:00")) == "2026-10-01T09:30:00.000Z" + with pytest.raises(ValueError): + langfuse.parse_time("yesterday") + + +# ---------------------------------------------------------------- CLI + +@pytest.fixture +def saved(tmp_path): + obs = ([gen(input=10, output=4, input_cached_tokens=5848, total=5862)] * 8 + + [gen(input=0, output=4, input_cache_creation=5848, input_cached_tokens=0, total=5862)] * 2 + + [gen(model="gpt-5", input=100, output=5)] + [gen(type_="AGENT")] + + [{"type": "GENERATION", "model": SONNET, "usageDetails": {}}]) + p = tmp_path / "obs.json" + p.write_text(json.dumps({"data": obs, "meta": {}})) + return p + + +def test_cli_file_report(saved, capsys): + assert cli.main(["langfuse", "--file", str(saved)]) == 0 + out = capsys.readouterr().out + assert f"{SONNET}: hit 80% over 11 calls (read 46784, write 11696, uncached 100, no usage 1)" in out + assert "input cost" in out and "lost to cache misses" in out + assert "gpt-5: hit 0% over 1 calls" in out and "no price for this model" in out + assert "Read 12 generations." in out + assert "2 generation(s): Langfuse's input count did not add up" in out + assert "1 generation(s) have no cache fields at all" in out + assert "1 generation(s) have no token counts." in out + assert "Ignored 1 non-generation observation(s)" in out + + +def test_cli_min_hit_exit_code(saved, capsys): + assert cli.main(["langfuse", "--file", str(saved), "--min-hit", "0.85"]) == 1 + assert "below threshold" in capsys.readouterr().out + + +def test_cli_json(saved, capsys): + assert cli.main(["--json", "langfuse", "--file", str(saved), "--by", "name"]) == 0 + data = json.loads(capsys.readouterr().out) + assert data["groups"]["agent"]["calls"] == 11 and data["groups"]["(no name)"]["calls"] == 1 and any("did not add up" in n for n in data["notes"]) + + +def test_cli_github_summary(saved, tmp_path, monkeypatch, capsys): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + assert cli.main(["--github", "langfuse", "--file", str(saved), "--min-hit", "0.85"]) == 1 + text = summary.read_text() + assert "### CacheCanary langfuse" in text and "| no usage |" in text and "did not add up" in text + assert "::error" in capsys.readouterr().out + + +def test_cli_empty(tmp_path, capsys): + p = tmp_path / "e.json" + p.write_text(json.dumps({"data": []})) + assert cli.main(["langfuse", "--file", str(p)]) == 0 + assert "No generations found" in capsys.readouterr().out + + +def test_cli_bad_file_exit_2(tmp_path, capsys): + assert cli.main(["langfuse", "--file", str(tmp_path / "nope.json")]) == 2 + assert "file not found" in capsys.readouterr().err + + +@pytest.mark.parametrize("argv", [["--since", "7"], ["--from", "soon"], ["--max-calls", "0"], + ["--since", "1d", "--from", "2026-10-01"], ["--by", "trace"]]) +def test_cli_rejects_bad_arguments(argv): + with pytest.raises(SystemExit) as exc: + cli.main(["langfuse", *argv]) + assert exc.value.code == 2 + + +def test_cli_from_after_to(monkeypatch, capsys): + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "a") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "b") + assert cli.main(["langfuse", "--from", "2026-10-10", "--to", "2026-10-01"]) == 2 + assert "--from must be earlier" in capsys.readouterr().err + + +def test_cli_without_keys(monkeypatch, capsys): + for var in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY"): + monkeypatch.delenv(var, raising=False) + assert cli.main(["langfuse"]) == 2 + assert "LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY" in capsys.readouterr().err + + +# ---------------------------------------------------------------- end to end against a local HTTP server + +class FakeLangfuse(BaseHTTPRequestHandler): + v2 = True + hits: list = [] + + def log_message(self, *a): + pass + + def do_GET(self): + url = urlparse(self.path) + q = parse_qs(url.query) + FakeLangfuse.hits.append((url.path, q, self.headers.get("Authorization"))) + if self.headers.get("Authorization") != "Basic " + base64.b64encode(b"pk:sk").decode(): + return self._send(401, {"message": "Invalid credentials"}) + if url.path == "/api/public/v2/observations": + if not FakeLangfuse.v2: + return self._send(404, {"message": "only available in a Langfuse v4 write mode"}) + if len(FakeLangfuse.hits) == 1: + return self._send(429, {"message": "rate limited"}, {"Retry-After": "0"}) + if "cursor" not in q: + return self._send(200, {"data": [gen()] * 3, "meta": {"cursor": "p2"}}) + return self._send(200, {"data": [gen(input=0, output=4, total=5862, input_cached_tokens=5848)], "meta": {}}) + if url.path == "/api/public/observations": + pg = int(q["page"][0]) + return self._send(200, {"data": [gen()] * (100 if pg == 1 else 1), + "meta": {"page": pg, "limit": 100, "totalItems": 101, "totalPages": 2}}) + self._send(404, {"message": "no route"}) + + def _send(self, code, body, headers=None): + raw = json.dumps(body).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + for k, v in (headers or {}).items(): + self.send_header(k, v) + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + +@pytest.fixture +def server(monkeypatch): + FakeLangfuse.hits, FakeLangfuse.v2 = [], True + httpd = HTTPServer(("127.0.0.1", 0), FakeLangfuse) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + monkeypatch.setattr(langfuse.time, "sleep", lambda s: None) + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk") + monkeypatch.delenv("LANGFUSE_HOST", raising=False) + monkeypatch.setenv("LANGFUSE_BASE_URL", f"http://127.0.0.1:{httpd.server_port}") + yield httpd + httpd.shutdown() + + +def test_e2e_v2_with_rate_limit_and_cursor(server, capsys): + assert cli.main(["langfuse", "--since", "24h", "--environment", "production"]) == 0 + out = capsys.readouterr().out + assert f"{SONNET}: hit" in out and "over 4 calls" in out + assert "(Langfuse API v2)" in out and "1 generation(s): Langfuse's input count did not add up" in out + paths = [h[0] for h in FakeLangfuse.hits] + assert paths == ["/api/public/v2/observations"] * 3 + assert FakeLangfuse.hits[-1][1]["environment"] == ["production"] + + +def test_e2e_v1_fallback(server, capsys): + FakeLangfuse.v2 = False + assert cli.main(["langfuse", "--from", "2026-10-01", "--to", "2026-10-08"]) == 0 + out = capsys.readouterr().out + assert "over 101 calls" in out and "(Langfuse API v1)" in out + assert "from 2026-10-01T00:00:00.000Z to 2026-10-08T00:00:00.000Z" in out + + +def test_e2e_host_flag_beats_env_and_bad_keys(server, monkeypatch, capsys): + monkeypatch.setenv("LANGFUSE_HOST", "https://wrong.example.com") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "wrong") + assert cli.main(["langfuse", "--host", f"http://127.0.0.1:{server.server_port}"]) == 2 + assert "rejected the API keys (HTTP 401)" in capsys.readouterr().err + + +def test_e2e_langfuse_host_beats_base_url(server, monkeypatch, capsys): + monkeypatch.setenv("LANGFUSE_HOST", f"http://127.0.0.1:{server.server_port}") + monkeypatch.setenv("LANGFUSE_BASE_URL", "https://wrong.example.com") + assert cli.main(["langfuse"]) == 0 + + +# ---------------------------------------------------------------- records captured from a real Langfuse 4.64.0 +# Each pair was produced by the real library calling a local stand-in for Bedrock Converse that returned the +# strands#3546 usage (call 1 writes 5848 tokens, call 2 reads them), traced into a local Langfuse 4.64.0 +# (docker compose, Oct 20 2026) and read back through /api/public/v2/observations. The right answer for +# every pair is read 5848, write 5848, uncached 20, hit 50%. + +REAL_RECORDS = { + "strands 1.61.1": [ + {"cache_write_input_tokens": 5848, "input": 10, "input_cache_creation": 5848, "input_cached_tokens": 0, "output": 4, "total": 5862}, + {"cache_write_input_tokens": 0, "input": 10, "input_cache_creation": 0, "input_cached_tokens": 5848, "output": 4, "total": 5862}, + ], + # Before 1.54 the write is only in the key Langfuse does not recognise, and the read call's input is 0. + "strands 1.53.0": [ + {"cache_write_input_tokens": 5848, "input": 10, "input_cached_tokens": 0, "output": 4, "total": 5862}, + {"cache_write_input_tokens": 0, "input": 0, "input_cached_tokens": 5848, "output": 4, "total": 5862}, + ], + # LiteLLM 1.106.0 `langfuse` callback (langfuse SDK 4.18.0): the write is also left inside input. + "litellm langfuse callback": [ + {"cache_creation_input_tokens": 5848, "cache_read_input_tokens": 0, "input": 5858, "output": 4, "total": 5862}, + {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 5848, "input": 10, "output": 4, "total": 5862}, + ], + "litellm langfuse_otel": [ + {"input": 10, "input_cache_creation": 5848, "output": 4, "total": 5862}, + {"input": 10, "input_cached_tokens": 5848, "output": 4, "total": 5862}, + ], + # langchain-aws 1.9.0 ChatBedrockConverse + langfuse 5.0.3 CallbackHandler + "langchain bedrock converse": [ + {"input": 10, "input_cache_creation": 5848, "input_cache_read": 0, "output": 4, "total": 5862}, + {"input": 10, "input_cache_creation": 0, "input_cache_read": 5848, "output": 4, "total": 5862}, + ], +} + + +@pytest.mark.parametrize("source", sorted(REAL_RECORDS)) +def test_real_records_give_the_true_hit_rate(source): + groups, _ = langfuse.aggregate([{"type": "GENERATION", "model": SONNET, "usageDetails": u} + for u in REAL_RECORDS[source]]) + u = groups[SONNET].usage + assert (u.cache_read, u.cache_write, u.uncached_input, u.output) == (5848, 5848, 20, 8) + assert u.hit_ratio == pytest.approx(0.5, abs=0.001) + + +def test_real_strands_1_53_without_the_passthrough_key_would_show_a_perfect_cache(): + # The bug the live run caught: ignoring cache_write_input_tokens drops the write and shows 100%. + write_call = dict(REAL_RECORDS["strands 1.53.0"][0]) + assert normalize_usage(write_call).usage.cache_write == 5848 + + +@pytest.mark.parametrize("raw,expected", [ + ("converse/us.anthropic.claude-sonnet-4-6", "us.anthropic.claude-sonnet-4-6"), # real LiteLLM callback record + ("bedrock/converse/global.anthropic.claude-sonnet-5-5", "global.anthropic.claude-sonnet-5-5"), + ("bedrock/invoke/us.anthropic.claude-haiku-5-5", "us.anthropic.claude-haiku-5-5"), + ("bedrock/us-west-2/anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic.claude-haiku-4-5-20251001-v1:0"), + ("bedrock/us-gov-west-1/anthropic.claude-sonnet-4-6", "anthropic.claude-sonnet-4-6"), + ("invoke/arn:aws:bedrock:us-east-1:111:inference-profile/global.anthropic.claude-opus-5-5", "global.anthropic.claude-opus-5-5"), + ("arn:aws:bedrock:us-east-2:111:inference-profile/global.anthropic.claude-sonnet-5-5", "global.anthropic.claude-sonnet-5-5"), + ("arn:aws:bedrock:us-east-1:111:application-inference-profile/abc", "arn:aws:bedrock:us-east-1:111:application-inference-profile/abc"), + ("bedrock/", "bedrock/"), ("gpt-5", "gpt-5"), (" us.anthropic.claude-sonnet-4-6 ", "us.anthropic.claude-sonnet-4-6"), + (None, None), ("", None), (" ", None), (42, None), +]) +def test_model_id(raw, expected): + assert langfuse.model_id(raw) == expected + + +def test_litellm_prefix_merges_groups_and_keeps_global_price(): + from cachecanary.pricing import rates_for + obs = [gen(model="converse/global.anthropic.claude-sonnet-5-5"), gen(model="global.anthropic.claude-sonnet-5-5")] + groups, _ = langfuse.aggregate(obs) + assert list(groups) == ["global.anthropic.claude-sonnet-5-5"] and groups["global.anthropic.claude-sonnet-5-5"].calls == 2 + assert rates_for(next(iter(groups["global.anthropic.claude-sonnet-5-5"].by_model))).rates.input == 2 # global, no +10% + + +def test_real_fixture_end_to_end(capsys): + assert cli.main(["langfuse", "--file", "tests/fixtures/langfuse_observations_2026-10.json"]) == 0 + out = capsys.readouterr().out + assert "us.anthropic.claude-sonnet-4-6: hit 50% over 10 calls (read 29240, write 29240, uncached 100, no usage 0)" in out + assert "2 generation(s): Langfuse's input count did not add up" in out