From b2057ee781f8dde4d79d77275590b3a241cef79b Mon Sep 17 00:00:00 2001 From: tjdonley <51825276+tjdonley@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:30:21 -0400 Subject: [PATCH 1/4] Estimate cost from tokens and rank report findings by waste Two gaps that both ended with reports users could not act on. Estimate span cost from token counts ------------------------------------ CostConfidence already declared an "estimated" tier that nothing produced. Normalization only read totalCost/costDetails.total, so an export carrying token counts and model names but no provider cost produced no cost at all, and every analyzer gates its waste math on cost_usd being present. Those traces reported $0.000000000 of waste no matter what they actually spent. Spans that arrive without a provider cost are now priced from their token counts and model name during normalization and stored with cost_confidence = estimated. Spans that already carry a provider cost are never touched, so estimation only adds attribution where there was none. Rates come from a `pricing` config block layered over a bundled default table. Model matching is exact first, then longest version-boundary prefix, so gpt-4o-mini prices gpt-4o-mini-2024-07-18 while gpt-4 never prices a gpt-4o span. Configured entries beat bundled defaults. Estimated spend is labelled everywhere rather than blended into provider cost: normalize reports estimated coverage and names models it had tokens for but no price, the ledger splits provider-reported from estimated cost, and reports carry a per-row confidence column plus a costs_by_confidence_usd summary field. `agentprof pricing list` shows the effective table. Rank findings by attributed waste --------------------------------- fetch_issues ordered by issue_id, which is "{kind}:{sha256[:16]}", so reports listed issues alphabetically by kind and then by hash digest. The demo report led with the multi-agent finding only because m sorts before r and s; on real data a $12 retry loop would sort below a $0.001 estimate. Issues are now ordered by attributed waste, then severity, then confidence, then issue_id for stability. The severity and confidence tiebreakers are what rank findings when a trace carries no cost data at all. Severity was also close to constant - multi-agent waste hardcoded "medium". It is now derived from each finding's own magnitude: retry-loop severity escalates with repeated wasted attempts, multi-agent severity tracks the cost multiple over baseline. Spec-violation severity already keyed off whether the span failed, so it is unchanged. Markdown and HTML reports now open with the Top Findings section the pop spec asks for, covering the three leading issues with their recommendation and one evidence line. The JSON payload exposes the same list as top_findings alongside summary.top_issue_kind. Demo totals are unchanged: the bundled fixture carries provider costs, so estimation does not touch it. Checked-in example reports are regenerated for the new section and ledger column. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Yyb1oKkpxm76dz7xGSjLz --- CHANGELOG.md | 6 + README.md | 49 ++- agentprof/analyze/multi_agent_waste.py | 14 +- agentprof/analyze/retry_loop.py | 14 +- agentprof/cli.py | 73 +++- agentprof/config.py | 36 +- agentprof/cost/pricing.py | 249 ++++++++++++++ agentprof/cost/runner.py | 6 + agentprof/cost/schema.py | 3 + agentprof/normalize/runner.py | 28 +- agentprof/normalize/schema.py | 7 + agentprof/report/runner.py | 167 ++++++++- agentprof/report/schema.py | 1 + agentprof/store/duckdb_store.py | 23 +- examples/reports/demo.html | 90 ++++- examples/reports/demo.json | 62 +++- examples/reports/demo.md | 36 +- .../langfuse_token_only_observations.json | 72 ++++ tests/test_multi_agent_waste.py | 17 + tests/test_pricing.py | 324 ++++++++++++++++++ tests/test_report.py | 209 +++++++++++ tests/test_retry_loop.py | 42 ++- 22 files changed, 1497 insertions(+), 31 deletions(-) create mode 100644 agentprof/cost/pricing.py create mode 100644 tests/fixtures/langfuse_token_only_observations.json create mode 100644 tests/test_pricing.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 49e2bad..dbfc6dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to AgentProf will be documented here. ### Added +- Token-based cost estimation for spans that arrive without provider cost fields, with a configurable `pricing` model rate table and a bundled default table. +- `agentprof pricing list` to show the effective model rates and where each came from. +- Estimated-versus-provider cost reporting in `agentprof normalize`, `agentprof cost ledger`, and report ledger tables. +- `Top Findings` section in Markdown and HTML reports, plus `top_findings` and `summary.top_issue_kind` in the JSON payload. - One-command `agentprof demo` workflow with bundled sample traces. - Static HTML report generation alongside Markdown and JSON reports. - Multi-agent waste SVG visual artifacts. @@ -14,5 +18,7 @@ All notable changes to AgentProf will be documented here. ### Changed +- Report issues are ranked by attributed waste, then severity and confidence, instead of by issue ID. +- Retry-loop severity now escalates with repeated wasted attempts, and multi-agent severity tracks the cost multiple over baseline, instead of being fixed. - README now leads with concrete trace-waste findings and the demo workflow. - CI now tests Python 3.11 and 3.12, runs demo smoke checks, builds the package, and validates wheel installation. diff --git a/README.md b/README.md index 9ce3e37..f2b9bae 100644 --- a/README.md +++ b/README.md @@ -118,12 +118,14 @@ AgentProf complements tracing tools by profiling exported traces locally and tur - Store raw and normalized trace data in a local DuckDB database. - Normalize Langfuse observations into canonical span and trace tables. - Show data-quality coverage for parent links, status, costs, tokens, models, and I/O hashes. +- Estimate span cost from token counts and model names when an export carries no provider cost fields. - Build an idempotent cost ledger from normalized span costs. - Print a status-based cost waterfall for successful, failed, and unknown span costs. - Detect retry loops where the same failing call repeats with the same input fingerprint and error signature. - Detect configured tool/spec contract violations from normalized redacted previews and error messages. - Estimate multi-agent orchestration overhead against configured or observed single-agent baselines. - Generate local Markdown, JSON, and static HTML reports from persisted issues, evidence, costs, and optional visuals. +- Rank report issues by attributed waste and lead with a top-findings section. - List and show generated reports from the local store. ## Current Status @@ -162,7 +164,7 @@ uv run agentprof report list uv run agentprof store stats ``` -The default fixture focuses on retry/spec behavior and does not include cost fields, so `agentprof cost ledger` will produce zero ledger entries for that sample. Use `tests/fixtures/langfuse_multi_agent_observations.json` for a costed multi-agent waste demo. +The default fixture focuses on retry/spec behavior and carries neither cost fields nor token counts, so `agentprof cost ledger` will produce zero ledger entries for that sample. Use `tests/fixtures/langfuse_multi_agent_observations.json` for a costed multi-agent waste demo, or `tests/fixtures/langfuse_token_only_observations.json` for an export that is priced from token counts alone. The workflow creates local files only. After `agentprof report generate`, the default report artifacts are written under `.agentprof/reports/` and can be inspected with `agentprof report show`. @@ -330,9 +332,47 @@ Observed mode matches costed successful single-agent traces by normalized root t | `agentprof report generate` | Generate Markdown, JSON, and HTML reports from persisted analysis results. | | `agentprof report list` | List generated reports recorded in the local store. | | `agentprof report show REPORT_ID` | Print a generated report's Markdown, JSON, or HTML artifact. | +| `agentprof pricing list` | Show the model rates used to estimate cost for uncosted spans. | | `agentprof store stats` | Show row counts for all store tables. | | `agentprof store reset --yes` | Delete and recreate the local DuckDB store. | +## Cost Estimation + +Many trace exports carry token counts and model names but no provider cost fields. Without a price, every analyzer reports $0.000000000, because waste attribution is built from span cost. + +AgentProf fills that gap during `agentprof normalize`: a span that has a model name and token counts but no provider cost is priced from a model rate table and stored with `cost_confidence = estimated`. Spans that already carry a provider cost are never overwritten, so estimation can only add attribution where there was none. + +Inspect the effective rates: + +```bash +uv run agentprof pricing list +``` + +Override any model, or add one AgentProf does not know, in `agentprof.yml`: + +```yaml +pricing: + enabled: true + use_default_table: true + models: + - model: house-fine-tune-v3 + input_per_1m_usd: '0.90' + output_per_1m_usd: '2.70' +``` + +- Rates are USD per 1,000,000 tokens. Quote them as strings to avoid float rounding. +- `model` matches the normalized model name exactly, or as a version-boundary prefix, so `gpt-4o-mini` prices `gpt-4o-mini-2024-07-18` while `gpt-4` never prices a `gpt-4o` span. Longer matches win, and configured entries beat bundled defaults. +- A provider prefix such as `anthropic/claude-sonnet-4-5` is matched with and without the prefix. +- Set `use_default_table: false` to price only from your own entries, or `enabled: false` to turn estimation off entirely. + +Estimated spend is always labelled rather than blended into provider-reported cost: + +- `agentprof normalize` reports estimated span coverage and names any model it had tokens for but no price. +- `agentprof cost ledger` splits provider-reported cost from estimated cost. +- Reports carry a `confidence` column per ledger row and a `costs_by_confidence_usd` summary field. + +The bundled rates are public list prices recorded on a fixed date, and provider pricing changes. Treat estimates as a starting point and override the models that matter to you before acting on a total. + ## Spec Contracts Spec-violation analysis is opt-in. Add contracts under `analyzers.spec_violations.contracts`: @@ -422,6 +462,13 @@ trace/span spend. The summary also includes gross and overlapping attribution fields so consumers can inspect how much per-issue waste was removed from the recoverable total. +Issues are ranked by attributed waste, so the most expensive finding leads the +report instead of whichever issue ID happened to sort first. Severity and +confidence break ties, which is what orders findings when a trace carries no cost +data at all. Every report opens with a `Top Findings` section covering the three +leading issues with their recommendation and one evidence line, and the JSON +payload exposes the same list as `top_findings` alongside `summary.top_issue_kind`. + When persisted `multi_agent_waste` issues exist, `agentprof report generate` also writes `-multi-agent-waste.svg` next to the Markdown/JSON/HTML files, embeds or links it from the Markdown and HTML reports, and records the artifact filename in `summary.artifacts.multi_agent_waste_svg`. List and inspect generated reports: diff --git a/agentprof/analyze/multi_agent_waste.py b/agentprof/analyze/multi_agent_waste.py index 123be86..38c45c3 100644 --- a/agentprof/analyze/multi_agent_waste.py +++ b/agentprof/analyze/multi_agent_waste.py @@ -29,6 +29,8 @@ AGENT_SPAN_TYPES = {"root", "agent"} BASELINE_MODES = {"estimated", "observed"} FAILURE_STATUSES = {"error", "timeout", "cancelled"} +HIGH_SEVERITY_COST_MULTIPLE = Decimal("3") +MEDIUM_SEVERITY_COST_MULTIPLE = Decimal("1.5") @dataclass(frozen=True) @@ -299,6 +301,16 @@ def _normalized_name(value: str | None) -> str: return " ".join((value or "").replace("_", " ").replace("-", " ").lower().split()) +def _severity_from_finding(finding: MultiAgentWasteFinding) -> str: + """Severity tracks how far the orchestration ran over its baseline.""" + + if finding.cost_multiple >= HIGH_SEVERITY_COST_MULTIPLE: + return "high" + if finding.cost_multiple >= MEDIUM_SEVERITY_COST_MULTIPLE: + return "medium" + return "low" + + def _issue_from_finding( finding: MultiAgentWasteFinding, *, affected_spans: int ) -> IssueRecord: @@ -307,7 +319,7 @@ def _issue_from_finding( issue_id=finding.issue_id, kind=ISSUE_KIND, title=f"Estimated orchestration overhead in {root_label}", - severity="medium", + severity=_severity_from_finding(finding), confidence=finding.confidence, first_seen=finding.first_seen, last_seen=finding.last_seen, diff --git a/agentprof/analyze/retry_loop.py b/agentprof/analyze/retry_loop.py index b36145f..64ee569 100644 --- a/agentprof/analyze/retry_loop.py +++ b/agentprof/analyze/retry_loop.py @@ -20,6 +20,8 @@ ATTRIBUTION_METHOD = "retry_loop" WASTED_COST_TYPE = "wasted_retry_cost" FAILURE_STATUSES = {"error", "timeout", "cancelled"} +HIGH_SEVERITY_WASTED_ATTEMPTS = 4 +MEDIUM_SEVERITY_WASTED_ATTEMPTS = 2 def analyze_retry_loops( @@ -144,12 +146,22 @@ def _finding_from_group( ) +def _severity_from_finding(finding: RetryLoopFinding) -> str: + """Severity tracks how many times the agent repeated a known-failing call.""" + + if finding.wasted_attempts >= HIGH_SEVERITY_WASTED_ATTEMPTS: + return "high" + if finding.wasted_attempts >= MEDIUM_SEVERITY_WASTED_ATTEMPTS: + return "medium" + return "medium" if finding.wasted_cost_usd > 0 else "low" + + def _issue_from_finding(finding: RetryLoopFinding) -> IssueRecord: return IssueRecord( issue_id=finding.issue_id, kind=ISSUE_KIND, title=f"Repeated failing call to {finding.name}", - severity="medium" if finding.wasted_cost_usd > 0 else "low", + severity=_severity_from_finding(finding), confidence="high", first_seen=finding.first_seen, last_seen=finding.last_seen, diff --git a/agentprof/cli.py b/agentprof/cli.py index 5d1f23c..9fbe097 100644 --- a/agentprof/cli.py +++ b/agentprof/cli.py @@ -28,6 +28,7 @@ write_workspace_gitignore, write_default_config, ) +from agentprof.cost.pricing import DEFAULT_PRICE_TABLE_AS_OF, build_pricing_table from agentprof.cost.runner import build_cost_ledger from agentprof.ingest.langfuse_export import ( LangfuseExportFormat, @@ -69,11 +70,13 @@ class MultiAgentBaselineMode(StrEnum): cost_app = typer.Typer(help="Analyze normalized trace costs.") analyze_app = typer.Typer(help="Run deterministic analyzers over normalized traces.") report_app = typer.Typer(help="Generate local AgentProf reports.") +pricing_app = typer.Typer(help="Inspect model pricing used for cost estimation.") app.add_typer(store_app, name="store") app.add_typer(import_app, name="import") app.add_typer(cost_app, name="cost") app.add_typer(analyze_app, name="analyze") app.add_typer(report_app, name="report") +app.add_typer(pricing_app, name="pricing") def _version_callback(value: bool) -> None: @@ -229,7 +232,7 @@ def _run_demo_pipeline( ) console.print(f" imported {import_result.observations_imported} sample observations") - normalize_store(store) + normalize_store(store, pricing=config.pricing) console.print(" normalized spans and traces") analyze_retry_loops(store) @@ -332,13 +335,18 @@ def normalize( config = _load_config_or_exit() store = DuckDBStore(config.store.path) - result = normalize_store(store, source=source) + result = normalize_store(store, source=source, pricing=config.pricing) quality = result.data_quality console.print("[green]Normalized imported spans.[/green]") console.print(f" raw spans seen: {result.raw_spans_seen}") console.print(f" normalized spans: {result.normalized_spans}") console.print(f" normalized traces: {result.normalized_traces}") + if result.estimated_cost_spans: + console.print( + f" estimated cost for {result.estimated_cost_spans} span(s) " + f"without provider cost: {_format_usd(result.estimated_cost_usd)}" + ) table = Table(title="Data quality") table.add_column("Metric") @@ -346,11 +354,25 @@ def normalize( table.add_row("Parent coverage", f"{quality.parent_coverage_pct:.1f}%") table.add_row("Status coverage", f"{quality.status_coverage_pct:.1f}%") table.add_row("Cost coverage", f"{quality.cost_coverage_pct:.1f}%") + table.add_row("Cost coverage (source)", f"{quality.source_cost_coverage_pct:.1f}%") + table.add_row( + "Cost coverage (estimated)", f"{quality.estimated_cost_coverage_pct:.1f}%" + ) table.add_row("Token coverage", f"{quality.token_coverage_pct:.1f}%") table.add_row("Model coverage", f"{quality.model_coverage_pct:.1f}%") table.add_row("I/O hash coverage", f"{quality.io_hash_coverage_pct:.1f}%") console.print(table) + if result.unpriced_models: + console.print( + "[yellow]No price configured for " + f"{', '.join(result.unpriced_models)}.[/yellow]" + ) + console.print( + " Add rates under `pricing.models` in agentprof.yml to attribute " + "their cost." + ) + @cost_app.command("ledger") def cost_ledger() -> None: @@ -365,6 +387,11 @@ def cost_ledger() -> None: console.print(f" ledger entries: {result.ledger_entries}") console.print(f" traces with cost: {result.traces_with_cost}") console.print(f" total cost: {_format_usd(result.total_cost_usd)}") + console.print(f" provider-reported cost: {_format_usd(result.source_cost_usd)}") + console.print( + f" estimated cost: {_format_usd(result.estimated_cost_usd)} " + f"across {result.estimated_entries} entry(s)" + ) table = Table(title="Cost waterfall") table.add_column("Cost type") @@ -584,6 +611,8 @@ def report_generate( console.print("[green]Generated AgentProf report.[/green]") console.print(f" report id: {result.report_id}") console.print(f" issues: {result.issues}") + if result.top_finding_title: + console.print(f" top finding: {result.top_finding_title}") console.print(f" evidence items: {result.evidence_items}") console.print(f" cost entries: {result.cost_entries}") console.print(f" total wasted cost: {_format_usd(result.total_wasted_cost_usd)}") @@ -658,6 +687,46 @@ def report_show( raise typer.Exit(code=2) from exc +@pricing_app.command("list") +def pricing_list() -> None: + """Show the model prices used to estimate cost for uncosted spans.""" + + config = _load_config_or_exit() + if not config.pricing.enabled: + console.print( + "[yellow]Cost estimation is disabled (`pricing.enabled: false`).[/yellow]" + ) + console.print(" Spans without provider cost fields stay unattributed.") + return + + table_prices = build_pricing_table(config.pricing).prices + if not table_prices: + console.print("No model prices are configured.") + console.print( + " Add rates under `pricing.models` in agentprof.yml, or set " + "`pricing.use_default_table: true`." + ) + return + + table = Table(title="Model prices (USD per 1M tokens)") + table.add_column("Model") + table.add_column("Input", justify="right") + table.add_column("Output", justify="right") + table.add_column("Source") + for price in sorted(table_prices, key=lambda item: (item.source, item.model)): + table.add_row( + price.model, + f"${price.input_per_1m_usd}", + f"${price.output_per_1m_usd}", + price.source, + ) + console.print(table) + console.print( + f" Bundled default rates were recorded on {DEFAULT_PRICE_TABLE_AS_OF}. " + "Verify them against your provider's current pricing." + ) + + @store_app.command("stats") def store_stats() -> None: """Show local store row counts.""" diff --git a/agentprof/config.py b/agentprof/config.py index da54493..ac960f1 100644 --- a/agentprof/config.py +++ b/agentprof/config.py @@ -1,10 +1,11 @@ from __future__ import annotations +from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any import yaml -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator APP_DIR = Path(".agentprof") @@ -43,6 +44,14 @@ spec_violations: contracts: [] +pricing: + # Estimated costs are only used for spans that arrive without provider cost + # fields. Verify rates against your provider's current pricing before you + # rely on estimated totals. + enabled: true + use_default_table: true + models: [] + store: path: .agentprof/data/agentprof.duckdb """ @@ -104,6 +113,30 @@ class AnalyzersConfig(BaseModel): ) +class ModelPriceConfig(BaseModel): + """Per-million-token rates used to estimate span cost for one model.""" + + model: str = Field(min_length=1) + input_per_1m_usd: Decimal = Field(ge=0) + output_per_1m_usd: Decimal = Field(ge=0) + + @field_validator("input_per_1m_usd", "output_per_1m_usd", mode="before") + @classmethod + def _exact_decimal(cls, value: Any) -> Any: + if isinstance(value, Decimal) or value is None: + return value + try: + return Decimal(str(value)) + except InvalidOperation as exc: + raise ValueError(f"{value!r} is not a valid price") from exc + + +class PricingConfig(BaseModel): + enabled: bool = True + use_default_table: bool = True + models: list[ModelPriceConfig] = Field(default_factory=list) + + class StoreConfig(BaseModel): path: Path = DEFAULT_STORE_PATH @@ -113,6 +146,7 @@ class AgentProfConfig(BaseModel): privacy: PrivacyConfig = Field(default_factory=PrivacyConfig) sources: SourcesConfig = Field(default_factory=SourcesConfig) analyzers: AnalyzersConfig = Field(default_factory=AnalyzersConfig) + pricing: PricingConfig = Field(default_factory=PricingConfig) store: StoreConfig = Field(default_factory=StoreConfig) diff --git a/agentprof/cost/pricing.py b/agentprof/cost/pricing.py new file mode 100644 index 0000000..bbee760 --- /dev/null +++ b/agentprof/cost/pricing.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from decimal import ROUND_HALF_UP, Decimal + +from agentprof.config import PricingConfig +from agentprof.normalize.schema import NormalizedSpan + + +COST_QUANTUM = Decimal("0.000000001") +TOKENS_PER_PRICE_UNIT = Decimal("1000000") +CONFIG_PRICE_SOURCE = "config" +DEFAULT_PRICE_SOURCE = "default" +PRICE_SOURCE_RANK = {CONFIG_PRICE_SOURCE: 0, DEFAULT_PRICE_SOURCE: 1} +BOUNDARY_CHARS = frozenset("-_.:@/ ") + +# Published list prices in USD per million tokens, recorded on this date. They +# drift: treat estimates built from this table as a starting point and override +# any model that matters with `pricing.models` in agentprof.yml. +DEFAULT_PRICE_TABLE_AS_OF = "2026-05-01" +DEFAULT_MODEL_PRICES: tuple[tuple[str, str, str], ...] = ( + # OpenAI + ("gpt-4.1", "2.00", "8.00"), + ("gpt-4.1-mini", "0.40", "1.60"), + ("gpt-4.1-nano", "0.10", "0.40"), + ("gpt-4o", "2.50", "10.00"), + ("gpt-4o-mini", "0.15", "0.60"), + ("gpt-3.5-turbo", "0.50", "1.50"), + ("o3-mini", "1.10", "4.40"), + # Anthropic + ("claude-3-opus", "15.00", "75.00"), + ("claude-3-5-sonnet", "3.00", "15.00"), + ("claude-3-5-haiku", "0.80", "4.00"), + ("claude-3-haiku", "0.25", "1.25"), + ("claude-haiku-4-5", "1.00", "5.00"), + ("claude-sonnet-4", "3.00", "15.00"), + ("claude-sonnet-4-5", "3.00", "15.00"), + ("claude-opus-4", "15.00", "75.00"), + ("claude-opus-4-1", "15.00", "75.00"), + # Google + ("gemini-1.5-flash", "0.075", "0.30"), + ("gemini-1.5-pro", "1.25", "5.00"), + ("gemini-2.0-flash", "0.10", "0.40"), +) + + +@dataclass(frozen=True) +class ModelPrice: + model: str + input_per_1m_usd: Decimal + output_per_1m_usd: Decimal + source: str + + +@dataclass(frozen=True) +class SpanCostEstimate: + cost_usd: Decimal + price: ModelPrice + + +@dataclass(frozen=True) +class CostEstimationResult: + """What estimation did to a batch of spans, for reporting back to the user.""" + + estimated_spans: int = 0 + estimated_cost_usd: Decimal = Decimal("0") + unpriced_models: tuple[str, ...] = () + priced_models: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PricingTable: + """Model prices ordered so the most specific configured match wins.""" + + prices: tuple[ModelPrice, ...] = () + _exact: dict[str, ModelPrice] = field(default_factory=dict, compare=False) + + def __bool__(self) -> bool: + return bool(self.prices) + + def lookup(self, model_name: str | None) -> ModelPrice | None: + if not model_name: + return None + + keys = _candidate_keys(model_name) + for key in keys: + exact = self._exact.get(key) + if exact is not None: + return exact + for key in keys: + for price in self.prices: + if _prefix_matches(key, price.model): + return price + return None + + +def build_pricing_table(config: PricingConfig | None) -> PricingTable: + """Merge configured prices over the bundled defaults.""" + + if config is None or not config.enabled: + return PricingTable() + + prices: list[ModelPrice] = [ + ModelPrice( + model=_normalize_model_name(entry.model), + input_per_1m_usd=entry.input_per_1m_usd, + output_per_1m_usd=entry.output_per_1m_usd, + source=CONFIG_PRICE_SOURCE, + ) + for entry in config.models + ] + if config.use_default_table: + prices.extend( + ModelPrice( + model=_normalize_model_name(model), + input_per_1m_usd=Decimal(input_rate), + output_per_1m_usd=Decimal(output_rate), + source=DEFAULT_PRICE_SOURCE, + ) + for model, input_rate, output_rate in DEFAULT_MODEL_PRICES + ) + + exact: dict[str, ModelPrice] = {} + for price in prices: + exact.setdefault(price.model, price) + + ordered = tuple( + sorted( + prices, + key=lambda price: ( + -len(price.model), + PRICE_SOURCE_RANK.get(price.source, len(PRICE_SOURCE_RANK)), + price.model, + ), + ) + ) + return PricingTable(prices=ordered, _exact=exact) + + +def estimate_span_cost( + span: NormalizedSpan, table: PricingTable +) -> SpanCostEstimate | None: + """Price one span from its token counts, or return None if it can't be priced.""" + + if span.cost_usd is not None: + return None + + input_tokens = span.input_tokens or 0 + output_tokens = span.output_tokens or 0 + if input_tokens <= 0 and output_tokens <= 0: + return None + + price = table.lookup(span.model_name) + if price is None: + return None + + cost = ( + Decimal(input_tokens) * price.input_per_1m_usd + + Decimal(output_tokens) * price.output_per_1m_usd + ) / TOKENS_PER_PRICE_UNIT + return SpanCostEstimate( + cost_usd=cost.quantize(COST_QUANTUM, rounding=ROUND_HALF_UP), + price=price, + ) + + +def apply_estimated_costs( + spans: Sequence[NormalizedSpan], table: PricingTable +) -> tuple[list[NormalizedSpan], CostEstimationResult]: + """Fill in missing span costs from token counts. + + Spans that already carry a provider cost are never touched, so estimation + can only add attribution where there was none. + """ + + if not table: + return list(spans), CostEstimationResult( + unpriced_models=_unpriced_model_names(spans, table) + ) + + updated: list[NormalizedSpan] = [] + estimated_spans = 0 + estimated_cost = Decimal("0") + priced_models: set[str] = set() + + for span in spans: + estimate = estimate_span_cost(span, table) + if estimate is None: + updated.append(span) + continue + + updated.append( + span.model_copy( + update={ + "cost_usd": estimate.cost_usd, + "cost_confidence": "estimated", + } + ) + ) + estimated_spans += 1 + estimated_cost += estimate.cost_usd + if span.model_name: + priced_models.add(_normalize_model_name(span.model_name)) + + return updated, CostEstimationResult( + estimated_spans=estimated_spans, + estimated_cost_usd=estimated_cost, + unpriced_models=_unpriced_model_names(spans, table), + priced_models=tuple(sorted(priced_models)), + ) + + +def _unpriced_model_names( + spans: Sequence[NormalizedSpan], table: PricingTable +) -> tuple[str, ...]: + """Models that had usable token counts but no matching price.""" + + names: set[str] = set() + for span in spans: + if span.cost_usd is not None or not span.model_name: + continue + if (span.input_tokens or 0) <= 0 and (span.output_tokens or 0) <= 0: + continue + if table.lookup(span.model_name) is None: + names.add(_normalize_model_name(span.model_name)) + return tuple(sorted(names)) + + +def _candidate_keys(model_name: str) -> tuple[str, ...]: + key = _normalize_model_name(model_name) + if not key: + return () + if "/" in key: + return (key, key.rsplit("/", 1)[-1]) + return (key,) + + +def _prefix_matches(key: str, model: str) -> bool: + """Match on a version-boundary so `gpt-4` never prices a `gpt-4o` span.""" + + if not key.startswith(model): + return False + remainder = key[len(model) :] + return not remainder or remainder[0] in BOUNDARY_CHARS + + +def _normalize_model_name(model_name: str) -> str: + return " ".join(model_name.split()).lower() diff --git a/agentprof/cost/runner.py b/agentprof/cost/runner.py index c06cf68..7ac2315 100644 --- a/agentprof/cost/runner.py +++ b/agentprof/cost/runner.py @@ -31,11 +31,17 @@ def build_cost_ledger(store: DuckDBStore) -> CostLedgerBuildResult: records, attribution_method=LEDGER_ATTRIBUTION_METHOD, ) + estimated = [record for record in records if record.confidence == "estimated"] return CostLedgerBuildResult( normalized_spans_seen=len(spans), ledger_entries=len(records), traces_with_cost=len({record.trace_id for record in records}), total_cost_usd=_sum_amounts(record.amount_usd for record in records), + source_cost_usd=_sum_amounts( + record.amount_usd for record in records if record.confidence == "source" + ), + estimated_cost_usd=_sum_amounts(record.amount_usd for record in estimated), + estimated_entries=len(estimated), waterfall=cost_waterfall(records), ) diff --git a/agentprof/cost/schema.py b/agentprof/cost/schema.py index f89b07a..6c35e9a 100644 --- a/agentprof/cost/schema.py +++ b/agentprof/cost/schema.py @@ -20,4 +20,7 @@ class CostLedgerBuildResult(BaseModel): ledger_entries: int traces_with_cost: int total_cost_usd: Decimal = Field(default=Decimal("0")) + source_cost_usd: Decimal = Field(default=Decimal("0")) + estimated_cost_usd: Decimal = Field(default=Decimal("0")) + estimated_entries: int = 0 waterfall: list[CostWaterfallRow] = Field(default_factory=list) diff --git a/agentprof/normalize/runner.py b/agentprof/normalize/runner.py index d7aeb0b..04fd53a 100644 --- a/agentprof/normalize/runner.py +++ b/agentprof/normalize/runner.py @@ -5,6 +5,8 @@ from decimal import Decimal from math import inf +from agentprof.config import PricingConfig +from agentprof.cost.pricing import apply_estimated_costs, build_pricing_table from agentprof.normalize.langfuse import map_langfuse_raw_span from agentprof.normalize.schema import ( DataQualityMetrics, @@ -15,9 +17,16 @@ from agentprof.store.duckdb_store import DuckDBStore, RawSpanRow -def normalize_store(store: DuckDBStore, *, source: str | None = None) -> NormalizationResult: +def normalize_store( + store: DuckDBStore, + *, + source: str | None = None, + pricing: PricingConfig | None = None, +) -> NormalizationResult: raw_rows = store.fetch_raw_spans(source=source) spans = [_map_raw_span(row) for row in raw_rows if _is_supported_source(row.source)] + # Estimate before the trace rollup so trace totals include estimated spans. + spans, estimation = apply_estimated_costs(spans, build_pricing_table(pricing)) traces = build_normalized_traces(spans) data_quality = compute_data_quality(spans, traces) store.replace_normalized(spans=spans, traces=traces) @@ -26,6 +35,9 @@ def normalize_store(store: DuckDBStore, *, source: str | None = None) -> Normali normalized_spans=len(spans), normalized_traces=len(traces), data_quality=data_quality, + estimated_cost_spans=estimation.estimated_spans, + estimated_cost_usd=estimation.estimated_cost_usd, + unpriced_models=list(estimation.unpriced_models), ) @@ -92,6 +104,16 @@ def compute_data_quality( ) spans_with_status = sum(1 for span in spans if span.status != "unknown") spans_with_cost = sum(1 for span in spans if span.cost_usd is not None) + spans_with_source_cost = sum( + 1 + for span in spans + if span.cost_usd is not None and span.cost_confidence == "source" + ) + spans_with_estimated_cost = sum( + 1 + for span in spans + if span.cost_usd is not None and span.cost_confidence == "estimated" + ) spans_with_token_counts = sum( 1 for span in spans if any(value is not None for value in _token_values(span)) ) @@ -105,12 +127,16 @@ def compute_data_quality( spans_with_valid_parent_links=valid_parent_links, spans_with_status=spans_with_status, spans_with_cost=spans_with_cost, + spans_with_source_cost=spans_with_source_cost, + spans_with_estimated_cost=spans_with_estimated_cost, spans_with_token_counts=spans_with_token_counts, spans_with_model=spans_with_model, spans_with_io_hashes=spans_with_io_hashes, parent_coverage_pct=_pct(valid_parent_links, total_spans), status_coverage_pct=_pct(spans_with_status, total_spans), cost_coverage_pct=_pct(spans_with_cost, total_spans), + source_cost_coverage_pct=_pct(spans_with_source_cost, total_spans), + estimated_cost_coverage_pct=_pct(spans_with_estimated_cost, total_spans), token_coverage_pct=_pct(spans_with_token_counts, total_spans), model_coverage_pct=_pct(spans_with_model, total_spans), io_hash_coverage_pct=_pct(spans_with_io_hashes, total_spans), diff --git a/agentprof/normalize/schema.py b/agentprof/normalize/schema.py index 755474d..616f461 100644 --- a/agentprof/normalize/schema.py +++ b/agentprof/normalize/schema.py @@ -92,12 +92,16 @@ class DataQualityMetrics(BaseModel): spans_with_valid_parent_links: int = 0 spans_with_status: int = 0 spans_with_cost: int = 0 + spans_with_source_cost: int = 0 + spans_with_estimated_cost: int = 0 spans_with_token_counts: int = 0 spans_with_model: int = 0 spans_with_io_hashes: int = 0 parent_coverage_pct: float = 0.0 status_coverage_pct: float = 0.0 cost_coverage_pct: float = 0.0 + source_cost_coverage_pct: float = 0.0 + estimated_cost_coverage_pct: float = 0.0 token_coverage_pct: float = 0.0 model_coverage_pct: float = 0.0 io_hash_coverage_pct: float = 0.0 @@ -108,3 +112,6 @@ class NormalizationResult(BaseModel): normalized_spans: int normalized_traces: int data_quality: DataQualityMetrics + estimated_cost_spans: int = 0 + estimated_cost_usd: Decimal = Decimal("0") + unpriced_models: list[str] = Field(default_factory=list) diff --git a/agentprof/report/runner.py b/agentprof/report/runner.py index 9d9ec35..8de22a1 100644 --- a/agentprof/report/runner.py +++ b/agentprof/report/runner.py @@ -28,6 +28,7 @@ MARKDOWN_WWW_LINK_RE = re.compile(r"(?i)\bwww\.") MULTI_AGENT_WASTE_KIND = "multi_agent_waste" TRACE_LEVEL_WASTE_COST_TYPES = frozenset({"estimated_multi_agent_overhead"}) +TOP_FINDING_LIMIT = 3 def generate_report( @@ -104,6 +105,7 @@ def generate_report( evidence_items=len(evidence), cost_entries=len(costs), total_wasted_cost_usd=attribution_totals["total_wasted_cost_usd"], + top_finding_title=issues[0].title if issues else None, report_md_path=markdown_path, report_json_path=json_path, report_html_path=html_path, @@ -128,9 +130,11 @@ def _summary( issue_kinds = Counter(issue.kind for issue in issues) severities = Counter(issue.severity for issue in issues) cost_types = defaultdict(Decimal) + cost_confidences = defaultdict(Decimal) for record in costs: if record.amount_usd is not None: cost_types[record.cost_type] += record.amount_usd + cost_confidences[record.confidence] += record.amount_usd return { "generated_at": _datetime_to_json(generated_at), @@ -149,6 +153,11 @@ def _summary( cost_type: _decimal_to_json(amount) for cost_type, amount in sorted(cost_types.items()) }, + "costs_by_confidence_usd": { + confidence: _decimal_to_json(amount) + for confidence, amount in sorted(cost_confidences.items()) + }, + "top_issue_kind": issues[0].kind if issues else None, "artifacts": dict(sorted(artifacts.items())), } @@ -297,18 +306,49 @@ def _json_payload( for item in evidence: evidence_by_issue[item.issue_id].append(item) + issue_payloads = [ + _issue_to_json(issue, evidence_by_issue[issue.issue_id]) for issue in issues + ] return { "report_id": report_id, "project": project, "generated_at": _datetime_to_json(generated_at), "summary": summary, - "issues": [ - _issue_to_json(issue, evidence_by_issue[issue.issue_id]) for issue in issues - ], + "top_findings": _top_findings(issue_payloads), + "issues": issue_payloads, "cost_ledger": [_cost_to_json(record) for record in costs], } +def _top_findings(issues: list[dict[str, Any]]) -> list[dict[str, Any]]: + """The leading issues, already ranked by the store's issue ordering.""" + + return [ + { + "issue_id": issue["issue_id"], + "kind": issue["kind"], + "title": issue["title"], + "severity": issue["severity"], + "confidence": issue["confidence"], + "wasted_cost_usd": issue["wasted_cost_usd"], + "potential_savings_usd": issue["potential_savings_usd"], + "recommendation": issue["recommendation"], + "evidence": _leading_evidence(issue), + } + for issue in issues[:TOP_FINDING_LIMIT] + ] + + +def _leading_evidence(issue: dict[str, Any]) -> dict[str, Any] | None: + for item in issue["evidence"]: + return { + "trace_id": item["trace_id"], + "span_id": item["span_id"], + "message": item["message"], + } + return None + + def _issue_to_json( issue: IssueRecord, evidence: list[IssueEvidenceRecord] ) -> dict[str, Any]: @@ -380,6 +420,8 @@ def _markdown_report(payload: dict[str, Any]) -> str: "", ] + lines.extend(_markdown_top_findings(payload["top_findings"])) + multi_agent_svg = summary.get("artifacts", {}).get("multi_agent_waste_svg") if multi_agent_svg: lines.extend( @@ -404,8 +446,8 @@ def _markdown_report(payload: dict[str, Any]) -> str: else: lines.extend( [ - "| Cost type | Amount | Attribution | Issue |", - "| --- | ---: | --- | --- |", + "| Cost type | Amount | Confidence | Attribution | Issue |", + "| --- | ---: | --- | --- | --- |", ] ) for record in payload["cost_ledger"]: @@ -413,6 +455,7 @@ def _markdown_report(payload: dict[str, Any]) -> str: "| " f"{_markdown_table_cell(record['cost_type'])} | " f"{_format_usd(record['amount_usd'])} | " + f"{_markdown_table_cell(record['confidence'])} | " f"{_markdown_table_cell(record['attribution_method'])} | " f"{_markdown_table_cell(record['issue_id'] or '')} |" ) @@ -421,6 +464,39 @@ def _markdown_report(payload: dict[str, Any]) -> str: return "\n".join(lines) +def _markdown_top_findings(top_findings: list[dict[str, Any]]) -> list[str]: + if not top_findings: + return [] + + lines = [ + "## Top Findings", + "", + "Ranked by attributed waste. Full details for every issue follow below.", + "", + ] + for position, finding in enumerate(top_findings, start=1): + lines.append( + f"{position}. **{_markdown_text(finding['title'])}** " + f"- {_format_usd(finding['wasted_cost_usd'])} wasted " + f"({_markdown_inline_code(finding['kind'])}, " + f"{_markdown_text(finding['severity'])} severity)" + ) + lines.append( + f" - Recommendation: {_markdown_text(finding['recommendation'])}" + ) + evidence = finding["evidence"] + if evidence: + location = ":".join( + part for part in (evidence["trace_id"], evidence["span_id"]) if part + ) + lines.append( + f" - Evidence: {_markdown_inline_code(location or 'unknown')} " + f"{_markdown_text(evidence['message'])}" + ) + lines.append("") + return lines + + def _markdown_issue(issue: dict[str, Any]) -> list[str]: lines = [ f"### {_markdown_text(issue['title'])}", @@ -515,6 +591,8 @@ def _html_report(payload: dict[str, Any]) -> str: " ", ] + lines.extend(_html_top_findings(payload["top_findings"])) + multi_agent_svg = summary.get("artifacts", {}).get("multi_agent_waste_svg") if multi_agent_svg: lines.extend( @@ -654,6 +732,39 @@ def _html_style() -> str: color: var(--muted); font-size: 0.92rem; } + .top-findings { + display: grid; + gap: 14px; + margin: 0; + padding-left: 20px; + } + .top-findings li { + padding: 18px; + border: 1px solid var(--line); + border-radius: 16px; + background: #fbfdff; + } + .top-findings li::marker { + color: var(--muted); + font-weight: 800; + } + .top-finding-head { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 16px; + } + .top-finding-head h3 { + margin: 0; + font-size: 1.1rem; + } + .top-finding-head strong { + font-size: 1.15rem; + white-space: nowrap; + } + .top-findings p { margin: 10px 0 0; } + .top-finding-evidence { color: var(--muted); font-size: 0.93rem; } + .top-findings .badges { justify-content: flex-start; margin-top: 10px; } .issue { margin-top: 16px; padding: 20px; @@ -752,6 +863,50 @@ def _html_style() -> str: }""" +def _html_top_findings(top_findings: list[dict[str, Any]]) -> list[str]: + if not top_findings: + return [] + + lines = [ + "", + '
', + "

Top Findings

", + '

Ranked by attributed waste. Full details for every issue follow below.

', + '
    ', + ] + for finding in top_findings: + lines.extend(_html_top_finding(finding)) + lines.extend(["
", "
"]) + return lines + + +def _html_top_finding(finding: dict[str, Any]) -> list[str]: + lines = [ + "
  • ", + '
    ', + f"

    {_html_text(finding['title'])}

    ", + f" {_html_text(_format_usd(finding['wasted_cost_usd']))}", + "
    ", + '
    ', + f" {_html_text(finding['kind'])}", + f" {_html_text(finding['severity'])}", + f" {_html_text(finding['confidence'])} confidence", + "
    ", + f"

    {_html_text(finding['recommendation'])}

    ", + ] + evidence = finding["evidence"] + if evidence: + location = ":".join( + part for part in (evidence["trace_id"], evidence["span_id"]) if part + ) + lines.append( + f'

    {_html_code(location or "unknown")} ' + f"{_html_text(evidence['message'])}

    " + ) + lines.append("
    • ") + return lines + + def _html_summary_card(label: str, value: Any) -> str: return ( '
    ' @@ -837,6 +992,7 @@ def _html_cost_ledger(records: list[dict[str, Any]]) -> list[str]: " Cost type", " Trace", " Span", + " Confidence", " Attribution", " Issue", " Amount", @@ -851,6 +1007,7 @@ def _html_cost_ledger(records: list[dict[str, Any]]) -> list[str]: f" {_html_text(record['cost_type'])}", f" {_html_code(record['trace_id'])}", f" {_html_code(record['span_id'] or '')}", + f" {_html_text(record['confidence'])}", f" {_html_text(record['attribution_method'])}", f" {_html_code(record['issue_id'] or '')}", f" {_format_usd(record['amount_usd'])}", diff --git a/agentprof/report/schema.py b/agentprof/report/schema.py index b782e44..ab4493f 100644 --- a/agentprof/report/schema.py +++ b/agentprof/report/schema.py @@ -13,6 +13,7 @@ class ReportBuildResult(BaseModel): evidence_items: int cost_entries: int total_wasted_cost_usd: Decimal = Field(default=Decimal("0")) + top_finding_title: str | None = None report_md_path: Path report_json_path: Path report_html_path: Path diff --git a/agentprof/store/duckdb_store.py b/agentprof/store/duckdb_store.py index 837a7c1..5a2eb23 100644 --- a/agentprof/store/duckdb_store.py +++ b/agentprof/store/duckdb_store.py @@ -24,6 +24,27 @@ "reports", ) +SEVERITY_ORDER = ("critical", "high", "medium", "low") +CONFIDENCE_ORDER = ("high", "medium", "low") + + +def _rank_case(column: str, values: Sequence[str]) -> str: + whens = " ".join( + f"WHEN '{value}' THEN {index}" for index, value in enumerate(values) + ) + return f"CASE {column} {whens} ELSE {len(values)} END" + + +# Issues are ranked by attributed waste first so the most expensive finding +# leads every report. Severity and confidence break ties, which is what ranks +# findings when a trace carries no cost data at all. issue_id keeps it stable. +ISSUE_RANK_ORDER_BY = ( + "COALESCE(wasted_cost_usd, 0) DESC, " + f"{_rank_case('severity', SEVERITY_ORDER)}, " + f"{_rank_case('confidence', CONFIDENCE_ORDER)}, " + "issue_id" +) + @dataclass(frozen=True) class Migration: @@ -864,7 +885,7 @@ def fetch_issues(self, *, kind: str | None = None) -> list[IssueRecord]: if kind: query += " WHERE kind = ?" params.append(kind) - query += " ORDER BY issue_id" + query += f" ORDER BY {ISSUE_RANK_ORDER_BY}" with self.connect() as connection: rows = connection.execute(query, params).fetchall() diff --git a/examples/reports/demo.html b/examples/reports/demo.html index 5b7b9a0..7507aaf 100644 --- a/examples/reports/demo.html +++ b/examples/reports/demo.html @@ -112,6 +112,39 @@ color: var(--muted); font-size: 0.92rem; } + .top-findings { + display: grid; + gap: 14px; + margin: 0; + padding-left: 20px; + } + .top-findings li { + padding: 18px; + border: 1px solid var(--line); + border-radius: 16px; + background: #fbfdff; + } + .top-findings li::marker { + color: var(--muted); + font-weight: 800; + } + .top-finding-head { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 16px; + } + .top-finding-head h3 { + margin: 0; + font-size: 1.1rem; + } + .top-finding-head strong { + font-size: 1.15rem; + white-space: nowrap; + } + .top-findings p { margin: 10px 0 0; } + .top-finding-evidence { color: var(--muted); font-size: 0.93rem; } + .top-findings .badges { justify-content: flex-start; margin-top: 10px; } .issue { margin-top: 16px; padding: 20px; @@ -217,7 +250,7 @@

    AgentProf Demo

    Report ID
    demo
    -
    Generated
    2026-05-30T14:11:08.868496Z
    +
    Generated
    2026-08-12T16:26:28.684493Z
    @@ -238,6 +271,52 @@

    Summary

    Total values deduplicate overlapping trace/span attributions. Gross values sum all issue estimates before deduplication.

    +
    +

    Top Findings

    +

    Ranked by attributed waste. Full details for every issue follow below.

    +
      +
    1. +
      +

      Estimated orchestration overhead in triage_agent

      + $0.042000000 +
      +
      + multi_agent_waste + medium + medium confidence +
      +

      Compare this multi-agent trace with a configured single-agent baseline before keeping the orchestration path.

      +

      trace-multi-agent-1:ma-root-1 Trace used 3 distinct agents; estimated orchestration overhead uses a configured single-agent baseline ratio.

      +
      2. +
    2. +
      +

      Repeated failing call to refund_policy_lookup

      + $0.006000000 +
      +
      + retry_loop + medium + high confidence +
      +

      Stop retrying deterministic failures until the input, schema, or tool precondition has changed.

      +

      trace-demo-retry:demo-tool-retry-1 Attempt 1 failed with missing required field region.

      +
      3. +
    3. +
      +

      Contract violation in refund_policy_lookup

      + $0.006000000 +
      +
      + spec_violation + medium + high confidence +
      +

      Validate tool inputs and outputs against the configured contract before continuing the agent trace.

      +

      trace-demo-retry:demo-tool-retry-2 refund_policy_lookup violated refund_policy_lookup: missing input fields: region.

      +
      4. +
    +
    +

    Visuals

    @@ -450,6 +529,7 @@

    Cost Ledger

    Cost type Trace Span + Confidence Attribution Issue Amount @@ -460,6 +540,7 @@

    Cost Ledger

    failed_span_cost trace-demo-retry demo-tool-retry-1 + source normalized_span_status $0.006000000 @@ -468,6 +549,7 @@

    Cost Ledger

    wasted_spec_violation_cost trace-demo-retry demo-tool-retry-1 + source spec_violation spec_violation:d62c2c3a59097026 $0.006000000 @@ -476,6 +558,7 @@

    Cost Ledger

    failed_span_cost trace-demo-retry demo-tool-retry-2 + source normalized_span_status $0.006000000 @@ -484,6 +567,7 @@

    Cost Ledger

    wasted_retry_cost trace-demo-retry demo-tool-retry-2 + source retry_loop retry_loop:14f469d943d86373 $0.006000000 @@ -492,6 +576,7 @@

    Cost Ledger

    wasted_spec_violation_cost trace-demo-retry demo-tool-retry-2 + source spec_violation spec_violation:ae3bd3cc2b213fc6 $0.006000000 @@ -500,6 +585,7 @@

    Cost Ledger

    successful_span_cost trace-multi-agent-1 ma-llm-policy + source normalized_span_status $0.052000000 @@ -508,6 +594,7 @@

    Cost Ledger

    successful_span_cost trace-multi-agent-1 ma-llm-research + source normalized_span_status $0.032000000 @@ -516,6 +603,7 @@

    Cost Ledger

    estimated_multi_agent_overhead trace-multi-agent-1 ma-root-1 + estimated multi_agent_waste multi_agent_waste:a7887662c9f550c6 $0.042000000 diff --git a/examples/reports/demo.json b/examples/reports/demo.json index 2f8709a..1d31177 100644 --- a/examples/reports/demo.json +++ b/examples/reports/demo.json @@ -73,7 +73,7 @@ "trace_id": "trace-multi-agent-1" } ], - "generated_at": "2026-05-30T14:11:08.868496Z", + "generated_at": "2026-08-12T16:26:28.684493Z", "issues": [ { "affected_spans": 5, @@ -251,6 +251,10 @@ "multi_agent_waste_svg": "demo-multi-agent-waste.svg" }, "cost_entry_count": 8, + "costs_by_confidence_usd": { + "estimated": "0.042000000", + "source": "0.114000000" + }, "costs_by_type_usd": { "estimated_multi_agent_overhead": "0.042000000", "failed_span_cost": "0.012000000", @@ -259,7 +263,9 @@ "wasted_spec_violation_cost": "0.012000000" }, "evidence_count": 5, - "generated_at": "2026-05-30T14:11:08.868496Z", + "generated_at": "2026-08-12T16:26:28.684493Z", + "gross_potential_savings_usd": "0.060000000", + "gross_wasted_cost_usd": "0.060000000", "issue_count": 4, "issues_by_kind": { "multi_agent_waste": 1, @@ -269,11 +275,57 @@ "issues_by_severity": { "medium": 4 }, - "gross_potential_savings_usd": "0.060000000", - "gross_wasted_cost_usd": "0.060000000", "overlapping_potential_savings_usd": "0.006000000", "overlapping_wasted_cost_usd": "0.006000000", + "top_issue_kind": "multi_agent_waste", "total_potential_savings_usd": "0.054000000", "total_wasted_cost_usd": "0.054000000" - } + }, + "top_findings": [ + { + "confidence": "medium", + "evidence": { + "message": "Trace used 3 distinct agents; estimated orchestration overhead uses a configured single-agent baseline ratio.", + "span_id": "ma-root-1", + "trace_id": "trace-multi-agent-1" + }, + "issue_id": "multi_agent_waste:a7887662c9f550c6", + "kind": "multi_agent_waste", + "potential_savings_usd": "0.042000000", + "recommendation": "Compare this multi-agent trace with a configured single-agent baseline before keeping the orchestration path.", + "severity": "medium", + "title": "Estimated orchestration overhead in triage_agent", + "wasted_cost_usd": "0.042000000" + }, + { + "confidence": "high", + "evidence": { + "message": "Attempt 1 failed with missing required field region.", + "span_id": "demo-tool-retry-1", + "trace_id": "trace-demo-retry" + }, + "issue_id": "retry_loop:14f469d943d86373", + "kind": "retry_loop", + "potential_savings_usd": "0.006000000", + "recommendation": "Stop retrying deterministic failures until the input, schema, or tool precondition has changed.", + "severity": "medium", + "title": "Repeated failing call to refund_policy_lookup", + "wasted_cost_usd": "0.006000000" + }, + { + "confidence": "high", + "evidence": { + "message": "refund_policy_lookup violated refund_policy_lookup: missing input fields: region.", + "span_id": "demo-tool-retry-2", + "trace_id": "trace-demo-retry" + }, + "issue_id": "spec_violation:ae3bd3cc2b213fc6", + "kind": "spec_violation", + "potential_savings_usd": "0.006000000", + "recommendation": "Validate tool inputs and outputs against the configured contract before continuing the agent trace.", + "severity": "medium", + "title": "Contract violation in refund_policy_lookup", + "wasted_cost_usd": "0.006000000" + } + ] } diff --git a/examples/reports/demo.md b/examples/reports/demo.md index 05a066d..2aa3ba4 100644 --- a/examples/reports/demo.md +++ b/examples/reports/demo.md @@ -1,7 +1,7 @@ # AgentProf Report: AgentProf Demo Report ID: `demo` -Generated at: `2026-05-30T14:11:08.868496Z` +Generated at: `2026-08-12T16:26:28.684493Z` ## Summary @@ -20,6 +20,20 @@ Generated at: `2026-05-30T14:11:08.868496Z` Total wasted cost and potential savings deduplicate overlapping trace/span attributions. Gross values sum all issue estimates before deduplication. +## Top Findings + +Ranked by attributed waste. Full details for every issue follow below. + +1. **Estimated orchestration overhead in triage_agent** - $0.042000000 wasted (`multi_agent_waste`, medium severity) + - Recommendation: Compare this multi-agent trace with a configured single-agent baseline before keeping the orchestration path. + - Evidence: `trace-multi-agent-1:ma-root-1` Trace used 3 distinct agents; estimated orchestration overhead uses a configured single-agent baseline ratio. +2. **Repeated failing call to refund_policy_lookup** - $0.006000000 wasted (`retry_loop`, medium severity) + - Recommendation: Stop retrying deterministic failures until the input, schema, or tool precondition has changed. + - Evidence: `trace-demo-retry:demo-tool-retry-1` Attempt 1 failed with missing required field region. +3. **Contract violation in refund_policy_lookup** - $0.006000000 wasted (`spec_violation`, medium severity) + - Recommendation: Validate tool inputs and outputs against the configured contract before continuing the agent trace. + - Evidence: `trace-demo-retry:demo-tool-retry-2` refund_policy_lookup violated refund_policy_lookup: missing input fields: region. + ## Visuals ![Multi-agent waste estimate](demo-multi-agent-waste.svg) @@ -113,13 +127,13 @@ Evidence: ## Cost Ledger -| Cost type | Amount | Attribution | Issue | -| --- | ---: | --- | --- | -| failed_span_cost | $0.006000000 | normalized_span_status | | -| wasted_spec_violation_cost | $0.006000000 | spec_violation | spec_violation:d62c2c3a59097026 | -| failed_span_cost | $0.006000000 | normalized_span_status | | -| wasted_retry_cost | $0.006000000 | retry_loop | retry_loop:14f469d943d86373 | -| wasted_spec_violation_cost | $0.006000000 | spec_violation | spec_violation:ae3bd3cc2b213fc6 | -| successful_span_cost | $0.052000000 | normalized_span_status | | -| successful_span_cost | $0.032000000 | normalized_span_status | | -| estimated_multi_agent_overhead | $0.042000000 | multi_agent_waste | multi_agent_waste:a7887662c9f550c6 | +| Cost type | Amount | Confidence | Attribution | Issue | +| --- | ---: | --- | --- | --- | +| failed_span_cost | $0.006000000 | source | normalized_span_status | | +| wasted_spec_violation_cost | $0.006000000 | source | spec_violation | spec_violation:d62c2c3a59097026 | +| failed_span_cost | $0.006000000 | source | normalized_span_status | | +| wasted_retry_cost | $0.006000000 | source | retry_loop | retry_loop:14f469d943d86373 | +| wasted_spec_violation_cost | $0.006000000 | source | spec_violation | spec_violation:ae3bd3cc2b213fc6 | +| successful_span_cost | $0.052000000 | source | normalized_span_status | | +| successful_span_cost | $0.032000000 | source | normalized_span_status | | +| estimated_multi_agent_overhead | $0.042000000 | estimated | multi_agent_waste | multi_agent_waste:a7887662c9f550c6 | diff --git a/tests/fixtures/langfuse_token_only_observations.json b/tests/fixtures/langfuse_token_only_observations.json new file mode 100644 index 0000000..759fc55 --- /dev/null +++ b/tests/fixtures/langfuse_token_only_observations.json @@ -0,0 +1,72 @@ +[ + { + "id": "tok-root-1", + "traceId": "trace-token-only-1", + "parentObservationId": null, + "type": "SPAN", + "name": "support_agent", + "startTime": "2026-05-04T09:00:00.000Z", + "endTime": "2026-05-04T09:00:06.000Z", + "input": "Summarize the open ticket backlog.", + "output": "Three tickets need a reply today.", + "level": "DEFAULT", + "statusMessage": "" + }, + { + "id": "tok-llm-summary", + "traceId": "trace-token-only-1", + "parentObservationId": "tok-root-1", + "type": "GENERATION", + "name": "summarize_backlog", + "startTime": "2026-05-04T09:00:01.000Z", + "endTime": "2026-05-04T09:00:03.000Z", + "model": "gpt-4o-mini", + "usageDetails": { + "input": 1000, + "output": 500, + "total": 1500 + }, + "input": "Summarize these tickets.", + "output": "Three tickets need a reply today.", + "level": "DEFAULT", + "statusMessage": "" + }, + { + "id": "tok-llm-draft", + "traceId": "trace-token-only-1", + "parentObservationId": "tok-root-1", + "type": "GENERATION", + "name": "draft_replies", + "startTime": "2026-05-04T09:00:03.000Z", + "endTime": "2026-05-04T09:00:05.000Z", + "model": "claude-sonnet-4-5-20250929", + "usageDetails": { + "input": 2000, + "output": 1000, + "total": 3000 + }, + "input": "Draft replies for the three tickets.", + "output": "Drafted three replies.", + "level": "DEFAULT", + "statusMessage": "" + }, + { + "id": "tok-llm-router", + "traceId": "trace-token-only-1", + "parentObservationId": "tok-root-1", + "type": "GENERATION", + "name": "route_request", + "startTime": "2026-05-04T09:00:05.000Z", + "endTime": "2026-05-04T09:00:06.000Z", + "model": "internal-router-v2", + "usageDetails": { + "input": 100, + "output": 50, + "total": 150 + }, + "input": "Which queue should own this?", + "output": "billing", + "level": "DEFAULT", + "statusMessage": "" + } +] diff --git a/tests/test_multi_agent_waste.py b/tests/test_multi_agent_waste.py index f6525b6..a811704 100644 --- a/tests/test_multi_agent_waste.py +++ b/tests/test_multi_agent_waste.py @@ -753,3 +753,20 @@ def _contract() -> SpecContractConfig: def _dt(value: str) -> datetime: return datetime.fromisoformat(value) + + +def test_multi_agent_severity_tracks_the_cost_multiple(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + spans = _multi_agent_spans() + store.replace_normalized(spans=spans, traces=build_normalized_traces(spans)) + + # A 0.25 baseline ratio puts the trace at 4.00x its single-agent baseline. + analyze_multi_agent_waste(store, baseline_ratio=Decimal("0.25")) + high = store.fetch_issues(kind="multi_agent_waste")[0] + + # A 0.80 baseline ratio puts the same trace at 1.25x. + analyze_multi_agent_waste(store, baseline_ratio=Decimal("0.80")) + low = store.fetch_issues(kind="multi_agent_waste")[0] + + assert high.severity == "high" + assert low.severity == "low" diff --git a/tests/test_pricing.py b/tests/test_pricing.py new file mode 100644 index 0000000..fc99337 --- /dev/null +++ b/tests/test_pricing.py @@ -0,0 +1,324 @@ +from __future__ import annotations + +from decimal import Decimal +from pathlib import Path + +from typer.testing import CliRunner + +from agentprof.cli import app +from agentprof.config import ( + AgentProfConfig, + ModelPriceConfig, + PricingConfig, + load_config, +) +from agentprof.cost.pricing import ( + apply_estimated_costs, + build_pricing_table, + estimate_span_cost, +) +from agentprof.cost.runner import build_cost_ledger +from agentprof.normalize.runner import normalize_store +from agentprof.normalize.schema import NormalizedSpan +from agentprof.store.duckdb_store import DuckDBStore + + +runner = CliRunner() +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_default_table_prices_a_known_model() -> None: + table = build_pricing_table(PricingConfig()) + + price = table.lookup("gpt-4o-mini") + + assert price is not None + assert price.source == "default" + assert price.input_per_1m_usd == Decimal("0.15") + assert price.output_per_1m_usd == Decimal("0.60") + + +def test_lookup_prefers_the_longest_matching_prefix() -> None: + table = build_pricing_table(PricingConfig()) + + dated = table.lookup("gpt-4o-mini-2024-07-18") + + assert dated is not None + assert dated.model == "gpt-4o-mini" + + +def test_lookup_respects_version_boundaries() -> None: + table = build_pricing_table( + PricingConfig( + use_default_table=False, + models=[ + ModelPriceConfig( + model="gpt-4", + input_per_1m_usd=Decimal("30"), + output_per_1m_usd=Decimal("60"), + ) + ], + ) + ) + + # `gpt-4o` is a different model and must not inherit the `gpt-4` rate. + assert table.lookup("gpt-4o") is None + assert table.lookup("gpt-4-turbo") is not None + assert table.lookup("gpt-4") is not None + + +def test_lookup_strips_a_provider_prefix() -> None: + table = build_pricing_table(PricingConfig()) + + price = table.lookup("anthropic/claude-sonnet-4-5") + + assert price is not None + assert price.model == "claude-sonnet-4-5" + + +def test_configured_prices_override_the_default_table() -> None: + table = build_pricing_table( + PricingConfig( + models=[ + ModelPriceConfig( + model="gpt-4o-mini", + input_per_1m_usd=Decimal("1"), + output_per_1m_usd=Decimal("2"), + ) + ] + ) + ) + + price = table.lookup("gpt-4o-mini") + + assert price is not None + assert price.source == "config" + assert price.input_per_1m_usd == Decimal("1") + + +def test_disabled_pricing_builds_an_empty_table() -> None: + table = build_pricing_table(PricingConfig(enabled=False)) + + assert not table + assert table.lookup("gpt-4o-mini") is None + + +def test_estimate_span_cost_uses_token_counts() -> None: + table = build_pricing_table(PricingConfig()) + + estimate = estimate_span_cost(_span(input_tokens=1000, output_tokens=500), table) + + assert estimate is not None + # 1000 * 0.15/1M + 500 * 0.60/1M + assert estimate.cost_usd == Decimal("0.000450000") + + +def test_estimate_span_cost_never_overwrites_a_provider_cost() -> None: + table = build_pricing_table(PricingConfig()) + span = _span(input_tokens=1000, output_tokens=500, cost_usd=Decimal("9")) + + assert estimate_span_cost(span, table) is None + + +def test_estimate_span_cost_skips_spans_without_usable_tokens() -> None: + table = build_pricing_table(PricingConfig()) + + assert estimate_span_cost(_span(), table) is None + assert estimate_span_cost(_span(input_tokens=0, output_tokens=0), table) is None + + +def test_estimate_span_cost_skips_unpriced_models() -> None: + table = build_pricing_table(PricingConfig()) + span = _span(model_name="internal-router-v2", input_tokens=100, output_tokens=50) + + assert estimate_span_cost(span, table) is None + + +def test_apply_estimated_costs_marks_confidence_and_reports_gaps() -> None: + table = build_pricing_table(PricingConfig()) + spans = [ + _span(span_id="priced", input_tokens=1000, output_tokens=500), + _span( + span_id="unpriced", + model_name="internal-router-v2", + input_tokens=100, + output_tokens=50, + ), + _span(span_id="already-costed", input_tokens=10, cost_usd=Decimal("0.5")), + ] + + updated, result = apply_estimated_costs(spans, table) + by_id = {span.span_id: span for span in updated} + + assert result.estimated_spans == 1 + assert result.estimated_cost_usd == Decimal("0.000450000") + assert result.unpriced_models == ("internal-router-v2",) + assert by_id["priced"].cost_confidence == "estimated" + assert by_id["priced"].cost_usd == Decimal("0.000450000") + assert by_id["unpriced"].cost_usd is None + assert by_id["unpriced"].cost_confidence == "unknown" + assert by_id["already-costed"].cost_usd == Decimal("0.5") + assert by_id["already-costed"].cost_confidence == "source" + + +def test_normalize_estimates_cost_for_token_only_export(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + _import_token_only_fixture(store) + + result = normalize_store(store, pricing=PricingConfig()) + ledger = build_cost_ledger(store) + + # 0.000450000 (gpt-4o-mini) + 0.021000000 (claude-sonnet-4-5) + assert result.estimated_cost_spans == 2 + assert result.estimated_cost_usd == Decimal("0.021450000") + assert result.unpriced_models == ["internal-router-v2"] + assert result.data_quality.spans_with_estimated_cost == 2 + assert result.data_quality.spans_with_source_cost == 0 + assert ledger.ledger_entries == 2 + assert ledger.total_cost_usd == Decimal("0.021450000") + assert ledger.estimated_cost_usd == Decimal("0.021450000") + assert ledger.source_cost_usd == Decimal("0") + + +def test_normalize_leaves_costs_untouched_when_pricing_is_disabled( + tmp_path: Path, +) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + _import_token_only_fixture(store) + + result = normalize_store(store, pricing=PricingConfig(enabled=False)) + ledger = build_cost_ledger(store) + + assert result.estimated_cost_spans == 0 + assert result.estimated_cost_usd == Decimal("0") + assert ledger.ledger_entries == 0 + assert ledger.total_cost_usd == Decimal("0") + + +def test_normalize_is_idempotent_with_estimated_costs(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + _import_token_only_fixture(store) + + first = normalize_store(store, pricing=PricingConfig()) + second = normalize_store(store, pricing=PricingConfig()) + + assert first == second + assert store.stats()["normalized_spans"] == 4 + + +def test_default_config_round_trips_pricing_defaults(tmp_path: Path) -> None: + with runner.isolated_filesystem(temp_dir=tmp_path): + assert runner.invoke(app, ["init"]).exit_code == 0 + config = load_config() + + assert config.pricing.enabled is True + assert config.pricing.use_default_table is True + assert config.pricing.models == [] + + +def test_config_parses_model_prices_without_float_drift(tmp_path: Path) -> None: + config_path = tmp_path / "agentprof.yml" + config_path.write_text( + "pricing:\n" + " models:\n" + " - model: house-model\n" + " input_per_1m_usd: 0.15\n" + " output_per_1m_usd: 0.60\n", + encoding="utf-8", + ) + + config = load_config(config_path) + + assert config.pricing.models[0].input_per_1m_usd == Decimal("0.15") + assert config.pricing.models[0].output_per_1m_usd == Decimal("0.60") + + +def test_cli_normalize_reports_estimated_cost_and_unpriced_models(monkeypatch) -> None: + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") + with runner.isolated_filesystem(): + assert runner.invoke(app, ["init"]).exit_code == 0 + import_result = runner.invoke( + app, + [ + "import", + "langfuse-export", + "--observations", + str(FIXTURES / "langfuse_token_only_observations.json"), + ], + ) + normalize_result = runner.invoke(app, ["normalize"]) + ledger_result = runner.invoke(app, ["cost", "ledger"]) + + assert import_result.exit_code == 0 + assert normalize_result.exit_code == 0 + assert ledger_result.exit_code == 0 + assert "estimated cost for 2 span(s)" in normalize_result.output + assert "No price configured for internal-router-v2" in normalize_result.output + assert "$0.021450000" in ledger_result.output + + +def test_cli_pricing_list_shows_default_and_configured_rates() -> None: + with runner.isolated_filesystem(): + assert runner.invoke(app, ["init"]).exit_code == 0 + Path("agentprof.yml").write_text( + "pricing:\n" + " models:\n" + " - model: house-model\n" + " input_per_1m_usd: '1.00'\n" + " output_per_1m_usd: '2.00'\n", + encoding="utf-8", + ) + result = runner.invoke(app, ["pricing", "list"]) + + assert result.exit_code == 0 + assert "house-model" in result.output + assert "gpt-4o-mini" in result.output + assert "Bundled default rates were recorded on" in result.output + + +def test_cli_pricing_list_reports_disabled_estimation() -> None: + with runner.isolated_filesystem(): + assert runner.invoke(app, ["init"]).exit_code == 0 + Path("agentprof.yml").write_text( + "pricing:\n enabled: false\n", encoding="utf-8" + ) + result = runner.invoke(app, ["pricing", "list"]) + + assert result.exit_code == 0 + assert "Cost estimation is disabled" in result.output + + +def _import_token_only_fixture(store: DuckDBStore) -> None: + from agentprof.ingest.langfuse_export import import_langfuse_export + + config = AgentProfConfig() + config.privacy.hash_inputs = False + import_langfuse_export( + observations_path=FIXTURES / "langfuse_token_only_observations.json", + store=store, + config=config, + ) + + +def _span( + *, + span_id: str = "llm", + model_name: str | None = "gpt-4o-mini", + input_tokens: int | None = None, + output_tokens: int | None = None, + cost_usd: Decimal | None = None, +) -> NormalizedSpan: + return NormalizedSpan( + trace_id="trace", + span_id=span_id, + parent_span_id="root", + source="langfuse", + name="generate", + span_type="llm", + model_name=model_name, + status="ok", + input_tokens=input_tokens, + output_tokens=output_tokens, + cost_usd=cost_usd, + cost_confidence="source" if cost_usd is not None else "unknown", + ) diff --git a/tests/test_report.py b/tests/test_report.py index f76ad54..30cc310 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -860,3 +860,212 @@ def _seed_markdown_injection_issue(store: DuckDBStore) -> None: def _dt(value: str) -> datetime: return datetime.fromisoformat(value).astimezone(UTC) + + +def test_fetch_issues_ranks_expensive_findings_first(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + _seed_ranked_issues(store) + + issues = store.fetch_issues() + + # Ordered by attributed waste, not by the issue_id hash. + assert [issue.issue_id for issue in issues] == [ + "retry_loop:expensive", + "multi_agent_waste:cheap", + "spec_violation:free", + ] + + +def test_fetch_issues_breaks_cost_ties_on_severity(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + store.replace_analysis_results( + issue_kind="spec_violation", + attribution_method="spec_violation", + issues=[ + _ranked_issue("spec_violation:aaa", "spec_violation", "low", Decimal("0")), + _ranked_issue("spec_violation:zzz", "spec_violation", "high", Decimal("0")), + ], + evidence=[], + cost_records=[], + ) + + issues = store.fetch_issues() + + assert [issue.issue_id for issue in issues] == [ + "spec_violation:zzz", + "spec_violation:aaa", + ] + + +def test_generate_report_renders_ranked_top_findings(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + _seed_ranked_issues(store) + + result = generate_report( + store, + project="tracer", + output_dir=tmp_path / "reports", + report_id="ranked-report", + generated_at=_dt("2026-04-26T12:00:00+00:00"), + ) + payload = json.loads(result.report_json_path.read_text(encoding="utf-8")) + markdown = result.report_md_path.read_text(encoding="utf-8") + html = result.report_html_path.read_text(encoding="utf-8") + + assert [finding["issue_id"] for finding in payload["top_findings"]] == [ + "retry_loop:expensive", + "multi_agent_waste:cheap", + "spec_violation:free", + ] + assert payload["summary"]["top_issue_kind"] == "retry_loop" + assert result.top_finding_title == "Costly retry loop" + assert payload["top_findings"][0]["wasted_cost_usd"] == "5.000000000" + assert payload["top_findings"][0]["evidence"]["span_id"] == "span-expensive" + + assert "## Top Findings" in markdown + assert markdown.index("## Top Findings") < markdown.index("## Issues") + assert "Costly retry loop" in markdown + + assert "

    Top Findings

    " in html + assert html.index("Top Findings") < html.index("

    Issues

    ") + + +def test_generate_report_caps_top_findings_at_three(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + store.replace_analysis_results( + issue_kind="spec_violation", + attribution_method="spec_violation", + issues=[ + _ranked_issue( + f"spec_violation:{index}", + "spec_violation", + "medium", + Decimal(index), + ) + for index in range(5) + ], + evidence=[], + cost_records=[], + ) + + result = generate_report( + store, + project="tracer", + output_dir=tmp_path / "reports", + report_id="capped-report", + generated_at=_dt("2026-04-26T12:00:00+00:00"), + ) + payload = json.loads(result.report_json_path.read_text(encoding="utf-8")) + + assert len(payload["top_findings"]) == 3 + assert payload["top_findings"][0]["issue_id"] == "spec_violation:4" + + +def test_generate_report_omits_top_findings_when_no_issues(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + + result = generate_report( + store, + project="tracer", + output_dir=tmp_path / "reports", + report_id="empty-top", + generated_at=_dt("2026-04-26T12:00:00+00:00"), + ) + payload = json.loads(result.report_json_path.read_text(encoding="utf-8")) + markdown = result.report_md_path.read_text(encoding="utf-8") + html = result.report_html_path.read_text(encoding="utf-8") + + assert payload["top_findings"] == [] + assert payload["summary"]["top_issue_kind"] is None + assert result.top_finding_title is None + assert "## Top Findings" not in markdown + assert "

    Top Findings

    " not in html + + +def test_report_records_cost_confidence(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + _seed_retry_issue(store) + store.replace_cost_ledger( + [ + CostLedgerRecord( + trace_id="trace-retry", + span_id="attempt-1", + issue_id=None, + cost_type="failed_span_cost", + amount_usd=Decimal("0.010"), + attribution_method="normalized_span_status", + confidence="estimated", + ) + ], + attribution_method="normalized_span_status", + ) + + result = generate_report( + store, + project="tracer", + output_dir=tmp_path / "reports", + report_id="confidence-report", + generated_at=_dt("2026-04-26T12:00:00+00:00"), + ) + payload = json.loads(result.report_json_path.read_text(encoding="utf-8")) + markdown = result.report_md_path.read_text(encoding="utf-8") + html = result.report_html_path.read_text(encoding="utf-8") + + assert payload["summary"]["costs_by_confidence_usd"] == { + "estimated": "0.010000000", + "source": "0.020000000", + } + assert "| Cost type | Amount | Confidence | Attribution | Issue |" in markdown + assert "Confidence" in html + assert html.count("estimated") == 1 + + +def _seed_ranked_issues(store: DuckDBStore) -> None: + for kind, issue_id, severity, wasted in ( + ("retry_loop", "retry_loop:expensive", "high", Decimal("5")), + ("multi_agent_waste", "multi_agent_waste:cheap", "medium", Decimal("0.001")), + ("spec_violation", "spec_violation:free", "low", Decimal("0")), + ): + issue = _ranked_issue(issue_id, kind, severity, wasted) + store.replace_analysis_results( + issue_kind=kind, + attribution_method=kind, + issues=[issue], + evidence=[ + IssueEvidenceRecord( + issue_id=issue_id, + trace_id="trace-ranked", + span_id=f"span-{issue_id.split(':')[1]}", + evidence_type=kind, + message=f"{kind} evidence line.", + attributes={}, + ) + ], + cost_records=[], + ) + + +def _ranked_issue( + issue_id: str, kind: str, severity: str, wasted: Decimal +) -> IssueRecord: + titles = { + "retry_loop:expensive": "Costly retry loop", + "multi_agent_waste:cheap": "Cheap orchestration overhead", + "spec_violation:free": "Uncosted contract violation", + } + return IssueRecord( + issue_id=issue_id, + kind=kind, + title=titles.get(issue_id, f"{kind} {issue_id}"), + severity=severity, + confidence="high", + first_seen=_dt("2026-04-26T10:00:00+00:00"), + last_seen=_dt("2026-04-26T10:00:02+00:00"), + affected_traces=1, + affected_spans=1, + total_cost_usd=wasted, + wasted_cost_usd=wasted, + potential_savings_usd=wasted, + recommendation=f"Fix the {kind} finding.", + recommended_tests=[], + ) diff --git a/tests/test_retry_loop.py b/tests/test_retry_loop.py index 3d123c8..c6ae1e3 100644 --- a/tests/test_retry_loop.py +++ b/tests/test_retry_loop.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime +from datetime import UTC, datetime from decimal import Decimal from pathlib import Path @@ -318,3 +318,43 @@ def _retry_loop_spans() -> list[NormalizedSpan]: def _dt(value: str) -> datetime: return datetime.fromisoformat(value) + + +def test_retry_loop_severity_escalates_with_repeated_attempts(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + spans = [ + NormalizedSpan( + trace_id="trace-severity", + span_id="root", + parent_span_id=None, + source="langfuse", + name="agent", + span_type="root", + status="error", + ), + *[ + NormalizedSpan( + trace_id="trace-severity", + span_id=f"attempt-{index}", + parent_span_id="root", + source="langfuse", + name="refund_policy_lookup", + span_type="tool", + status="error", + status_message="missing required field region", + error_signature="missing required field region", + input_retry_fingerprint="same-input", + start_time=datetime(2026, 4, 26, 10, 0, index, tzinfo=UTC), + cost_usd=Decimal("0.001"), + cost_confidence="source", + ) + for index in range(1, 7) + ], + ] + store.replace_normalized(spans=spans, traces=build_normalized_traces(spans)) + + result = analyze_retry_loops(store) + issues = store.fetch_issues(kind="retry_loop") + + assert result.findings[0].wasted_attempts == 5 + assert issues[0].severity == "high" From 50b8d245d842e419b2e4f2c4389aa0568b225c61 Mon Sep 17 00:00:00 2001 From: tjdonley <51825276+tjdonley@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:39:44 -0400 Subject: [PATCH 2/4] Protect provider cost and scope the confidence breakdown Addresses Codex review on #27. Never estimate inside a provider-costed branch (P1) --------------------------------------------------- Cost attribution counts only leaf costs, dropping a costed ancestor as soon as a descendant carries cost. Estimating a token-bearing child underneath a provider-costed parent therefore swapped the authoritative amount for the estimate: a $1.00 provider-reported trace with one uncosted 1500-token child collapsed to $0.00045 in trace totals, the ledger, and every analyzer. Estimation now skips any span that shares a trace path with a provider-costed span, walking both up to ancestors and down to descendants. Uncosted sibling branches are still priced, so the fix costs no coverage where provider spend is absent. Blocked spans are also excluded from the unpriced-model hint, which was inviting users to configure rates that would never be used. Scope the confidence breakdown to measured spend (P2) ----------------------------------------------------- costs_by_confidence_usd summed every ledger row, so analyzer attributions that re-point at spend the span ledger already measured were counted a second time as provider-reported, and multi-agent overhead - an analytical estimate, not a token estimate - was reported as estimated spend. The demo's $0.096 of provider-backed span spend was published as $0.114 source plus $0.042 estimated. The field is now restricted to the normalized span ledger, the only attribution method that measures spend rather than re-describing it, and renamed to span_costs_by_confidence_usd to say so. The demo now reports $0.096000000 source and nothing estimated, matching its span ledger exactly. Show only effective rates in pricing list (P2) ---------------------------------------------- PricingTable.prices keeps configured and default entries together so lookup can resolve by specificity, so overriding a bundled model printed both the override and the shadowed default with no indication which one applied. Added PricingTable.effective_prices(), which drops defaults shadowed by an exact configured name, and pointed the command at it. Shadowing stays exact-name only, so configuring gpt-4o-mini does not hide gpt-4o. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Yyb1oKkpxm76dz7xGSjLz --- README.md | 4 +- agentprof/cli.py | 2 +- agentprof/cost/pricing.py | 88 ++++++++++++++++++++++-- agentprof/report/runner.py | 19 ++++-- examples/reports/demo.html | 2 +- examples/reports/demo.json | 11 ++- examples/reports/demo.md | 2 +- tests/test_pricing.py | 134 ++++++++++++++++++++++++++++++++++++- tests/test_report.py | 6 +- 9 files changed, 243 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index f2b9bae..c18f3d3 100644 --- a/README.md +++ b/README.md @@ -369,7 +369,9 @@ Estimated spend is always labelled rather than blended into provider-reported co - `agentprof normalize` reports estimated span coverage and names any model it had tokens for but no price. - `agentprof cost ledger` splits provider-reported cost from estimated cost. -- Reports carry a `confidence` column per ledger row and a `costs_by_confidence_usd` summary field. +- Reports carry a `confidence` column per ledger row and a `span_costs_by_confidence_usd` summary field. That field covers the normalized span ledger only, because analyzer attributions re-point at spend the span ledger already measured and carry their own notion of confidence. + +Provider-reported spend is never displaced. A span is left alone both when it already carries a cost and when it shares a trace path with a span that does, because cost attribution counts only leaf costs: estimating a token-bearing child underneath a provider-costed parent would drop the parent's authoritative amount in favour of the estimate. The bundled rates are public list prices recorded on a fixed date, and provider pricing changes. Treat estimates as a starting point and override the models that matter to you before acting on a total. diff --git a/agentprof/cli.py b/agentprof/cli.py index 9fbe097..86d1120 100644 --- a/agentprof/cli.py +++ b/agentprof/cli.py @@ -699,7 +699,7 @@ def pricing_list() -> None: console.print(" Spans without provider cost fields stay unattributed.") return - table_prices = build_pricing_table(config.pricing).prices + table_prices = build_pricing_table(config.pricing).effective_prices() if not table_prices: console.print("No model prices are configured.") console.print( diff --git a/agentprof/cost/pricing.py b/agentprof/cost/pricing.py index bbee760..b044c1d 100644 --- a/agentprof/cost/pricing.py +++ b/agentprof/cost/pricing.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections import defaultdict from collections.abc import Sequence from dataclasses import dataclass, field from decimal import ROUND_HALF_UP, Decimal @@ -79,6 +80,20 @@ class PricingTable: def __bool__(self) -> bool: return bool(self.prices) + def effective_prices(self) -> tuple[ModelPrice, ...]: + """The rates `lookup` can actually return, with shadowed defaults dropped.""" + + configured = { + price.model + for price in self.prices + if price.source == CONFIG_PRICE_SOURCE + } + return tuple( + price + for price in self.prices + if price.source == CONFIG_PRICE_SOURCE or price.model not in configured + ) + def lookup(self, model_name: str | None) -> ModelPrice | None: if not model_name: return None @@ -170,13 +185,15 @@ def apply_estimated_costs( ) -> tuple[list[NormalizedSpan], CostEstimationResult]: """Fill in missing span costs from token counts. - Spans that already carry a provider cost are never touched, so estimation - can only add attribution where there was none. + Provider-reported spend is never displaced: a span is skipped both when it + already carries a cost and when it shares a trace path with a span that + does. See `_provider_costed_branch_keys` for why the second rule matters. """ + blocked = _provider_costed_branch_keys(spans) if not table: return list(spans), CostEstimationResult( - unpriced_models=_unpriced_model_names(spans, table) + unpriced_models=_unpriced_model_names(spans, table, blocked) ) updated: list[NormalizedSpan] = [] @@ -185,6 +202,10 @@ def apply_estimated_costs( priced_models: set[str] = set() for span in spans: + if _span_key(span) in blocked: + updated.append(span) + continue + estimate = estimate_span_cost(span, table) if estimate is None: updated.append(span) @@ -206,13 +227,68 @@ def apply_estimated_costs( return updated, CostEstimationResult( estimated_spans=estimated_spans, estimated_cost_usd=estimated_cost, - unpriced_models=_unpriced_model_names(spans, table), + unpriced_models=_unpriced_model_names(spans, table, blocked), priced_models=tuple(sorted(priced_models)), ) +def _provider_costed_branch_keys( + spans: Sequence[NormalizedSpan], +) -> set[tuple[str, str]]: + """Spans on a trace path that already carries provider-reported cost. + + Cost attribution counts only leaf costs, dropping a costed ancestor as soon + as a descendant carries cost. Estimating a token-bearing child underneath a + provider-costed parent would therefore silently swap the authoritative + amount for an estimate, so nothing on such a path is estimated. + """ + + by_key = {_span_key(span): span for span in spans} + children: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list) + for span in spans: + if span.parent_span_id is not None: + children[(span.trace_id, span.parent_span_id)].append(_span_key(span)) + + blocked: set[tuple[str, str]] = set() + for span in spans: + if span.cost_usd is None: + continue + + key = _span_key(span) + blocked.add(key) + + parent_id = span.parent_span_id + seen: set[str] = set() + while ( + parent_id + and (span.trace_id, parent_id) in by_key + and parent_id not in seen + ): + seen.add(parent_id) + blocked.add((span.trace_id, parent_id)) + parent_id = by_key[(span.trace_id, parent_id)].parent_span_id + + stack = list(children.get(key, ())) + visited: set[tuple[str, str]] = set() + while stack: + current = stack.pop() + if current in visited: + continue + visited.add(current) + blocked.add(current) + stack.extend(children.get(current, ())) + + return blocked + + +def _span_key(span: NormalizedSpan) -> tuple[str, str]: + return span.trace_id, span.span_id + + def _unpriced_model_names( - spans: Sequence[NormalizedSpan], table: PricingTable + spans: Sequence[NormalizedSpan], + table: PricingTable, + blocked: set[tuple[str, str]], ) -> tuple[str, ...]: """Models that had usable token counts but no matching price.""" @@ -220,6 +296,8 @@ def _unpriced_model_names( for span in spans: if span.cost_usd is not None or not span.model_name: continue + if _span_key(span) in blocked: + continue if (span.input_tokens or 0) <= 0 and (span.output_tokens or 0) <= 0: continue if table.lookup(span.model_name) is None: diff --git a/agentprof/report/runner.py b/agentprof/report/runner.py index 8de22a1..9d672bd 100644 --- a/agentprof/report/runner.py +++ b/agentprof/report/runner.py @@ -10,6 +10,7 @@ from typing import Any from agentprof.config import APP_DIR +from agentprof.cost.runner import LEDGER_ATTRIBUTION_METHOD from agentprof.report.schema import ReportBuildResult from agentprof.store.duckdb_store import ( CostLedgerRecord, @@ -130,11 +131,17 @@ def _summary( issue_kinds = Counter(issue.kind for issue in issues) severities = Counter(issue.severity for issue in issues) cost_types = defaultdict(Decimal) - cost_confidences = defaultdict(Decimal) + span_cost_confidences = defaultdict(Decimal) for record in costs: - if record.amount_usd is not None: - cost_types[record.cost_type] += record.amount_usd - cost_confidences[record.confidence] += record.amount_usd + if record.amount_usd is None: + continue + cost_types[record.cost_type] += record.amount_usd + # Only the span ledger measures spend. Analyzer attributions re-point at + # the same spend and carry their own notion of confidence, so folding + # them in here would double count and blur token estimates together + # with analytical estimates such as multi-agent overhead. + if record.attribution_method == LEDGER_ATTRIBUTION_METHOD: + span_cost_confidences[record.confidence] += record.amount_usd return { "generated_at": _datetime_to_json(generated_at), @@ -153,9 +160,9 @@ def _summary( cost_type: _decimal_to_json(amount) for cost_type, amount in sorted(cost_types.items()) }, - "costs_by_confidence_usd": { + "span_costs_by_confidence_usd": { confidence: _decimal_to_json(amount) - for confidence, amount in sorted(cost_confidences.items()) + for confidence, amount in sorted(span_cost_confidences.items()) }, "top_issue_kind": issues[0].kind if issues else None, "artifacts": dict(sorted(artifacts.items())), diff --git a/examples/reports/demo.html b/examples/reports/demo.html index 7507aaf..287169f 100644 --- a/examples/reports/demo.html +++ b/examples/reports/demo.html @@ -250,7 +250,7 @@

    AgentProf Demo

    Report ID
    demo
    -
    Generated
    2026-08-12T16:26:28.684493Z
    +
    Generated
    2026-08-12T16:39:23.702668Z
    diff --git a/examples/reports/demo.json b/examples/reports/demo.json index 1d31177..4a89376 100644 --- a/examples/reports/demo.json +++ b/examples/reports/demo.json @@ -73,7 +73,7 @@ "trace_id": "trace-multi-agent-1" } ], - "generated_at": "2026-08-12T16:26:28.684493Z", + "generated_at": "2026-08-12T16:39:23.702668Z", "issues": [ { "affected_spans": 5, @@ -251,10 +251,6 @@ "multi_agent_waste_svg": "demo-multi-agent-waste.svg" }, "cost_entry_count": 8, - "costs_by_confidence_usd": { - "estimated": "0.042000000", - "source": "0.114000000" - }, "costs_by_type_usd": { "estimated_multi_agent_overhead": "0.042000000", "failed_span_cost": "0.012000000", @@ -263,7 +259,7 @@ "wasted_spec_violation_cost": "0.012000000" }, "evidence_count": 5, - "generated_at": "2026-08-12T16:26:28.684493Z", + "generated_at": "2026-08-12T16:39:23.702668Z", "gross_potential_savings_usd": "0.060000000", "gross_wasted_cost_usd": "0.060000000", "issue_count": 4, @@ -277,6 +273,9 @@ }, "overlapping_potential_savings_usd": "0.006000000", "overlapping_wasted_cost_usd": "0.006000000", + "span_costs_by_confidence_usd": { + "source": "0.096000000" + }, "top_issue_kind": "multi_agent_waste", "total_potential_savings_usd": "0.054000000", "total_wasted_cost_usd": "0.054000000" diff --git a/examples/reports/demo.md b/examples/reports/demo.md index 2aa3ba4..d108219 100644 --- a/examples/reports/demo.md +++ b/examples/reports/demo.md @@ -1,7 +1,7 @@ # AgentProf Report: AgentProf Demo Report ID: `demo` -Generated at: `2026-08-12T16:26:28.684493Z` +Generated at: `2026-08-12T16:39:23.702668Z` ## Summary diff --git a/tests/test_pricing.py b/tests/test_pricing.py index fc99337..0554e99 100644 --- a/tests/test_pricing.py +++ b/tests/test_pricing.py @@ -18,7 +18,7 @@ estimate_span_cost, ) from agentprof.cost.runner import build_cost_ledger -from agentprof.normalize.runner import normalize_store +from agentprof.normalize.runner import build_normalized_traces, normalize_store from agentprof.normalize.schema import NormalizedSpan from agentprof.store.duckdb_store import DuckDBStore @@ -303,6 +303,7 @@ def _import_token_only_fixture(store: DuckDBStore) -> None: def _span( *, span_id: str = "llm", + parent_span_id: str | None = "root", model_name: str | None = "gpt-4o-mini", input_tokens: int | None = None, output_tokens: int | None = None, @@ -311,7 +312,7 @@ def _span( return NormalizedSpan( trace_id="trace", span_id=span_id, - parent_span_id="root", + parent_span_id=parent_span_id, source="langfuse", name="generate", span_type="llm", @@ -322,3 +323,132 @@ def _span( cost_usd=cost_usd, cost_confidence="source" if cost_usd is not None else "unknown", ) + + +def test_estimation_never_displaces_a_provider_costed_ancestor() -> None: + table = build_pricing_table(PricingConfig()) + parent = _span( + span_id="parent", + parent_span_id=None, + model_name=None, + cost_usd=Decimal("1.00"), + ) + child = _span(span_id="child", parent_span_id="parent", input_tokens=1000, output_tokens=500) + + updated, result = apply_estimated_costs([parent, child], table) + trace = build_normalized_traces(updated)[0] + + # Cost attribution drops a costed ancestor once a descendant is costed, so + # estimating the child here would replace $1.00 of provider spend with an + # estimate two orders of magnitude smaller. + assert result.estimated_spans == 0 + assert trace.total_cost_usd == Decimal("1.00") + assert {span.span_id: span.cost_usd for span in updated} == { + "parent": Decimal("1.00"), + "child": None, + } + + +def test_estimation_skips_spans_above_a_provider_costed_descendant() -> None: + table = build_pricing_table(PricingConfig()) + parent = _span(span_id="parent", parent_span_id=None, input_tokens=1000, output_tokens=500) + child = _span( + span_id="child", + parent_span_id="parent", + model_name=None, + cost_usd=Decimal("0.75"), + ) + + updated, result = apply_estimated_costs([parent, child], table) + trace = build_normalized_traces(updated)[0] + + assert result.estimated_spans == 0 + assert trace.total_cost_usd == Decimal("0.75") + + +def test_estimation_still_prices_sibling_branches_without_provider_cost() -> None: + table = build_pricing_table(PricingConfig()) + root = _span(span_id="root", parent_span_id=None, model_name=None) + costed = _span( + span_id="costed-branch", + parent_span_id="root", + model_name=None, + cost_usd=Decimal("0.50"), + ) + uncosted = _span( + span_id="token-branch", + parent_span_id="root", + input_tokens=1000, + output_tokens=500, + ) + + updated, result = apply_estimated_costs([root, costed, uncosted], table) + by_id = {span.span_id: span for span in updated} + + # The root is blocked because it sits above provider spend, but an + # uncosted sibling branch is still fair game. + assert result.estimated_spans == 1 + assert by_id["token-branch"].cost_usd == Decimal("0.000450000") + assert by_id["root"].cost_usd is None + assert build_normalized_traces(updated)[0].total_cost_usd == Decimal("0.500450000") + + +def test_blocked_spans_are_not_reported_as_unpriced_models() -> None: + table = build_pricing_table(PricingConfig()) + parent = _span( + span_id="parent", + parent_span_id=None, + model_name=None, + cost_usd=Decimal("1.00"), + ) + child = _span( + span_id="child", + parent_span_id="parent", + model_name="internal-router-v2", + input_tokens=100, + output_tokens=50, + ) + + _, result = apply_estimated_costs([parent, child], table) + + assert result.unpriced_models == () + + +def test_effective_prices_drop_shadowed_defaults() -> None: + table = build_pricing_table( + PricingConfig( + models=[ + ModelPriceConfig( + model="gpt-4o-mini", + input_per_1m_usd=Decimal("1"), + output_per_1m_usd=Decimal("2"), + ) + ] + ) + ) + + effective = [price for price in table.effective_prices() if price.model == "gpt-4o-mini"] + + assert len(effective) == 1 + assert effective[0].source == "config" + # A configured entry only shadows its exact name. + assert any(price.model == "gpt-4o" for price in table.effective_prices()) + + +def test_cli_pricing_list_shows_one_row_per_overridden_model() -> None: + with runner.isolated_filesystem(): + assert runner.invoke(app, ["init"]).exit_code == 0 + Path("agentprof.yml").write_text( + "pricing:\n" + " models:\n" + " - model: gpt-4o-mini\n" + " input_per_1m_usd: '9.99'\n" + " output_per_1m_usd: '8.88'\n", + encoding="utf-8", + ) + result = runner.invoke(app, ["pricing", "list"]) + + assert result.exit_code == 0 + assert "9.99" in result.output + # The shadowed bundled rate must not appear alongside the override. + assert "0.15" not in result.output diff --git a/tests/test_report.py b/tests/test_report.py index 30cc310..2e26bcf 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1011,9 +1011,11 @@ def test_report_records_cost_confidence(tmp_path: Path) -> None: markdown = result.report_md_path.read_text(encoding="utf-8") html = result.report_html_path.read_text(encoding="utf-8") - assert payload["summary"]["costs_by_confidence_usd"] == { + # Only the span ledger counts. The retry analyzer's $0.020 attribution + # re-points at spend the span ledger already measured, so folding it in + # would double count it as provider-reported. + assert payload["summary"]["span_costs_by_confidence_usd"] == { "estimated": "0.010000000", - "source": "0.020000000", } assert "| Cost type | Amount | Confidence | Attribution | Issue |" in markdown assert "Confidence" in html From 91d7501fa39247bd14199ecf5fd73775068ea92d Mon Sep 17 00:00:00 2001 From: tjdonley <51825276+tjdonley@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:47:57 -0400 Subject: [PATCH 3/4] Price only real version suffixes and reject negative token counts Addresses the second Codex review round on #27. Named variants no longer inherit a base model rate -------------------------------------------------- Boundary matching stopped `gpt-4` pricing a `gpt-4o` span, but not `gpt-4o` pricing a `gpt-4o-mini` span, because `-mini` starts with a delimiter like any version suffix. Against a table configuring only `gpt-4o`, a `gpt-4o-mini` span resolved to $2.50/$10.00 per 1M instead of $0.15/$0.60, overstating its input cost by more than sixteen times. A suffix now has to look like a version - a digit, or `v` followed by a digit - so date and release suffixes still resolve (`gpt-4o-mini-2024-07-18`, `claude-opus-4-1-20250805`, `gemini-2.0-flash-001`) while named variants such as `-mini`, `-nano`, and `-turbo` do not. Unmatched models fall through to the unpriced-model hint, so a missing rate surfaces as a prompt to configure one rather than as a confidently wrong number. This also corrects a wrong expectation in the original tests, which asserted that `gpt-4-turbo` should inherit the `gpt-4` rate. It is a separate model priced differently and now stays unpriced until configured. Negative token counts are unusable, not cheap --------------------------------------------- Nothing constrains the sign of imported token counts, and the usable-token check only rejected spans where both counts were non-positive. A span with -100000 input and 10 output tokens priced out at -$0.014994, putting a negative amount into the ledger, into trace totals, and now into the issue ranking this PR introduced. Spans carrying a negative count on either component are left unpriced, and excluded from the unpriced-model hint since the problem is the usage record rather than a missing rate. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Yyb1oKkpxm76dz7xGSjLz --- README.md | 4 +- agentprof/cost/pricing.py | 50 ++++++++++++++++++----- tests/test_pricing.py | 85 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 128 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c18f3d3..d468221 100644 --- a/README.md +++ b/README.md @@ -361,7 +361,9 @@ pricing: ``` - Rates are USD per 1,000,000 tokens. Quote them as strings to avoid float rounding. -- `model` matches the normalized model name exactly, or as a version-boundary prefix, so `gpt-4o-mini` prices `gpt-4o-mini-2024-07-18` while `gpt-4` never prices a `gpt-4o` span. Longer matches win, and configured entries beat bundled defaults. +- `model` matches the normalized model name exactly, or as a prefix followed by a version or date suffix, so `gpt-4o-mini` prices `gpt-4o-mini-2024-07-18` and `claude-opus-4-1` prices `claude-opus-4-1-20250805`. Longer matches win, and configured entries beat bundled defaults. +- A named variant never inherits a base model's rate: `gpt-4o` does not price a `gpt-4o-mini` span, and `gpt-4` does not price `gpt-4o` or `gpt-4-turbo`. Those models are reported as unpriced instead, so a missing rate shows up as a prompt to configure one rather than as a wrong number. +- Spans with a negative token count are left unpriced, since a negative component would subtract from the other and can drive a cost below zero. - A provider prefix such as `anthropic/claude-sonnet-4-5` is matched with and without the prefix. - Set `use_default_table: false` to price only from your own entries, or `enabled: false` to turn estimation off entirely. diff --git a/agentprof/cost/pricing.py b/agentprof/cost/pricing.py index b044c1d..b9ef066 100644 --- a/agentprof/cost/pricing.py +++ b/agentprof/cost/pricing.py @@ -158,14 +158,11 @@ def estimate_span_cost( ) -> SpanCostEstimate | None: """Price one span from its token counts, or return None if it can't be priced.""" - if span.cost_usd is not None: + if span.cost_usd is not None or not _has_usable_tokens(span): return None input_tokens = span.input_tokens or 0 output_tokens = span.output_tokens or 0 - if input_tokens <= 0 and output_tokens <= 0: - return None - price = table.lookup(span.model_name) if price is None: return None @@ -285,6 +282,22 @@ def _span_key(span: NormalizedSpan) -> tuple[str, str]: return span.trace_id, span.span_id +def _has_usable_tokens(span: NormalizedSpan) -> bool: + """Whether token counts can price this span. + + Nothing constrains the sign of imported token counts, and a negative count + on one component subtracts from the other, which can understate a span or + drive its cost below zero. A negative count means the usage record is + unusable, not cheap, so the span is left unpriced. + """ + + input_tokens = span.input_tokens or 0 + output_tokens = span.output_tokens or 0 + if input_tokens < 0 or output_tokens < 0: + return False + return input_tokens > 0 or output_tokens > 0 + + def _unpriced_model_names( spans: Sequence[NormalizedSpan], table: PricingTable, @@ -296,9 +309,7 @@ def _unpriced_model_names( for span in spans: if span.cost_usd is not None or not span.model_name: continue - if _span_key(span) in blocked: - continue - if (span.input_tokens or 0) <= 0 and (span.output_tokens or 0) <= 0: + if _span_key(span) in blocked or not _has_usable_tokens(span): continue if table.lookup(span.model_name) is None: names.add(_normalize_model_name(span.model_name)) @@ -315,12 +326,33 @@ def _candidate_keys(model_name: str) -> tuple[str, ...]: def _prefix_matches(key: str, model: str) -> bool: - """Match on a version-boundary so `gpt-4` never prices a `gpt-4o` span.""" + """Match a priced model against its own release and date suffixes only. + + A prefix match exists to price `gpt-4o-mini-2024-07-18` from `gpt-4o-mini`. + It must not price `gpt-4o-mini` from `gpt-4o`, which is a different model + an order of magnitude more expensive. So the suffix has to look like a + version - a digit, or a `v` followed by a digit - and never a named variant + such as `-mini`, `-nano`, or `-turbo`. Unmatched models are reported as + unpriced, which asks the user for a rate instead of inventing a wrong one. + """ if not key.startswith(model): return False + remainder = key[len(model) :] - return not remainder or remainder[0] in BOUNDARY_CHARS + if not remainder: + return True + if remainder[0] not in BOUNDARY_CHARS: + return False + return _is_version_suffix(remainder[1:]) + + +def _is_version_suffix(suffix: str) -> bool: + if not suffix: + return False + if suffix[0].isdigit(): + return True + return suffix[0] in "vV" and len(suffix) > 1 and suffix[1].isdigit() def _normalize_model_name(model_name: str) -> str: diff --git a/tests/test_pricing.py b/tests/test_pricing.py index 0554e99..0ca09e4 100644 --- a/tests/test_pricing.py +++ b/tests/test_pricing.py @@ -63,8 +63,11 @@ def test_lookup_respects_version_boundaries() -> None: # `gpt-4o` is a different model and must not inherit the `gpt-4` rate. assert table.lookup("gpt-4o") is None - assert table.lookup("gpt-4-turbo") is not None + # Nor is `gpt-4-turbo` a version of `gpt-4`; it is a separate model priced + # differently, so it stays unpriced until a rate is configured for it. + assert table.lookup("gpt-4-turbo") is None assert table.lookup("gpt-4") is not None + assert table.lookup("gpt-4-0613") is not None def test_lookup_strips_a_provider_prefix() -> None: @@ -452,3 +455,83 @@ def test_cli_pricing_list_shows_one_row_per_overridden_model() -> None: assert "9.99" in result.output # The shadowed bundled rate must not appear alongside the override. assert "0.15" not in result.output + + +def test_named_variants_never_inherit_a_base_model_rate() -> None: + table = build_pricing_table( + PricingConfig( + use_default_table=False, + models=[ + ModelPriceConfig( + model="gpt-4o", + input_per_1m_usd=Decimal("2.50"), + output_per_1m_usd=Decimal("10.00"), + ) + ], + ) + ) + + # gpt-4o-mini costs a fraction of gpt-4o. Inheriting the base rate would + # overstate it by more than an order of magnitude, so it stays unpriced + # and gets reported as a model needing a configured rate. + assert table.lookup("gpt-4o-mini") is None + assert table.lookup("gpt-4o-nano") is None + assert table.lookup("gpt-4o-turbo") is None + + +def test_version_and_date_suffixes_still_match_their_base_model() -> None: + table = build_pricing_table(PricingConfig()) + + for model_name, expected in ( + ("gpt-4o-mini-2024-07-18", "gpt-4o-mini"), + ("gpt-4o-2024-11-20", "gpt-4o"), + ("claude-3-5-sonnet-20241022", "claude-3-5-sonnet"), + ("claude-opus-4-1-20250805", "claude-opus-4-1"), + ("gemini-2.0-flash-001", "gemini-2.0-flash"), + ): + price = table.lookup(model_name) + assert price is not None, model_name + assert price.model == expected, model_name + + +def test_version_suffixes_accept_a_v_prefix() -> None: + table = build_pricing_table( + PricingConfig( + use_default_table=False, + models=[ + ModelPriceConfig( + model="house-model", + input_per_1m_usd=Decimal("1"), + output_per_1m_usd=Decimal("2"), + ) + ], + ) + ) + + assert table.lookup("house-model-v2") is not None + assert table.lookup("house-model-v1.5") is not None + assert table.lookup("house-model-experimental") is None + + +def test_negative_token_counts_are_never_priced() -> None: + table = build_pricing_table(PricingConfig()) + + # A negative count subtracts from the other component and can drive a span + # below zero, which would understate spend and distort issue ranking. + assert estimate_span_cost(_span(input_tokens=-1000, output_tokens=500), table) is None + assert estimate_span_cost(_span(input_tokens=-100000, output_tokens=10), table) is None + assert estimate_span_cost(_span(input_tokens=1000, output_tokens=-500), table) is None + assert estimate_span_cost(_span(input_tokens=-1, output_tokens=-1), table) is None + + +def test_spans_with_negative_tokens_are_not_reported_as_unpriced() -> None: + table = build_pricing_table(PricingConfig()) + span = _span(model_name="internal-router-v2", input_tokens=-100, output_tokens=50) + + updated, result = apply_estimated_costs([span], table) + + # The usage record is unusable, so this is not a missing-rate problem and + # must not send the user off to configure one. + assert result.estimated_spans == 0 + assert result.unpriced_models == () + assert updated[0].cost_usd is None From fb842e583e61932f71c0861451bddfecd08eeec6 Mon Sep 17 00:00:00 2001 From: tjdonley <51825276+tjdonley@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:55:44 -0400 Subject: [PATCH 4/4] Restrict prefix pricing to build stamps and leaf-attribute the estimate Addresses the third Codex review round on #27. Numeric releases no longer inherit a base rate ---------------------------------------------- Requiring a digit-leading suffix stopped `-mini` and `-turbo`, but still accepted the short numeric components that mark a separate release. Against a table configuring `gpt-4`, both `gpt-4.1` and `gpt-4.1-mini` resolved to the `gpt-4` rate - $30/$60 per 1M against gpt-4.1's $2.00/$8.00, a fifteenfold overstatement - and a configured `claude-sonnet-4` absorbed `claude-sonnet-4-5`. A suffix now has to be a date or a build number: `\d{4}-\d{2}-\d{2}` or three or more digits. Dated releases still resolve without configuration (`gpt-4o-mini-2024-07-18`, `claude-3-5-sonnet-20241022`, `claude-opus-4-1-20250805`, `gemini-2.0-flash-001`, `gpt-4-0613`), while short components such as `.1` and `-5` do not. This drops the `v2` form accepted in the previous round, which had the same ambiguity: a `v` bump is a new release, not a build of the existing one. Report the estimate the ledger will carry ----------------------------------------- When an estimated span sat above another estimated span, both amounts were added to the reported total while trace and ledger rollups keep leaf costs only, so `agentprof normalize` announced more estimated spend than the import actually produced - $0.001350000 against a $0.000900000 ledger. The returned total now applies the same leaf-only rule as the persisted rollups. Spans still each carry their own estimate in normalized_spans; only the reported total is leaf-attributed, and a test asserts it equals both the trace total and the ledger total for a nested pair. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Yyb1oKkpxm76dz7xGSjLz --- README.md | 2 +- agentprof/cost/pricing.py | 80 ++++++++++++++++++++++++++++----------- tests/test_pricing.py | 54 ++++++++++++++++++++++---- 3 files changed, 106 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index d468221..34eb6a9 100644 --- a/README.md +++ b/README.md @@ -362,7 +362,7 @@ pricing: - Rates are USD per 1,000,000 tokens. Quote them as strings to avoid float rounding. - `model` matches the normalized model name exactly, or as a prefix followed by a version or date suffix, so `gpt-4o-mini` prices `gpt-4o-mini-2024-07-18` and `claude-opus-4-1` prices `claude-opus-4-1-20250805`. Longer matches win, and configured entries beat bundled defaults. -- A named variant never inherits a base model's rate: `gpt-4o` does not price a `gpt-4o-mini` span, and `gpt-4` does not price `gpt-4o` or `gpt-4-turbo`. Those models are reported as unpriced instead, so a missing rate shows up as a prompt to configure one rather than as a wrong number. +- Only a date or build stamp counts as the same model. A separately priced release does not inherit a base rate: `gpt-4` does not price `gpt-4.1`, `gpt-4o` does not price `gpt-4o-mini`, and `claude-sonnet-4` does not price `claude-sonnet-4-5`. Those models are reported as unpriced instead, so a missing rate shows up as a prompt to configure one rather than as a wrong number. - Spans with a negative token count are left unpriced, since a negative component would subtract from the other and can drive a cost below zero. - A provider prefix such as `anthropic/claude-sonnet-4-5` is matched with and without the prefix. - Set `use_default_table: false` to price only from your own entries, or `enabled: false` to turn estimation off entirely. diff --git a/agentprof/cost/pricing.py b/agentprof/cost/pricing.py index b9ef066..5e3f62f 100644 --- a/agentprof/cost/pricing.py +++ b/agentprof/cost/pricing.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from collections import defaultdict from collections.abc import Sequence from dataclasses import dataclass, field @@ -15,6 +16,10 @@ DEFAULT_PRICE_SOURCE = "default" PRICE_SOURCE_RANK = {CONFIG_PRICE_SOURCE: 0, DEFAULT_PRICE_SOURCE: 1} BOUNDARY_CHARS = frozenset("-_.:@/ ") +# A dated or numbered build of the same model: `20250805`, `2024-07-18`, `001`. +# Deliberately excludes short components such as `1` or `5`, which mark a new +# release that is priced on its own. +BUILD_STAMP_RE = re.compile(r"\d{4}-\d{2}-\d{2}|\d{3,}") # Published list prices in USD per million tokens, recorded on this date. They # drift: treat estimates built from this table as a starting point and override @@ -194,8 +199,7 @@ def apply_estimated_costs( ) updated: list[NormalizedSpan] = [] - estimated_spans = 0 - estimated_cost = Decimal("0") + estimated_keys: set[tuple[str, str]] = set() priced_models: set[str] = set() for span in spans: @@ -216,19 +220,54 @@ def apply_estimated_costs( } ) ) - estimated_spans += 1 - estimated_cost += estimate.cost_usd + estimated_keys.add(_span_key(span)) if span.model_name: priced_models.add(_normalize_model_name(span.model_name)) + # Report what the ledger will actually carry. Trace and ledger rollups keep + # leaf costs only, so an estimated span with an estimated descendant is + # dropped there, and counting it here would overstate the import. + leaf_keys = _cost_leaf_keys(updated) + estimated_cost = sum( + ( + span.cost_usd + for span in updated + if span.cost_usd is not None + and _span_key(span) in estimated_keys + and _span_key(span) in leaf_keys + ), + Decimal("0"), + ) + return updated, CostEstimationResult( - estimated_spans=estimated_spans, + estimated_spans=len(estimated_keys), estimated_cost_usd=estimated_cost, unpriced_models=_unpriced_model_names(spans, table, blocked), priced_models=tuple(sorted(priced_models)), ) +def _cost_leaf_keys(spans: Sequence[NormalizedSpan]) -> set[tuple[str, str]]: + """Costed spans that survive the leaf-only rollup used by cost attribution.""" + + by_key = {_span_key(span): span for span in spans} + costed = [span for span in spans if span.cost_usd is not None] + ancestors: set[tuple[str, str]] = set() + for span in costed: + parent_id = span.parent_span_id + seen: set[str] = set() + while ( + parent_id + and (span.trace_id, parent_id) in by_key + and parent_id not in seen + ): + seen.add(parent_id) + ancestors.add((span.trace_id, parent_id)) + parent_id = by_key[(span.trace_id, parent_id)].parent_span_id + + return {_span_key(span) for span in costed if _span_key(span) not in ancestors} + + def _provider_costed_branch_keys( spans: Sequence[NormalizedSpan], ) -> set[tuple[str, str]]: @@ -326,14 +365,19 @@ def _candidate_keys(model_name: str) -> tuple[str, ...]: def _prefix_matches(key: str, model: str) -> bool: - """Match a priced model against its own release and date suffixes only. - - A prefix match exists to price `gpt-4o-mini-2024-07-18` from `gpt-4o-mini`. - It must not price `gpt-4o-mini` from `gpt-4o`, which is a different model - an order of magnitude more expensive. So the suffix has to look like a - version - a digit, or a `v` followed by a digit - and never a named variant - such as `-mini`, `-nano`, or `-turbo`. Unmatched models are reported as - unpriced, which asks the user for a rate instead of inventing a wrong one. + """Match a priced model against its own build and date stamps only. + + A prefix match exists so `gpt-4o-mini` prices `gpt-4o-mini-2024-07-18` + without the user configuring every dated release. Everything else is a + different model that happens to share a prefix, and pricing it from the + base rate invents a number: `gpt-4o-mini` costs a fraction of `gpt-4o`, + and `gpt-4.1` a fraction of `gpt-4`. + + Only a date or a build number is treated as the same model. A short + numeric component is a release of its own - `gpt-4` to `gpt-4.1`, + `claude-sonnet-4` to `claude-sonnet-4-5` - and is rejected along with + named variants like `-mini` and `-turbo`. Rejected models surface in the + unpriced-model hint, which asks for a rate instead of guessing one. """ if not key.startswith(model): @@ -344,15 +388,7 @@ def _prefix_matches(key: str, model: str) -> bool: return True if remainder[0] not in BOUNDARY_CHARS: return False - return _is_version_suffix(remainder[1:]) - - -def _is_version_suffix(suffix: str) -> bool: - if not suffix: - return False - if suffix[0].isdigit(): - return True - return suffix[0] in "vV" and len(suffix) > 1 and suffix[1].isdigit() + return BUILD_STAMP_RE.fullmatch(remainder[1:]) is not None def _normalize_model_name(model_name: str) -> str: diff --git a/tests/test_pricing.py b/tests/test_pricing.py index 0ca09e4..9e74bc0 100644 --- a/tests/test_pricing.py +++ b/tests/test_pricing.py @@ -494,22 +494,32 @@ def test_version_and_date_suffixes_still_match_their_base_model() -> None: assert price.model == expected, model_name -def test_version_suffixes_accept_a_v_prefix() -> None: +def test_numeric_releases_do_not_inherit_a_base_model_rate() -> None: table = build_pricing_table( PricingConfig( use_default_table=False, models=[ ModelPriceConfig( - model="house-model", - input_per_1m_usd=Decimal("1"), - output_per_1m_usd=Decimal("2"), - ) + model="gpt-4", + input_per_1m_usd=Decimal("30"), + output_per_1m_usd=Decimal("60"), + ), + ModelPriceConfig( + model="claude-sonnet-4", + input_per_1m_usd=Decimal("3"), + output_per_1m_usd=Decimal("15"), + ), ], ) ) - assert table.lookup("house-model-v2") is not None - assert table.lookup("house-model-v1.5") is not None + # A short numeric component marks a separately priced release, not a build + # of the same model: gpt-4.1 costs a fraction of gpt-4. + assert table.lookup("gpt-4.1") is None + assert table.lookup("gpt-4.1-mini") is None + assert table.lookup("claude-sonnet-4-5") is None + # A build or date stamp is still the same model. + assert table.lookup("gpt-4-0613") is not None assert table.lookup("house-model-experimental") is None @@ -535,3 +545,33 @@ def test_spans_with_negative_tokens_are_not_reported_as_unpriced() -> None: assert result.estimated_spans == 0 assert result.unpriced_models == () assert updated[0].cost_usd is None + + +def test_reported_estimate_matches_the_leaf_rollup(tmp_path: Path) -> None: + table = build_pricing_table(PricingConfig()) + parent = _span(span_id="parent", parent_span_id=None, input_tokens=1000, output_tokens=500) + child = _span(span_id="child", parent_span_id="parent", input_tokens=2000, output_tokens=1000) + + updated, result = apply_estimated_costs([parent, child], table) + trace = build_normalized_traces(updated)[0] + + # Both spans are priced, but trace and ledger rollups keep leaf costs only, + # so the reported total has to drop the ancestor the same way. + assert result.estimated_spans == 2 + assert result.estimated_cost_usd == trace.total_cost_usd + assert result.estimated_cost_usd == Decimal("0.000900000") + + +def test_reported_estimate_matches_the_ledger_for_nested_spans(tmp_path: Path) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + spans = [ + _span(span_id="parent", parent_span_id=None, input_tokens=1000, output_tokens=500), + _span(span_id="child", parent_span_id="parent", input_tokens=2000, output_tokens=1000), + ] + updated, result = apply_estimated_costs(spans, build_pricing_table(PricingConfig())) + store.replace_normalized(spans=updated, traces=build_normalized_traces(updated)) + + ledger = build_cost_ledger(store) + + assert result.estimated_cost_usd == ledger.total_cost_usd + assert ledger.estimated_cost_usd == result.estimated_cost_usd