From ee70f2897839d1089ac14376883767898272b377 Mon Sep 17 00:00:00 2001 From: Mac Date: Thu, 12 Nov 2026 17:31:15 +0000 Subject: [PATCH] fix(chatgpt): honest context-window metadata + usage-corrected throughput MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Context windows: - CatalogModel::context_window defaulted to 272'000, so any slug whose /models row omitted or mis-typed the field displayed "272k" as if the endpoint had reported it. Default is now 0 (unknown) — the picker shows "auto" and the compaction ladder takes its conservative default; a value the endpoint DID report is used verbatim. - Verified against the live Codex catalog (via the new codex_catalog_probe) and codex-rs 0.167.0's embedded snapshot: gpt-6.1-sol is served at 500k per the live endpoint (the client snapshot still says 272k), gpt-6.1-terra 500k, everything else 272k. The endpoint also carries a separate max_context_window (872k) agentty does not consume — documented, not invented. Throughput: - The billed output_tokens figure includes reasoning tokens; the stats panel's single "Output rate" conflated billed with watched. It now records two series: "Generated rate" (incl. reasoning, labelled) and "Visible rate" (output − reasoning). - The status-bar sparkline samples streamed bytes / 4 mid-stream (usage only arrives at stream end); finalize_turn now rewrites the ring's last sample with real tokens over real generation time, so the frozen post-stream readout is authoritative. - Zero-duration, interrupted and unmeasured turns stay excluded, never zero-filled; a turn whose reasoning tokens are not a strict subset of output_tokens records no visible sample at all rather than a negative one. Tests: metadata_throughput_test (9 cases), codex_catalog_probe (manual, live dump), updated chatgpt parse expectations, and the stats golden re-pinned for the rate split. --- cmake/AgenttyTests.cmake | 6 +- include/agentty/domain/stats/facts.hpp | 8 +- .../agentty/provider/chatgpt/responses.hpp | 7 +- src/domain/stats/extract.cpp | 17 +- src/domain/stats/fold.cpp | 13 ++ src/runtime/app/update/stream.cpp | 19 ++ .../status_bar/token_stream_sparkline.cpp | 6 + tests/chatgpt_bundled_models_test.cpp | 4 +- tests/codex_catalog_probe.cpp | 32 +++ tests/metadata_throughput_test.cpp | 218 ++++++++++++++++++ tests/stats_golden_test.cpp | 6 +- 11 files changed, 328 insertions(+), 8 deletions(-) create mode 100644 tests/codex_catalog_probe.cpp create mode 100644 tests/metadata_throughput_test.cpp diff --git a/cmake/AgenttyTests.cmake b/cmake/AgenttyTests.cmake index 28ce9c9e6..d1a250328 100644 --- a/cmake/AgenttyTests.cmake +++ b/cmake/AgenttyTests.cmake @@ -26,7 +26,7 @@ set(_AGENTTY_CONSOLIDATED slash_menu_test effort_auto_test settings_multiproc_test wire_shared_test complexity_test copilot_token_test kimi_token_test - chatgpt_bundled_models_test settings_default_test + chatgpt_bundled_models_test metadata_throughput_test settings_default_test turn_provenance_test subagent_pin_test program_hooks_test login_cancel_test subs_key_test reducer_effects_test sim_turn_test view_purity_test @@ -564,6 +564,10 @@ agentty_finalize_fold( # "this looks nice" asserts nothing. Build it explicitly: # cmake --build build --target stats_visual && ./build/stats_visual agentty_test(stats_visual MODE standalone NO_TEST) +# Live /models catalog dump for a signed-in ChatGPT account (needs real +# credentials; never runs in CI). Human compares its rows against the +# codex-rs embedded catalog to audit the picker's context windows. +agentty_test(codex_catalog_probe MODE standalone NO_TEST) # NO_TEST too: it prints timings for a human. An assertion on microseconds # would be a flaky test of the machine it runs on, not of the code. agentty_test(stats_refresh_bench MODE standalone NO_TEST) diff --git a/include/agentty/domain/stats/facts.hpp b/include/agentty/domain/stats/facts.hpp index 28af0be15..c102cbff4 100644 --- a/include/agentty/domain/stats/facts.hpp +++ b/include/agentty/domain/stats/facts.hpp @@ -309,7 +309,13 @@ struct Facts { // fast on average"; a sparkline answers "is it getting worse", // which is the question someone opens this tab with. std::vector ttft_series; - std::vector rate_series; // output tokens per second + std::vector rate_series; // total generated tokens per second (incl. reasoning) + // Visible output only: (output − reasoning) per second of generation + // time. On thinking models the billed output is dominated by + // reasoning tokens the user never sees, so the two series can + // legitimately differ by several ×; showing one number labelled + // "output" conflates them. Empty for turns without reasoning. + std::vector visible_rate_series; } stream; struct ContextF { diff --git a/include/agentty/provider/chatgpt/responses.hpp b/include/agentty/provider/chatgpt/responses.hpp index 175d8cd1c..9f6e53ddb 100644 --- a/include/agentty/provider/chatgpt/responses.hpp +++ b/include/agentty/provider/chatgpt/responses.hpp @@ -26,7 +26,12 @@ namespace agentty::provider::chatgpt { struct CatalogModel { std::string slug; // wire id sent as body.model std::string display_name; // human label for the picker - int context_window = 272000; + // 0 = the server did not report a window for this slug. The default was + // once 272'000 — that fabricated precision for every model whose row + // omitted (or mis-typed) the field, and the picker printed "272k" as if + // the endpoint had said so. 0 flows into ModelInfo and the picker shows + // "auto"; the compaction ladder falls back to its conservative default. + int context_window = 0; bool is_default = false; int priority = 0; }; diff --git a/src/domain/stats/extract.cpp b/src/domain/stats/extract.cpp index 97a463e20..3f788de62 100644 --- a/src/domain/stats/extract.cpp +++ b/src/domain/stats/extract.cpp @@ -547,12 +547,25 @@ void stream_rates(const Facts& f, std::vector& out) { double sum = 0; for (double v : f.stream.rate_series) sum += v; out.push_back(Metric{ - .label = "Output rate", + .label = "Generated rate", .unit = Unit::Rate, .value = sum / static_cast(f.stream.rate_series.size()), - .detail = "tok/s", + // `output_tokens` is the billed figure and INCLUDES reasoning + // tokens, so say so — on a thinking turn this number is mostly + // tokens nobody watched appear. + .detail = "tok/s incl. reasoning", .series = f.stream.rate_series}); } + if (!f.stream.visible_rate_series.empty()) { + double sum = 0; + for (double v : f.stream.visible_rate_series) sum += v; + out.push_back(Metric{ + .label = "Visible rate", + .unit = Unit::Rate, + .value = sum / static_cast(f.stream.visible_rate_series.size()), + .detail = "tok/s", + .series = f.stream.visible_rate_series}); + } } void stream_timing(const Facts& f, std::vector& out) { diff --git a/src/domain/stats/fold.cpp b/src/domain/stats/fold.cpp index a74aa1b2f..3e42d7315 100644 --- a/src/domain/stats/fold.cpp +++ b/src/domain/stats/fold.cpp @@ -190,6 +190,17 @@ void Facts::fold(const Message& msg) { push_capped(stream.rate_series, static_cast(t.output_tokens) / (static_cast(t.stream_ms) / 1000.0)); + // Same denominator, visible tokens only. `output_tokens` is the BILLED + // figure and includes reasoning on every provider that reports it + // separately, so a thinking turn's raw rate overstates what the user + // watched appear by exactly the reasoning share. Guarded on reasoning + // < output: a provider that double-counts (reasoning outside output) + // must not produce a negative rate. + if (t.stream_ms && t.reasoning_tokens + && t.reasoning_tokens < t.output_tokens) + push_capped(stream.visible_rate_series, + static_cast(t.output_tokens - t.reasoning_tokens) + / (static_cast(t.stream_ms) / 1000.0)); } void Facts::merge(const Facts& o) { @@ -249,6 +260,8 @@ void Facts::merge(const Facts& o) { stream.bytes.merge(o.stream.bytes); for (double x : o.stream.ttft_series) push_capped(stream.ttft_series, x); for (double x : o.stream.rate_series) push_capped(stream.rate_series, x); + for (double x : o.stream.visible_rate_series) + push_capped(stream.visible_rate_series, x); context.compactions += o.context.compactions; if (o.context.peak_input > context.peak_input) diff --git a/src/runtime/app/update/stream.cpp b/src/runtime/app/update/stream.cpp index 2932d0648..3eae87aa0 100644 --- a/src/runtime/app/update/stream.cpp +++ b/src/runtime/app/update/stream.cpp @@ -545,6 +545,25 @@ Cmd finalize_turn(Model& m, StopReason stop_reason) { t.wire_bytes = a->live_delta_bytes > 0xFFFFFFFFull ? 0xFFFFFFFFu : static_cast(a->live_delta_bytes); + // Correct the sparkline's frozen tail with the AUTHORITATIVE + // rate. The live samples were the streamed-bytes÷4 estimate + // (the usage frame only arrives at stream end, so mid-stream + // there is nothing else to sample); once stream_ms and the + // billed output are both known, the last sample becomes real + // tokens over real generation time — the number that stays + // on screen after the stream stops. A zero-duration stream + // (interrupted before any interval sample) keeps the + // estimate rather than dividing by nothing. + if (t.stream_ms && t.output_tokens + && (m.s.rate_history_full || m.s.rate_history_pos > 0)) { + const float authoritative = + static_cast(t.output_tokens) + / (static_cast(t.stream_ms) / 1000.0f); + const std::size_t last_idx = + (m.s.rate_history_pos + StreamState::kRateSamples - 1) + % StreamState::kRateSamples; + m.s.rate_history[last_idx] = authoritative; + } } } // Drain any text still in the smoothing buffer before committing diff --git a/src/runtime/view/status_bar/token_stream_sparkline.cpp b/src/runtime/view/status_bar/token_stream_sparkline.cpp index 17be8c1b9..170ec99c6 100644 --- a/src/runtime/view/status_bar/token_stream_sparkline.cpp +++ b/src/runtime/view/status_bar/token_stream_sparkline.cpp @@ -44,6 +44,12 @@ maya::TokenStreamSparkline::Config token_stream_sparkline_config(const Model& m) ? std::chrono::duration_cast( now - a->first_delta_at).count() : 0; + // ÷4 is the usual ASCII tokens-per-byte heuristic, so this TOTAL and + // every rate sample in the ring are ESTIMATES — the authoritative usage + // frame only lands at stream end, and finalize_turn then rewrites the + // ring's last sample with real tokens over real generation time. The + // stats panel's rates are usage-derived from the start; this chip is + // the live view, and mid-stream an estimate is all that exists. double approx_tok = a ? static_cast(a->live_delta_bytes) / 4.0 : 0.0; // Pick the displayed rate: live during streaming (after a 250 ms diff --git a/tests/chatgpt_bundled_models_test.cpp b/tests/chatgpt_bundled_models_test.cpp index 7b7ce2e34..9618354b8 100644 --- a/tests/chatgpt_bundled_models_test.cpp +++ b/tests/chatgpt_bundled_models_test.cpp @@ -44,7 +44,7 @@ TEST_CASE("chatgpt catalog parsing filters and orders models") { CHECK(models[0].slug == "first"); CHECK(models[0].is_default); CHECK(models[1].slug == "fallback"); - CHECK(models[1].context_window == 272000); + CHECK(models[1].context_window == 0); // absent/invalid on the wire → unknown, never a fabricated 272k CHECK(models[2].slug == "other"); CHECK(models[3].slug == "later"); CHECK(models[3].context_window == 12345); @@ -69,7 +69,7 @@ TEST_CASE("chatgpt catalog prioritizes valid priorities before missing or invali CHECK(models[2].slug == "last-valid"); CHECK(models[3].slug == "missing"); CHECK(models[4].slug == "unsigned-overflow"); - CHECK(models[4].context_window == 272000); + CHECK(models[4].context_window == 0); // u64-overflow on the wire → unknown, never a fabricated 272k CHECK(models[5].slug == "invalid"); CHECK(models[6].slug == "out-of-range"); } diff --git a/tests/codex_catalog_probe.cpp b/tests/codex_catalog_probe.cpp new file mode 100644 index 000000000..dc12c7aae --- /dev/null +++ b/tests/codex_catalog_probe.cpp @@ -0,0 +1,32 @@ +// codex_catalog_probe — dumps the signed-in account's LIVE ChatGPT (Codex) +// /models catalog: slug, display_name, context_window, priority, default. +// +// NO_TEST on purpose: it prints server-reported metadata for a human to +// compare against the bundled fallback and the codex-rs embedded catalog. +// It requires real ChatGPT OAuth credentials, so it can never run in CI. +// Build and run explicitly: +// cmake --build build --target codex_catalog_probe && ./build/codex_catalog_probe + +#include + +#include "agentty/provider/chatgpt/responses.hpp" +#include "agentty/util/io.hpp" + +namespace cc = agentty::provider::chatgpt; + +int main() { + auto models = cc::fetch_models(::agentty::IoAccess::grant(), + ::maya::test_root()); + if (models.empty()) { + std::printf("live catalog unavailable (not signed in or offline)\n"); + return 1; + } + std::printf("%-24s %-16s %10s %8s %s\n", "slug", "display", "ctx_window", + "priority", "default"); + for (const auto& m : models) { + std::printf("%-24s %-16s %10d %8d %s\n", m.slug.c_str(), + m.display_name.c_str(), m.context_window, m.priority, + m.is_default ? "yes" : ""); + } + return 0; +} diff --git a/tests/metadata_throughput_test.cpp b/tests/metadata_throughput_test.cpp new file mode 100644 index 000000000..5a7391604 --- /dev/null +++ b/tests/metadata_throughput_test.cpp @@ -0,0 +1,218 @@ +// metadata_throughput_test — two audit regressions, in one place because +// they share a shape: a number shown to the user must never be MORE precise +// than the evidence behind it. +// +// 1. CONTEXT WINDOWS. The live Codex /models catalog is parsed into +// CatalogModel rows whose context_window used to DEFAULT to 272'000. +// Any slug whose server row omitted (or mis-typed) the field then +// displayed "272k" as if the endpoint had reported it — fabricated +// precision. The authoritative shape (codex-rs 0.167.0) is optional: +// a slug can legitimately have no served window. Missing/invalid must +// parse to 0 (unknown → the picker shows "auto"); valid values pass +// through untouched; out-of-range values are rejected, not clamped +// into a plausible-looking wrong number. +// +// 2. THROUGHPUT. The billed `output_tokens` figure INCLUDES reasoning +// tokens on every provider that reports them separately. One rate +// labelled "output" conflated "tokens the user watched appear" with +// "tokens billed" — on a thinking turn those differ by several ×. +// The fold now records both series and the stats panel labels them +// distinctly ("Generated rate" incl. reasoning, "Visible rate"). +// +// Purity: both halves are functions of their input payload/telemetry and +// nothing else — no Model, no wire, no clock. + +#include +#include + +#include "agtest.hpp" + +#include "agentty/domain/stats/facts.hpp" +#include "agentty/domain/stats/tabs.hpp" +#include "agentty/provider/chatgpt/responses.hpp" +#include "agentty/runtime/view/helpers.hpp" + +using namespace agentty; +namespace cc = agentty::provider::chatgpt; +namespace st = agentty::stats; + +// ── 1. Catalog metadata ────────────────────────────────────────────────── + +TEST_CASE("chatgpt catalog: absent or invalid context windows stay unknown") { + const auto models = cc::parse_models_payload_for_test(R"({"models":[ + {"slug":"no-window"}, + {"slug":"zero", "context_window":0}, + {"slug":"neg", "context_window":-5}, + {"slug":"string", "context_window":"abc"}, + {"slug":"null", "context_window":null}, + {"slug":"huge", "context_window":25000000}, + {"slug":"ok", "context_window":272000}, + {"slug":"terra", "context_window":500000} + ]})"); + REQUIRE(models.size() == 8); + // Every row the endpoint declined to describe truthfully is UNKNOWN (0), + // which the picker renders as "auto" — never a hard-coded 272k. + for (const auto& slug : {"no-window", "zero", "neg", "string", + "null", "huge"}) { + bool found = false; + for (const auto& m : models) + if (m.slug == slug) { + found = true; + CAPTURE(m.slug); + CHECK(m.context_window == 0); + } + CHECK(found); + } + // A value the endpoint DID report passes through untouched. + const auto win = [&](std::string_view slug) -> int { + for (const auto& m : models) + if (m.slug == slug) return m.context_window; + return -1; + }; + CHECK(win("ok") == 272000); + CHECK(win("terra") == 500000); +} + +TEST_CASE("chatgpt catalog: unknown windows render as unknown, not invented") { + // The picker's context column: 0 → "auto" (names the state), a real + // window → the same k/M formatting the status bar uses. + CHECK(ui::context_window_label(0) == "auto"); + CHECK(ui::context_window_label(272'000) == "272k"); + CHECK(ui::context_window_label(500'000) == "500k"); + CHECK(ui::context_window_label(872'000) == "872k"); + CHECK(ui::context_window_label(1'000'000) == "1M"); +} + +TEST_CASE("context ladder: an unknown chatgpt window falls back conservatively") { + // With the fabricated 272k default gone, a slug the server did not + // describe falls through the ladder to the conservative default. That + // is correct on both sides of the asymmetry: too LOW only compacts + // earlier than strictly necessary; too HIGH fails the turn on the wire. + store::Settings s; + CHECK(ui::resolve_context_window("chatgpt", "gpt-9-unknown-slug", 0, s) + == ui::kDefaultContextWindow); + // And a window the server DID report is used verbatim (rung 2 outranks + // every static source). + CHECK(ui::resolve_context_window("chatgpt", "gpt-6.1-terra", 500'000, s) + == 500'000); +} + +// ── 2. Throughput accounting ───────────────────────────────────────────── + +namespace { + +// An assistant message with no telemetry — the "not measured" case (a turn +// that predates the field, or was interrupted before any accounting). +Message unrouted(const char* model) { + Message m; + m.role = Role::Assistant; + m.served_model = ModelId{model}; + return m; +} + +Message measured(std::uint32_t out, std::uint32_t reasoning, + std::uint32_t stream_ms) { + Message m; + m.role = Role::Assistant; + m.served_model = ModelId{"m1"}; + Message::Telemetry t; + t.output_tokens = out; + t.reasoning_tokens = reasoning; + t.stream_ms = stream_ms; + m.telemetry = t; + return m; +} + +st::Facts folded(std::vector msgs) { + Thread th; + th.id = agentty::ThreadId{"t1"}; + th.messages = std::move(msgs); + st::Projection p; + return p.refresh(th); +} + +// The "Rates" section of the stream tab: stream_rates() is only reachable +// through the section table. +std::vector rates_of(const st::Facts& f) { + std::vector out; + for (const auto& sec : st::sections::stream) + if (sec.heading == "Rates") sec.extract(f, out); + return out; +} + +const st::Metric* find_metric(const std::vector& ms, + std::string_view label) { + for (const auto& m : ms) + if (m.label == label) return &m; + return nullptr; +} + +} // namespace + +TEST_CASE("throughput: total generated rate includes reasoning; visible does not") { + // 400 visible + 600 reasoning over 2 s of generation time. Billed + // output = 1000 → 500 tok/s; the user watched 400 appear → 200 tok/s. + const auto f = folded({measured(1000, 600, 2000)}); + REQUIRE(f.stream.rate_series.size() == 1); + CHECK(f.stream.rate_series[0] == doctest::Approx(500.0)); + REQUIRE(f.stream.visible_rate_series.size() == 1); + CHECK(f.stream.visible_rate_series[0] == doctest::Approx(200.0)); + + const auto ms = rates_of(f); + const auto* gen = find_metric(ms, "Generated rate"); + const auto* vis = find_metric(ms, "Visible rate"); + REQUIRE(gen != nullptr); + REQUIRE(vis != nullptr); + CHECK(gen->value == doctest::Approx(500.0)); + CHECK(vis->value == doctest::Approx(200.0)); + // The labels carry the distinction, not just the numbers. + CHECK(gen->detail.find("reasoning") != std::string::npos); + CHECK(vis->detail.find("reasoning") == std::string::npos); +} + +TEST_CASE("throughput: a non-thinking turn has exactly one rate") { + const auto f = folded({measured(300, 0, 1000)}); + REQUIRE(f.stream.rate_series.size() == 1); + CHECK(f.stream.rate_series[0] == doctest::Approx(300.0)); + CHECK(f.stream.visible_rate_series.empty()); + const auto ms = rates_of(f); + CHECK(find_metric(ms, "Generated rate") != nullptr); + CHECK(find_metric(ms, "Visible rate") == nullptr); +} + +TEST_CASE("throughput: a provider double-counting reasoning cannot go negative") { + // If reasoning arrives OUTSIDE output_tokens (a broken wire), the + // visible rate must simply be absent — never negative, never folded in. + const auto f = folded({measured(400, 900, 1000)}); + REQUIRE(f.stream.rate_series.size() == 1); // billed rate is still real + CHECK(f.stream.visible_rate_series.empty()); +} + +TEST_CASE("throughput: zero-duration and token-less turns fold nothing") { + // stream_ms == 0 (interrupted before the first byte, or a clock that + // refused to advance) divides by nothing — the turn is excluded from + // the averages, not counted as zero. + const auto f = folded({measured(500, 100, 0)}); + CHECK(f.stream.rate_series.empty()); + CHECK(f.stream.visible_rate_series.empty()); + const auto f2 = folded({measured(0, 0, 1000)}); + CHECK(f2.stream.rate_series.empty()); +} + +TEST_CASE("throughput: unmeasured turns are excluded, not zeroed") { + const auto f = folded({measured(1000, 600, 2000), + unrouted("m1")}); // no telemetry at all + REQUIRE(f.stream.rate_series.size() == 1); + CHECK(f.stream.rate_series[0] == doctest::Approx(500.0)); + REQUIRE(f.stream.visible_rate_series.size() == 1); + CHECK(f.stream.visible_rate_series[0] == doctest::Approx(200.0)); +} + +TEST_CASE("throughput: merge carries the visible series across projections") { + auto f = folded({measured(1000, 600, 2000)}); + st::Facts g; + g.fold(measured(2000, 0, 1000)); + f.merge(g); + REQUIRE(f.stream.rate_series.size() == 2); + CHECK(f.stream.visible_rate_series.size() == 1); +} diff --git a/tests/stats_golden_test.cpp b/tests/stats_golden_test.cpp index 256c74c8e..704f45769 100644 --- a/tests/stats_golden_test.cpp +++ b/tests/stats_golden_test.cpp @@ -446,7 +446,11 @@ TEST_CASE("stats_golden: every tab renders byte-identically") { // width pressure like every other panel. Only the footer line changed; // the clip scan, the overrun check and every feature assertion stayed // green across the sweep. - const std::uint64_t kGoldenHash = 0x024089ffde43e434ull; + // Moved 2026-11 (again): the stream tab's single "Output rate" split + // into "Generated rate" (billed output, incl. reasoning) + "Visible + // rate" (output − reasoning), with the incl-reasoning note in the + // detail. See metadata_throughput_test for the accounting behind it. + const std::uint64_t kGoldenHash = 0x815fad5537bf222full; const std::uint64_t got = fnv1a(out); if (kGoldenHash == 0) {