Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion cmake/AgenttyTests.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ set(_AGENTTY_CONSOLIDATED
slash_menu_test
effort_auto_test settings_multiproc_test
wire_shared_test complexity_test copilot_token_test kimi_token_test
chatgpt_bundled_models_test settings_default_test
chatgpt_bundled_models_test metadata_throughput_test settings_default_test
turn_provenance_test subagent_pin_test
program_hooks_test login_cancel_test subs_key_test reducer_effects_test
sim_turn_test view_purity_test
Expand Down Expand Up @@ -564,6 +564,10 @@ agentty_finalize_fold(
# "this looks nice" asserts nothing. Build it explicitly:
# cmake --build build --target stats_visual && ./build/stats_visual
agentty_test(stats_visual MODE standalone NO_TEST)
# Live /models catalog dump for a signed-in ChatGPT account (needs real
# credentials; never runs in CI). Human compares its rows against the
# codex-rs embedded catalog to audit the picker's context windows.
agentty_test(codex_catalog_probe MODE standalone NO_TEST)
# NO_TEST too: it prints timings for a human. An assertion on microseconds
# would be a flaky test of the machine it runs on, not of the code.
agentty_test(stats_refresh_bench MODE standalone NO_TEST)
Expand Down
8 changes: 7 additions & 1 deletion include/agentty/domain/stats/facts.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,13 @@ struct Facts {
// fast on average"; a sparkline answers "is it getting worse",
// which is the question someone opens this tab with.
std::vector<double> ttft_series;
std::vector<double> rate_series; // output tokens per second
std::vector<double> rate_series; // total generated tokens per second (incl. reasoning)
// Visible output only: (output − reasoning) per second of generation
// time. On thinking models the billed output is dominated by
// reasoning tokens the user never sees, so the two series can
// legitimately differ by several ×; showing one number labelled
// "output" conflates them. Empty for turns without reasoning.
std::vector<double> visible_rate_series;
} stream;

struct ContextF {
Expand Down
7 changes: 6 additions & 1 deletion include/agentty/provider/chatgpt/responses.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@ namespace agentty::provider::chatgpt {
struct CatalogModel {
std::string slug; // wire id sent as body.model
std::string display_name; // human label for the picker
int context_window = 272000;
// 0 = the server did not report a window for this slug. The default was
// once 272'000 — that fabricated precision for every model whose row
// omitted (or mis-typed) the field, and the picker printed "272k" as if
// the endpoint had said so. 0 flows into ModelInfo and the picker shows
// "auto"; the compaction ladder falls back to its conservative default.
int context_window = 0;
bool is_default = false;
int priority = 0;
};
Expand Down
17 changes: 15 additions & 2 deletions src/domain/stats/extract.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -547,12 +547,25 @@ void stream_rates(const Facts& f, std::vector<Metric>& out) {
double sum = 0;
for (double v : f.stream.rate_series) sum += v;
out.push_back(Metric{
.label = "Output rate",
.label = "Generated rate",
.unit = Unit::Rate,
.value = sum / static_cast<double>(f.stream.rate_series.size()),
.detail = "tok/s",
// `output_tokens` is the billed figure and INCLUDES reasoning
// tokens, so say so — on a thinking turn this number is mostly
// tokens nobody watched appear.
.detail = "tok/s incl. reasoning",
.series = f.stream.rate_series});
}
if (!f.stream.visible_rate_series.empty()) {
double sum = 0;
for (double v : f.stream.visible_rate_series) sum += v;
out.push_back(Metric{
.label = "Visible rate",
.unit = Unit::Rate,
.value = sum / static_cast<double>(f.stream.visible_rate_series.size()),
.detail = "tok/s",
.series = f.stream.visible_rate_series});
}
}

void stream_timing(const Facts& f, std::vector<Metric>& out) {
Expand Down
13 changes: 13 additions & 0 deletions src/domain/stats/fold.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,17 @@ void Facts::fold(const Message& msg) {
push_capped(stream.rate_series,
static_cast<double>(t.output_tokens)
/ (static_cast<double>(t.stream_ms) / 1000.0));
// Same denominator, visible tokens only. `output_tokens` is the BILLED
// figure and includes reasoning on every provider that reports it
// separately, so a thinking turn's raw rate overstates what the user
// watched appear by exactly the reasoning share. Guarded on reasoning
// < output: a provider that double-counts (reasoning outside output)
// must not produce a negative rate.
if (t.stream_ms && t.reasoning_tokens
&& t.reasoning_tokens < t.output_tokens)
push_capped(stream.visible_rate_series,
static_cast<double>(t.output_tokens - t.reasoning_tokens)
/ (static_cast<double>(t.stream_ms) / 1000.0));
}

void Facts::merge(const Facts& o) {
Expand Down Expand Up @@ -249,6 +260,8 @@ void Facts::merge(const Facts& o) {
stream.bytes.merge(o.stream.bytes);
for (double x : o.stream.ttft_series) push_capped(stream.ttft_series, x);
for (double x : o.stream.rate_series) push_capped(stream.rate_series, x);
for (double x : o.stream.visible_rate_series)
push_capped(stream.visible_rate_series, x);

context.compactions += o.context.compactions;
if (o.context.peak_input > context.peak_input)
Expand Down
19 changes: 19 additions & 0 deletions src/runtime/app/update/stream.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -545,6 +545,25 @@ Cmd finalize_turn(Model& m, StopReason stop_reason) {
t.wire_bytes = a->live_delta_bytes > 0xFFFFFFFFull
? 0xFFFFFFFFu
: static_cast<std::uint32_t>(a->live_delta_bytes);
// Correct the sparkline's frozen tail with the AUTHORITATIVE
// rate. The live samples were the streamed-bytes÷4 estimate
// (the usage frame only arrives at stream end, so mid-stream
// there is nothing else to sample); once stream_ms and the
// billed output are both known, the last sample becomes real
// tokens over real generation time — the number that stays
// on screen after the stream stops. A zero-duration stream
// (interrupted before any interval sample) keeps the
// estimate rather than dividing by nothing.
if (t.stream_ms && t.output_tokens
&& (m.s.rate_history_full || m.s.rate_history_pos > 0)) {
const float authoritative =
static_cast<float>(t.output_tokens)
/ (static_cast<float>(t.stream_ms) / 1000.0f);
const std::size_t last_idx =
(m.s.rate_history_pos + StreamState::kRateSamples - 1)
% StreamState::kRateSamples;
m.s.rate_history[last_idx] = authoritative;
}
}
}
// Drain any text still in the smoothing buffer before committing
Expand Down
6 changes: 6 additions & 0 deletions src/runtime/view/status_bar/token_stream_sparkline.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@ maya::TokenStreamSparkline::Config token_stream_sparkline_config(const Model& m)
? std::chrono::duration_cast<std::chrono::milliseconds>(
now - a->first_delta_at).count()
: 0;
// ÷4 is the usual ASCII tokens-per-byte heuristic, so this TOTAL and
// every rate sample in the ring are ESTIMATES — the authoritative usage
// frame only lands at stream end, and finalize_turn then rewrites the
// ring's last sample with real tokens over real generation time. The
// stats panel's rates are usage-derived from the start; this chip is
// the live view, and mid-stream an estimate is all that exists.
double approx_tok = a ? static_cast<double>(a->live_delta_bytes) / 4.0 : 0.0;

// Pick the displayed rate: live during streaming (after a 250 ms
Expand Down
4 changes: 2 additions & 2 deletions tests/chatgpt_bundled_models_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ TEST_CASE("chatgpt catalog parsing filters and orders models") {
CHECK(models[0].slug == "first");
CHECK(models[0].is_default);
CHECK(models[1].slug == "fallback");
CHECK(models[1].context_window == 272000);
CHECK(models[1].context_window == 0); // absent/invalid on the wire → unknown, never a fabricated 272k
CHECK(models[2].slug == "other");
CHECK(models[3].slug == "later");
CHECK(models[3].context_window == 12345);
Expand All @@ -69,7 +69,7 @@ TEST_CASE("chatgpt catalog prioritizes valid priorities before missing or invali
CHECK(models[2].slug == "last-valid");
CHECK(models[3].slug == "missing");
CHECK(models[4].slug == "unsigned-overflow");
CHECK(models[4].context_window == 272000);
CHECK(models[4].context_window == 0); // u64-overflow on the wire → unknown, never a fabricated 272k
CHECK(models[5].slug == "invalid");
CHECK(models[6].slug == "out-of-range");
}
Expand Down
32 changes: 32 additions & 0 deletions tests/codex_catalog_probe.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// codex_catalog_probe — dumps the signed-in account's LIVE ChatGPT (Codex)
// /models catalog: slug, display_name, context_window, priority, default.
//
// NO_TEST on purpose: it prints server-reported metadata for a human to
// compare against the bundled fallback and the codex-rs embedded catalog.
// It requires real ChatGPT OAuth credentials, so it can never run in CI.
// Build and run explicitly:
// cmake --build build --target codex_catalog_probe && ./build/codex_catalog_probe

#include <cstdio>

#include "agentty/provider/chatgpt/responses.hpp"
#include "agentty/util/io.hpp"

namespace cc = agentty::provider::chatgpt;

int main() {
auto models = cc::fetch_models(::agentty::IoAccess::grant(),
::maya::test_root());
if (models.empty()) {
std::printf("live catalog unavailable (not signed in or offline)\n");
return 1;
}
std::printf("%-24s %-16s %10s %8s %s\n", "slug", "display", "ctx_window",
"priority", "default");
for (const auto& m : models) {
std::printf("%-24s %-16s %10d %8d %s\n", m.slug.c_str(),
m.display_name.c_str(), m.context_window, m.priority,
m.is_default ? "yes" : "");
}
return 0;
}
Loading