diff --git a/CHANGELOG.md b/CHANGELOG.md index e63ab2e..6632e75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,55 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),\ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.9.2] - 2026-07-15 + +### Fixed + +- **Deep Rescan can no longer shrink sealed history.** The windowed rebuild + used to overwrite sealed relay days unconditionally; when the raw JSONL + behind a day had aged out (or a file read truncated mid-stream), the rebuilt + day was smaller and the loss was permanent — this shrank a sealed + 2026-06-12 on 2026-07-12, and by 2026-07-15 four more days (2.4B tokens) + would have gone the same way. Sealed days are now only replaced when the + rebuild carries at least as much data, per provider (`shouldKeepSealedDay`); + `/api/rescan?force=true` remains the explicit override for parser-fix + rescans where shrinking is the point. +- Per-file fail-soft read errors in the codex and claude-code parsers (and + the shared `readJsonlFile`/`readJsonlFileFromOffset` helpers) now surface + as provider warnings tagged `partial` instead of silently truncating a + scan. Partial faults deliberately do NOT abort the trailing gap-fill (one + permanently unreadable file must not block sealing forever); a forced deep + rescan refuses to shrink a sealed day for a provider that hit a truncated + read during that rebuild. +- The sealed-day guard also protects frozen cost: a rebuild that re-derives + identical tokens but lower cost (kosha unreadable/offline during the + rescan, a model since pruned from the registry) keeps the sealed day. +- Provider warnings no longer accumulate without bound in the warm daemon: + each refresh tick and each deep rescan replaces the previous cycle's + provider warnings instead of appending duplicates forever. +- Summary cache schema bumped to 3 so cached summaries holding the old + even-split model rows rebuild instead of being served post-upgrade. +- The trailing gap-fill no longer seals days BEFORE the gap from the partial + slice a fresh-mtime multi-day file happens to carry — interior holes stay + open for an explicit rescan instead of freezing a partial day forever. +- Model summaries now use the exact per-project (provider, model) cross-cut + instead of even-splitting a day-level bucket across providers — one model + used by codex and codex-desktop in the same day no longer shows two + identical phantom rows with fractional half-tokens. +- The web dashboard's "Today" is now actually today: the last daily entry is + only presented as today when its date matches the local calendar date, + instead of relabeling the most recent active day. +- `ensure-summary` no longer bakes an arbitrarily stale cached summary into + the static web build: it prefers the live daemon, accepts the cache only if + it was scanned today, and rescans otherwise. + +### Added + +- **Today, hour by hour** panel on the web dashboard: today's cost per local + hour as a 24-bar chart with a custom hover tooltip (cost, tokens, records, + busiest model per hour), current-hour marker, and an honest whole-series + switch to token bars when the day has no priced usage. + ## [1.9.1] - 2026-07-10 ### Added diff --git a/README.md b/README.md index ba8cceb..841d37b 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@

Token Usage Tracker for AI Coding Agents

+ release npm node license diff --git a/packages/cli/package.json b/packages/cli/package.json index 84c71e6..40e7127 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@sriinnu/tokmeter-cli", - "version": "1.9.1", + "version": "1.9.2", "private": true, "description": "Token usage tracking CLI and automation helpers", "type": "module", diff --git a/packages/core/package.json b/packages/core/package.json index a1ada1f..65a174e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@sriinnu/tokmeter-core", - "version": "1.9.1", + "version": "1.9.2", "private": true, "description": "Token usage tracking core — session parsers, aggregation, and pricing", "type": "module", diff --git a/packages/core/src/aggregate-consumers.ts b/packages/core/src/aggregate-consumers.ts index f3a6c04..8ff2bb5 100644 --- a/packages/core/src/aggregate-consumers.ts +++ b/packages/core/src/aggregate-consumers.ts @@ -389,11 +389,31 @@ export function computeModelCostsFromState( return modelTotalsToSummaries(totals.values(), grandTotalCost); } - // Unfiltered fast path: day-level model buckets. Multi-provider single-model - // days even-split (best we can do at this granularity). + // Unfiltered path. Exact (provider, model) pairing lives in the per-project + // cross-cut buckets — use them whenever the day carries them. Only + // pre-enrichment day files (modelBuckets backfilled empty by + // forwardMigrateAggregate) fall back to the day-level even-split + // approximation, which fabricates identical half-token rows when one model + // ran under two providers in a day (codex + codex-desktop sharing + // gpt-5.6-sol showed 190,725,623.5 tokens EACH). for (const day of iterateAllDays(aggregates, todayAccumulator)) { if (!dayInWindow(day, opts, todayKey)) continue; grandTotalCost += day.cost; + // The cross-cut must be COMPLETE for the day, not merely present: a + // partially corrupt/hand-edited file with one bucket-less project would + // otherwise silently drop that project's models from the rows. Every + // record folds into exactly one (project, model) bucket, so a complete + // cross-cut's record count equals the day's. + let bucketRecords = 0; + for (const project of Object.values(day.projects)) { + for (const mb of Object.values(project.modelBuckets)) bucketRecords += mb.recordCount; + } + if (bucketRecords >= day.recordCount && day.recordCount > 0) { + for (const project of Object.values(day.projects)) { + for (const mb of Object.values(project.modelBuckets)) foldCrossCut(totals, mb); + } + continue; + } for (const model of Object.values(day.models)) { const n = model.providers.length; for (const provider of model.providers) { diff --git a/packages/core/src/aggregate-migration-parity.test.ts b/packages/core/src/aggregate-migration-parity.test.ts index 30dec98..1d041fe 100644 --- a/packages/core/src/aggregate-migration-parity.test.ts +++ b/packages/core/src/aggregate-migration-parity.test.ts @@ -492,3 +492,44 @@ describe("Phase 3 parity — provider filter on aggregate getters", () => { expect(fresh).toEqual(legacy); }); }); + +describe("getModelCosts — one model under two providers stays exactly attributed", () => { + // codex + codex-desktop both ran gpt-5.6-sol on the same day with very + // different volumes. The old day-level even-split fabricated two identical + // half-token rows (…623.5 each); the per-project cross-cut is exact. + const now = Date.now(); + const records = [ + r({ + timestamp: now, + provider: "codex" as TokenRecord["provider"], + model: "gpt-5.6-sol", + project: "alpha", + inputTokens: 1000, + outputTokens: 500, + cost: 2, + }), + r({ + timestamp: now + 1, + provider: "codex-desktop" as TokenRecord["provider"], + model: "gpt-5.6-sol", + project: "alpha", + inputTokens: 9000, + outputTokens: 100, + cost: 0, + }), + ]; + const { aggregates, todayAcc } = projectToState(records, now); + + test("matches legacy exact pairing — no even-split, no fractional tokens", () => { + const legacy = aggregateByModel(records); + const fresh = computeModelCostsFromState(aggregates, todayAcc); + expect(fresh).toEqual(legacy); + const codexRow = fresh.find((m) => m.provider === "codex"); + const desktopRow = fresh.find((m) => m.provider === "codex-desktop"); + expect(codexRow?.inputTokens).toBe(1000); + expect(desktopRow?.inputTokens).toBe(9000); + for (const row of fresh) { + expect(Number.isInteger(row.totalTokens)).toBe(true); + } + }); +}); diff --git a/packages/core/src/aggregates.ts b/packages/core/src/aggregates.ts index 73ac389..083667c 100644 Binary files a/packages/core/src/aggregates.ts and b/packages/core/src/aggregates.ts differ diff --git a/packages/core/src/config-service.ts b/packages/core/src/config-service.ts index 63d9a84..c829058 100644 --- a/packages/core/src/config-service.ts +++ b/packages/core/src/config-service.ts @@ -40,6 +40,8 @@ export type DefaultRange = "all" | "today" | "week" | "month" | "year"; export type DefaultSort = "cost" | "tokens" | "activeDays"; export type MenubarColorSource = "off" | "context" | "block" | "budget"; const MENUBAR_COLOR_SOURCES: readonly MenubarColorSource[] = ["off", "context", "block", "budget"]; +export type WeekChartStyle = "line" | "bars" | "area"; +const WEEK_CHART_STYLES: readonly WeekChartStyle[] = ["line", "bars", "area"]; export interface UserConfig { version: 1; @@ -55,6 +57,9 @@ export interface UserConfig { * - "off": no coloring. */ menubarColorSource: MenubarColorSource; + /** Rendering style for the popover's LAST 7 DAYS chart (Swift mirrors + * this in HubConfigStore.WeekChartStyle — keep raw values identical). */ + weekChartStyle: WeekChartStyle; }; daemon: { /** Advisory: seconds between full rescans inside the daemon. */ @@ -107,7 +112,7 @@ export interface UserConfig { */ export const DEFAULT_CONFIG: UserConfig = { version: 1, - bar: { refreshSeconds: 30, menubarColorSource: "context" }, + bar: { refreshSeconds: 30, menubarColorSource: "context", weekChartStyle: "line" }, daemon: { scanIntervalSeconds: 60, antigravityLivePolling: false }, cli: { defaultRange: "all", defaultSort: "cost" }, alerts: { dailyCostThreshold: null }, @@ -188,6 +193,9 @@ function normalizeConfig(raw: Partial): UserConfig { ) ? (raw.bar?.menubarColorSource as MenubarColorSource) : d.bar.menubarColorSource, + weekChartStyle: WEEK_CHART_STYLES.includes(raw.bar?.weekChartStyle as WeekChartStyle) + ? (raw.bar?.weekChartStyle as WeekChartStyle) + : d.bar.weekChartStyle, }, daemon: { scanIntervalSeconds: clampPositiveInt( diff --git a/packages/core/src/parsers/claude-code.ts b/packages/core/src/parsers/claude-code.ts index d25cc02..4256418 100644 --- a/packages/core/src/parsers/claude-code.ts +++ b/packages/core/src/parsers/claude-code.ts @@ -116,11 +116,16 @@ export class ClaudeCodeParser implements SessionParser { const cwd = decodeClaudeSlugDir(file); const isSubagent = file.includes("/subagents/"); - // Append mode: only parse new bytes from where we left off + // Append mode: only parse new bytes from where we left off. Read faults + // fail soft but surface via onWarning so a rebuild knows it's partial. + const readFault = (error: unknown) => + opts?.onWarning?.( + `failed read of ${file}: ${error instanceof Error ? error.message : error}` + ); const lines = cacheResult.appendOffset > 0 - ? await readJsonlFileFromOffset(file, cacheResult.appendOffset) - : await readJsonlFile(file); + ? await readJsonlFileFromOffset(file, cacheResult.appendOffset, readFault) + : await readJsonlFile(file, readFault); const newRecords: TokenRecord[] = []; // Dedup: using both usage values and a stable per-message discriminator so diff --git a/packages/core/src/parsers/codex.ts b/packages/core/src/parsers/codex.ts index 28a74a8..0d68cbd 100644 --- a/packages/core/src/parsers/codex.ts +++ b/packages/core/src/parsers/codex.ts @@ -449,7 +449,11 @@ function foldCodexEvent( * read. Either way the caller gets just this file's records — to fold and drop * — so peak memory is bounded to a single file, not the whole corpus. */ -export async function parseCodexFile(file: string, sizeBytes: number): Promise { +export async function parseCodexFile( + file: string, + sizeBytes: number, + onWarning?: (message: string) => void +): Promise { const out: TokenRecord[] = []; const state = defaultState(); if (sizeBytes >= CODEX_LARGE_FILE_BYTES) { @@ -469,19 +473,24 @@ export async function parseCodexFile(file: string, sizeBytes: number): Promise(file); + const events = await readJsonlFile(file, (error) => { + onWarning?.(`failed read of ${file}: ${error instanceof Error ? error.message : error}`); + }); for (const evt of events) foldCodexEvent(evt, state, file, out); } return out; @@ -588,12 +597,12 @@ export class CodexParser implements SessionParser { // Big ones: strictly one at a time. for (const f of large) { - await onFile(await parseCodexFile(f.file, f.size)); + await onFile(await parseCodexFile(f.file, f.size, opts?.onWarning)); } // Small ones: a bounded concurrent batch. onFile's fold is synchronous, so // interleaved awaits here can't corrupt the caller's accumulators. await mapWithConcurrency(small, CODEX_SCAN_CONCURRENCY, async (f) => { - await onFile(await parseCodexFile(f.file, f.size)); + await onFile(await parseCodexFile(f.file, f.size, opts?.onWarning)); }); } } diff --git a/packages/core/src/parsers/utils.ts b/packages/core/src/parsers/utils.ts index 67f5be7..0affca0 100644 --- a/packages/core/src/parsers/utils.ts +++ b/packages/core/src/parsers/utils.ts @@ -341,8 +341,14 @@ export function clearRecordCache(): void { } catch {} } -/** Read only the tail of a file from a byte offset (for append-only parsing). */ -export async function readJsonlFileFromOffset(path: string, offsetBytes: number): Promise { +/** Read only the tail of a file from a byte offset (for append-only parsing). + * A read failure returns [] (fail-soft) — pass `onError` to distinguish that + * from an empty tail when the caller seals history. */ +export async function readJsonlFileFromOffset( + path: string, + offsetBytes: number, + onError?: (error: unknown) => void +): Promise { const { open } = await import("node:fs/promises"); let fd: Awaited> | null = null; try { @@ -379,7 +385,8 @@ export async function readJsonlFileFromOffset(path: string, offsetBytes: numb } catch {} } return results; - } catch { + } catch (error) { + onError?.(error); return []; } finally { // Guarantee the fd is released even if read() threw mid-buffer. Without @@ -576,8 +583,13 @@ export async function readJsonFile(path: string): Promise { } } -/** Read JSONL file and parse each line. */ -export async function readJsonlFile(path: string): Promise { +/** Read JSONL file and parse each line. Malformed lines are skipped; a read + * failure returns [] (fail-soft) — pass `onError` to distinguish that from a + * genuinely empty file when the caller seals history. */ +export async function readJsonlFile( + path: string, + onError?: (error: unknown) => void +): Promise { try { const raw = await readFile(path, "utf-8"); const lines = raw.split("\n").filter((l: string) => l.trim()); @@ -590,7 +602,8 @@ export async function readJsonlFile(path: string): Promise { } } return results; - } catch { + } catch (error) { + onError?.(error); return []; } } diff --git a/packages/core/src/relay-loader.test.ts b/packages/core/src/relay-loader.test.ts index b35395d..57de751 100644 --- a/packages/core/src/relay-loader.test.ts +++ b/packages/core/src/relay-loader.test.ts @@ -129,3 +129,170 @@ describe("refreshFromRelay — bounded trailing-gap fill (no full rescan on a no expect(state.aggregates.has(yKey)).toBe(true); }); }); + +// ─── Deep Rescan sealed-day guard (the 2026-07-12 history-shrink regression) ── +// +// rebuildRecentWindow re-derives the window from raw and used to overwrite +// sealed days unconditionally — a partial raw corpus (cleaned-up JSONL, a +// truncated read) permanently shrank sealed history. The guard: a sealed day +// is only replaced when the rebuild carries at least as much data, per +// provider. These tests drive the REAL codex parser against a fixture corpus +// in a temp home (codex keeps no record cache, so nothing leaks outside it). + +import { mkdirSync, writeFileSync } from "node:fs"; +import { type DailyAggregate, shouldKeepSealedDay } from "./aggregates.js"; +import { rebuildRecentWindow } from "./relay-loader.js"; +import type { ScanWarning } from "./types.js"; + +// Local-time ISO (no Z) so day keys are TZ-stable under both bun and vitest. +const codexEvents = (day: string, tokens: number[]): string => + [ + JSON.stringify({ + timestamp: `${day}T12:00:00`, + type: "session_meta", + payload: { cwd: "/tmp/demo" }, + }), + JSON.stringify({ + timestamp: `${day}T12:00:01`, + type: "turn_context", + payload: { model: "gpt-5" }, + }), + ...tokens.map((n, i) => + JSON.stringify({ + timestamp: `${day}T12:00:0${2 + i}`, + type: "event_msg", + payload: { + type: "token_count", + info: { last_token_usage: { input_tokens: n, output_tokens: 0 } }, + }, + }) + ), + ].join("\n"); + +function writeCodexFixture(homeDir: string, day: string, tokens: number[]): void { + const [y, m, d] = day.split("-"); + const dir = join(homeDir, ".codex/sessions", y, m, d); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, `rollout-${day}-fixture.jsonl`), codexEvents(day, tokens)); +} + +const codexRecord = (ts: number, inputTokens: number): TokenRecord => + ({ + timestamp: ts, + provider: "codex", + model: "gpt-5", + project: "demo", + inputTokens, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + reasoningTokens: 0, + cost: 0, + }) as TokenRecord; + +describe("rebuildRecentWindow — sealed days never shrink without force", () => { + let home: string; + const DAY_KEY = "2026-06-13"; + const dayTs = Date.parse(`${DAY_KEY}T12:00:00`); // local noon + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "tokmeter-rescan-")); + }); + afterEach(() => { + rmSync(home, { recursive: true, force: true }); + }); + + function sealDay(tokenCounts: number[]): void { + const [day] = aggregateRecordsByDay(tokenCounts.map((n, i) => codexRecord(dayTs + i, n))); + writeDayFile(home, day); + } + + test("partial raw (fewer tokens than sealed) → sealed day kept, warning pushed", async () => { + sealDay([100, 200]); // sealed: 300 tokens + writeCodexFixture(home, DAY_KEY, [100]); // raw now holds only 100 + const warnings: ScanWarning[] = []; + const state = await rebuildRecentWindow(ctxFor(home), REF, warnings, 7); + expect(state.aggregates.get(DAY_KEY)?.totalTokens).toBe(300); + expect(warnings.some((w) => w.scope === "history" && w.message.includes(DAY_KEY))).toBe(true); + }); + + test("force=true replaces the sealed day even when the rebuild is smaller", async () => { + sealDay([100, 200]); + writeCodexFixture(home, DAY_KEY, [100]); + const warnings: ScanWarning[] = []; + const state = await rebuildRecentWindow(ctxFor(home), REF, warnings, 7, true); + expect(state.aggregates.get(DAY_KEY)?.totalTokens).toBe(100); + }); + + test("richer raw (more tokens than sealed) → replaced without force", async () => { + sealDay([100]); // sealed: 100 tokens + writeCodexFixture(home, DAY_KEY, [100, 200]); // raw grew (e.g. parser fix found more) + const warnings: ScanWarning[] = []; + const state = await rebuildRecentWindow(ctxFor(home), REF, warnings, 7); + expect(state.aggregates.get(DAY_KEY)?.totalTokens).toBe(300); + }); + + test("raw fully deleted → sealed day untouched (relay's core promise)", async () => { + sealDay([100, 200]); + const warnings: ScanWarning[] = []; + const state = await rebuildRecentWindow(ctxFor(home), REF, warnings, 7); + expect(state.aggregates.get(DAY_KEY)?.totalTokens).toBe(300); + }); +}); + +describe("shouldKeepSealedDay — per-provider no-shrink contract", () => { + const dayFrom = (records: TokenRecord[]): DailyAggregate => aggregateRecordsByDay(records)[0]; + const ts = Date.parse("2026-06-13T12:00:00"); + + test("day grows overall but one provider bucket shrinks → keep sealed", () => { + const sealed = dayFrom([codexRecord(ts, 100), r(ts + 1, 0)]); // codex 100 + claude 120 + const rebuilt = dayFrom([codexRecord(ts, 50), r(ts + 1, 0), r(ts + 2, 0)]); // codex 50, claude 240 + expect(rebuilt.totalTokens).toBeGreaterThan(sealed.totalTokens); + expect(shouldKeepSealedDay(sealed, rebuilt, { force: false })).toBe(true); + }); + + test("identical rebuild → replace (costByHour backfill must work)", () => { + const sealed = dayFrom([codexRecord(ts, 100)]); + const rebuilt = dayFrom([codexRecord(ts, 100)]); + expect(shouldKeepSealedDay(sealed, rebuilt, { force: false })).toBe(false); + }); + + test("same tokens but lower cost (kosha offline during rescan) → keep sealed", () => { + const sealed = dayFrom([{ ...codexRecord(ts, 100), cost: 2.5 } as TokenRecord]); + const rebuilt = dayFrom([codexRecord(ts, 100)]); // cost 0 — pricing unavailable + expect(rebuilt.totalTokens).toBe(sealed.totalTokens); + expect(shouldKeepSealedDay(sealed, rebuilt, { force: false })).toBe(true); + expect(shouldKeepSealedDay(sealed, rebuilt, { force: true })).toBe(false); + }); + + test("provider missing entirely from rebuild → keep sealed", () => { + const sealed = dayFrom([codexRecord(ts, 100), r(ts + 1, 0)]); + const rebuilt = dayFrom([r(ts + 1, 0)]); // codex vanished + expect(shouldKeepSealedDay(sealed, rebuilt, { force: false })).toBe(true); + }); +}); + +describe("refreshFromRelay — gap fill never seals days before the gap", () => { + let home: string; + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "tokmeter-gapfix-")); + }); + afterEach(() => { + rmSync(home, { recursive: true, force: true }); + }); + + test("pre-gap records admitted by the mtime watermark are not sealed as partial days", async () => { + // Relay is current through 2026-06-10; the gap starts 06-11. A fresh-mtime + // multi-day file also carries 06-09 records (a still-active old session). + // 06-09 is an interior hole — sealing it from this one file's slice would + // freeze a partial day forever. + const seeded = seedDay(home, Date.parse("2026-06-10T12:00:00"), 1.0); + expect(seeded).toBe("2026-06-10"); + writeCodexFixture(home, "2026-06-09", [500]); + writeCodexFixture(home, "2026-06-14", [700]); + const warnings: ScanWarning[] = []; + const state = await refreshFromRelay(ctxFor(home), REF, warnings, false); + expect(state.aggregates.has("2026-06-09")).toBe(false); // interior hole stays open + expect(state.aggregates.get("2026-06-14")?.totalTokens).toBe(700); // gap day sealed + expect(state.aggregates.get("2026-06-10")?.cost).toBeCloseTo(1.0, 10); // untouched + }); +}); diff --git a/packages/core/src/relay-loader.ts b/packages/core/src/relay-loader.ts index facbbf7..7d08815 100644 --- a/packages/core/src/relay-loader.ts +++ b/packages/core/src/relay-loader.ts @@ -16,7 +16,7 @@ import { migrateMonolithSnapshotIfNeeded, writeDayFile, } from "./aggregates-store.js"; -import { type DailyAggregate, aggregateRecordsByDay } from "./aggregates.js"; +import { type DailyAggregate, aggregateRecordsByDay, shouldKeepSealedDay } from "./aggregates.js"; import { isBeforeToday, localDateKey, startOfLocalDay, yesterdayDateKey } from "./date-utils.js"; import { getParsers } from "./parsers/index.js"; import { enrichCosts, toErrorMessage } from "./pricing-enrichment.js"; @@ -73,15 +73,24 @@ export async function refreshFromRelay( const raw = await scanRawRecords(ctx, undefined, "history", warnings, floorMs); const todayKey = localDateKey(referenceTimestamp); - // A partial gap scan (provider crash mid-fill) writes nothing — a healthy - // re-run will fill it correctly. Better to retry than freeze a degraded day. - const gapDegraded = warnings.slice(warnBefore).some((w) => w.scope === "provider"); + // A crashed provider mid-fill writes nothing — a healthy re-run will fill + // it correctly. Better to retry than freeze a degraded day. Per-file + // `partial` faults deliberately do NOT abort: a permanently unreadable file + // would otherwise block the gap fill on every cold start until the raw + // JSONL behind the gap ages out — losing whole days to protect one file's + // tail. + const gapDegraded = warnings.slice(warnBefore).some((w) => w.scope === "provider" && !w.partial); if (gapDegraded) return { aggregates, historySource: "extended" }; const onDiskSet = new Set(onDisk); for (const day of aggregateRecordsByDay(raw)) { // Never seal today; never overwrite an already-sealed (immutable) day. - if (day.date >= todayKey || onDiskSet.has(day.date)) continue; + // Never seal a day BEFORE the gap either: the mtime watermark admits + // multi-day session files whose early records predate gapStart, but those + // records are only the slice living in still-fresh files — sealing an + // interior-hole day from them freezes a partial day permanently. Interior + // holes are recovered by an explicit rescan, not the gap fill. + if (day.date >= todayKey || day.date < gapStart || onDiskSet.has(day.date)) continue; try { writeDayFile(ctx.homeDir, day); aggregates.set(day.date, day); @@ -121,7 +130,6 @@ async function foldRawIntoDays( floorMs?: number ): Promise> { const todayKey = localDateKey(referenceTimestamp); - const scanOpts = floorMs !== undefined ? { modifiedSinceMs: floorMs } : undefined; const dayAccs = new Map(); const foldFile = async (records: TokenRecord[]): Promise => { @@ -147,6 +155,21 @@ async function foldRawIntoDays( }; for (const parser of getParsers(undefined)) { + // Per-file fail-soft truncations surface here for visibility. They don't + // gate the rebuild — rebuildRecentWindow's overwrite-vs-keep decision is + // shouldKeepSealedDay's size/cost comparison, which catches the truncated + // result by its own shrinkage. `partial: true` keeps the gap fill's + // whole-provider abort from firing on a single bad file. + const scanOpts = { + ...(floorMs !== undefined ? { modifiedSinceMs: floorMs } : {}), + onWarning: (message: string) => + warnings.push({ + scope: "provider", + provider: parser.providerId, + message: `${parser.providerId} partial scan: ${message}`, + partial: true, + }), + }; try { if (parser.scanStreaming) { await parser.scanStreaming(ctx.homeDir, scanOpts, foldFile); @@ -204,19 +227,55 @@ async function rebuildHistoricalFromScratch( * day sealed by buggy code, WITHOUT the multi-GB full-history parse that OOM'd * the box. Older days' curves are never read by pace, so re-deriving them is * pure waste — we skip it. + * + * Sealed days are only REPLACED when the rebuild carries at least as much + * data — day total and every provider bucket (see shouldKeepSealedDay). A + * shrunken rebuild means missing data: raw JSONL cleaned up, a truncated + * per-file read (now surfaced via onWarning), or a provider crash. This is + * the guard whose absence let the 2026-07-12 deep rescan shrink a sealed + * 2026-06-12. `force` overrides for an explicit "replace it anyway" rescan. */ export async function rebuildRecentWindow( ctx: ScanContext, referenceTimestamp: number, warnings: ScanWarning[], - windowDays: number + windowDays: number, + force = false ): Promise { migrateV2IfNeeded(ctx.homeDir); const floorMs = startOfLocalDay(referenceTimestamp - windowDays * 86_400_000); + const warnBefore = warnings.length; const rebuilt = await foldRawIntoDays(ctx, referenceTimestamp, warnings, floorMs); + // Providers whose scan soft-failed on a file during THIS fold. `force` + // consents to an intentional shrink (a parser fix), not to a broken read — + // a day that shrank for a partial-scan provider is kept even when forced, + // so a transient read fault during a forced rescan can't reproduce the + // 2026-07-12 loss with the user's own consent as the murder weapon. + const partialProviders = new Set( + warnings + .slice(warnBefore) + .filter((w) => w.scope === "provider" && w.partial && w.provider) + .map((w) => w.provider) + ); // Start from what's already sealed, overwrite ONLY the window's days. const aggregates = loadAggregates(ctx.homeDir); for (const [date, day] of rebuilt) { + const existing = aggregates.get(date); + const forcedButTruncated = + force && + existing && + [...partialProviders].some( + (p) => + p !== undefined && + (day.providers[p]?.totalTokens ?? 0) < (existing.providers[p]?.totalTokens ?? 0) + ); + if (existing && (forcedButTruncated || shouldKeepSealedDay(existing, day, { force }))) { + warnings.push({ + scope: "history", + message: `Kept sealed day ${date} — rebuild carried less data (rebuilt ${day.totalTokens} vs sealed ${existing.totalTokens} tokens${forcedButTruncated ? "; a truncated file read made the forced rebuild untrustworthy for it — retry" : ""}).`, + }); + continue; + } try { writeDayFile(ctx.homeDir, day); aggregates.set(date, day); diff --git a/packages/core/src/scan-pipeline.ts b/packages/core/src/scan-pipeline.ts index aad6608..49fb310 100644 --- a/packages/core/src/scan-pipeline.ts +++ b/packages/core/src/scan-pipeline.ts @@ -59,11 +59,23 @@ export async function scanRawRecords( } const parsers = getParsers(providers); - const scanOpts = modifiedSinceMs !== undefined ? { modifiedSinceMs } : undefined; const results = await Promise.all( parsers.map(async (parser) => { try { - return await parser.scan(ctx.homeDir, scanOpts); + return await parser.scan(ctx.homeDir, { + ...(modifiedSinceMs !== undefined ? { modifiedSinceMs } : {}), + // Per-file fail-soft truncations surface as provider warnings so + // consumers can see the scan was partial. `partial: true` keeps the + // gap fill's abort reserved for whole-provider crashes — one + // persistently bad file must not block gap sealing forever. + onWarning: (message) => + warnings.push({ + scope: "provider", + provider: parser.providerId, + message: `${parser.providerId} partial scan: ${message}`, + partial: true, + }), + }); } catch (error) { warnings.push({ scope: "provider", diff --git a/packages/core/src/summary-cache.ts b/packages/core/src/summary-cache.ts index 5b7ad11..2671046 100644 --- a/packages/core/src/summary-cache.ts +++ b/packages/core/src/summary-cache.ts @@ -20,7 +20,10 @@ export interface LoadedSummaryCache { warnings: ScanWarning[]; } -const SUMMARY_CACHE_VERSION = 2; +// 3 — model summaries switched from day-level even-split to exact per-project +// (provider, model) attribution; cached summaries hold the old phantom +// half-token rows and must rebuild. +const SUMMARY_CACHE_VERSION = 3; const SUMMARY_CACHE_DIR_NAME = ".cache/tokmeter"; const SUMMARY_CACHE_FILE_NAME = "summary-cache.json"; diff --git a/packages/core/src/tokmeter-core.ts b/packages/core/src/tokmeter-core.ts index f06f6e3..b60eceb 100644 --- a/packages/core/src/tokmeter-core.ts +++ b/packages/core/src/tokmeter-core.ts @@ -276,7 +276,15 @@ export class TokmeterCore { ...this.scanMeta, todayState: resolveTodayState(this.recentRecords, todayWarnings, false), lastScanAt: Date.now(), - warnings: [...this.scanMeta.warnings.filter((w) => w.scope !== "today"), ...todayWarnings], + // Drop the previous tick's today AND provider warnings before merging — + // this tick's todayWarnings carries both kinds afresh. Without the + // provider filter, a persistently unreadable file added one identical + // warning per 12s tick, growing scanMeta.warnings (and every + // /api/summary payload) without bound. + warnings: [ + ...this.scanMeta.warnings.filter((w) => w.scope !== "today" && w.scope !== "provider"), + ...todayWarnings, + ], }; // No saveSummaryCache here: that's an offline fallback, not per-tick — @@ -353,14 +361,23 @@ export class TokmeterCore { * can exhaust memory. Older days (which pace never reads) are left as-is. * Streaming + windowed, so peak memory stays bounded. Today is untouched. */ - async rebuildRecentDays(windowDays: number, now: number = Date.now()): Promise { + async rebuildRecentDays( + windowDays: number, + now: number = Date.now(), + force = false + ): Promise { const warnings: ScanWarning[] = []; - const relay = await rebuildRecentWindow(this.ctx(), now, warnings, windowDays); + const relay = await rebuildRecentWindow(this.ctx(), now, warnings, windowDays, force); this.aggregates = relay.aggregates; this.scanMeta = { ...this.scanMeta, lastScanAt: Date.now(), - warnings: [...this.scanMeta.warnings.filter((w) => w.scope !== "history"), ...warnings], + // The rebuild emits history AND provider warnings — replace both kinds + // rather than stacking a fresh copy per rescan. + warnings: [ + ...this.scanMeta.warnings.filter((w) => w.scope !== "history" && w.scope !== "provider"), + ...warnings, + ], }; } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index af0c186..451b8f0 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -237,6 +237,14 @@ export interface ScanWarning { provider?: ProviderId; /** Human-readable warning message. */ message: string; + /** + * True for a per-file fail-soft fault (one truncated/unreadable file) as + * opposed to a whole-provider crash. The gap fill aborts only on full + * crashes — a single permanently-bad file (iCloud-evicted, EACCES) must + * not block sealing forever, or the gap grows until the raw JSONL behind + * it ages out and those days are lost for every provider. + */ + partial?: boolean; } /** Scan metadata describing the stable-history/live-today composition state. */ @@ -520,6 +528,15 @@ export interface TokmeterConfig { export interface ScanFilterOptions { /** Skip files whose mtime is strictly before this epoch-ms watermark. */ modifiedSinceMs?: number; + /** + * Sink for per-file fail-soft conditions (a mid-stream read fault, a file + * that vanished between stat and read). Parsers deliberately return partial + * results rather than abort the provider — but a windowed relay rebuild must + * KNOW the result is partial, or it overwrites good sealed days with the + * truncated one. Callers that seal history should treat any invocation as + * "this provider's scan is incomplete". + */ + onWarning?: (message: string) => void; } /** The interface every session parser must implement. */ diff --git a/packages/macos-bar/Sources/TokmeterBar/DataSections.swift b/packages/macos-bar/Sources/TokmeterBar/DataSections.swift index 1227844..fdca192 100644 --- a/packages/macos-bar/Sources/TokmeterBar/DataSections.swift +++ b/packages/macos-bar/Sources/TokmeterBar/DataSections.swift @@ -46,8 +46,16 @@ struct ModelsSection: View { .foregroundColor(theme.backgroundMode.secondaryTextColor) } else { let maxCost = activeModels.first?.cost ?? 1 + // The same model can appear under two providers (codex + + // codex-desktop) — those rows are otherwise byte-identical + // (glyph and label derive from the model name), so surface + // the provider on exactly the ambiguous ones. + let dupNames = Set( + Dictionary(grouping: activeModels, by: \.model) + .filter { $0.value.count > 1 }.keys + ) ForEach(activeModels) { model in - modelRow(model, maxCost: maxCost) + modelRow(model, maxCost: maxCost, showProvider: dupNames.contains(model.model)) } } } @@ -86,17 +94,25 @@ struct ModelsSection: View { model.cost == 0 } - private func modelRow(_ model: ModelUsage, maxCost: Double) -> some View { + private func modelRow(_ model: ModelUsage, maxCost: Double, showProvider: Bool = false) -> some View { let activityOnly = isActivityOnly(model) return HStack(spacing: 8) { HStack(spacing: 3) { Image(systemName: providerGlyph(for: model.model)) .font(.system(size: 10)) .foregroundColor(c.accent) - Text(Fmt.shortModel(model.model)) - .font(.system(size: 11, weight: .medium, design: .monospaced)) - .foregroundColor(theme.backgroundMode.primaryTextColor) - .lineLimit(1) + VStack(alignment: .leading, spacing: 0) { + Text(Fmt.shortModel(model.model)) + .font(.system(size: 11, weight: .medium, design: .monospaced)) + .foregroundColor(theme.backgroundMode.primaryTextColor) + .lineLimit(1) + if showProvider { + Text(model.provider) + .font(.system(size: 8, design: theme.fonts.bodyDesign)) + .foregroundColor(theme.backgroundMode.secondaryTextColor) + .lineLimit(1) + } + } } .frame(width: 110, alignment: .leading) GeometryReader { geo in @@ -218,10 +234,13 @@ struct ModelsSection: View { // MARK: - Week section -/// Last-7-days line chart. Data changes animate with a spring curve so new -/// values glide into place instead of snapping. +/// Last-7-days chart. Style (line / bars / area) comes from +/// `bar.weekChartStyle` in config.json, switchable from the settings popover. +/// Data changes animate with a spring curve so new values glide into place +/// instead of snapping. struct WeekSection: View { @ObservedObject var loader: TokmeterLoader + @ObservedObject private var configStore = HubConfigStore.shared let theme: AppTheme /// 0→1 over ~0.9s on first appear. Drives a leading-edge mask so the @@ -229,6 +248,7 @@ struct WeekSection: View { @State private var drawProgress: CGFloat = 0 private var c: ThemeColors { theme.colors } + private var style: WeekChartStyle { configStore.config.chartStyle } var body: some View { VStack(alignment: .leading, spacing: 8) { @@ -238,24 +258,54 @@ struct WeekSection: View { ShimmerBar(width: 340, height: 60, breathToggle: true) } else { Chart(loader.recentDaily) { day in - LineMark( - x: .value("Date", String(day.date.suffix(5))), - y: .value("Cost", day.cost) - ) - .foregroundStyle(LinearGradient( - colors: [c.accent, c.warm], - startPoint: .leading, endPoint: .trailing)) - .interpolationMethod(.catmullRom) - .lineStyle(StrokeStyle(lineWidth: 2.5, lineCap: .round, lineJoin: .round)) - - AreaMark( - x: .value("Date", String(day.date.suffix(5))), - y: .value("Cost", day.cost) - ) - .foregroundStyle(LinearGradient( - colors: [c.accent.opacity(0.3), .clear], - startPoint: .top, endPoint: .bottom)) - .interpolationMethod(.catmullRom) + if style == .bars { + BarMark( + x: .value("Date", String(day.date.suffix(5))), + y: .value("Cost", day.cost) + ) + .foregroundStyle(LinearGradient( + colors: [c.accent, c.accent.opacity(0.45)], + startPoint: .top, endPoint: .bottom)) + .cornerRadius(3) + } else if style == .area { + AreaMark( + x: .value("Date", String(day.date.suffix(5))), + y: .value("Cost", day.cost) + ) + .foregroundStyle(LinearGradient( + colors: [c.accent.opacity(0.55), c.accent.opacity(0.06)], + startPoint: .top, endPoint: .bottom)) + .interpolationMethod(.catmullRom) + + LineMark( + x: .value("Date", String(day.date.suffix(5))), + y: .value("Cost", day.cost) + ) + .foregroundStyle(LinearGradient( + colors: [c.accent, c.warm], + startPoint: .leading, endPoint: .trailing)) + .interpolationMethod(.catmullRom) + .lineStyle(StrokeStyle(lineWidth: 1.5, lineCap: .round, lineJoin: .round)) + } else { + LineMark( + x: .value("Date", String(day.date.suffix(5))), + y: .value("Cost", day.cost) + ) + .foregroundStyle(LinearGradient( + colors: [c.accent, c.warm], + startPoint: .leading, endPoint: .trailing)) + .interpolationMethod(.catmullRom) + .lineStyle(StrokeStyle(lineWidth: 2.5, lineCap: .round, lineJoin: .round)) + + AreaMark( + x: .value("Date", String(day.date.suffix(5))), + y: .value("Cost", day.cost) + ) + .foregroundStyle(LinearGradient( + colors: [c.accent.opacity(0.3), .clear], + startPoint: .top, endPoint: .bottom)) + .interpolationMethod(.catmullRom) + } // Today is the still-accumulating last point — usually a // flat tail next to full days, so give it a marker + its @@ -266,7 +316,7 @@ struct WeekSection: View { y: .value("Cost", day.cost) ) .foregroundStyle(c.warm) - .symbolSize(28) + .symbolSize(style == .bars ? 0 : 28) .annotation(position: .top, spacing: 3) { Text(Fmt.cost(day.cost)) .font(.system(size: 8, weight: .semibold, design: .rounded)) @@ -295,6 +345,7 @@ struct WeekSection: View { } ) .animation(.spring(response: 0.7, dampingFraction: 0.80), value: loader.recentDaily.map(\.cost)) + .animation(.spring(response: 0.5, dampingFraction: 0.80), value: style) .onAppear { withAnimation(.easeOut(duration: 0.9)) { drawProgress = 1.0 } } diff --git a/packages/macos-bar/Sources/TokmeterBar/HubConfigStore.swift b/packages/macos-bar/Sources/TokmeterBar/HubConfigStore.swift index 8bea514..75b5b79 100644 --- a/packages/macos-bar/Sources/TokmeterBar/HubConfigStore.swift +++ b/packages/macos-bar/Sources/TokmeterBar/HubConfigStore.swift @@ -65,12 +65,30 @@ enum MenubarColorSource: String, Codable, CaseIterable, Identifiable { } } +/// Rendering style for the popover's LAST 7 DAYS chart. Mirrors +/// WeekChartStyle in packages/core/src/config-service.ts — keep raw values +/// identical. +enum WeekChartStyle: String, Codable, CaseIterable, Identifiable { + case line, bars, area + var id: String { rawValue } + var label: String { + switch self { + case .line: return "Line" + case .bars: return "Bars" + case .area: return "Area" + } + } +} + struct HubUserConfig: Codable { struct BarConfig: Codable { var refreshSeconds: Int /// Optional so a config.json written before this field existed still /// decodes; nil means the default (context). Use `colorSource` to read. var menubarColorSource: MenubarColorSource? + /// Optional for the same decode-tolerance reason; nil means line. + /// Use `chartStyle` to read. + var weekChartStyle: WeekChartStyle? } struct DaemonConfig: Codable { var scanIntervalSeconds: Int @@ -79,7 +97,12 @@ struct HubUserConfig: Codable { /// internal status RPC — real enough to run unsupervised and /// indefinitely in the background that it needs an explicit, /// durable opt-in (see config-service.ts for the full rationale). - var antigravityLivePolling: Bool + /// Optional so a config.json written before this field existed still + /// strict-decodes (same tolerance as the bar's optional fields); + /// nil means off — read through `antigravityPollingEnabled`. + var antigravityLivePolling: Bool? + + var antigravityPollingEnabled: Bool { antigravityLivePolling ?? false } } struct CliConfig: Codable { var defaultRange: ConfigDefaultRange @@ -98,16 +121,21 @@ struct HubUserConfig: Codable { /// this yet — edit config.json by hand — but it must still round-trip: /// without this field, saving any *other* Hub setting would silently /// drop a hand-edited providerPaths entry on the next disk write. - var providerPaths: [String: [String]] + /// Optional so a config.json from before the field strict-decodes; a + /// missing key means "none configured" and encodes back as absent. + var providerPaths: [String: [String]]? var modifiedBy: ConfigModifiedBy var modifiedAt: String /// Menubar color source with the nil-safe default applied. var colorSource: MenubarColorSource { bar.menubarColorSource ?? .context } + /// 7-day chart style with the nil-safe default applied. + var chartStyle: WeekChartStyle { bar.weekChartStyle ?? .line } + static let defaults = HubUserConfig( version: 1, - bar: .init(refreshSeconds: 30, menubarColorSource: .context), + bar: .init(refreshSeconds: 30, menubarColorSource: .context, weekChartStyle: .line), daemon: .init(scanIntervalSeconds: 60, antigravityLivePolling: false), cli: .init(defaultRange: .all, defaultSort: .cost), alerts: .init(dailyCostThreshold: nil), @@ -175,9 +203,13 @@ final class HubConfigStore: ObservableObject { return nil } // Tolerant decode: a missing field falls back to its default so schema - // growth on the CLI side doesn't brick the Swift side. + // growth on the CLI side doesn't brick the Swift side. The strict path + // MUST clamp too — the optional fields mean legacy files now succeed + // here instead of falling through to merge(), and a hand-edited + // "refreshSeconds": 0 would otherwise drive a near-continuous refresh + // loop (merge() was the only place the [5,3600] clamp lived). if let decoded = try? JSONDecoder().decode(HubUserConfig.self, from: data) { - return decoded + return sanitize(decoded) } // Partial / old schema — try a lenient Any decode and merge over defaults. if let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any] { @@ -186,6 +218,17 @@ final class HubConfigStore: ObservableObject { return nil } + /// Range-clamp the numeric knobs, mirroring config-service.ts. Applied to + /// every strict decode; merge() applies the same bounds on the lenient path. + private static func sanitize(_ cfg: HubUserConfig) -> HubUserConfig { + var out = cfg + let d = HubUserConfig.defaults + out.bar.refreshSeconds = clamp(out.bar.refreshSeconds, min: 5, max: 3600, def: d.bar.refreshSeconds) + out.daemon.scanIntervalSeconds = clamp( + out.daemon.scanIntervalSeconds, min: 10, max: 3600, def: d.daemon.scanIntervalSeconds) + return out + } + private func saveToDisk(_ cfg: HubUserConfig) { let dir = (Self.filePath as NSString).deletingLastPathComponent try? FileManager.default.createDirectory(atPath: dir, withIntermediateDirectories: true) @@ -224,6 +267,10 @@ final class HubConfigStore: ObservableObject { let src = MenubarColorSource(rawValue: cs) { out.bar.menubarColorSource = src } + if let ws = bar["weekChartStyle"] as? String, + let style = WeekChartStyle(rawValue: ws) { + out.bar.weekChartStyle = style + } } if let daemon = raw["daemon"] as? [String: Any] { // Independent `if let`s, same reasoning as the `bar` block above — diff --git a/packages/macos-bar/Sources/TokmeterBar/HubSettings.swift b/packages/macos-bar/Sources/TokmeterBar/HubSettings.swift index 33da9c1..169ef03 100644 --- a/packages/macos-bar/Sources/TokmeterBar/HubSettings.swift +++ b/packages/macos-bar/Sources/TokmeterBar/HubSettings.swift @@ -404,7 +404,7 @@ struct HubSettingsPanel: View { + "Antigravity didn't publish for this — an unsupervised, indefinite " + "background job, so it's opt-in only.", isOn: Binding( - get: { store.config.daemon.antigravityLivePolling }, + get: { store.config.daemon.antigravityPollingEnabled }, set: { v in store.update { $0.daemon.antigravityLivePolling = v } } ), theme: theme diff --git a/packages/macos-bar/Sources/TokmeterBar/Models.swift b/packages/macos-bar/Sources/TokmeterBar/Models.swift index 547d70f..80a2c1c 100644 --- a/packages/macos-bar/Sources/TokmeterBar/Models.swift +++ b/packages/macos-bar/Sources/TokmeterBar/Models.swift @@ -18,9 +18,11 @@ struct DailyUsage: Identifiable, Equatable { } struct ModelUsage: Identifiable, Equatable { - // Model name is the natural stable ID — SwiftUI diffs rows when models - // are added/removed or reorder, rather than rebuilding them all. - var id: String { model } + // Provider-qualified ID: the same model can legitimately appear under two + // providers in one day (codex + codex-desktop both running gpt-5.6-*). + // Keying by model name alone collided those rows, so SwiftUI rendered the + // first provider's cost twice. + var id: String { "\(provider)::\(model)" } let model: String let provider: String let cost: Double diff --git a/packages/macos-bar/Sources/TokmeterBar/SettingsPopover.swift b/packages/macos-bar/Sources/TokmeterBar/SettingsPopover.swift index 2cace38..2e4a3a8 100644 --- a/packages/macos-bar/Sources/TokmeterBar/SettingsPopover.swift +++ b/packages/macos-bar/Sources/TokmeterBar/SettingsPopover.swift @@ -13,6 +13,7 @@ struct SettingsPopover: View { /// @AppStorage and therefore every themed view in the tree. @Binding var theme: AppTheme @ObservedObject var loader: TokmeterLoader + @ObservedObject private var configStore = HubConfigStore.shared var body: some View { VStack(alignment: .leading, spacing: 14) { @@ -23,6 +24,10 @@ struct SettingsPopover: View { Divider() + chartStyleRow + + Divider() + HStack { Text("Refresh interval") .font(.system(size: 11, design: .rounded)) @@ -54,6 +59,30 @@ struct SettingsPopover: View { .frame(width: 320) } + // MARK: - 7-day chart style row + + /// Segmented line/bars/area picker for the popover's LAST 7 DAYS chart. + /// Writes through HubConfigStore so the choice persists in config.json + /// and survives restarts (and cross-machine restores, via modifiedBy). + private var chartStyleRow: some View { + HStack { + Text("7-day chart") + .font(.system(size: 11, design: .rounded)) + Spacer() + Picker("7-day chart style", selection: Binding( + get: { configStore.config.chartStyle }, + set: { style in configStore.update { $0.bar.weekChartStyle = style } } + )) { + ForEach(WeekChartStyle.allCases) { s in + Text(s.label).tag(s) + } + } + .pickerStyle(.segmented) + .labelsHidden() + .frame(width: 160) + } + } + // MARK: - Pricing refresh row private var pricingRefreshRow: some View { diff --git a/packages/macos-bar/bundle.sh b/packages/macos-bar/bundle.sh index 6cabee7..0aa72c7 100755 --- a/packages/macos-bar/bundle.sh +++ b/packages/macos-bar/bundle.sh @@ -61,7 +61,7 @@ RESOURCES_DIR="${CONTENTS}/Resources" FRAMEWORKS_DIR="${CONTENTS}/Frameworks" ENTITLEMENTS="entitlements.plist" SHORT_VERSION="${CFBundleShortVersionString:-1.9.2}" -BUILD_VERSION="${CFBundleVersion:-43}" +BUILD_VERSION="${CFBundleVersion:-44}" SUFEED_URL="${SUFEED_URL:-https://raw.githubusercontent.com/sriinnu/tokmeter/main/packages/macos-bar/appcast.xml}" SUPUBLIC_KEY="${SUPUBLIC_KEY:-}" # populated below if private key is present diff --git a/packages/mcp/package.json b/packages/mcp/package.json index 6faf5bb..477886d 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@sriinnu/drishti", - "version": "1.9.1", + "version": "1.9.2", "description": "दृष्टि — MCP server + live token observatory for AI coding agents", "type": "module", "bin": { diff --git a/packages/mcp/src/daemon/server.ts b/packages/mcp/src/daemon/server.ts index e96b399..35998fc 100644 --- a/packages/mcp/src/daemon/server.ts +++ b/packages/mcp/src/daemon/server.ts @@ -1363,9 +1363,15 @@ function startHttpApi(): void { if (_rescanInFlight) { json(res, { ok: true, started: false, alreadyRunning: true }); } else { + // `?force=true` bypasses the sealed-day shrink guard — an explicit + // "replace even if the rebuild carries less data" rescan. The + // default deep rescan can only grow or match sealed days. + const qi = url.indexOf("?"); + const force = + qi >= 0 && new URLSearchParams(url.slice(qi + 1)).get("force") === "true"; _rescanInFlight = true; void core - .rebuildRecentDays(DEEP_RESCAN_WINDOW_DAYS) + .rebuildRecentDays(DEEP_RESCAN_WINDOW_DAYS, Date.now(), force) .then(() => { _httpCore = { core, ts: Date.now() }; }) diff --git a/packages/tokmeter/package.json b/packages/tokmeter/package.json index c6af2fc..deb6f73 100644 --- a/packages/tokmeter/package.json +++ b/packages/tokmeter/package.json @@ -1,6 +1,6 @@ { "name": "@sriinnu/tokmeter", - "version": "1.9.1", + "version": "1.9.2", "description": "Token usage tracking for AI coding agents — parsers, CLI, and TUI", "type": "module", "main": "dist/core/index.js", diff --git a/packages/tui/package.json b/packages/tui/package.json index 130ea7a..c28de42 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,6 +1,6 @@ { "name": "@sriinnu/tokmeter-tui", - "version": "1.9.1", + "version": "1.9.2", "private": true, "description": "Token usage tracking TUI \u2014 interactive terminal UI with charts", "type": "module", diff --git a/packages/web/package.json b/packages/web/package.json index c91cd5c..6c77cbf 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,6 +1,6 @@ { "name": "@sriinnu/tokmeter-web", - "version": "1.9.1", + "version": "1.9.2", "private": true, "description": "Token usage tracking web dashboard \u2014 React + Plotly", "type": "module", diff --git a/packages/web/scripts/ensure-summary.ts b/packages/web/scripts/ensure-summary.ts index 6e50ff4..82893d9 100644 --- a/packages/web/scripts/ensure-summary.ts +++ b/packages/web/scripts/ensure-summary.ts @@ -2,22 +2,55 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { TokmeterCore, loadSummaryCache, saveSummaryCache } from "../../core/src/index.ts"; +import { + TokmeterCore, + type TokmeterSummary, + loadSummaryCache, + saveSummaryCache, +} from "../../core/src/index.ts"; const scriptDir = dirname(fileURLToPath(import.meta.url)); const publicDir = join(scriptDir, "..", "public"); const publicDataPath = join(publicDir, "data.json"); +const DAEMON_SUMMARY_URL = "http://127.0.0.1:9877/api/summary"; +const DAEMON_FETCH_TIMEOUT_MS = 3_000; + +/** Local calendar date key, matching core's localDateKey convention. */ +function localDateKey(ts: number): string { + const d = new Date(ts); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; +} + /** * Ensures the web package always has a fallback `public/data.json` summary. - * Uses the persisted summary cache when possible and builds it if missing. + * + * Source order: the warm daemon (always fresh, zero scan cost) → the persisted + * summary cache ONLY if it was scanned today → a fresh scan. The cache used to + * be trusted unconditionally, which baked days-old data into a static build + * that the dashboard then polled every 15s as if it were live. */ async function main(): Promise { const homeDir = homedir(); const existingFile = existsSync(publicDataPath); - const cached = loadSummaryCache(homeDir); - let summary = cached.summary; + // The freshness gate applies to BOTH shortcut sources: the daemon's + // /api/summary itself falls back to a persisted cache when its scan throws + // (it answers 200 either way), so a daemon response is not inherently + // fresh — an ungated fetch would bake days-old data right back into the + // static build. + const scannedToday = (s: TokmeterSummary | null): s is TokmeterSummary => + s !== null && localDateKey(s.meta?.lastScanAt ?? 0) === localDateKey(Date.now()); + + let summary = await fetchDaemonSummary(); + if (!scannedToday(summary)) summary = null; + + if (!summary) { + const cached = loadSummaryCache(homeDir); + if (scannedToday(cached.summary)) { + summary = cached.summary; + } + } if (!summary) { try { @@ -41,6 +74,18 @@ async function main(): Promise { writeFileSync(publicDataPath, `${JSON.stringify(summary, null, 2)}\n`, "utf-8"); } +async function fetchDaemonSummary(): Promise { + try { + const res = await fetch(DAEMON_SUMMARY_URL, { + signal: AbortSignal.timeout(DAEMON_FETCH_TIMEOUT_MS), + }); + if (!res.ok) return null; + return (await res.json()) as TokmeterSummary; + } catch { + return null; // daemon offline — fall back to cache/scan + } +} + function toErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/packages/web/src/charts/TodayHourlyChart.tsx b/packages/web/src/charts/TodayHourlyChart.tsx new file mode 100644 index 0000000..b051513 --- /dev/null +++ b/packages/web/src/charts/TodayHourlyChart.tsx @@ -0,0 +1,253 @@ +import type { CSSProperties } from "react"; +import { useState } from "react"; +import type { DashboardTodayHour } from "../pages/dashboard/buildDashboardInsights.js"; +import { + formatDashboardCurrency, + formatDashboardNumber, +} from "../pages/dashboard/dashboardFormatters.js"; +import { webTheme, withAlpha } from "../theme.js"; + +interface TodayHourlyChartProps { + hours: DashboardTodayHour[]; +} + +/** + * Today's activity by local hour — 24 bars with a hand-rolled hover tooltip. + * + * Bars encode ONE measure. Cost is the default; when every record today is + * unpriced (subscription/local models) the whole series switches to tokens and + * says so, rather than silently mixing dollars and token counts in one axis. + * Hover targets span the full column height so empty hours are inspectable too. + */ +export function TodayHourlyChart({ hours }: TodayHourlyChartProps) { + const [hovered, setHovered] = useState(null); + + const usesTokens = hours.every((h) => h.cost === 0) && hours.some((h) => h.totalTokens > 0); + const values = hours.map((h) => (usesTokens ? h.totalTokens : h.cost)); + const max = Math.max(...values); + const nowHour = new Date().getHours(); + const active = hovered !== null ? hours[hovered] : null; + + return ( +

+ {active !== null && hovered !== null && ( +
= 21 + ? { right: 0 } + : { + left: `${((hovered + 0.5) / 24) * 100}%`, + transform: "translateX(-50%)", + }), + }} + > +
+ {formatHourLabel(active.hour)} – {formatHourLabel(active.hour + 1)} +
+ {active.records > 0 ? ( + <> +
+ Cost + {formatDashboardCurrency(active.cost)} +
+
+ Tokens + {formatDashboardNumber(active.totalTokens)} +
+
+ Records + {active.records} +
+ {active.topModel && ( +
+ Top model + {active.topModel} +
+ )} + + ) : ( +
No activity
+ )} +
+ )} + +
setHovered(null)}> + {hours.map((bucket, index) => { + const value = values[index]; + const heightPct = max > 0 ? (value / max) * 100 : 0; + const isNow = bucket.hour === nowHour; + const isHovered = hovered === index; + return ( +
setHovered(index)} + role="presentation" + > +
0 ? `max(${heightPct}%, 3px)` : "2px", + background: + value > 0 + ? isHovered + ? webTheme.colors.cream + : withAlpha(webTheme.colors.olive, 0.92) + : withAlpha(webTheme.colors.cream, 0.14), + boxShadow: isNow ? `0 0 0 2px ${withAlpha(webTheme.colors.cream, 0.45)}` : "none", + }} + /> +
+ ); + })} +
+ +
+ {[0, 6, 12, 18].map((hour) => ( + + {formatHourLabel(hour)} + + ))} + {formatHourLabel(24)} +
+ +
+ + {usesTokens + ? "Bars show tokens per hour (no priced usage today)" + : "Bars show cost per hour"} + + + + current hour + +
+
+ ); +} + +function formatHourLabel(hour: number): string { + const h = hour % 24; + if (h === 0) return "12am"; + if (h === 12) return "12pm"; + return h < 12 ? `${h}am` : `${h - 12}pm`; +} + +const rootStyle: CSSProperties = { + position: "relative", + display: "grid", + gap: webTheme.spacing.sm, +}; + +const plotStyle: CSSProperties = { + alignItems: "flex-end", + borderBottom: `1px solid ${webTheme.charts.grid}`, + display: "flex", + gap: 2, + height: 180, +}; + +const columnStyle: CSSProperties = { + alignItems: "flex-end", + cursor: "default", + display: "flex", + flex: 1, + height: "100%", +}; + +const barStyle: CSSProperties = { + borderRadius: "3px 3px 0 0", + transition: `height ${webTheme.motion.duration.fast} ${webTheme.motion.easing.default}, background ${webTheme.motion.duration.fast} ${webTheme.motion.easing.default}`, + width: "100%", +}; + +const axisRowStyle: CSSProperties = { + color: webTheme.charts.axis, + fontSize: webTheme.typography.micro.size, + height: 14, + position: "relative", +}; + +const axisTickStyle: CSSProperties = { + position: "absolute", + top: 0, +}; + +const footRowStyle: CSSProperties = { + color: webTheme.text.muted, + display: "flex", + fontSize: webTheme.typography.micro.size, + justifyContent: "space-between", +}; + +const nowLegendStyle: CSSProperties = { + alignItems: "center", + display: "inline-flex", + gap: 6, +}; + +const nowDotStyle: CSSProperties = { + background: withAlpha(webTheme.colors.olive, 0.92), + borderRadius: 2, + boxShadow: `0 0 0 2px ${withAlpha(webTheme.colors.cream, 0.45)}`, + display: "inline-block", + height: 8, + width: 8, +}; + +const tooltipStyle: CSSProperties = { + background: `linear-gradient(180deg, ${withAlpha(webTheme.colors.pine, 0.98)}, ${withAlpha( + webTheme.colors.teal, + 0.94 + )})`, + border: `1px solid ${withAlpha(webTheme.colors.cream, 0.24)}`, + borderRadius: webTheme.radii.lg, + boxShadow: webTheme.elevation.high, + bottom: "calc(100% + 6px)", + minWidth: 168, + padding: webTheme.spacing.md, + pointerEvents: "none", + position: "absolute", + zIndex: 4, +}; + +const tooltipTitleStyle: CSSProperties = { + color: webTheme.text.muted, + fontSize: webTheme.typography.micro.size, + fontWeight: 700, + letterSpacing: "0.08em", + marginBottom: webTheme.spacing.xs, + textTransform: "uppercase", +}; + +const tooltipRowStyle: CSSProperties = { + display: "flex", + fontSize: webTheme.typography.caption.size, + gap: webTheme.spacing.md, + justifyContent: "space-between", + lineHeight: 1.7, +}; + +const tooltipLabelStyle: CSSProperties = { + color: webTheme.text.secondary, +}; + +const tooltipValueStyle: CSSProperties = { + color: webTheme.text.primary, + fontWeight: 700, + maxWidth: 200, + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap", +}; + +const tooltipEmptyStyle: CSSProperties = { + color: webTheme.text.secondary, + fontSize: webTheme.typography.caption.size, +}; diff --git a/packages/web/src/pages/dashboard/DashboardOverviewSection.tsx b/packages/web/src/pages/dashboard/DashboardOverviewSection.tsx index 3d4061a..cdbb378 100644 --- a/packages/web/src/pages/dashboard/DashboardOverviewSection.tsx +++ b/packages/web/src/pages/dashboard/DashboardOverviewSection.tsx @@ -4,11 +4,12 @@ import React from "react"; import Plot from "react-plotly.js"; import { ContributionHeatmap } from "../../charts/ContributionHeatmap.js"; import { ProviderPieChart } from "../../charts/ProviderPieChart.js"; +import { TodayHourlyChart } from "../../charts/TodayHourlyChart.js"; import type { LiveData } from "../../hooks/useLiveData.js"; import type { TokmeterData } from "../../hooks/useTokmeterData.js"; import { webTheme, withAlpha } from "../../theme.js"; import { DashboardPanel } from "./DashboardPanel.js"; -import type { DashboardInsights } from "./buildDashboardInsights.js"; +import type { DashboardInsights, DashboardModelInsight } from "./buildDashboardInsights.js"; import { formatDashboardCostPerMillion, formatDashboardCurrency, @@ -81,6 +82,20 @@ export const DashboardOverviewSection = memo(function DashboardOverviewSection({ ))}
+ + } + > + + +
0 + ? rankedByCost + : (insights.todayModels.reduce( + (best, m) => (!best || m.totalTokens > best.totalTokens ? m : best), + null + ) ?? null); + const hourlyRecords = today.hourly.reduce((sum, h) => sum + h.records, 0); + + if (!today.hasActivity) { + return ( + + ); + } + + return ( +
+
+
+
Cost today
+
{formatDashboardCurrency(today.cost)}
+
+
+
Tokens today
+
{formatDashboardNumber(today.totalTokens)}
+
+
+
Records
+
{formatDashboardNumber(today.records)}
+
+
+
Top model
+
{topModel ? topModel.model : "—"}
+ {topModel && ( +
+ {topModel.cost > 0 + ? `${formatDashboardCurrency(topModel.cost)} today` + : `${formatDashboardNumber(topModel.totalTokens)} tokens today (unpriced)`} +
+ )} +
+
+ + {hourlyRecords > 0 ? ( + + ) : ( + // Totals can exist without an hourly curve: the raw records window is + // what feeds the buckets, and a summary without records (stripped + // static export, writer-TZ mismatch) would otherwise render 24 dead + // bars under a real headline — reads as broken, not quiet. + + )} +
+ ); +} + function ActivityTrendPanel({ data, insights, diff --git a/packages/web/src/pages/dashboard/buildDashboardInsights.ts b/packages/web/src/pages/dashboard/buildDashboardInsights.ts index 13c8e74..c93cfe3 100644 --- a/packages/web/src/pages/dashboard/buildDashboardInsights.ts +++ b/packages/web/src/pages/dashboard/buildDashboardInsights.ts @@ -88,6 +88,26 @@ export interface DashboardActivityHighlight { helper: string; } +/** One local-hour bucket of today's activity, derived from the raw records window. */ +export interface DashboardTodayHour { + hour: number; + cost: number; + totalTokens: number; + records: number; + topModel: string | null; +} + +export interface DashboardTodayInsight { + /** Local calendar date this "today" refers to (browser timezone). */ + date: string; + cost: number; + totalTokens: number; + records: number; + hasActivity: boolean; + /** Always 24 entries, hour 0–23; empty hours carry zeros. */ + hourly: DashboardTodayHour[]; +} + export interface DashboardInsights { spotlight: DashboardSpotlight; heroMetrics: DashboardHeroMetric[]; @@ -97,6 +117,7 @@ export interface DashboardInsights { topProjects: DashboardProjectInsight[]; topModels: DashboardModelInsight[]; todayModels: DashboardModelInsight[]; + today: DashboardTodayInsight; recentDays: DashboardRecentDayInsight[]; trendWindow: TokmeterDailyEntry[]; } @@ -114,7 +135,13 @@ export function buildDashboardInsights(data: TokmeterData, liveData: LiveData): const trendWindow = daily.slice(-TREND_WINDOW_DAYS); const recentWindow = trendWindow.length > 0 ? trendWindow : daily; - const today = daily[daily.length - 1] ?? null; + // "Today" must actually BE today. The last daily entry is just the most + // recent active day — on a fresh morning or against a stale summary it's a + // previous day, and presenting it as today misreports spend (the + // display-vs-ledger scare class). No entry for the local date → no today. + const todayKey = rawLocalDateKey(Date.now()); + const latestDay = daily[daily.length - 1] ?? null; + const today = latestDay && latestDay.date === todayKey ? latestDay : null; const peakDay = daily.reduce((best, entry) => { if (!best || entry.cost > best.cost) { return entry; @@ -144,7 +171,8 @@ export function buildDashboardInsights(data: TokmeterData, liveData: LiveData): percentageOfTotal: model.percentageOfTotal, })); - const todayModels = buildTodayModels(records, today?.date ?? null); + const todayModels = buildTodayModels(records, todayKey); + const todayInsight = buildTodayInsight(records, todayKey, today); const recentDays = [...daily] .slice(-MAX_RECENT_DAYS) @@ -312,11 +340,89 @@ export function buildDashboardInsights(data: TokmeterData, liveData: LiveData): topProjects, topModels, todayModels, + today: todayInsight, recentDays, trendWindow, }; } +/** + * Bucket today's raw records into 24 local-hour slots for the hourly chart. + * Headline totals prefer the daily entry when one exists for today (it runs + * through the accumulator's dedup); the hourly curve itself only exists in the + * raw records window. + */ +function buildTodayInsight( + rawRecords: Array>, + todayKey: string, + todayEntry: TokmeterDailyEntry | null +): DashboardTodayInsight { + const hourly: DashboardTodayHour[] = Array.from({ length: 24 }, (_, hour) => ({ + hour, + cost: 0, + totalTokens: 0, + records: 0, + topModel: null, + })); + const modelsByHour: Array> = Array.from( + { length: 24 }, + () => new Map() + ); + + let cost = 0; + let totalTokens = 0; + let records = 0; + for (const raw of rawRecords) { + const r = raw as unknown as RawRecord; + if (rawLocalDateKey(r.timestamp) !== todayKey) continue; + const hour = new Date(r.timestamp).getHours(); + const tokens = + (r.inputTokens ?? 0) + + (r.outputTokens ?? 0) + + (r.cacheReadTokens ?? 0) + + (r.cacheWriteTokens ?? 0) + + (r.reasoningTokens ?? 0); + const bucket = hourly[hour]; + bucket.cost += r.cost ?? 0; + bucket.totalTokens += tokens; + bucket.records += 1; + cost += r.cost ?? 0; + totalTokens += tokens; + records += 1; + const models = modelsByHour[hour]; + const entry = models.get(r.model) ?? { cost: 0, tokens: 0 }; + entry.cost += r.cost ?? 0; + entry.tokens += tokens; + models.set(r.model, entry); + } + + // Top model per hour: cost first, token volume strictly as the tie-break. + // Never a mixed-unit sum — a cache-heavy unpriced model shouldn't outrank a + // model that genuinely cost money that hour. + for (const bucket of hourly) { + let top: string | null = null; + let topCost = 0; + let topTokens = 0; + for (const [model, m] of modelsByHour[bucket.hour]) { + if (m.cost > topCost || (m.cost === topCost && m.tokens > topTokens)) { + top = model; + topCost = m.cost; + topTokens = m.tokens; + } + } + bucket.topModel = top; + } + + return { + date: todayKey, + cost: todayEntry?.cost ?? cost, + totalTokens: todayEntry?.totalTokens ?? totalTokens, + records: todayEntry?.records ?? records, + hasActivity: records > 0 || (todayEntry?.totalTokens ?? 0) > 0, + hourly, + }; +} + function aggregateProviderInsights(projects: TokmeterProjectSummary[]): DashboardProviderInsight[] { const providerMap = new Map< string, diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 2b86f15..bd3b130 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -9,6 +9,7 @@ # Touches: # - all packages/*/package.json "version" field # - packages/macos-bar/bundle.sh SHORT_VERSION default (+ bumps BUILD_VERSION) +# - README.md release badge version # - CHANGELOG.md inserts a dated "## [X.Y.Z]" skeleton if absent # # The macOS bar BUILD_VERSION (CFBundleVersion) is monotonic — every run @@ -49,7 +50,14 @@ if [[ -f "$BUNDLE" ]]; then echo " bundle.sh SHORT_VERSION=${VERSION} BUILD_VERSION ${cur_build} -> ${new_build}" fi -# 3. CHANGELOG.md — insert a dated skeleton entry directly above the newest +# 3. README.md — release badge tracks the tag being cut. +README="README.md" +if [[ -f "$README" ]]; then + perl -i -pe 's/(badge\/release-v)[0-9.]+(-)/${1}'"$VERSION"'${2}/' "$README" + echo " README.md release badge -> v${VERSION}" +fi + +# 4. CHANGELOG.md — insert a dated skeleton entry directly above the newest # existing release heading, but only if this version isn't already present. CHANGELOG="CHANGELOG.md" if [[ -f "$CHANGELOG" ]] && ! grep -q "## \[${VERSION}\]" "$CHANGELOG"; then diff --git a/scripts/release.sh b/scripts/release.sh index 6ea47de..612557d 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -78,7 +78,7 @@ run "bun run lint" # ── 5. commit + tag (Titan-signed) ─────────────────────────────────────────── say "5/10 commit + tag (signed)" -run "git add -A packages/*/package.json packages/macos-bar/bundle.sh CHANGELOG.md" +run "git add -A packages/*/package.json packages/macos-bar/bundle.sh CHANGELOG.md README.md" run "git commit -S -m 'chore(release): ${TAG}'" run "git tag -s '${TAG}' -m '${TAG}'"