From 85c896db41f20ec669bbdfb1432e42c27be0bb2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Gonz=C3=A1lez=20Barrera?= <150662051+dagoaie@users.noreply.github.com> Date: Sun, 13 Sep 2026 12:10:41 +0200 Subject: [PATCH] feat(runs): add --json output and a usage report over run history `convoy runs --json` prints the run history as a JSON array of durable facts (no liveness probe results, no workspace paths), and `convoy runs stats` aggregates the phases' recorded usage by pipeline, model, step or day as a table or as JSON rows. Both accept `--since d|h|YYYY-MM-DD` and `--pipeline `; tokens are first-class since subscription models record a cost of 0. - change run-history-report - fix(runs): address review findings on the history report - fix(runs): harden run history report after adversarial review - cover run history reporting behavior - neutralize terminal controls in run usage tables - align run history reporting with repository patterns Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0187tW4PcQWpmyDk3Wf6S8p2 --- README.md | 9 ++ .../.openspec.yaml | 2 + .../2026-09-13-run-history-report/design.md | 106 ++++++++++++++ .../2026-09-13-run-history-report/proposal.md | 30 ++++ .../specs/run-history-report/spec.md | 132 +++++++++++++++++ .../2026-09-13-run-history-report/tasks.md | 26 ++++ openspec/specs/run-history-report/spec.md | 133 ++++++++++++++++++ src/cli.ts | 84 +++++++++-- src/run-history-render.ts | 67 +++++++++ src/run-history-report.ts | 113 +++++++++++++++ src/runs.ts | 77 ++++++++++ test/cli-parser.test.ts | 13 ++ test/cli-regression.test.ts | 68 +++++++++ test/run-history-render.test.ts | 69 +++++++++ test/run-history-report.test.ts | 79 +++++++++++ test/runs.test.ts | 61 +++++++- 16 files changed, 1059 insertions(+), 10 deletions(-) create mode 100644 openspec/changes/archive/2026-09-13-run-history-report/.openspec.yaml create mode 100644 openspec/changes/archive/2026-09-13-run-history-report/design.md create mode 100644 openspec/changes/archive/2026-09-13-run-history-report/proposal.md create mode 100644 openspec/changes/archive/2026-09-13-run-history-report/specs/run-history-report/spec.md create mode 100644 openspec/changes/archive/2026-09-13-run-history-report/tasks.md create mode 100644 openspec/specs/run-history-report/spec.md create mode 100644 src/run-history-render.ts create mode 100644 src/run-history-report.ts create mode 100644 test/run-history-render.test.ts create mode 100644 test/run-history-report.test.ts diff --git a/README.md b/README.md index fb6005d..7639237 100644 --- a/README.md +++ b/README.md @@ -562,6 +562,15 @@ convoy --resume 20260519-103045-x7q2 convoy runs convoy runs 20260519-103045-x7q2 +# export durable run history as JSON (newest first), optionally filtered +convoy runs --json --since 7d | jq 'map({runID, pipeline, tokens: ([.phases[].tokens.total // 0] | add)})' + +# summarize phase usage; group by pipeline (default), model, step, or day +convoy runs stats --since 7d +convoy runs stats --pipeline implement --group-by step + +# Subscription-billed models record a cost of 0; compare their token totals. + # view and edit the global (~/.convoy) and current project config in a TUI: # two tabs (Global / Project), pick models with autocomplete, edit pipelines # and steps, or initialize a starter config when none exists. diff --git a/openspec/changes/archive/2026-09-13-run-history-report/.openspec.yaml b/openspec/changes/archive/2026-09-13-run-history-report/.openspec.yaml new file mode 100644 index 0000000..2b596d1 --- /dev/null +++ b/openspec/changes/archive/2026-09-13-run-history-report/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-12 diff --git a/openspec/changes/archive/2026-09-13-run-history-report/design.md b/openspec/changes/archive/2026-09-13-run-history-report/design.md new file mode 100644 index 0000000..6ca829b --- /dev/null +++ b/openspec/changes/archive/2026-09-13-run-history-report/design.md @@ -0,0 +1,106 @@ +## Context + +`convoy runs` (`src/runs.ts`) is the only reader of the run history. `listRuns()` merges the runs that still have a workspace under `~/.convoy/runs` with the cleanup-surviving run records under `~/.convoy/run-records`, and `loadRunEntry` derives, per run, the pipeline name, a status summary, the executor/advisor cost split (from `metadata.json` plus the attempt logs `readAdvisorSplit` reads) and a `RunPhaseInfo` per phase (`name`, `status`, `durationMs`, `cost`, `advisorCost`, `model`). That `RunEntry` also carries process state probed at read time — `live`, `waiting`, `serverUrl` — and the workspace path. The two consumers are the browser (`src/runs-tui.ts`) and `printRunList`, the plain listing used when no terminal is attached. + +`parseCommand` (`src/cli.ts`) accepts `convoy runs [run-id]` and nothing else: a second argument is a usage error. The repository has no shared flag parser; `parseWorktreesArgs` (`src/worktree-commands.ts`) is the reference pattern — a `Map`, a `knownFlags` set and a usage error carrying the help text. There is no `--json` flag anywhere in Convoy's own CLI; every `--json` in `src/` is an argument passed to `gh` or `openspec`. + +Per phase, `PhaseMetadata` (`src/metadata.ts`) records `tokens` (`ProgressTokens`), `logicalModel`, `startedAt` and `endedAt` in addition to what `RunPhaseInfo` surfaces. Subscription-billed models record `cost: 0` — a fact, not an absence — so a report that only shows cost shows `$0.00` for such users; tokens are the figure that varies. `metadata.phases` also holds rows that are not pipeline steps: hook rows such as `Compact run` and goal-fragment invocations. Run IDs begin with a local-time `YYYYMMDD-HHMMSS` stamp, which is the only creation time a workspace-less run record retains. + +Vocabulary (from the contribution glossary): *run history* is everything Convoy knows about, workspaces and run records alike; *phase usage* is what one phase recorded; the *usage report* is the history's phase usage aggregated by one dimension. + +## Goals / Non-Goals + +**Goals** + +- A stable, script-friendly JSON view of the run history, limited to durable facts and reusing `RunEntry`'s field names. +- A usage report by pipeline, model, step or day, readable as a table and consumable as JSON, with tokens as a first-class figure. +- One filter vocabulary (`--since`, `--pipeline`) shared by both outputs, parsed once. +- Pure, disk-free aggregation and rendering, tested with in-memory `RunEntry` fixtures. + +**Non-Goals** + +- A CSV export (`--json` piped through `jq -r '@csv'` covers it; a dedicated `export` can follow if asked). +- Any change to the runs browser, the plain listing, the dashboard, `SUMMARY.md` or `metadata.json`. +- Averages, percentiles or trends (derivable from the sums; not part of the contract). +- A shared flag parser for the whole CLI (an opportunistic refactor). +- Reading attempt logs beyond what `loadRunEntry` already reads. + +## Decisions + +### D1: `RunHistoryRecord` — the durable projection of `RunEntry`, in `src/runs.ts` + +```ts +export type RunHistoryPhase = RunPhaseInfo // name, status, durationMs?, cost?, advisorCost?, model?, plus: +// tokens?: ProgressTokens; logicalModel?: string; startedAt?: number; endedAt?: number +export type RunHistoryRecord = Pick & { phases: RunHistoryPhase[] } +export function runHistoryRecord(entry: RunEntry): RunHistoryRecord +``` + +`phaseInfos` gains the four fields straight from `PhaseMetadata`; they are optional and no existing consumer reads them. `runHistoryRecord` copies the listed keys only when the entry holds them (`undefined` values are dropped, so `JSON.stringify` never prints `null` and absent facts stay absent). `live`, `waiting`, `serverUrl` and `dir` are deliberately not in the pick: they are process state or internals, and a public contract must not promise the result of a socket probe. + +Alternative rejected: emitting `RunEntry` as is. It would freeze `dir` and the liveness probe as a contract and make the output non-reproducible across invocations. + +### D2: History filter in `src/run-history-report.ts` + +```ts +export type RunHistoryFilter = { since?: number; pipeline?: string } +export function parseSince(value: string, now: number): number // "7d" | "36h" | "YYYY-MM-DD" → epoch ms; throws on anything else +export function runCreatedAt(entry: Pick): number | undefined // createdAt ?? stamp parsed from the run ID +export function filterRunHistory>(entries: T[], filter: RunHistoryFilter): T[] +``` + +`YYYY-MM-DD` resolves to local midnight (`new Date(y, m - 1, d)`), matching how a person reads "since Monday". The run-ID fallback parses `YYYYMMDD-HHMMSS` as local time, which is how `newRunID` stamps it. A run with neither is kept when no `--since` is given and dropped when one is (nothing says it is inside the window). `--pipeline` is exact string equality on `pipeline`; a run without a pipeline name never matches a filter. + +### D3: Usage report aggregation in `src/run-history-report.ts` + +```ts +export type UsageReportDimension = "pipeline" | "model" | "step" | "day" +export type UsageReportRow = { + key: string // the group value, "(none)" when the fact is missing + runs: number // distinct runs contributing to the row + phases: number + completed: number // runs (pipeline, day) or phases (model, step) + failed: number + tokens: ProgressTokens + cost: number // executor + advisor + advisorCost: number + durationMs: number +} +export function usageReport(records: RunHistoryRecord[], dimension: UsageReportDimension): UsageReportRow[] +``` + +The aggregation flattens `(record, phase)` pairs, computes the key per pair (`record.pipeline`, `phase.model`, `phase.name`, or the local `YYYY-MM-DD` of `runCreatedAt(record)`), and folds with `addTokens`/`safeCost` from `src/usage.ts`. Phase cost is `phase.cost + phase.advisorCost` when recorded, so `cost` matches what `convoy runs` shows as the run's cost when every phase has a cost; `advisorCost` is the advisor part alone. Workspace-less runs contribute to `runs`, `completed`/`failed` on run-grained rows and nothing else. Outcome counting follows the dimension (glossary ADR *usage report grain*): `statusKind === "completed"` / `"failed"` per distinct run on `pipeline`/`day`; `status === "completed"` / `"failed"` per phase on `model`/`step` (`skipped` and `pending` count in `phases` only). Ordering: `tokens.total` descending, ties by key; `day` ascending by key. The `total` row is a rendering concern (D4): the JSON rows are the groups, nothing else, so a script can sum them without special-casing. + +Alternative rejected: one schema per grain (two output shapes to document) and phase-only counting everywhere (makes the `pipeline` row answer a question nobody asks). + +### D4: Rendering in `src/run-history-render.ts` + +```ts +export function renderUsageReportTable(rows: UsageReportRow[], dimension: UsageReportDimension): string +export function renderJson(value: unknown): string // JSON.stringify(value, null, 2) + "\n" +``` + +Columns: ` | runs | ok | failed | tokens | cache read | cost | advisor | duration`, widths from content, numbers right-aligned, no colors, one `total` row last (sums of every column; `runs` in the total is the number of distinct runs, not the sum of the rows' `runs`, which would double count on phase grains). Formats: tokens `n < 1000` as is, `k` and `M` with one decimal (`412k`, `3.3M`); cost `$` + `toFixed(4)` (the format the hook variables and `SUMMARY.md` use; the plain listing's two decimals would flatten cheap runs); duration `Xh Ym`, `Xm Ys` or `Xs` with no fraction. An empty report renders the header and a zeroed total. + +### D5: CLI surface in `src/cli.ts` + +``` +convoy runs [run-id] +convoy runs --json [--since d|h|YYYY-MM-DD] [--pipeline ] +convoy runs stats [--group-by pipeline|model|step|day] [--since …] [--pipeline …] [--json] +``` + +`CliCommand`'s `runs` variant becomes `{ type: "runs"; mode: "browse"; runID?: string } | { type: "runs"; mode: "json"; filter } | { type: "runs"; mode: "stats"; filter; dimension; json: boolean }`, produced by a `parseRunsArgs(rest)` that mirrors `parseWorktreesArgs` (flag map, known flags, usage error with the three forms). `--since` is validated at parse time so an invalid value fails before any I/O. Dispatch: `browse` → `openRunsBrowser` unchanged; `json`/`stats` → `listRuns()` → `runHistoryRecord` → `filterRunHistory` → (`usageReport`) → render → `process.stdout.write`. `listRuns()` still probes liveness per run before the filter applies; `--since` narrows the output, not the loading (see Risks). The help text's `runs` entry and the examples block list the new forms. + +### D6: README + +Under the `convoy runs` examples: `convoy runs --json --since 7d | jq 'map({runID, pipeline, tokens: ([.phases[].tokens.total // 0] | add)})'`, `convoy runs stats --since 7d` and `convoy runs stats --pipeline implement --group-by step`, plus one sentence that subscription models record a cost of `0`, so tokens are the figure to compare. + +## Risks / Trade-offs + +- **`RunEntry` field names become a contract.** Mitigated by exposing a picked subset; a rename inside `RunEntry` would now need a projection change rather than silently changing the output. +- **Liveness probes on every run.** `listRuns()` was written for an interactive list; for a 68-run history the probes cost a few hundred milliseconds. Filtering before probing would require restructuring `listRuns`, which is out of scope; a probe-free loader for the report modes is the follow-up if long histories make it noticeable. +- **Hook rows and goal fragments appear as steps.** The `step` dimension groups by phase name as recorded; `Compact run` and `goal-measure-1-…` rows are phases too (glossary) and an operator can drop them with `jq`. Distinguishing configured steps from auxiliary rows would need the frozen pipeline per run and a second vocabulary; deferred. +- **Overlap with the `hook-run-usage` change (PR #113).** That change sums the in-memory store's phases for post-hooks; this one aggregates `RunEntry` phases across runs. Neither depends on the other; if both land, no line is shared or duplicated. diff --git a/openspec/changes/archive/2026-09-13-run-history-report/proposal.md b/openspec/changes/archive/2026-09-13-run-history-report/proposal.md new file mode 100644 index 0000000..8a44430 --- /dev/null +++ b/openspec/changes/archive/2026-09-13-run-history-report/proposal.md @@ -0,0 +1,30 @@ +## Why + +Every run records, per phase, its status, executor cost, tokens, model and duration in `metadata.json` (`PhaseMetadata` in `src/metadata.ts`), and `convoy runs` already derives each run's cost and executor/advisor split from it (`loadRunEntry` in `src/runs.ts`). But that knowledge only reaches an operator through the runs browser or the plain text listing, one run at a time. "How much did `implement` cost this week", "which step fails most often" or "which model burns the most cache" all require scripting over `~/.convoy/runs/*/metadata.json` and re-deriving the aggregation Convoy performs internally. With subscription models the recorded cost is `0`, so the tokens — which nothing exposes outside the dashboard — are the figure that matters. + +## What Changes + +- Add `convoy runs --json`: the run history as a JSON array (newest first) of durable run facts — pipeline, status, cost split, goal outcome, finalization evidence, feature link and every phase's recorded usage (tokens, model, duration, timestamps). Process state (`live`, `waiting`, server URL, run directory) is never emitted. +- Add `convoy runs stats`: the usage report — the whole history's phase usage aggregated by `--group-by pipeline|model|step|day` into rows with run and phase counts, outcomes, tokens by kind, cost, advisor cost and duration; a plain text table by default, raw rows with `--json`. +- Give both outputs the same history filters: `--since d|h|YYYY-MM-DD` and `--pipeline `. +- Document the flags and three `jq`/`stats` examples in the README's `convoy runs` block. +- The runs browser, the plain listing, `metadata.json`, the dashboard and `SUMMARY.md` are untouched; Convoy still never sends anything anywhere. + +## Capabilities + +### New Capabilities + +- `run-history-report`: `convoy runs` exposes the run history as JSON and as a usage report aggregated by pipeline, model, step or day, so an operator can answer spend and failure questions across runs without parsing run files. + +### Modified Capabilities + + + +## Impact + +- `src/runs.ts` (`RunPhaseInfo` gains the recorded usage fields; the durable `RunHistoryRecord` projection). +- `src/run-history-report.ts` (new: history filter and usage report aggregation, pure). +- `src/run-history-render.ts` (new: table and JSON rendering). +- `src/cli.ts` (`runs` argument parser, command modes, dispatch, help text). +- `README.md` (`convoy runs` block). +- No harness protocol, control protocol, config schema or persisted-state change. One CLI surface addition: two flags on `convoy runs` and one subcommand. diff --git a/openspec/changes/archive/2026-09-13-run-history-report/specs/run-history-report/spec.md b/openspec/changes/archive/2026-09-13-run-history-report/specs/run-history-report/spec.md new file mode 100644 index 0000000..ab0a836 --- /dev/null +++ b/openspec/changes/archive/2026-09-13-run-history-report/specs/run-history-report/spec.md @@ -0,0 +1,132 @@ +## Purpose + +Expose the run history — every run Convoy knows about, with the usage each phase recorded — as machine-readable JSON and as a usage report aggregated by pipeline, model, step or day, so an operator or a script can answer "what did this pipeline cost this week", "which step fails most" or "which model consumes the most cache" from `convoy runs` alone, without parsing run workspaces or re-deriving totals Convoy already computes. + +## ADDED Requirements + +### Requirement: `convoy runs --json` prints the run history as JSON + +`convoy runs --json` SHALL write the run history to stdout as a JSON array ordered newest first and SHALL never open the runs browser, whether or not stdin and stdout are terminals. Each element SHALL carry only durable facts, under the same field names `convoy runs` uses internally: `runID`, `title`, `pipeline`, `targetDir`, `status`, `statusKind`, `createdAt`, `cost`, `executorCost`, `advisorCost`, `goal`, `finalization`, `feature` and `phases`. Process state — whether the run is live, whether it waits on a gate, its server URL, its workspace path — SHALL NOT be emitted. When no run matches, the output SHALL be an empty array and the exit status zero. + +#### Scenario: Two recorded runs + +- **WHEN** the history holds a completed `implement` run and a failed `review` run, and `convoy runs --json` is invoked +- **THEN** stdout is a two-element array, the `implement` run first if it is newer, each element carrying `pipeline`, `statusKind`, `cost` and a `phases` array with every recorded phase + +#### Scenario: A terminal is attached + +- **WHEN** `convoy runs --json` is invoked from an interactive terminal +- **THEN** the JSON is printed and the runs browser never opens + +#### Scenario: A live run + +- **WHEN** one run's server is still up +- **THEN** its element carries the same durable fields as every other run and no `live`, `waiting` or `serverUrl` key + +#### Scenario: Empty history + +- **WHEN** the runs directory holds no run +- **THEN** stdout is `[]` and the exit status is zero + +### Requirement: Phases carry their recorded usage and nothing invented + +Each element of `phases` SHALL carry the phase's `name` and `status`, plus `durationMs`, `startedAt`, `endedAt`, `cost`, `advisorCost`, `tokens` (`input`, `output`, `reasoning`, `cacheRead`, `cacheWrite`, `total`), `model` and `logicalModel` exactly when the run's metadata recorded them. A fact the metadata did not record SHALL be an absent key — never `null`, never zero — and a recorded cost of `0` SHALL be emitted as the fact it is. + +#### Scenario: A hook row without usage + +- **WHEN** a run's `Compact run` phase was skipped and recorded a duration but no cost, tokens or model +- **THEN** its element carries `name`, `status` and `durationMs` and no `cost`, `tokens` or `model` key + +#### Scenario: A subscription model + +- **WHEN** a phase ran on a model whose recorded cost is `0` with 1,200,000 tokens +- **THEN** its element carries `cost: 0` and `tokens.total: 1200000` + +### Requirement: Runs without a workspace stay in the history + +A run whose workspace was deleted but whose run record survives SHALL appear in the JSON with `phases: []`, the `status` and `statusKind` the run record implies and its `finalization` evidence; the keys for facts the record does not hold (`createdAt`, `cost`, `pipeline`) SHALL be absent. + +#### Scenario: A compacted run whose workspace was cleaned + +- **WHEN** `~/.convoy/run-records/.json` exists with a `producedSha` and `~/.convoy/runs/` does not +- **THEN** the JSON element for `` carries `phases: []`, `statusKind: "completed"` and `finalization.producedSha`, and no `cost` or `createdAt` key + +### Requirement: History filters apply to both outputs + +`convoy runs --json` and `convoy runs stats` SHALL accept `--since ` and `--pipeline `. `--since` SHALL accept a relative window in days (`7d`) or hours (`36h`) and an absolute local date (`YYYY-MM-DD`, midnight); a run SHALL pass when its `createdAt` is at or after the boundary, or — when the run has no `createdAt` — when the timestamp encoded in its run ID is. `--pipeline` SHALL keep only runs whose recorded pipeline name equals the value exactly. Filters SHALL apply before any aggregation. + +#### Scenario: A relative window + +- **WHEN** the history holds a run created 8 days ago and one created 2 days ago, and `--since 7d` is passed +- **THEN** only the 2-day-old run is included + +#### Scenario: A run record without a creation time + +- **WHEN** a workspace-less run's ID begins with a timestamp older than the `--since` boundary +- **THEN** it is excluded, and it is included when the timestamp is inside the window + +#### Scenario: A pipeline filter before aggregation + +- **WHEN** the history holds `implement` and `review` runs and `convoy runs stats --pipeline implement --group-by step` is invoked +- **THEN** every row counts phases of `implement` runs only + +### Requirement: `convoy runs stats` prints the usage report + +`convoy runs stats` SHALL aggregate the filtered history's phases into one row per value of the `--group-by` dimension — `pipeline` (default), `model`, `step` or `day` — and print the rows as a plain text table on stdout, with or without a terminal, or as a JSON array of raw rows when `--json` is passed. Every row SHALL carry the group key, `runs` (distinct runs contributing to the row), `phases`, `completed`, `failed`, `tokens` (by kind and `total`), `cost` (executor plus advisor), `advisorCost` and `durationMs` (sum of the phases' recorded durations). On the run-grained dimensions `pipeline` and `day`, `completed` and `failed` SHALL count runs by their status kind; on the phase-grained dimensions `model` and `step`, they SHALL count phases by their status. A run without a recorded pipeline name, or a phase without a recorded model, SHALL fall into a `(none)` row. Rows SHALL be ordered by `tokens.total` descending, except `day`, ordered ascending by date. The table SHALL end with a `total` row; the JSON SHALL NOT include one. + +#### Scenario: Grouped by pipeline + +- **WHEN** the history holds two `implement` runs (one completed, one failed) and one completed `review` run, and `convoy runs stats` is invoked +- **THEN** the `implement` row reads `runs 2`, `completed 1`, `failed 1` with its phases' tokens, cost and duration summed, and the `review` row reads `runs 1`, `completed 1`, `failed 0` + +#### Scenario: Grouped by step + +- **WHEN** the same history is reported with `--group-by step` +- **THEN** the `implementer` row counts every `implementer` phase across the two `implement` runs, with `completed` and `failed` counting phase statuses and `runs` counting the distinct runs that ran the step + +#### Scenario: Grouped by model + +- **WHEN** phases on `openai/gpt-5.6-sol#medium` recorded 400,000 and 600,000 cache-read tokens +- **THEN** that model's row carries `tokens.cacheRead: 1000000` + +#### Scenario: Grouped by day + +- **WHEN** runs were created on two different local dates +- **THEN** the rows are one per `YYYY-MM-DD`, oldest first + +#### Scenario: Nothing matches + +- **WHEN** the filters exclude every run +- **THEN** the table shows the header and a zeroed `total` row with exit status zero, and `--json` prints `[]` + +### Requirement: Table values are compact and JSON values are raw + +In the table, token counts SHALL be rendered compactly (`412k`, `3.3M`), costs with a dollar sign and four fractional digits (`$0.0000`), and durations humanized (`4m23s`). With `--json` the same figures SHALL be plain numbers: integer token counts, unrounded costs and integer milliseconds. + +#### Scenario: One row both ways + +- **WHEN** a row sums 3,289,185 tokens, a cost of 0.1234567 USD and 263,135 ms +- **THEN** the table shows `3.3M`, `$0.1235` and `4m23s`, and `--json` shows `3289185`, `0.1234567` and `263135` + +### Requirement: Usage errors are explicit + +`convoy runs` SHALL reject, with a usage message that names the accepted forms and a non-zero exit status and before opening anything, a run ID combined with `--json` or `stats`, an unparseable `--since` value, an unknown `--group-by` dimension, a filter or `--group-by` given without `--json` or `stats`, and any unknown flag. + +#### Scenario: A run ID with `--json` + +- **WHEN** `convoy runs 20260911-190652-lrv2 --json` is invoked +- **THEN** the command fails with the usage message and the browser never opens + +#### Scenario: An unknown dimension + +- **WHEN** `convoy runs stats --group-by week` is invoked +- **THEN** the command fails naming the accepted dimensions + +### Requirement: The browser and the plain listing are unchanged + +`convoy runs` and `convoy runs ` without flags SHALL behave exactly as before: the runs browser on a terminal, the plain listing otherwise. + +#### Scenario: Bare invocation without a terminal + +- **WHEN** `convoy runs` is invoked with stdout piped +- **THEN** the plain text listing is printed, not JSON diff --git a/openspec/changes/archive/2026-09-13-run-history-report/tasks.md b/openspec/changes/archive/2026-09-13-run-history-report/tasks.md new file mode 100644 index 0000000..4fdc337 --- /dev/null +++ b/openspec/changes/archive/2026-09-13-run-history-report/tasks.md @@ -0,0 +1,26 @@ +## 1. History records + +- [x] 1.1 In `src/runs.ts`, extend `RunPhaseInfo` with `tokens?`, `logicalModel?`, `startedAt?` and `endedAt?` (copied from `PhaseMetadata` in `phaseInfos`), and add `RunHistoryRecord`, `RunHistoryPhase` and `runHistoryRecord(entry)` picking only the durable keys and dropping `undefined` values. Verification: `test/runs.test.ts` shows phases carrying recorded tokens/model/timestamps, a recorded `cost: 0` emitted, absent facts absent (no `null`), an index-only run yielding `phases: []` with `finalization`, and no `live`/`waiting`/`serverUrl`/`dir` key in the record. + +## 2. Filter and report + +- [x] 2.1 In `src/run-history-report.ts`, add `RunHistoryFilter`, `parseSince(value, now)` (`d`, `h`, `YYYY-MM-DD` local midnight; throws on anything else), `runCreatedAt` (metadata `createdAt`, else the run ID's local-time stamp) and `filterRunHistory`. Verification: `test/run-history-report.test.ts` covers each accepted form and the boundary (at-or-after passes), an invalid value throwing, a workspace-less run filtered by its ID stamp, a run with neither kept only without `--since`, and `--pipeline` exact match. +- [x] 2.2 In `src/run-history-report.ts`, add `UsageReportDimension`, `UsageReportRow` and `usageReport(records, dimension)`: flatten `(record, phase)`, key by pipeline/model/step/day with `(none)` for a missing fact, fold tokens with `addTokens`, costs with `safeCost` (`cost` = executor + advisor, `advisorCost` apart), durations, distinct `runs`, `phases`, and `completed`/`failed` by run status kind on `pipeline`/`day` and by phase status on `model`/`step`; order by `tokens.total` descending (`day` ascending). Verification: the test file covers a two-pipeline history (run outcomes on `pipeline`, phase outcomes on `step`, distinct runs on both), cache-read sums on `model`, `day` ordering, `(none)` rows, a workspace-less run counted on run grains only, and NaN-safe costs. + +## 3. Render + +- [x] 3.1 In `src/run-history-render.ts`, add `renderUsageReportTable(rows, dimension)` (columns ` | runs | ok | failed | tokens | cache read | cost | advisor | duration`; compact tokens `412k`/`3.3M`; `$` with four fractional digits; humanized durations; right-aligned numbers; a final `total` row whose `runs` is the distinct count; header plus zeroed total for an empty report) and `renderJson(value)`. Verification: `test/run-history-render.test.ts` snapshots a three-row table and the empty table, and checks each formatter at its boundaries (`999`, `1000`, `1_000_000`; `59s`, `60s`, `3600s`). + +## 4. CLI + +- [x] 4.1 In `src/cli.ts`, replace the `runs` branch of `parseCommand` with `parseRunsArgs(rest)` producing the `browse`/`json`/`stats` modes (flag map and known-flag set in the style of `parseWorktreesArgs`; `--since` validated at parse time; usage error listing the three forms), extend `CliCommand`, dispatch `json`/`stats` through `listRuns` → `runHistoryRecord` → `filterRunHistory` → (`usageReport`) → renderer → stdout, and extend the help text's `runs` entry and examples. Verification: `test/cli-parser.test.ts` parses every form, rejects a run ID with `--json` or `stats`, an unknown `--group-by`, filters without a mode, an invalid `--since` and unknown flags, and shows bare `runs`/`runs ` unchanged; `test/cli.test.ts` drives the `json` and `stats` dispatch against a temporary `CONVOY_HOME` history and asserts stdout is parseable JSON / a table with a `total` row and that no TUI module is loaded. + +## 5. Documentation + +- [x] 5.1 In `README.md`, extend the `convoy runs` block with the three forms, the two filters, one `--json | jq` example and two `stats` examples, and the note that subscription models record a cost of `0`. + +## 6. Verify + +- [x] 6.1 `bun run typecheck` and `bun test` pass; coverage stays above the `verify.yml` threshold. +- [x] 6.2 `openspec validate run-history-report --strict`. +- [x] 6.3 Manual check against the real history: `convoy runs --json --since 2d | jq length`, `convoy runs stats --since 7d`, `convoy runs stats --pipeline implement-gpt --group-by step` and `convoy runs stats --group-by model --json` print figures consistent with `convoy runs` and the runs' `metadata.json`. diff --git a/openspec/specs/run-history-report/spec.md b/openspec/specs/run-history-report/spec.md new file mode 100644 index 0000000..4720045 --- /dev/null +++ b/openspec/specs/run-history-report/spec.md @@ -0,0 +1,133 @@ +# run-history-report Specification + +## Purpose +Expose the run history — every run Convoy knows about, with the usage each phase recorded — as machine-readable JSON and as a usage report aggregated by pipeline, model, step or day, so an operator or a script can answer "what did this pipeline cost this week", "which step fails most" or "which model consumes the most cache" from `convoy runs` alone, without parsing run workspaces or re-deriving totals Convoy already computes. + +## Requirements + +### Requirement: `convoy runs --json` prints the run history as JSON + +`convoy runs --json` SHALL write the run history to stdout as a JSON array ordered newest first and SHALL never open the runs browser, whether or not stdin and stdout are terminals. Each element SHALL carry only durable facts, under the same field names `convoy runs` uses internally: `runID`, `title`, `pipeline`, `targetDir`, `status`, `statusKind`, `createdAt`, `cost`, `executorCost`, `advisorCost`, `goal`, `finalization`, `feature` and `phases`. Process state — whether the run is live, whether it waits on a gate, its server URL, its workspace path — SHALL NOT be emitted. When no run matches, the output SHALL be an empty array and the exit status zero. + +#### Scenario: Two recorded runs + +- **WHEN** the history holds a completed `implement` run and a failed `review` run, and `convoy runs --json` is invoked +- **THEN** stdout is a two-element array, the `implement` run first if it is newer, each element carrying `pipeline`, `statusKind`, `cost` and a `phases` array with every recorded phase + +#### Scenario: A terminal is attached + +- **WHEN** `convoy runs --json` is invoked from an interactive terminal +- **THEN** the JSON is printed and the runs browser never opens + +#### Scenario: A live run + +- **WHEN** one run's server is still up +- **THEN** its element carries the same durable fields as every other run and no `live`, `waiting` or `serverUrl` key + +#### Scenario: Empty history + +- **WHEN** the runs directory holds no run +- **THEN** stdout is `[]` and the exit status is zero + +### Requirement: Phases carry their recorded usage and nothing invented + +Each element of `phases` SHALL carry the phase's `name` and `status`, plus `durationMs`, `startedAt`, `endedAt`, `cost`, `advisorCost`, `tokens` (`input`, `output`, `reasoning`, `cacheRead`, `cacheWrite`, `total`), `model` and `logicalModel` exactly when the run's metadata recorded them. A fact the metadata did not record SHALL be an absent key — never `null`, never zero — and a recorded cost of `0` SHALL be emitted as the fact it is. + +#### Scenario: A hook row without usage + +- **WHEN** a run's `Compact run` phase was skipped and recorded a duration but no cost, tokens or model +- **THEN** its element carries `name`, `status` and `durationMs` and no `cost`, `tokens` or `model` key + +#### Scenario: A subscription model + +- **WHEN** a phase ran on a model whose recorded cost is `0` with 1,200,000 tokens +- **THEN** its element carries `cost: 0` and `tokens.total: 1200000` + +### Requirement: Runs without a workspace stay in the history + +A run whose workspace was deleted but whose run record survives SHALL appear in the JSON with `phases: []`, the `status` and `statusKind` the run record implies and its `finalization` evidence; the keys for facts the record does not hold (`createdAt`, `cost`, `pipeline`) SHALL be absent. + +#### Scenario: A compacted run whose workspace was cleaned + +- **WHEN** `~/.convoy/run-records/.json` exists with a `producedSha` and `~/.convoy/runs/` does not +- **THEN** the JSON element for `` carries `phases: []`, `statusKind: "completed"` and `finalization.producedSha`, and no `cost` or `createdAt` key + +### Requirement: History filters apply to both outputs + +`convoy runs --json` and `convoy runs stats` SHALL accept `--since ` and `--pipeline `. `--since` SHALL accept a relative window in days (`7d`) or hours (`36h`) and an absolute local date (`YYYY-MM-DD`, midnight); a run SHALL pass when its `createdAt` is at or after the boundary, or — when the run has no `createdAt` — when the timestamp encoded in its run ID is. `--pipeline` SHALL keep only runs whose recorded pipeline name equals the value exactly. Filters SHALL apply before any aggregation. + +#### Scenario: A relative window + +- **WHEN** the history holds a run created 8 days ago and one created 2 days ago, and `--since 7d` is passed +- **THEN** only the 2-day-old run is included + +#### Scenario: A run record without a creation time + +- **WHEN** a workspace-less run's ID begins with a timestamp older than the `--since` boundary +- **THEN** it is excluded, and it is included when the timestamp is inside the window + +#### Scenario: A pipeline filter before aggregation + +- **WHEN** the history holds `implement` and `review` runs and `convoy runs stats --pipeline implement --group-by step` is invoked +- **THEN** every row counts phases of `implement` runs only + +### Requirement: `convoy runs stats` prints the usage report + +`convoy runs stats` SHALL aggregate the filtered history's phases into one row per value of the `--group-by` dimension — `pipeline` (default), `model`, `step` or `day` — and print the rows as a plain text table on stdout, with or without a terminal, or as a JSON array of raw rows when `--json` is passed. Every row SHALL carry the group key, `runs` (distinct runs contributing to the row), `phases`, `completed`, `failed`, `tokens` (by kind and `total`), `cost` (executor plus advisor), `advisorCost` and `durationMs` (sum of the phases' recorded durations). On the run-grained dimensions `pipeline` and `day`, `completed` and `failed` SHALL count runs by their status kind; on the phase-grained dimensions `model` and `step`, they SHALL count phases by their status. A run without a recorded pipeline name, or a phase without a recorded model, SHALL fall into a `(none)` row. Rows SHALL be ordered by `tokens.total` descending, except `day`, ordered ascending by date. The table SHALL end with a `total` row; the JSON SHALL NOT include one. + +#### Scenario: Grouped by pipeline + +- **WHEN** the history holds two `implement` runs (one completed, one failed) and one completed `review` run, and `convoy runs stats` is invoked +- **THEN** the `implement` row reads `runs 2`, `completed 1`, `failed 1` with its phases' tokens, cost and duration summed, and the `review` row reads `runs 1`, `completed 1`, `failed 0` + +#### Scenario: Grouped by step + +- **WHEN** the same history is reported with `--group-by step` +- **THEN** the `implementer` row counts every `implementer` phase across the two `implement` runs, with `completed` and `failed` counting phase statuses and `runs` counting the distinct runs that ran the step + +#### Scenario: Grouped by model + +- **WHEN** phases on `openai/gpt-5.6-sol#medium` recorded 400,000 and 600,000 cache-read tokens +- **THEN** that model's row carries `tokens.cacheRead: 1000000` + +#### Scenario: Grouped by day + +- **WHEN** runs were created on two different local dates +- **THEN** the rows are one per `YYYY-MM-DD`, oldest first + +#### Scenario: Nothing matches + +- **WHEN** the filters exclude every run +- **THEN** the table shows the header and a zeroed `total` row with exit status zero, and `--json` prints `[]` + +### Requirement: Table values are compact and JSON values are raw + +In the table, token counts SHALL be rendered compactly (`412k`, `3.3M`), costs with a dollar sign and four fractional digits (`$0.0000`), and durations humanized (`4m23s`). With `--json` the same figures SHALL be plain numbers: integer token counts, unrounded costs and integer milliseconds. + +#### Scenario: One row both ways + +- **WHEN** a row sums 3,289,185 tokens, a cost of 0.1234567 USD and 263,135 ms +- **THEN** the table shows `3.3M`, `$0.1235` and `4m23s`, and `--json` shows `3289185`, `0.1234567` and `263135` + +### Requirement: Usage errors are explicit + +`convoy runs` SHALL reject, with a usage message that names the accepted forms and a non-zero exit status and before opening anything, a run ID combined with `--json` or `stats`, an unparseable `--since` value, an unknown `--group-by` dimension, a filter or `--group-by` given without `--json` or `stats`, and any unknown flag. + +#### Scenario: A run ID with `--json` + +- **WHEN** `convoy runs 20260911-190652-lrv2 --json` is invoked +- **THEN** the command fails with the usage message and the browser never opens + +#### Scenario: An unknown dimension + +- **WHEN** `convoy runs stats --group-by week` is invoked +- **THEN** the command fails naming the accepted dimensions + +### Requirement: The browser and the plain listing are unchanged + +`convoy runs` and `convoy runs ` without flags SHALL behave exactly as before: the runs browser on a terminal, the plain listing otherwise. + +#### Scenario: Bare invocation without a terminal + +- **WHEN** `convoy runs` is invoked with stdout piped +- **THEN** the plain text listing is printed, not JSON diff --git a/src/cli.ts b/src/cli.ts index ecc653b..5bc3e39 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -14,7 +14,9 @@ import { confirmRunPlan, renderRunPlan } from "./run-review" import { loadPrdHistoryPreview } from "./prd-history" import { loadOpenSpecBundle, openSpecPromptFor } from "./openspec" import { isModelGateway, modelGatewayChoices, modelGateways, type ModelGateway } from "./model-routing" -import { browseRuns, isControlLive, isServerLive } from "./runs" +import { browseRuns, isControlLive, isServerLive, listRuns, runHistoryRecord } from "./runs" +import { filterRunHistory, parseSince, usageReport, type RunHistoryFilter, type UsageReportDimension } from "./run-history-report" +import { renderJson, renderUsageReportTable } from "./run-history-render" import { browseSpecs, buildIterateSessionInput, loadSpecsView, type SpecsResolution, type SpecsResumeSelection } from "./specs" import { deleteKeychainSecret, keychainAvailable, storeKeychainSecret } from "./secrets" import type { Pipeline, RunOptions, RunPlan } from "./types" @@ -89,7 +91,9 @@ export type InitOptions = { export type CliCommand = | { type: "help"; text: string } | { type: "run"; options: RunOptions } - | { type: "runs"; runID?: string } + | { type: "runs"; mode: "browse"; runID?: string } + | { type: "runs"; mode: "json"; runID?: undefined; filter: RunHistoryFilter } + | { type: "runs"; mode: "stats"; runID?: undefined; filter: RunHistoryFilter; dimension: UsageReportDimension; json: boolean } | { type: "specs"; targetDir: string } | { type: "spin"; options: SpinOptions } | { type: "opencode-install" } @@ -135,7 +139,15 @@ export async function parseAndRun(argv: string[]) { return } if (command.type === "runs") { - await openRunsBrowser(command.runID) + if (command.mode === "browse") await openRunsBrowser(command.runID) + else { + const history = filterRunHistory((await listRuns()).map(runHistoryRecord), command.filter) + if (command.mode === "json") process.stdout.write(renderJson(history)) + else { + const report = usageReport(history, command.dimension) + process.stdout.write(command.json ? renderJson(report.rows) : renderUsageReportTable(report)) + } + } return } if (command.type === "specs") { @@ -1865,10 +1877,7 @@ export async function parseCommand(argv: string[]): Promise { throw new Error("usage: convoy auth [status] | convoy auth openrouter [--remove]") } if (argv[0] === "runs") { - const rest = argv.slice(1) - if (rest.length > 1) throw new Error("usage: convoy runs [run-id]") - if (rest[0] !== undefined && !isValidRunID(rest[0])) throw new Error(`invalid run id: ${rest[0]}`) - return { type: "runs", runID: rest[0] } + return parseRunsArgs(argv.slice(1)) } if (argv[0] === "specs") { // No positionals or flags yet — the viewer reads the whole OpenSpec state. @@ -2029,6 +2038,61 @@ export async function parseCommand(argv: string[]): Promise { return { type: "run", options } } +const runsUsage = "usage: convoy runs [run-id]\n convoy runs --json [--since d|h|YYYY-MM-DD] [--pipeline ]\n convoy runs stats [--group-by pipeline|model|step|day] [--since ] [--pipeline ] [--json]" +const runsFlags = new Set(["--json", "--since", "--pipeline", "--group-by"]) +const usageReportDimensions = new Set(["pipeline", "model", "step", "day"]) + +function parseRunsArgs(rest: string[]): Extract { + if (rest.length === 0) return { type: "runs", mode: "browse" } + if (rest[0] !== "stats" && !rest[0]!.startsWith("-")) { + if (rest.length !== 1) throw new Error(runsUsage) + if (!isValidRunID(rest[0]!)) throw new Error(`invalid run id: ${rest[0]}`) + return { type: "runs", mode: "browse", runID: rest[0] } + } + + const stats = rest[0] === "stats" + const args = stats ? rest.slice(1) : rest + if (!stats && !args.includes("--json")) throw new Error(runsUsage) + const values = new Map() + let json = false + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]! + if (!runsFlags.has(arg)) throw new Error(runsUsage) + if (arg === "--json") { + if (json) throw new Error(runsUsage) + json = true + continue + } + const value = args[index + 1] + if (value === undefined || value.startsWith("-") || values.has(arg)) throw new Error(runsUsage) + values.set(arg, value) + index += 1 + } + if (!stats && values.has("--group-by")) throw new Error(runsUsage) + const since = values.get("--since") + const filter: RunHistoryFilter = { + ...(since !== undefined ? { since: parseRunsSince(since) } : {}), + ...(values.get("--pipeline") !== undefined ? { pipeline: values.get("--pipeline") } : {}), + } + if (!stats) return { type: "runs", mode: "json", filter } + const rawDimension = values.get("--group-by") ?? "pipeline" + if (!isUsageReportDimension(rawDimension)) throw new Error(`unknown --group-by dimension "${rawDimension}"; use pipeline, model, step, or day\n${runsUsage}`) + return { type: "runs", mode: "stats", filter, dimension: rawDimension, json } +} + +/** Every `convoy runs` usage error names the accepted forms, the `--since` reason included. */ +function parseRunsSince(value: string): number { + try { + return parseSince(value, Date.now()) + } catch (error) { + throw new Error(`${error instanceof Error ? error.message : String(error)}\n${runsUsage}`) + } +} + +function isUsageReportDimension(value: string): value is UsageReportDimension { + return usageReportDimensions.has(value as UsageReportDimension) +} + type ParsedInitArgs = InitOptions & { help?: boolean } /** `convoy spin [--change ] [--prefix ]` — no positionals. */ @@ -2492,6 +2556,8 @@ Usage: convoy agents eject convoy update [--check] convoy runs [run-id] + convoy runs --json [--since d|h|YYYY-MM-DD] [--pipeline ] + convoy runs stats [--group-by pipeline|model|step|day] [--since ] [--pipeline ] [--json] convoy specs convoy worktrees [--help] convoy spin @@ -2509,8 +2575,8 @@ Commands: override it ("convoy agents" lists the available ones) update [--check] Check GitHub Releases for a newer official binary, or install it (source checkouts are never modified) - runs [run-id] Browse run history: resume a run, read its summary/reports, - or open a subshell in its run dir (under ~/.convoy/runs) + runs [run-id] Browse run history; --json emits durable records and + stats reports usage by pipeline, model, step, or day worktrees The worktree control center: every registered checkout with its independent Git/OpenSpec facts, plus guarded fetch, sync, push, archive, remove, delete-branch, and recover actions diff --git a/src/run-history-render.ts b/src/run-history-render.ts new file mode 100644 index 0000000..5a2f461 --- /dev/null +++ b/src/run-history-render.ts @@ -0,0 +1,67 @@ +import { addTokens, emptyTokens } from "./usage" +import type { UsageReport, UsageReportRow } from "./run-history-report" + +/** + * JSON.stringify escapes C0 controls only: C1 controls (CSI, OSC, ST), bidi + * formats and line separators survive literally, so a persisted title could + * drive or reorder the terminal that shows the output. Escape them as \uXXXX; + * parsed values are unchanged. + */ +const terminalActiveCharacters = /[\u0080-\u009f\u061c\u200e\u200f\u2028\u2029\u202a-\u202e\u2066-\u2069]/g + +export function renderJson(value: unknown): string { + const json = JSON.stringify(value, null, 2).replace(terminalActiveCharacters, (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`) + return `${json}\n` +} + +export function renderUsageReportTable({ dimension, runs, rows }: UsageReport): string { + const emptyTotal: UsageReportRow = { + key: "total", + runs, + phases: 0, + completed: 0, + failed: 0, + tokens: emptyTokens(), + cost: 0, + advisorCost: 0, + durationMs: 0, + } + const total = rows.reduce((sum, row) => ({ + key: "total", + runs, + phases: sum.phases + row.phases, + completed: sum.completed + row.completed, + failed: sum.failed + row.failed, + tokens: addTokens(sum.tokens, row.tokens), + cost: sum.cost + row.cost, + advisorCost: sum.advisorCost + row.advisorCost, + durationMs: sum.durationMs + row.durationMs, + }), emptyTotal) + const header = [dimension, "runs", "phases", "ok", "failed", "tokens", "cache read", "cost", "advisor", "duration"] + const data = [...rows, total].map((row) => [terminalSafeCell(row.key), String(row.runs), String(row.phases), String(row.completed), String(row.failed), compact(row.tokens.total), compact(row.tokens.cacheRead), `$${row.cost.toFixed(4)}`, `$${row.advisorCost.toFixed(4)}`, duration(row.durationMs)]) + const widths = header.map((label, index) => Math.max(label.length, ...data.map((row) => row[index]!.length))) + const format = (row: string[], isHeader = false) => row.map((cell, index) => index === 0 || isHeader ? cell.padEnd(widths[index]!) : cell.padStart(widths[index]!)).join(" | ") + return `${format(header, true)}\n${widths.map((width) => "-".repeat(width)).join("-|-")}\n${data.map((row) => format(row)).join("\n")}\n` +} + +/** Persisted pipeline, model, and step names must never emit terminal commands. */ +function terminalSafeCell(value: string): string { + return value + .replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, " ") + .replace(/[\u202a-\u202e\u2066-\u2069]/g, "") + .replace(/\s+/gu, " ") + .trim() +} + +function compact(value: number): string { + if (value < 1_000) return String(value) + if (value < 1_000_000) return `${(value / 1_000).toFixed(1).replace(/\.0$/, "")}k` + return `${(value / 1_000_000).toFixed(1).replace(/\.0$/, "")}M` +} + +function duration(milliseconds: number): string { + const seconds = Math.floor(milliseconds / 1_000) + if (seconds >= 3_600) return `${Math.floor(seconds / 3_600)}h${Math.floor((seconds % 3_600) / 60)}m` + if (seconds >= 60) return `${Math.floor(seconds / 60)}m${seconds % 60}s` + return `${seconds}s` +} diff --git a/src/run-history-report.ts b/src/run-history-report.ts new file mode 100644 index 0000000..58db8d3 --- /dev/null +++ b/src/run-history-report.ts @@ -0,0 +1,113 @@ +import { addTokens, emptyTokens, safeCost } from "./usage" +import type { ProgressTokens } from "./progress" +import type { RunEntry, RunHistoryRecord } from "./runs" + +export type RunHistoryFilter = { since?: number; pipeline?: string } +export type UsageReportDimension = "pipeline" | "model" | "step" | "day" + +export type UsageReportRow = { + key: string + runs: number + phases: number + completed: number + failed: number + tokens: ProgressTokens + cost: number + advisorCost: number + durationMs: number +} + +/** The rows of one dimension plus the distinct runs behind them, which the table's `total` row reports. */ +export type UsageReport = { dimension: UsageReportDimension; runs: number; rows: UsageReportRow[] } + +export function parseSince(value: string, now: number): number { + const relative = /^(\d+)([dh])$/.exec(value) + if (relative) { + const amount = Number(relative[1]) + const milliseconds = amount * (relative[2] === "d" ? 86_400_000 : 3_600_000) + const boundary = now - milliseconds + if (Number.isSafeInteger(amount) && Number.isSafeInteger(milliseconds) && Number.isSafeInteger(boundary)) return boundary + } + const date = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value) + if (date) { + const parsed = new Date(Number(date[1]), Number(date[2]) - 1, Number(date[3])) + if (parsed.getFullYear() === Number(date[1]) && parsed.getMonth() === Number(date[2]) - 1 && parsed.getDate() === Number(date[3])) return parsed.getTime() + } + throw new Error(`invalid --since value "${value}"; use d, h, or YYYY-MM-DD`) +} + +export function runCreatedAt(entry: Pick): number | undefined { + if (typeof entry.createdAt === "number" && Number.isFinite(entry.createdAt)) return entry.createdAt + const match = /^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})/.exec(entry.runID) + if (!match) return undefined + const parts = match.slice(1).map(Number) + const value = new Date(parts[0]!, parts[1]! - 1, parts[2]!, parts[3]!, parts[4]!, parts[5]!) + const valid = value.getFullYear() === parts[0] + && value.getMonth() === parts[1]! - 1 + && value.getDate() === parts[2] + && value.getHours() === parts[3] + && value.getMinutes() === parts[4] + && value.getSeconds() === parts[5] + return valid ? value.getTime() : undefined +} + +export function filterRunHistory>(entries: T[], filter: RunHistoryFilter): T[] { + return entries.filter((entry) => { + if (filter.pipeline !== undefined && entry.pipeline !== filter.pipeline) return false + if (filter.since === undefined) return true + const createdAt = runCreatedAt(entry) + return createdAt !== undefined && createdAt >= filter.since + }) +} + +type Accumulator = UsageReportRow & { runIDs: Set } + +export function usageReport(records: RunHistoryRecord[], dimension: UsageReportDimension): UsageReport { + const groups = new Map() + const runIDs = new Set() + for (const record of records) { + const runKey = dimension === "pipeline" ? record.pipeline ?? "(none)" : dimension === "day" ? dayFor(record) : undefined + if (runKey !== undefined) { + const row = getRow(groups, runKey) + row.runIDs.add(record.runID) + runIDs.add(record.runID) + if (record.statusKind === "completed") row.completed += 1 + if (record.statusKind === "failed") row.failed += 1 + } + for (const phase of record.phases) { + const key = dimension === "pipeline" ? runKey! : dimension === "model" ? phase.model ?? "(none)" : dimension === "step" ? phase.name : dayFor(record) + const row = getRow(groups, key) + row.runIDs.add(record.runID) + runIDs.add(record.runID) + row.phases += 1 + if (dimension === "model" || dimension === "step") { + if (phase.status === "completed") row.completed += 1 + if (phase.status === "failed") row.failed += 1 + } + row.tokens = addTokens(row.tokens, phase.tokens ?? emptyTokens()) + row.cost += safeCost(phase.cost) + safeCost(phase.advisorCost) + row.advisorCost += safeCost(phase.advisorCost) + row.durationMs += safeCost(phase.durationMs) + } + } + const rows = [...groups.values()] + .map(({ runIDs: rowRunIDs, ...row }) => ({ ...row, runs: rowRunIDs.size })) + .sort((left, right) => dimension === "day" ? left.key.localeCompare(right.key) : right.tokens.total - left.tokens.total || left.key.localeCompare(right.key)) + return { dimension, runs: runIDs.size, rows } +} + +function getRow(groups: Map, key: string): Accumulator { + let row = groups.get(key) + if (!row) { + row = { key, runs: 0, phases: 0, completed: 0, failed: 0, tokens: emptyTokens(), cost: 0, advisorCost: 0, durationMs: 0, runIDs: new Set() } + groups.set(key, row) + } + return row +} + +function dayFor(record: Pick): string { + const created = runCreatedAt(record) + if (created === undefined) return "(none)" + const date = new Date(created) + return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}` +} diff --git a/src/runs.ts b/src/runs.ts index 0d65fff..37d8538 100644 --- a/src/runs.ts +++ b/src/runs.ts @@ -9,6 +9,7 @@ import { readRunMetadata, type GoalRunState, type PhaseMetadataStatus, type RunM import { isValidRunID, convoyHome, runsRoot } from "./workspace" import { readAdvisorSplit } from "./advisor-report" import type { TuiRoute } from "./tui-session" +import type { ProgressTokens } from "./progress" export type RunStatusKind = "completed" | "failed" | "incomplete" | "empty" | "unknown" @@ -19,6 +20,78 @@ export type RunPhaseInfo = { cost?: number advisorCost?: number model?: string + tokens?: ProgressTokens + logicalModel?: string + startedAt?: number + endedAt?: number +} + +/** Durable, script-safe projection of a run history entry. */ +export type RunHistoryPhase = RunPhaseInfo +export type RunHistoryRecord = Pick< + RunEntry, + "runID" | "title" | "pipeline" | "targetDir" | "status" | "statusKind" | "createdAt" | "cost" | "executorCost" | "advisorCost" | "goal" | "finalization" | "feature" +> & { phases: RunHistoryPhase[] } + +/** Excludes transient liveness and workspace-path observations from public history JSON. */ +export function runHistoryRecord(entry: RunEntry): RunHistoryRecord { + return { + runID: entry.runID, + title: entry.title, + status: entry.status, + statusKind: entry.statusKind, + phases: entry.phases.map((phase) => ({ + name: phase.name, + status: phase.status, + ...(finiteNumber(phase.durationMs) ? { durationMs: phase.durationMs } : {}), + ...(finiteNumber(phase.cost) ? { cost: phase.cost } : {}), + ...(finiteNumber(phase.advisorCost) ? { advisorCost: phase.advisorCost } : {}), + ...(isString(phase.model) ? { model: phase.model } : {}), + ...(validTokens(phase.tokens) ? { tokens: copyTokens(phase.tokens) } : {}), + ...(isString(phase.logicalModel) ? { logicalModel: phase.logicalModel } : {}), + ...(finiteNumber(phase.startedAt) ? { startedAt: phase.startedAt } : {}), + ...(finiteNumber(phase.endedAt) ? { endedAt: phase.endedAt } : {}), + })), + ...(isString(entry.pipeline) ? { pipeline: entry.pipeline } : {}), + ...(isString(entry.targetDir) ? { targetDir: entry.targetDir } : {}), + ...(finiteNumber(entry.createdAt) ? { createdAt: entry.createdAt } : {}), + ...(finiteNumber(entry.cost) ? { cost: entry.cost } : {}), + ...(finiteNumber(entry.executorCost) ? { executorCost: entry.executorCost } : {}), + ...(finiteNumber(entry.advisorCost) ? { advisorCost: entry.advisorCost } : {}), + ...(entry.goal !== undefined ? { goal: entry.goal } : {}), + ...(entry.finalization !== undefined ? { finalization: entry.finalization } : {}), + ...(entry.feature !== undefined ? { feature: entry.feature } : {}), + } +} + +function finiteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) +} + +function isString(value: unknown): value is string { + return typeof value === "string" +} + +function validTokens(tokens: unknown): tokens is ProgressTokens { + if (typeof tokens !== "object" || tokens === null) return false + const value = tokens as Record + return finiteNumber(value.input) + && finiteNumber(value.output) + && finiteNumber(value.reasoning) + && finiteNumber(value.cacheRead) + && finiteNumber(value.cacheWrite) + && finiteNumber(value.total) +} + +function copyTokens(tokens: ProgressTokens): ProgressTokens { + return { + input: tokens.input, + output: tokens.output, + reasoning: tokens.reasoning, + cacheRead: tokens.cacheRead, + cacheWrite: tokens.cacheWrite, + total: tokens.total, + } } export type RunEntry = { @@ -608,6 +681,10 @@ function phaseInfos(metadata: RunMetadata | undefined): RunPhaseInfo[] { cost: phase.cost, advisorCost: phase.advisor?.cost, model: phase.model, + tokens: phase.tokens, + logicalModel: phase.logicalModel, + startedAt: phase.startedAt, + endedAt: phase.endedAt, })) } diff --git a/test/cli-parser.test.ts b/test/cli-parser.test.ts index b04dc82..95085f4 100644 --- a/test/cli-parser.test.ts +++ b/test/cli-parser.test.ts @@ -107,6 +107,19 @@ describe("resolveRunOptions", () => { await expect(parseCommand(["control", "extra"])).rejects.toThrow("usage: convoy control") }) + test("parseCommand parses run history JSON and stats modes", async () => { + expect(await parseCommand(["runs", "--json", "--since", "7d", "--pipeline", "implement"])).toMatchObject({ type: "runs", mode: "json", filter: { pipeline: "implement" } }) + expect(await parseCommand(["runs", "stats", "--group-by", "step", "--json"])).toMatchObject({ type: "runs", mode: "stats", dimension: "step", json: true }) + await expect(parseCommand(["runs", "stats", "--group-by", "week"])).rejects.toThrow(/unknown --group-by[\s\S]*usage: convoy runs/) + await expect(parseCommand(["runs", "--json", "--since", "yesterday"])).rejects.toThrow(/invalid --since value "yesterday"[\s\S]*usage: convoy runs/) + await expect(parseCommand(["runs", "--json", "--bogus"])).rejects.toThrow("usage: convoy runs") + await expect(parseCommand(["runs", "--json", "--pipeline", "-x"])).rejects.toThrow("usage: convoy runs") + await expect(parseCommand(["runs", "stats", "--since", "-7d"])).rejects.toThrow("usage: convoy runs") + await expect(parseCommand(["runs", "stats", "--group-by"])).rejects.toThrow("usage: convoy runs") + await expect(parseCommand(["runs", "--since", "7d"])).rejects.toThrow("usage: convoy runs") + await expect(parseCommand(["runs", "bad-id", "--json"])).rejects.toThrow("usage: convoy runs") + }) + test("the retired --feature flag fails before any plan is built", () => { // Feature-ID selectors are retired (capability feature-lifecycle): the // refusal happens in the parser, before plan review or any side effect. diff --git a/test/cli-regression.test.ts b/test/cli-regression.test.ts index 1b46d7b..44e86c5 100644 --- a/test/cli-regression.test.ts +++ b/test/cli-regression.test.ts @@ -411,3 +411,71 @@ describe("parseAndRun init and agents effects", () => { expect(await readFile(prompt, "utf8")).toContain("# Implementer") }) }) + +// `convoy runs --json` and `convoy runs stats` read the history under +// CONVOY_HOME (relocated by beforeEach) and print to stdout without opening +// the runs browser, so the dispatch is observable headlessly. +describe("parseAndRun run history outputs", () => { + async function writeRun(runID: string, metadata: Omit): Promise { + const dir = join(process.env.CONVOY_HOME!, ".convoy", "runs", runID) + await mkdir(dir, { recursive: true }) + await writeFile(join(dir, "prd.md"), `# ${runID}\n`) + await writeFile(join(dir, "metadata.json"), JSON.stringify({ ...metadata, runID, updatedAt: metadata.createdAt, control: { state: "running" } })) + } + + async function captureStdout(run: () => Promise): Promise { + const chunks: string[] = [] + const original = process.stdout.write + process.stdout.write = ((chunk: string | Uint8Array) => { + chunks.push(String(chunk)) + return true + }) as typeof process.stdout.write + try { + await run() + } finally { + process.stdout.write = original + } + return chunks.join("") + } + + const tokens = { input: 10, output: 5, reasoning: 0, cacheRead: 100, cacheWrite: 0, total: 115 } + + beforeEach(async () => { + await writeRun("20260910-090000-old1", { + schemaVersion: 3, + targetDir: "/tmp/repo", + createdAt: new Date(2026, 8, 10, 9).getTime(), + pipeline: { name: "review", steps: [] }, + phases: { reviewer: { status: "failed", cost: 0.5, model: "gpt", tokens, durationMs: 1_000 } }, + }) + await writeRun("20260912-090000-new1", { + schemaVersion: 3, + targetDir: "/tmp/repo", + createdAt: new Date(2026, 8, 12, 9).getTime(), + pipeline: { name: "implement", steps: [] }, + phases: { implementer: { status: "completed", cost: 0, model: "gpt", tokens, durationMs: 263_135 } }, + }) + }) + + test("runs --json prints durable records newest first and honors the filters", async () => { + const all = JSON.parse(await captureStdout(() => parseAndRun(["runs", "--json"]))) + expect(all.map((entry: { runID: string }) => entry.runID)).toEqual(["20260912-090000-new1", "20260910-090000-old1"]) + expect(all[0]).toMatchObject({ pipeline: "implement", statusKind: "completed", cost: 0, phases: [{ name: "implementer", status: "completed", cost: 0, tokens, model: "gpt" }] }) + expect(all[0]).not.toHaveProperty("live") + expect(all[0]).not.toHaveProperty("dir") + + const filtered = JSON.parse(await captureStdout(() => parseAndRun(["runs", "--json", "--pipeline", "review", "--since", "2026-09-10"]))) + expect(filtered.map((entry: { runID: string }) => entry.runID)).toEqual(["20260910-090000-old1"]) + expect(JSON.parse(await captureStdout(() => parseAndRun(["runs", "--json", "--since", "2026-09-13"])))).toEqual([]) + }) + + test("runs stats prints a table with a total row, or raw rows as JSON", async () => { + const table = await captureStdout(() => parseAndRun(["runs", "stats"])) + expect(table).toMatch(/^pipeline\s+\| runs \| phases \| ok \| failed \| tokens/) + expect(table).toMatch(/^implement\s+\|\s+1\s+\|\s+1\s+\|\s+1\s+\|\s+0\s+\|\s+115\s+\|\s+100\s+\|\s+\$0\.0000\s+\|\s+\$0\.0000\s+\|\s+4m23s$/m) + expect(table).toMatch(/^total\s+\|\s+2\s+\|\s+2\s+\|\s+1\s+\|\s+1\s+\|\s+230\s+\|\s+200\s+\|\s+\$0\.5000\s+\|\s+\$0\.0000\s+\|\s+4m24s$/m) + + const rows = JSON.parse(await captureStdout(() => parseAndRun(["runs", "stats", "--group-by", "model", "--json"]))) + expect(rows).toEqual([{ key: "gpt", runs: 2, phases: 2, completed: 1, failed: 1, tokens: { ...tokens, input: 20, output: 10, cacheRead: 200, total: 230 }, cost: 0.5, advisorCost: 0, durationMs: 264_135 }]) + }) +}) diff --git a/test/run-history-render.test.ts b/test/run-history-render.test.ts new file mode 100644 index 0000000..0f393fc --- /dev/null +++ b/test/run-history-render.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, test } from "bun:test" + +import { renderJson, renderUsageReportTable } from "../src/run-history-render" +import type { UsageReport, UsageReportRow } from "../src/run-history-report" + +const row: UsageReportRow = { key: "implement", runs: 1, phases: 1, completed: 1, failed: 0, tokens: { input: 1, output: 2, reasoning: 0, cacheRead: 1_000, cacheWrite: 0, total: 1_000 }, cost: 0.1234567, advisorCost: 0, durationMs: 263_135 } + +function report(dimension: UsageReport["dimension"], rows: UsageReportRow[], runs = rows.length): UsageReport { + return { dimension, runs, rows } +} + +describe("usage report rendering", () => { + test("renders compact table values and a total row", () => { + const table = renderUsageReportTable(report("pipeline", [row])) + expect(table).toContain("1k") + expect(table).toContain("$0.1235") + expect(table).toContain("phases") + expect(table).toMatch(/implement\s+\|\s+1\s+\|\s+1\s+\|/) + expect(table).toContain("4m23s") + expect(table).toContain("total") + }) + + test("reports the distinct run count on the total row, not the sum of the rows", () => { + const table = renderUsageReportTable(report("step", [row, { ...row, key: "tests" }], 1)) + expect(table).toMatch(/^total\s+\|\s+1\s+\|\s+2\s+\|/m) + }) + + test("renders a zeroed table and indented JSON", () => { + const empty = renderUsageReportTable(report("step", [])) + expect(empty).toMatch(/^step\s+\| runs/) + expect(empty).toMatch(/^total\s+\|\s+0\s+\|\s+0\s+\|\s+0\s+\|\s+0\s+\|\s+0\s+\|\s+0\s+\|\s+\$0\.0000\s+\|\s+\$0\.0000\s+\|\s+0s$/m) + expect(renderJson([{ value: 1 }])).toBe('[\n {\n "value": 1\n }\n]\n') + }) + + test("formats token and duration boundaries without changing JSON values", () => { + const table = renderUsageReportTable(report("step", [ + { ...row, key: "under", tokens: { ...row.tokens, total: 999, cacheRead: 999 }, durationMs: 59_999 }, + { ...row, key: "thousand", tokens: { ...row.tokens, total: 1_000, cacheRead: 1_000 }, durationMs: 60_000 }, + { ...row, key: "million", tokens: { ...row.tokens, total: 1_000_000, cacheRead: 1_000_000 }, durationMs: 3_600_000 }, + ])) + + expect(table).toContain("999") + expect(table).toContain("1k") + expect(table).toContain("1M") + expect(table).toContain("59s") + expect(table).toContain("1m0s") + expect(table).toContain("1h0m") + expect(JSON.parse(renderJson([{ tokens: 1_000_000, durationMs: 3_600_000 }]))).toEqual([{ tokens: 1_000_000, durationMs: 3_600_000 }]) + }) + + test("escapes terminal-active characters in JSON without changing parsed values", () => { + // C1 CSI/OSC/ST, bidi formats and line separators — everything JSON.stringify leaves literal. + const title = "build\u009b31m\u009d52;c;ZmFrZQ==\u009c\u202e\u200f\u2028spoof\u2066" + const json = renderJson([{ title }]) + expect(json).not.toMatch(/[\u007f-\u009f\u061c\u200e\u200f\u2028\u2029\u202a-\u202e\u2066-\u2069]/) + expect(json).toContain("\\u009b") + expect(json).toContain("\\u202e") + expect(JSON.parse(json)).toEqual([{ title }]) + }) + + test("neutralizes terminal control sequences in table group names", () => { + const unsafe = { ...row, key: "build\u202e\nspoof\u001b]52;c;ZmFrZQ==\u0007\u009b31m" } + const table = renderUsageReportTable(report("step", [unsafe])) + + expect(table).toContain("build spoof ]52;c;ZmFrZQ== 31m") + expect(table).not.toMatch(/[\u0000-\u0009\u000b-\u001f\u007f-\u009f]/) + expect(table).not.toMatch(/[\u202a-\u202e\u2066-\u2069]/) + }) +}) diff --git a/test/run-history-report.test.ts b/test/run-history-report.test.ts new file mode 100644 index 0000000..008deac --- /dev/null +++ b/test/run-history-report.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, test } from "bun:test" + +import { filterRunHistory, parseSince, runCreatedAt, usageReport } from "../src/run-history-report" +import type { RunHistoryRecord } from "../src/runs" + +const tokens = { input: 2, output: 3, reasoning: 0, cacheRead: 5, cacheWrite: 0, total: 10 } +const run = (overrides: Partial = {}): RunHistoryRecord => ({ + runID: "20260910-120000-abcd", title: "run", pipeline: "implement", status: "completed", statusKind: "completed", phases: [ + { name: "implementer", status: "completed", model: "gpt", tokens, cost: 0, advisorCost: 0.25, durationMs: 2_000 }, + ], ...overrides, +}) + +describe("run history filters", () => { + test("parses relative and local-date boundaries", () => { + expect(parseSince("7d", 1_000_000)).toBe(1_000_000 - 604_800_000) + expect(parseSince("36h", 1_000_000)).toBe(1_000_000 - 129_600_000) + expect(parseSince("2026-09-10", 0)).toBe(new Date(2026, 8, 10).getTime()) + expect(() => parseSince("soon", 0)).toThrow("invalid --since") + expect(() => parseSince(`${Number.MAX_SAFE_INTEGER}d`, 0)).toThrow("invalid --since") + }) + + test("filters exact pipeline names and falls back to the local run-id stamp", () => { + const entries = [run({ createdAt: new Date(2026, 8, 10, 12).getTime() }), run({ pipeline: "review", createdAt: new Date(2026, 8, 1).getTime() })] + expect(filterRunHistory(entries, { pipeline: "implement" })).toHaveLength(1) + expect(filterRunHistory(entries, { since: new Date(2026, 8, 5).getTime() })).toHaveLength(1) + expect(filterRunHistory([run({ createdAt: undefined })], { since: new Date(2026, 8, 10).getTime() })).toHaveLength(1) + }) + + test("keeps unknown dates only when no since filter is requested and includes the boundary", () => { + const boundary = new Date(2026, 8, 10, 12).getTime() + const atBoundary = run({ createdAt: boundary }) + const unknown = run({ runID: "legacy-run", createdAt: undefined }) + + expect(runCreatedAt(unknown)).toBeUndefined() + expect(runCreatedAt(run({ runID: "20261340-256199-bad", createdAt: undefined }))).toBeUndefined() + expect(filterRunHistory([atBoundary, unknown], {})).toEqual([atBoundary, unknown]) + expect(filterRunHistory([atBoundary, unknown], { since: boundary })).toEqual([atBoundary]) + }) +}) + +describe("usage reports", () => { + test("aggregates phase usage and uses run outcomes for pipeline rows", () => { + const report = usageReport([run(), run({ runID: "20260911-120000-efgh", statusKind: "failed", phases: [{ name: "implementer", status: "failed", model: "gpt", tokens, cost: Number.NaN, durationMs: 3_000 }] })], "pipeline") + expect(report).toMatchObject({ dimension: "pipeline", runs: 2 }) + expect(report.rows).toEqual([expect.objectContaining({ key: "implement", runs: 2, phases: 2, completed: 1, failed: 1, cost: 0.25, advisorCost: 0.25, durationMs: 5_000, tokens: { ...tokens, input: 4, output: 6, cacheRead: 10, total: 20 } })]) + }) + + test("uses phase outcomes and distinct runs for step rows", () => { + const report = usageReport([run(), run({ phases: [{ name: "implementer", status: "failed" }, { name: "tests", status: "skipped" }] })], "step") + expect(report.rows[0]).toMatchObject({ key: "implementer", runs: 1, phases: 2, completed: 1, failed: 1 }) + expect(report.rows[1]).toMatchObject({ key: "tests", runs: 1, phases: 1, completed: 0, failed: 0 }) + // The report's run count is distinct across rows, not the sum of each row's `runs`. + expect(report.runs).toBe(1) + }) + + test("creates missing-fact and date groups", () => { + expect(usageReport([run({ pipeline: undefined, phases: [{ name: "x", status: "completed" }] })], "pipeline").rows[0]).toMatchObject({ key: "(none)", runs: 1 }) + expect(usageReport([run({ createdAt: new Date(2026, 8, 10).getTime() })], "day").rows[0]?.key).toBe("2026-09-10") + }) + + test("sums cache usage by model, keeps workspace-less runs, and orders days chronologically", () => { + const records = [ + run({ runID: "20260911-120000-next", createdAt: new Date(2026, 8, 11).getTime(), phases: [{ name: "measure", status: "completed", model: "gpt", tokens: { ...tokens, cacheRead: 600_000 } }] }), + run({ runID: "20260910-120000-prev", createdAt: new Date(2026, 8, 10).getTime(), phases: [{ name: "measure", status: "skipped", model: "gpt", tokens: { ...tokens, cacheRead: 400_000 } }] }), + run({ runID: "20260912-120000-clean", pipeline: undefined, statusKind: "failed", phases: [] }), + ] + + const byModel = usageReport(records, "model") + expect(byModel.rows).toEqual([expect.objectContaining({ key: "gpt", runs: 2, phases: 2, completed: 1, failed: 0, tokens: expect.objectContaining({ cacheRead: 1_000_000 }) })]) + // The workspace-less run has no phase, so it contributes to no phase-grained row. + expect(byModel.runs).toBe(2) + const byPipeline = usageReport(records, "pipeline") + expect(byPipeline.rows).toEqual(expect.arrayContaining([ + expect.objectContaining({ key: "(none)", runs: 1, phases: 0, completed: 0, failed: 1 }), + ])) + expect(byPipeline.runs).toBe(3) + expect(usageReport(records, "day").rows.map((row) => row.key)).toEqual(["2026-09-10", "2026-09-11", "2026-09-12"]) + }) +}) diff --git a/test/runs.test.ts b/test/runs.test.ts index 9d75887..41a5d2c 100644 --- a/test/runs.test.ts +++ b/test/runs.test.ts @@ -5,8 +5,9 @@ import { join } from "node:path" import { afterAll, beforeAll, describe, expect, test } from "bun:test" -import { isServerLive, listRuns, probeRunWaiting, refreshRunWaiting } from "../src/runs" +import { isServerLive, listRuns, probeRunWaiting, refreshRunWaiting, runHistoryRecord } from "../src/runs" import { startControlServer, type ControlServer } from "../src/control-server" +import type { ProgressTokens } from "../src/progress" function listen(): Promise<{ port: number; close: () => void }> { return new Promise((resolve, reject) => { @@ -69,6 +70,64 @@ afterAll(async () => { }) describe("run history listing", () => { + test("projects only durable phase facts for JSON history", () => { + const record = runHistoryRecord({ + runID: "20260912-120000-proj", dir: "/private", title: "Projection", pipeline: "implement", status: "completed", statusKind: "completed", live: true, waiting: "review", serverUrl: "http://private", cost: 0, + phases: [{ name: "implementer", status: "completed", cost: 0, tokens: { input: 1, output: 2, reasoning: 0, cacheRead: 0, cacheWrite: 0, total: 3 }, logicalModel: "gpt", startedAt: 1, endedAt: 2 }], + }) + expect(record).toMatchObject({ cost: 0, phases: [{ cost: 0, logicalModel: "gpt", startedAt: 1, endedAt: 2 }] }) + expect(record).not.toHaveProperty("dir") + expect(record).not.toHaveProperty("live") + expect(record).not.toHaveProperty("waiting") + expect(record).not.toHaveProperty("serverUrl") + }) + + test("omits unrecorded usage facts while preserving recorded zero cost", () => { + const record = runHistoryRecord({ + runID: "20260912-120001-proj", + dir: "/private", + title: "Projection", + status: "completed", + statusKind: "completed", + live: false, + phases: [ + { name: "Compact run", status: "skipped", durationMs: 25 }, + { name: "implementer", status: "completed", cost: 0, tokens: { input: 1, output: 2, reasoning: 0, cacheRead: 0, cacheWrite: 0, total: 3 } }, + ], + }) + + expect(record.phases[0]).toEqual({ name: "Compact run", status: "skipped", durationMs: 25 }) + expect(record.phases[1]).toMatchObject({ name: "implementer", cost: 0, tokens: { total: 3 } }) + expect(JSON.stringify(record)).not.toContain("null") + }) + + test("omits malformed optional numeric facts instead of serializing null", () => { + const record = runHistoryRecord({ + runID: "20260912-120002-proj", + dir: "/private", + title: "Projection", + status: "completed", + statusKind: "completed", + live: false, + cost: null as unknown as number, + createdAt: Number.NaN, + phases: [{ + name: "implementer", + status: "completed", + durationMs: Number.NaN, + model: null as unknown as string, + tokens: null as unknown as ProgressTokens, + }], + }) + + expect(record).not.toHaveProperty("cost") + expect(record).not.toHaveProperty("createdAt") + expect(record.phases[0]).not.toHaveProperty("durationMs") + expect(record.phases[0]).not.toHaveProperty("model") + expect(record.phases[0]).not.toHaveProperty("tokens") + expect(JSON.stringify(record)).not.toContain("null") + }) + test("lists valid runs newest first with metadata details", async () => { const runs = await listRuns(root)