From ae63ffe683bcfd365a59011366045bf97808a950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Gonz=C3=A1lez=20Barrera?= <150662051+dagoaie@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:14:17 +0200 Subject: [PATCH] feat(hooks): expose run cost, tokens and duration to post-hooks Post-hooks receive the run's aggregated usage in their environment: CONVOY_RUN_COST (executor plus advisor), CONVOY_RUN_ADVISOR_COST, CONVOY_RUN_TOKENS_* and CONVOY_RUN_DURATION_MS, summed from the metadata the runner already holds, on success and on failure. Each variable appears only when a phase recorded that fact; nothing is reported as zero. The README documents the formats with a PR-comment hook and an awk budget guard. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BG8wiaLWbUzGXBm3rtzN41 --- README.md | 16 +++- .../2026-09-11-hook-run-usage/.openspec.yaml | 2 + .../2026-09-11-hook-run-usage/design.md | 83 +++++++++++++++++++ .../2026-09-11-hook-run-usage/proposal.md | 29 +++++++ .../specs/hook-run-usage/spec.md | 80 ++++++++++++++++++ .../2026-09-11-hook-run-usage/tasks.md | 22 +++++ openspec/specs/hook-run-usage/spec.md | 81 ++++++++++++++++++ src/hooks.ts | 39 ++++++++- src/metadata.ts | 7 +- src/runner.ts | 4 + src/usage.ts | 61 ++++++++++++++ test/hooks.test.ts | 62 ++++++++++++++ test/metadata.test.ts | 41 +++++++++ test/runner-hosted.test.ts | 24 ++++++ test/usage.test.ts | 25 +++++- 15 files changed, 572 insertions(+), 4 deletions(-) create mode 100644 openspec/changes/archive/2026-09-11-hook-run-usage/.openspec.yaml create mode 100644 openspec/changes/archive/2026-09-11-hook-run-usage/design.md create mode 100644 openspec/changes/archive/2026-09-11-hook-run-usage/proposal.md create mode 100644 openspec/changes/archive/2026-09-11-hook-run-usage/specs/hook-run-usage/spec.md create mode 100644 openspec/changes/archive/2026-09-11-hook-run-usage/tasks.md create mode 100644 openspec/specs/hook-run-usage/spec.md diff --git a/README.md b/README.md index fb6005d..7bf0be6 100644 --- a/README.md +++ b/README.md @@ -375,6 +375,20 @@ hooks: fi ``` +Post-hooks also receive the run's recorded usage without parsing `metadata.json`: `CONVOY_RUN_COST` (executor plus advisor spend) and `CONVOY_RUN_ADVISOR_COST` are USD decimals with exactly four fractional digits; `CONVOY_RUN_TOKENS_INPUT`, `CONVOY_RUN_TOKENS_OUTPUT`, `CONVOY_RUN_TOKENS_REASONING`, `CONVOY_RUN_TOKENS_CACHE_READ`, `CONVOY_RUN_TOKENS_CACHE_WRITE`, and `CONVOY_RUN_TOKENS_TOTAL` are integer token counts; and `CONVOY_RUN_DURATION_MS` is the integer sum of recorded phase durations. Cost appears only after a phase reports executor or advisor spend, token variables appear together only after a phase reports executor usage, advisor cost appears only when advisor spend is positive, and duration appears only after a phase records one — absent facts are not represented as zero. For example, a post-hook can comment the result on its PR and enforce a decimal budget: + +```yaml +hooks: + post: + - name: report run usage + command: gh pr comment --body "Convoy run: \$${CONVOY_RUN_COST} in ${CONVOY_RUN_DURATION_MS}ms" + - name: enforce budget + when: always + command: 'awk "BEGIN { exit !(${CONVOY_RUN_COST:-0} <= 5.0000) }"' +``` + +Use `awk` for the budget guard because shell integer comparisons cannot compare decimal USD amounts. A failing post-hook fails the run unless `continueOnError: true` is set. + The dashboard shows the goal, the current iteration, and the trajectory (`◆ convoy · goal 90 · iter 2/4 · 71 → …`), and when the cycle ends — goal met, plateau, iteration cap, no score, or a failure — the dashboard holds its finish screen **once**, with the verdict in place of the live goal readout (`✓ goal 92/100`, `plateau 86/100`, `cap 88/100`, `no score`, or `✗ run failed`) and the full trajectory (`71 → 84 → 92`); the terminal prints the trajectory and why it stopped after the dashboard closes. Goal fragment phases appear under the parent pipeline with their iteration-qualified names (for example `goal-measure-1-score-report`); the whole cycle runs in one run, so the dashboard never remounts between rounds. ## Requirements @@ -837,7 +851,7 @@ The rules: - **Resume is frozen**: the resolved pipeline is persisted in the run's `metadata.json`; `--resume` replays it even if the config changed since. - **Dirty-tree recovery**: a writable phase interrupted before its commit (Ctrl+C, a failed commit step, a killed process) leaves uncommitted work in the tree, which normally blocks `--resume`. In an interactive terminal, resume offers to commit that work as the interrupted phase (`convoy(): …` with the resumed run's `Convoy-Run` trailer), mark it done, and continue with the following phases. If the interrupted phase had already accepted a structured commit description through `write_report`, recovery reuses it; otherwise the message describes the staged paths or says plainly what happened. Read-only phases are never recoverable as agent output: preserved changes must be resolved manually, and resume also verifies their recorded HEAD/branch baseline. Decline (or a non-TTY resume) keeps the old "commit/stash first" behavior. - **Permissions are additive**: `permissions.deny` extends the hard denylist, `permissions.allow` extends the allowlist, deny always wins, and there is deliberately no way for a repo to grant itself `--yolo`. -- **Hooks are trusted local shell commands**: `hooks.pre` runs after the run workspace/dashboard is initialized and before the pipeline starts (pre-hooks are skipped on `--resume`); `hooks.post` runs at the end according to `when`. Top-level hooks apply to every pipeline, and `hooks.pipelines.` entries are appended for that pipeline. Hooks run via `$SHELL -lc` from the target repo by default, receive `CONVOY_RUN_ID`, `CONVOY_RUN_DIR`, `CONVOY_TARGET_DIR`, `CONVOY_PIPELINE`, `CONVOY_PROMPT_FILE`, and post-hooks also receive `CONVOY_RUN_STATUS`, plus `CONVOY_RUN_SCORE` on a scored pipeline and `CONVOY_GOAL_REACHED`/`CONVOY_GOAL_SCORE`/`CONVOY_GOAL_TARGET` when a [goal loop](#goal-mode) ran (in which case post-hooks run once, after the loop, not once per iteration). A failing hook fails the run unless `continueOnError: true` is set. Each hook is also a row in the dashboard pipeline — pre-hooks ahead of the steps, post-hooks after — with live running/✓/✗/skipped status, and the tail of its output lands in that row's `logs` tab; the rows are recorded in the run metadata, so re-opened runs show them too. +- **Hooks are trusted local shell commands**: `hooks.pre` runs after the run workspace/dashboard is initialized and before the pipeline starts (pre-hooks are skipped on `--resume`); `hooks.post` runs at the end according to `when`. Top-level hooks apply to every pipeline, and `hooks.pipelines.` entries are appended for that pipeline. Hooks run via `$SHELL -lc` from the target repo by default, receive `CONVOY_RUN_ID`, `CONVOY_RUN_DIR`, `CONVOY_TARGET_DIR`, `CONVOY_PIPELINE`, `CONVOY_PROMPT_FILE`, and post-hooks also receive `CONVOY_RUN_STATUS`, plus `CONVOY_RUN_SCORE` on a scored pipeline, `CONVOY_GOAL_REACHED`/`CONVOY_GOAL_SCORE`/`CONVOY_GOAL_TARGET` when a [goal loop](#goal-mode) ran, and recorded run usage (`CONVOY_RUN_COST`, `CONVOY_RUN_ADVISOR_COST`, `CONVOY_RUN_TOKENS_*`, and `CONVOY_RUN_DURATION_MS`; formats and presence rules are in [Goal mode](#goal-mode)). Pre-hooks never receive usage variables. A failing hook fails the run unless `continueOnError: true` is set. Each hook is also a row in the dashboard pipeline — pre-hooks ahead of the steps, post-hooks after — with live running/✓/✗/skipped status, and the tail of its output lands in that row's `logs` tab; the rows are recorded in the run metadata, so re-opened runs show them too. ## Global configuration diff --git a/openspec/changes/archive/2026-09-11-hook-run-usage/.openspec.yaml b/openspec/changes/archive/2026-09-11-hook-run-usage/.openspec.yaml new file mode 100644 index 0000000..515eaae --- /dev/null +++ b/openspec/changes/archive/2026-09-11-hook-run-usage/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-11 diff --git a/openspec/changes/archive/2026-09-11-hook-run-usage/design.md b/openspec/changes/archive/2026-09-11-hook-run-usage/design.md new file mode 100644 index 0000000..b2a7e77 --- /dev/null +++ b/openspec/changes/archive/2026-09-11-hook-run-usage/design.md @@ -0,0 +1,83 @@ +## Context + +`runHookCommand` (`src/hooks.ts`) builds the hook environment from a `RunHookContext` the runner assembles at two post-hook call sites: the success path after the summary is written and the failure path in the runner's catch block. The context carries `status`, an optional `score` and an optional `goal` outcome; each optional field becomes a conditional spread into `env`, so a hook sees a variable only when Convoy actually knows the value. + +The numbers a hook would want already exist in memory at both call sites. `openRunMetadata` returns a `RunMetadataStore` whose `data.phases` holds, per phase, the executor usage (`cost`, `tokens`, written together by `recalculate` from a `PhaseUsage` accumulator), the advisor aggregate (`advisor.cost`, from `phaseAdvisorEvent`) and `durationMs` (`endedAt - startedAt`, set by `phaseEnded` before its first `await`, so a failed phase already carries it when the failure hooks run). Hook rows are phases too: pre-hooks get a duration, never a cost. The store exposes phases only by name (`snapshot(name)`), and no code path today sums them for a whole run in memory: `convoy runs` (`totalCost` in `src/runs.ts`) and `SUMMARY.md` (`readAdvisorSplit` in `src/advisor-report.ts`) both aggregate from disk, mixing `metadata.json` with attempt logs, and the dashboard sums its own `PhaseState` list. + +The vocabulary is already settled by the code: *usage* is cost plus tokens (`ProgressUsage`, `PhaseUsage`, `phaseStepUsage`, `phaseUsageTotal`); the executor's usage and the advisor's are recorded apart and only `convoy runs` adds them up. + +## Goals / Non-Goals + +**Goals** + +- Post-hooks — success and failure — receive the run's aggregated cost, advisor cost, tokens and duration from the in-memory metadata store. +- The aggregate follows the store's facts: nothing recorded means no variable, never a fabricated zero. +- One aggregation, unit-testable without a store, reused by the store. +- The README documents every variable and its format, and shows the `gh pr comment` and budget-guard use cases. + +**Non-Goals** + +- Pre-hooks (nothing has been recorded when they run). +- Per-step hooks or per-phase variables (there are no per-step hooks). +- Changing how `convoy runs`, the dashboard or `SUMMARY.md` compute or display cost and duration, including the `--resume` duration caveat below. +- Any remote publication by Convoy itself; the hook owns whatever happens with the numbers. + +## Decisions + +### D1: `sumRunUsage` in `src/usage.ts`, a pure fold over recorded phases + +```ts +export type RunUsage = { + /** Executor plus advisor cost in USD; present when at least one phase recorded either. */ + cost?: number + /** Advisor cost in USD; present only when the summed advisor spend is above zero. */ + advisorCost?: number + /** Summed executor tokens; present when at least one phase recorded executor usage. */ + tokens?: ProgressTokens + /** Sum of the recorded phases' durations; present when at least one phase recorded one. */ + durationMs?: number +} + +export function sumRunUsage(phases: Iterable): RunUsage | undefined +``` + +`RunUsagePhase` is a structural type — `{ cost?: number; tokens?: ProgressTokens; durationMs?: number; advisor?: { cost: number } }` — so `usage.ts` keeps depending only on `progress.ts` and does not import `PhaseMetadata` (which would close a `usage → metadata → usage` cycle). The fold reuses `emptyTokens`, `addTokens` and `safeCost`. `tokens` is set when any phase has a numeric `cost` (the same marker `totalCost` uses, and the invariant `recalculate` guarantees that tokens travel with it); `cost` when any phase has a numeric executor or advisor cost, so advisor-only spend still yields a total, as in `convoy runs`; `advisorCost` when the advisor sum is above zero; `durationMs` when any phase has one. The function returns `undefined` when none of the groups has data, so callers can spread it like `goal`. + +Alternative considered: a store method `phases()` and the fold at the call site. Rejected: it exposes the store's internal record shape for a single consumer. + +### D2: `RunMetadataStore.runUsage(): RunUsage | undefined` + +The store delegates to `sumRunUsage(Object.values(data.phases))`. No I/O, no cache: `data` is the live record, and the call happens once per post-hook stage. Its name matches the glossary term (run usage) rather than an implementation-flavoured `usageTotals`. + +Alternative considered: re-reading `metadata.json` (or attempt logs, as `readAdvisorSplit` does) from `hooks.ts`. Rejected: it adds I/O and a second source of truth inside the process that already holds the data, and the attempt logs would double count phases the metadata has totals for. + +### D3: `RunHookContext.usage?: RunUsage` and the environment variables + +`hooks.ts` adds the optional field and a `usageEnv(usage)` helper that returns the variables, spread into `env` after `CONVOY_GOAL_*`: + +| Variable | Present when | Format | +|---|---|---| +| `CONVOY_RUN_COST` | `usage.cost !== undefined` | `toFixed(4)` | +| `CONVOY_RUN_TOKENS_INPUT/OUTPUT/REASONING/CACHE_READ/CACHE_WRITE/TOTAL` | `usage.tokens !== undefined` | integer | +| `CONVOY_RUN_ADVISOR_COST` | `usage.advisorCost !== undefined` | `toFixed(4)` | +| `CONVOY_RUN_DURATION_MS` | `usage.durationMs !== undefined` | integer (`Math.round`) | + +Four fractional digits match the advisor split in `SUMMARY.md`; the dashboard's two are a display concession, and a hook that aggregates runs would lose the cheap ones. No `CONVOY_RUN_EXECUTOR_COST`: two numbers are enough and the third is a subtraction. Formatting lives in `hooks.ts` so `test/hooks.test.ts` pins the textual contract the README documents. + +### D4: Both post-hook call sites pass the aggregate + +The success and the failure call sites in `src/runner.ts` add `...(usage ? { usage } : {})` with `usage = metadata?.runUsage()`. The failure path uses optional access because the run may fail before `openRunMetadata` returned. The pre-hook call site is untouched. The post-hook row itself is `running` at that moment and carries no duration, so it does not count itself. + +### D5: README + +The hooks paragraph lists the new variables after `CONVOY_GOAL_*`, states the formats and that the duration is a sum of phase durations, and the goal-mode example gains a sibling: a post-hook that comments cost and duration on the PR with `gh pr comment`, and a one-line `when: always` budget guard using `awk` (a shell `[ -gt ]` cannot compare a decimal), with the note that a failing post-hook fails the run. + +## Risks / Trade-offs + +- A phase interrupted and resumed keeps its original `startedAt` (`phaseStarted` uses `??=`), so its `durationMs` — and therefore `CONVOY_RUN_DURATION_MS` — includes the time the run sat between the two processes. This is pre-existing `durationMs` behavior shared with `convoy runs` and the dashboard; changing it belongs to a separate change. +- A Claude Code phase reports cost and tokens like an OpenCode one (`src/claude-code.ts`), so the cost group does not go missing on mixed pipelines; a phase whose session never reported usage simply contributes nothing. +- Advisor consultations billed at zero (subscription models) leave `CONVOY_RUN_ADVISOR_COST` unset by design: the variable reports spend, not activity. + +## Migration + +None. Additive environment variables; no CLI, config, protocol or persisted-state change. diff --git a/openspec/changes/archive/2026-09-11-hook-run-usage/proposal.md b/openspec/changes/archive/2026-09-11-hook-run-usage/proposal.md new file mode 100644 index 0000000..07337a7 --- /dev/null +++ b/openspec/changes/archive/2026-09-11-hook-run-usage/proposal.md @@ -0,0 +1,29 @@ +## Why + +Post-hooks receive `CONVOY_RUN_STATUS`, `CONVOY_RUN_SCORE` and the `CONVOY_GOAL_*` outcome (`src/hooks.ts`, `runHookCommand`), but nothing about what the run spent or how long it worked. The runner invokes them while it still holds the run's metadata store in memory, and that store already carries every phase's executor cost, tokens, advisor cost and duration (`PhaseMetadata` in `src/metadata.ts`). A hook that wants to comment "this run cost $3.20 in 14 min" on the pull request, post the figure to a webhook, or fail a CI job above a budget has to re-open `$CONVOY_RUN_DIR/metadata.json` and reimplement the aggregation `convoy runs` performs — and every such hook does it slightly differently. + +## What Changes + +- Add a run-level usage aggregate to the metadata store: total cost (executor plus advisor), advisor cost, tokens and run duration, summed over every recorded phase in memory, with the same "absent when nothing was recorded" rule `CONVOY_RUN_SCORE` follows. +- Pass that aggregate to post-hooks — on success and on failure — as `CONVOY_RUN_COST`, `CONVOY_RUN_ADVISOR_COST`, `CONVOY_RUN_TOKENS_{INPUT,OUTPUT,REASONING,CACHE_READ,CACHE_WRITE,TOTAL}` and `CONVOY_RUN_DURATION_MS`, with a fixed, documented format. +- Document the variables in the README hooks paragraph with a post-hook that comments cost and duration on the PR and a one-line budget guard. +- Pre-hooks, the dashboard, `convoy runs`, `SUMMARY.md` and the metadata file format are untouched; Convoy still never publishes anything itself — the hook decides what to do with the numbers. + +## Capabilities + +### New Capabilities + +- `hook-run-usage`: post-hooks receive the run's aggregated usage — cost, advisor cost, tokens and duration — from the run's own metadata, so a project can report or gate on spend without parsing run files. + +### Modified Capabilities + + + +## Impact + +- `src/usage.ts` (run-level sum over recorded phases). +- `src/metadata.ts` (`RunMetadataStore.runUsage()`). +- `src/hooks.ts` (`RunHookContext.usage` and the new environment variables). +- `src/runner.ts` (both post-hook call sites pass the aggregate). +- `README.md` (hooks paragraph and example). +- No CLI surface, harness protocol, control protocol, config schema or persisted-state change. diff --git a/openspec/changes/archive/2026-09-11-hook-run-usage/specs/hook-run-usage/spec.md b/openspec/changes/archive/2026-09-11-hook-run-usage/specs/hook-run-usage/spec.md new file mode 100644 index 0000000..b106010 --- /dev/null +++ b/openspec/changes/archive/2026-09-11-hook-run-usage/specs/hook-run-usage/spec.md @@ -0,0 +1,80 @@ +## Purpose + +Give post-hooks the run's aggregated usage — what it cost, what it consumed and how long it worked — straight from the run's own metadata, so a project can report or gate on spend in a hook without parsing run files or re-deriving totals that Convoy already knows. + +## ADDED Requirements + +### Requirement: Post-hooks receive the run's aggregated usage + +When Convoy runs post-hooks, on success and on failure alike, it SHALL export the run's aggregated usage in the hook environment, summed over every phase recorded in the run's metadata (pipeline steps, goal-fragment invocations and hook rows). `CONVOY_RUN_COST` SHALL be the total cost in USD — executor cost plus advisor cost, the same figure `convoy runs` reports as `cost`. `CONVOY_RUN_TOKENS_INPUT`, `CONVOY_RUN_TOKENS_OUTPUT`, `CONVOY_RUN_TOKENS_REASONING`, `CONVOY_RUN_TOKENS_CACHE_READ`, `CONVOY_RUN_TOKENS_CACHE_WRITE` and `CONVOY_RUN_TOKENS_TOTAL` SHALL be the summed token counts, where `TOTAL` follows the same definition as the phase totals the dashboard shows. `CONVOY_RUN_DURATION_MS` SHALL be the sum of the recorded phases' durations — the time the pipeline spent working, not the wall clock since the run was created. The aggregate SHALL come from the metadata the runner holds in memory, never from re-reading run files. + +#### Scenario: A successful two-phase run + +- **WHEN** two phases recorded usage of `0.5 USD / 1000 input / 200 output` and `0.25 USD / 400 input / 100 output`, with durations of 60000 ms and 30000 ms, and the run succeeds +- **THEN** the success post-hooks see `CONVOY_RUN_COST=0.7500`, `CONVOY_RUN_TOKENS_INPUT=1400`, `CONVOY_RUN_TOKENS_OUTPUT=300` and `CONVOY_RUN_DURATION_MS=90000` + +#### Scenario: A run that fails in its second phase + +- **WHEN** the first phase recorded usage, the second phase fails after recording some usage, and the failure post-hooks run +- **THEN** they see the cost, tokens and duration accumulated up to and including the failed phase, and `CONVOY_RUN_STATUS=failure` as before + +#### Scenario: A goal cycle + +- **WHEN** a goal cycle ran two measure rounds and one improve round before settling +- **THEN** the post-hooks, which run once after the whole cycle, see totals that include every fragment invocation of every round + +### Requirement: Usage variables are omitted when nothing was recorded + +Convoy SHALL NOT invent a zero: `CONVOY_RUN_COST` SHALL be exported only when at least one recorded phase carries an executor or advisor cost, the token variables SHALL be exported together only when at least one recorded phase carries executor usage, and `CONVOY_RUN_DURATION_MS` SHALL be exported only when at least one recorded phase carries a duration. When none is available no usage variable SHALL be set, mirroring how `CONVOY_RUN_SCORE` is absent on an unscored pipeline. + +#### Scenario: A run that fails before any session reported usage + +- **WHEN** a pre-hook fails after running for a while, so the failure post-hooks run with phases that have durations but no cost +- **THEN** `CONVOY_RUN_DURATION_MS` is set and `CONVOY_RUN_COST` and every `CONVOY_RUN_TOKENS_*` variable are unset + +#### Scenario: A run whose only recorded spend is the advisor's + +- **WHEN** a phase recorded an advisor cost of `0.20 USD` but its executor never reported usage before the run failed +- **THEN** the failure post-hooks see `CONVOY_RUN_COST=0.2000` and `CONVOY_RUN_ADVISOR_COST=0.2000`, and every `CONVOY_RUN_TOKENS_*` variable is unset + +#### Scenario: A run that fails before any phase started + +- **WHEN** the run fails before any phase recorded a start +- **THEN** no `CONVOY_RUN_COST`, `CONVOY_RUN_TOKENS_*` or `CONVOY_RUN_DURATION_MS` variable is set + +### Requirement: Advisor spend is reported on its own + +When any recorded phase carries advisor usage with a cost above zero, Convoy SHALL export `CONVOY_RUN_ADVISOR_COST` with the summed advisor cost, so a hook can tell the executor's share from the advisor's. `CONVOY_RUN_COST` SHALL already include that amount. When no advisor spend was recorded the variable SHALL be absent. + +#### Scenario: A run with an advisor + +- **WHEN** one phase recorded an executor cost of `1.00 USD` and an advisor cost of `0.20 USD` +- **THEN** the post-hooks see `CONVOY_RUN_COST=1.2000` and `CONVOY_RUN_ADVISOR_COST=0.2000` + +#### Scenario: A run without an advisor + +- **WHEN** no phase recorded advisor usage +- **THEN** `CONVOY_RUN_ADVISOR_COST` is unset and `CONVOY_RUN_COST` is the executor cost alone + +### Requirement: Values use a fixed, documented format + +Cost variables SHALL be decimal USD with exactly four fractional digits and no currency sign; token and duration variables SHALL be non-negative integers with no separators. The README SHALL document each variable, its format, and that the duration is a sum of phase durations. + +#### Scenario: Formatting a fractional cost and a large token count + +- **WHEN** the aggregated cost is `3.2` USD and the summed input tokens are `1234567` +- **THEN** the hook sees `CONVOY_RUN_COST=3.2000` and `CONVOY_RUN_TOKENS_INPUT=1234567` + +### Requirement: Pre-hooks and existing hook variables are unchanged + +Pre-hooks SHALL NOT receive any usage variable — nothing has been recorded when they run — and every variable post-hooks received before this change SHALL keep its name, presence rule and value. + +#### Scenario: A pre-hook inspects its environment + +- **WHEN** a pre-hook runs +- **THEN** `CONVOY_RUN_COST`, `CONVOY_RUN_ADVISOR_COST`, every `CONVOY_RUN_TOKENS_*` variable and `CONVOY_RUN_DURATION_MS` are unset + +#### Scenario: An existing post-hook keeps working + +- **WHEN** a post-hook written against `CONVOY_RUN_STATUS`, `CONVOY_RUN_SCORE` and `CONVOY_GOAL_*` runs after this change +- **THEN** it sees those variables exactly as before, with the usage variables added alongside diff --git a/openspec/changes/archive/2026-09-11-hook-run-usage/tasks.md b/openspec/changes/archive/2026-09-11-hook-run-usage/tasks.md new file mode 100644 index 0000000..7bce2fd --- /dev/null +++ b/openspec/changes/archive/2026-09-11-hook-run-usage/tasks.md @@ -0,0 +1,22 @@ +## 1. Aggregate run usage + +- [x] 1.1 In `src/usage.ts`, add `RunUsage`, the structural `RunUsagePhase`, and `sumRunUsage(phases)` folding cost (when any phase has a numeric executor or advisor cost), tokens (when any phase has a numeric executor `cost`), advisor cost (when the sum is above zero) and duration (when any phase has one), returning `undefined` when no group has data. Verification: `test/usage.test.ts` covers mixed phases, phases without usage, a duration-only run, an advisor-bearing run, a zero-cost advisor, NaN-safety, and the `undefined` case. +- [x] 1.2 In `src/metadata.ts`, add `runUsage(): RunUsage | undefined` to `RunMetadataStore`, delegating to `sumRunUsage(Object.values(data.phases))`. Verification: `test/metadata.test.ts` shows `undefined` on a fresh store, totals after `phaseStepUsage`/`phaseUsageTotal`/`phaseAdvisorEvent`/`phaseEnded` across two phases plus a hook row, and a failed phase's duration included. + +## 2. Hook environment + +- [x] 2.1 In `src/hooks.ts`, add `usage?: RunUsage` to `RunHookContext` and a `usageEnv` helper spread into `env` after `CONVOY_GOAL_*`: `CONVOY_RUN_COST` (four fractional digits) with `CONVOY_RUN_TOKENS_INPUT/OUTPUT/REASONING/CACHE_READ/CACHE_WRITE/TOTAL` (integers), `CONVOY_RUN_ADVISOR_COST` (four fractional digits) and `CONVOY_RUN_DURATION_MS` (integer). Verification: `test/hooks.test.ts` asserts the exact strings for a full aggregate, the duration-only case, the no-advisor case, that an absent `usage` sets none of them, and that pre-hooks never receive them. + +## 3. Runner wiring + +- [x] 3.1 In `src/runner.ts`, pass `usage: metadata?.runUsage()` (spread conditionally) at both post-hook call sites — the success path and the failure path — leaving the pre-hook call untouched. Verification: a `test/runner-hosted.test.ts` run that fails before any usage sees `CONVOY_RUN_STATUS=failure`, `CONVOY_RUN_COST`/`CONVOY_RUN_TOKENS_TOTAL` unset and `CONVOY_RUN_DURATION_MS` set; the success path is exercised by the headless smoke in 5.3 (a `run()` test cannot fake the `claude` CLI: `Bun.spawn` resolves binaries against the process's original PATH). + +## 4. Documentation + +- [x] 4.1 In `README.md`, extend the "Hooks are trusted local shell commands" paragraph with the new variables, their formats and the sum-of-phases definition of the duration, and add a post-hook example next to the goal-mode one: `gh pr comment` with cost and duration, plus a one-line `when: always` budget guard using `awk`, noting that a failing post-hook fails the run. + +## 5. Verify + +- [x] 5.1 `bun run typecheck` and `bun test` pass; coverage stays above the `verify.yml` threshold. +- [x] 5.2 `openspec validate hook-run-usage --strict`. +- [x] 5.3 Headless smoke: a pipeline with a post-hook `env | grep '^CONVOY_RUN_' > "$CONVOY_RUN_DIR/hook-env.txt"` run with `--no-tui` produces cost, token and duration values consistent with the run's `metadata.json` and `convoy runs`. diff --git a/openspec/specs/hook-run-usage/spec.md b/openspec/specs/hook-run-usage/spec.md new file mode 100644 index 0000000..e73f5d9 --- /dev/null +++ b/openspec/specs/hook-run-usage/spec.md @@ -0,0 +1,81 @@ +# hook-run-usage Specification + +## Purpose +Give post-hooks the run's aggregated usage — what it cost, what it consumed and how long it worked — straight from the run's own metadata, so a project can report or gate on spend in a hook without parsing run files or re-deriving totals that Convoy already knows. + +## Requirements + +### Requirement: Post-hooks receive the run's aggregated usage + +When Convoy runs post-hooks, on success and on failure alike, it SHALL export the run's aggregated usage in the hook environment, summed over every phase recorded in the run's metadata (pipeline steps, goal-fragment invocations and hook rows). `CONVOY_RUN_COST` SHALL be the total cost in USD — executor cost plus advisor cost, the same figure `convoy runs` reports as `cost`. `CONVOY_RUN_TOKENS_INPUT`, `CONVOY_RUN_TOKENS_OUTPUT`, `CONVOY_RUN_TOKENS_REASONING`, `CONVOY_RUN_TOKENS_CACHE_READ`, `CONVOY_RUN_TOKENS_CACHE_WRITE` and `CONVOY_RUN_TOKENS_TOTAL` SHALL be the summed token counts, where `TOTAL` follows the same definition as the phase totals the dashboard shows. `CONVOY_RUN_DURATION_MS` SHALL be the sum of the recorded phases' durations — the time the pipeline spent working, not the wall clock since the run was created. The aggregate SHALL come from the metadata the runner holds in memory, never from re-reading run files. + +#### Scenario: A successful two-phase run + +- **WHEN** two phases recorded usage of `0.5 USD / 1000 input / 200 output` and `0.25 USD / 400 input / 100 output`, with durations of 60000 ms and 30000 ms, and the run succeeds +- **THEN** the success post-hooks see `CONVOY_RUN_COST=0.7500`, `CONVOY_RUN_TOKENS_INPUT=1400`, `CONVOY_RUN_TOKENS_OUTPUT=300` and `CONVOY_RUN_DURATION_MS=90000` + +#### Scenario: A run that fails in its second phase + +- **WHEN** the first phase recorded usage, the second phase fails after recording some usage, and the failure post-hooks run +- **THEN** they see the cost, tokens and duration accumulated up to and including the failed phase, and `CONVOY_RUN_STATUS=failure` as before + +#### Scenario: A goal cycle + +- **WHEN** a goal cycle ran two measure rounds and one improve round before settling +- **THEN** the post-hooks, which run once after the whole cycle, see totals that include every fragment invocation of every round + +### Requirement: Usage variables are omitted when nothing was recorded + +Convoy SHALL NOT invent a zero: `CONVOY_RUN_COST` SHALL be exported only when at least one recorded phase carries an executor or advisor cost, the token variables SHALL be exported together only when at least one recorded phase carries executor usage, and `CONVOY_RUN_DURATION_MS` SHALL be exported only when at least one recorded phase carries a duration. When none is available no usage variable SHALL be set, mirroring how `CONVOY_RUN_SCORE` is absent on an unscored pipeline. + +#### Scenario: A run that fails before any session reported usage + +- **WHEN** a pre-hook fails after running for a while, so the failure post-hooks run with phases that have durations but no cost +- **THEN** `CONVOY_RUN_DURATION_MS` is set and `CONVOY_RUN_COST` and every `CONVOY_RUN_TOKENS_*` variable are unset + +#### Scenario: A run whose only recorded spend is the advisor's + +- **WHEN** a phase recorded an advisor cost of `0.20 USD` but its executor never reported usage before the run failed +- **THEN** the failure post-hooks see `CONVOY_RUN_COST=0.2000` and `CONVOY_RUN_ADVISOR_COST=0.2000`, and every `CONVOY_RUN_TOKENS_*` variable is unset + +#### Scenario: A run that fails before any phase started + +- **WHEN** the run fails before any phase recorded a start +- **THEN** no `CONVOY_RUN_COST`, `CONVOY_RUN_TOKENS_*` or `CONVOY_RUN_DURATION_MS` variable is set + +### Requirement: Advisor spend is reported on its own + +When any recorded phase carries advisor usage with a cost above zero, Convoy SHALL export `CONVOY_RUN_ADVISOR_COST` with the summed advisor cost, so a hook can tell the executor's share from the advisor's. `CONVOY_RUN_COST` SHALL already include that amount. When no advisor spend was recorded the variable SHALL be absent. + +#### Scenario: A run with an advisor + +- **WHEN** one phase recorded an executor cost of `1.00 USD` and an advisor cost of `0.20 USD` +- **THEN** the post-hooks see `CONVOY_RUN_COST=1.2000` and `CONVOY_RUN_ADVISOR_COST=0.2000` + +#### Scenario: A run without an advisor + +- **WHEN** no phase recorded advisor usage +- **THEN** `CONVOY_RUN_ADVISOR_COST` is unset and `CONVOY_RUN_COST` is the executor cost alone + +### Requirement: Values use a fixed, documented format + +Cost variables SHALL be decimal USD with exactly four fractional digits and no currency sign; token and duration variables SHALL be non-negative integers with no separators. The README SHALL document each variable, its format, and that the duration is a sum of phase durations. + +#### Scenario: Formatting a fractional cost and a large token count + +- **WHEN** the aggregated cost is `3.2` USD and the summed input tokens are `1234567` +- **THEN** the hook sees `CONVOY_RUN_COST=3.2000` and `CONVOY_RUN_TOKENS_INPUT=1234567` + +### Requirement: Pre-hooks and existing hook variables are unchanged + +Pre-hooks SHALL NOT receive any usage variable — nothing has been recorded when they run — and every variable post-hooks received before this change SHALL keep its name, presence rule and value. + +#### Scenario: A pre-hook inspects its environment + +- **WHEN** a pre-hook runs +- **THEN** `CONVOY_RUN_COST`, `CONVOY_RUN_ADVISOR_COST`, every `CONVOY_RUN_TOKENS_*` variable and `CONVOY_RUN_DURATION_MS` are unset + +#### Scenario: An existing post-hook keeps working + +- **WHEN** a post-hook written against `CONVOY_RUN_STATUS`, `CONVOY_RUN_SCORE` and `CONVOY_GOAL_*` runs after this change +- **THEN** it sees those variables exactly as before, with the usage variables added alongside diff --git a/src/hooks.ts b/src/hooks.ts index 02a6ab2..c400020 100644 --- a/src/hooks.ts +++ b/src/hooks.ts @@ -5,6 +5,7 @@ import { log } from "./log" import type { ProgressUI } from "./progress" import type { HookSet, HookSpec, HookWhen, HooksConfig } from "./types" +import type { RunUsage } from "./usage" import type { Workspace } from "./workspace" export type HookStage = "pre" | "post" @@ -22,6 +23,8 @@ export type RunHookContext = { score?: number /** Outcome of the goal loop, when these post-hooks run after one. */ goal?: GoalHookOutcome + /** Aggregated phase facts available only to post-hooks. */ + usage?: RunUsage } /** @@ -148,7 +151,20 @@ async function runHookCommand(stage: HookStage, hook: HookSpec, context: RunHook // The loopback bridge's bearer credential is for the OpenCode custom tools // only. Hooks are arbitrary project commands and do not need either value; // do not let them inherit a live capability to call the bridge. - const { [advisorUrlEnv]: _advisorUrl, [advisorTokenEnv]: _advisorToken, ...parentEnv } = process.env + const { + [advisorUrlEnv]: _advisorUrl, + [advisorTokenEnv]: _advisorToken, + CONVOY_RUN_COST: _runCost, + CONVOY_RUN_ADVISOR_COST: _runAdvisorCost, + CONVOY_RUN_TOKENS_INPUT: _runTokensInput, + CONVOY_RUN_TOKENS_OUTPUT: _runTokensOutput, + CONVOY_RUN_TOKENS_REASONING: _runTokensReasoning, + CONVOY_RUN_TOKENS_CACHE_READ: _runTokensCacheRead, + CONVOY_RUN_TOKENS_CACHE_WRITE: _runTokensCacheWrite, + CONVOY_RUN_TOKENS_TOTAL: _runTokensTotal, + CONVOY_RUN_DURATION_MS: _runDuration, + ...parentEnv + } = process.env const env = { ...parentEnv, CONVOY_HOOK_STAGE: stage, @@ -167,6 +183,7 @@ async function runHookCommand(stage: HookStage, hook: HookSpec, context: RunHook ...(context.goal.score !== undefined ? { CONVOY_GOAL_SCORE: String(context.goal.score) } : {}), } : {}), + ...(stage === "post" ? usageEnv(context.usage) : {}), } const proc = Bun.spawn([shell, "-lc", hook.command], { @@ -219,6 +236,26 @@ async function runHookCommand(stage: HookStage, hook: HookSpec, context: RunHook } } +/** Converts in-memory usage into the stable text contract exposed to post-hooks. */ +function usageEnv(usage: RunUsage | undefined): Record { + if (!usage) return {} + return { + ...(usage.cost !== undefined ? { CONVOY_RUN_COST: usage.cost.toFixed(4) } : {}), + ...(usage.advisorCost !== undefined ? { CONVOY_RUN_ADVISOR_COST: usage.advisorCost.toFixed(4) } : {}), + ...(usage.tokens + ? { + CONVOY_RUN_TOKENS_INPUT: String(usage.tokens.input), + CONVOY_RUN_TOKENS_OUTPUT: String(usage.tokens.output), + CONVOY_RUN_TOKENS_REASONING: String(usage.tokens.reasoning), + CONVOY_RUN_TOKENS_CACHE_READ: String(usage.tokens.cacheRead), + CONVOY_RUN_TOKENS_CACHE_WRITE: String(usage.tokens.cacheWrite), + CONVOY_RUN_TOKENS_TOTAL: String(usage.tokens.total), + } + : {}), + ...(usage.durationMs !== undefined ? { CONVOY_RUN_DURATION_MS: String(Math.round(usage.durationMs)) } : {}), + } +} + async function readOutput(promise: Promise, timeoutMs: number | undefined): Promise { if (timeoutMs === undefined) return promise return Promise.race([ diff --git a/src/metadata.ts b/src/metadata.ts index a6d7a5e..9627d0a 100644 --- a/src/metadata.ts +++ b/src/metadata.ts @@ -16,7 +16,7 @@ import type { import type { QualityScore } from "./quality-score" import type { FeaturePlanLink, Pipeline } from "./types" import type { ModelGateway } from "./model-routing" -import { PhaseUsage } from "./usage" +import { PhaseUsage, sumRunUsage, type RunUsage } from "./usage" import { aggregateAdvisorEvents, type AdvisorEvent, type AdvisorPhaseAggregate } from "./advisor-events" import { readCommitLedger, readFinalizationRecord, readRunBoundary, type CommitLedgerEntry, type FinalizationRecord, type RunBoundary } from "./finalization/types" import { resolveRunTitleFor } from "./run-title" @@ -117,6 +117,8 @@ export type RunMetadataStore = { pipeline: Pipeline snapshot(name: string): ProgressPhaseSnapshot | undefined phaseStatus(name: string): PhaseMetadataStatus | undefined + /** Current aggregate of phase facts recorded during this run. */ + runUsage(): RunUsage | undefined /** The durable goal-cycle record, when the run has reached a goal checkpoint. */ goalState(): GoalRunState | undefined /** Persists a goal checkpoint after a stage boundary, score promotion, or settlement. */ @@ -316,6 +318,9 @@ export async function openRunMetadata( phaseStatus(name) { return data.phases[name]?.status }, + runUsage() { + return sumRunUsage(Object.values(data.phases)) + }, goalState() { return data.goal }, diff --git a/src/runner.ts b/src/runner.ts index f47d0ac..036986a 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -873,6 +873,7 @@ export async function run(options: RunOptions, deps: RunDeps = defaultRunDeps) { // along: CONVOY_GOAL_REACHED distinguishes "cleared the bar" from "gave up // short of it" in a way `when: success` alone cannot). postHooksStarted = true + const usage = metadata.runUsage() await runHooks("post", hookSet.post, { workspace, targetDir: options.targetDir, @@ -891,6 +892,7 @@ export async function run(options: RunOptions, deps: RunDeps = defaultRunDeps) { }, } : {}), + ...(usage ? { usage } : {}), }) // Automatic run finalization (capability run-finalization): the terminal @@ -995,6 +997,7 @@ export async function run(options: RunOptions, deps: RunDeps = defaultRunDeps) { // gates on CONVOY_GOAL_REACHED correctly stays inert. if (!postHooksStarted && !isUserAbortError(failure)) { postHooksStarted = true + const usage = metadata?.runUsage() try { await runHooks("post", hookSet.post, { workspace, @@ -1004,6 +1007,7 @@ export async function run(options: RunOptions, deps: RunDeps = defaultRunDeps) { status: "failure", progress, signal: shutdown.signal, + ...(usage ? { usage } : {}), }) } catch (hookError) { failure = new Error(`${formatSdkError(error)}; post-hook failed: ${formatSdkError(hookError)}`) diff --git a/src/usage.ts b/src/usage.ts index f900726..d85551f 100644 --- a/src/usage.ts +++ b/src/usage.ts @@ -1,5 +1,25 @@ import type { ProgressStepUsage, ProgressTokens, ProgressUsage } from "./progress" +/** Aggregated usage available to post-run hooks when the corresponding facts exist. */ +export type RunUsage = { + /** Executor plus advisor cost in USD, present when any phase recorded either. */ + cost?: number + /** Advisor cost in USD, present only when advisor spend was positive. */ + advisorCost?: number + /** Summed executor tokens, present when any phase recorded executor usage. */ + tokens?: ProgressTokens + /** Sum of recorded phase durations. */ + durationMs?: number +} + +/** The persisted phase facts required to calculate a run-level usage aggregate. */ +export type RunUsagePhase = { + cost?: number + tokens?: ProgressTokens + durationMs?: number + advisor?: { cost: number } +} + /** A zeroed token tally; the canonical empty value for every accumulator. */ export function emptyTokens(): ProgressTokens { return { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0, total: 0 } @@ -25,6 +45,47 @@ export function safeCost(cost: number | undefined): number { return typeof cost === "number" && Number.isFinite(cost) ? cost : 0 } +/** + * Sums recorded phase facts without fabricating zero-valued usage groups. + * + * Tokens follow executor usage because metadata records them with the + * executor cost. The total cost also counts advisor spend, so a phase whose + * advisor was priced still yields a cost, matching the run-history total. + */ +export function sumRunUsage(phases: Iterable): RunUsage | undefined { + let executorCost = 0 + let advisorCost = 0 + let tokens = emptyTokens() + let durationMs = 0 + let hasExecutorUsage = false + let hasAdvisorCost = false + let hasDuration = false + + for (const phase of phases) { + if (typeof phase.cost === "number" && Number.isFinite(phase.cost)) { + hasExecutorUsage = true + executorCost += safeCost(phase.cost) + if (phase.tokens) tokens = addTokens(tokens, phase.tokens) + } + if (typeof phase.advisor?.cost === "number" && Number.isFinite(phase.advisor.cost)) { + hasAdvisorCost = true + advisorCost += safeCost(phase.advisor.cost) + } + if (typeof phase.durationMs === "number" && Number.isFinite(phase.durationMs)) { + hasDuration = true + durationMs += phase.durationMs + } + } + + const usage: RunUsage = { + ...(hasExecutorUsage || hasAdvisorCost ? { cost: executorCost + advisorCost } : {}), + ...(advisorCost > 0 ? { advisorCost } : {}), + ...(hasExecutorUsage ? { tokens } : {}), + ...(hasDuration ? { durationMs } : {}), + } + return Object.keys(usage).length > 0 ? usage : undefined +} + /** Normalizes opencode's `{ input, output, reasoning, cache: { read, write } }` token shape into ProgressTokens. */ export function tokensFromValue(value: unknown): ProgressTokens | undefined { if (!value || typeof value !== "object") return undefined diff --git a/test/hooks.test.ts b/test/hooks.test.ts index 505175d..f154bd0 100644 --- a/test/hooks.test.ts +++ b/test/hooks.test.ts @@ -11,6 +11,17 @@ import type { HooksConfig } from "../src/types" import type { Workspace } from "../src/workspace" const dirs: string[] = [] +const runUsageEnvNames = [ + "CONVOY_RUN_COST", + "CONVOY_RUN_ADVISOR_COST", + "CONVOY_RUN_TOKENS_INPUT", + "CONVOY_RUN_TOKENS_OUTPUT", + "CONVOY_RUN_TOKENS_REASONING", + "CONVOY_RUN_TOKENS_CACHE_READ", + "CONVOY_RUN_TOKENS_CACHE_WRITE", + "CONVOY_RUN_TOKENS_TOTAL", + "CONVOY_RUN_DURATION_MS", +] as const afterAll(async () => { await Promise.all(dirs.map((dir) => rm(dir, { recursive: true, force: true }))) @@ -121,6 +132,57 @@ describe("hooks", () => { expect(await readFile(join(context.targetDir, "nogoal.out"), "utf8")).toBe("unset") }) + test("post hooks receive formatted run usage while pre-hooks do not", async () => { + const context = await hookContext() + const usage = { + cost: 3.2, + advisorCost: 0.125, + tokens: { input: 1_234_567, output: 20, reasoning: 3, cacheRead: 4, cacheWrite: 5, total: 1_234_599 }, + durationMs: 90_000.4, + } + const command = 'printf "%s:%s:%s:%s:%s:%s:%s:%s:%s" "$CONVOY_RUN_COST" "$CONVOY_RUN_ADVISOR_COST" "$CONVOY_RUN_TOKENS_INPUT" "$CONVOY_RUN_TOKENS_OUTPUT" "$CONVOY_RUN_TOKENS_REASONING" "$CONVOY_RUN_TOKENS_CACHE_READ" "$CONVOY_RUN_TOKENS_CACHE_WRITE" "$CONVOY_RUN_TOKENS_TOTAL" "$CONVOY_RUN_DURATION_MS" > usage.out' + + await runHooks("post", [{ command, when: "always" }], { ...context, status: "success", usage }) + expect(await readFile(join(context.targetDir, "usage.out"), "utf8")).toBe("3.2000:0.1250:1234567:20:3:4:5:1234599:90000") + + await runHooks("pre", [{ command: 'printf "%s" "${CONVOY_RUN_COST-unset}" > pre-usage.out' }], { ...context, usage }) + expect(await readFile(join(context.targetDir, "pre-usage.out"), "utf8")).toBe("unset") + }) + + test("post hooks omit unavailable usage groups", async () => { + const context = await hookContext() + const command = 'printf "%s:%s:%s" "${CONVOY_RUN_COST-unset}" "${CONVOY_RUN_ADVISOR_COST-unset}" "${CONVOY_RUN_DURATION_MS-unset}" > partial-usage.out' + + await runHooks("post", [{ command, when: "always" }], { ...context, status: "failure", usage: { durationMs: 25 } }) + expect(await readFile(join(context.targetDir, "partial-usage.out"), "utf8")).toBe("unset:unset:25") + + // Advisor-only spend reaches the total without inventing executor tokens. + const advisorOnly = 'printf "%s:%s:%s" "$CONVOY_RUN_COST" "$CONVOY_RUN_ADVISOR_COST" "${CONVOY_RUN_TOKENS_TOTAL-unset}" > advisor-usage.out' + await runHooks("post", [{ command: advisorOnly, when: "always" }], { ...context, status: "failure", usage: { cost: 0.2, advisorCost: 0.2 } }) + expect(await readFile(join(context.targetDir, "advisor-usage.out"), "utf8")).toBe("0.2000:0.2000:unset") + }) + + test("hooks do not inherit run usage from the parent environment", async () => { + const context = await hookContext() + const previous = runUsageEnvNames.map((name) => [name, process.env[name]] as const) + const expansions = runUsageEnvNames.map((name) => `\${${name}-unset}`).join(":") + const expected = runUsageEnvNames.map(() => "unset").join(":") + for (const name of runUsageEnvNames) process.env[name] = "stale" + + try { + await runHooks("pre", [{ command: `printf "%s" "${expansions}" > inherited-pre-usage.out` }], { ...context, usage: { durationMs: 25 } }) + await runHooks("post", [{ command: `printf "%s" "${expansions}" > inherited-post-usage.out`, when: "always" }], { ...context, status: "success" }) + + expect(await readFile(join(context.targetDir, "inherited-pre-usage.out"), "utf8")).toBe(expected) + expect(await readFile(join(context.targetDir, "inherited-post-usage.out"), "utf8")).toBe(expected) + } finally { + for (const [name, value] of previous) { + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + } + }) + test("fails on a non-zero hook unless continueOnError is true", async () => { const context = await hookContext() diff --git a/test/metadata.test.ts b/test/metadata.test.ts index 58f923b..5944df5 100644 --- a/test/metadata.test.ts +++ b/test/metadata.test.ts @@ -306,6 +306,7 @@ describe("openRunMetadata", () => { try { expect(typeof store.snapshot).toBe("function") expect(typeof store.phaseStatus).toBe("function") + expect(typeof store.runUsage).toBe("function") expect(typeof store.serverStarted).toBe("function") expect(typeof store.serverStopped).toBe("function") expect(typeof store.phaseStarted).toBe("function") @@ -541,6 +542,45 @@ describe("openRunMetadata", () => { } }) + test("runUsage aggregates recorded executor, advisor, and duration facts", async () => { + const { ws, cleanup } = await withDir("run-usage") + const store = await openRunMetadata(ws, "/target", validPipeline([validAgentStep("design"), validAgentStep("test")])) + try { + expect(store.runUsage()).toBeUndefined() + + store.phaseStepUsage("design", { + cost: 0.5, + tokens: { input: 1_000, output: 200, reasoning: 0, cacheRead: 0, cacheWrite: 0, total: 1_200 }, + }) + store.phaseUsageTotal("test", { + cost: 0.25, + tokens: { input: 400, output: 100, reasoning: 0, cacheRead: 0, cacheWrite: 0, total: 500 }, + }) + store.phaseAdvisorEvent("design", { + ...advisorRequestedEvent("usage-advisor"), + type: "advisor.completed", + model: "advisor-model", + latencyMs: 10, + usage: { cost: 0.2, tokens: { input: 1, output: 2, reasoning: 0, cacheRead: 0, cacheWrite: 0 }, model: "advisor-model" }, + adviceChars: 0, + }) + await store.phaseStarted("design") + await store.phaseEnded("design", "completed") + await store.phaseStarted("test") + await store.phaseEnded("test", "failed") + await store.phaseStarted("post-hook: notify") + await store.phaseEnded("post-hook: notify", "completed") + + const usage = store.runUsage() + expect(usage?.cost).toBeCloseTo(0.95) + expect(usage?.advisorCost).toBeCloseTo(0.2) + expect(usage?.tokens).toEqual({ input: 1_400, output: 300, reasoning: 0, cacheRead: 0, cacheWrite: 0, total: 1_700 }) + expect(usage?.durationMs).toBeGreaterThanOrEqual(0) + } finally { + await cleanup() + } + }) + test("default modelRouting gateway is configured", async () => { const { dir, ws, cleanup } = await withDir("gwdef") const store = await openRunMetadata(ws, "/target", validPipeline([validAgentStep("design")])) @@ -890,6 +930,7 @@ describe("recordProgress", () => { pipeline: validPipeline([]), snapshot: () => undefined, phaseStatus: () => undefined, + runUsage: () => undefined, goalState: () => undefined, checkpointGoal: () => Promise.resolve(), boundary: () => undefined, diff --git a/test/runner-hosted.test.ts b/test/runner-hosted.test.ts index 7c915c9..3c8038b 100644 --- a/test/runner-hosted.test.ts +++ b/test/runner-hosted.test.ts @@ -343,6 +343,30 @@ describe("run() with a hosted progress", () => { } }) + test("failure post-hooks omit cost and tokens when no phase recorded usage", async () => { + const repo = await cleanRepo() + try { + let failure: unknown + try { + await run(makeOptions(repo, { + hooks: { + pre: [{ name: "fail-before-session", command: "exit 1" }], + post: [{ name: "record-usage", when: "always", command: 'printf "%s:%s:%s:%s" "$CONVOY_RUN_STATUS" "${CONVOY_RUN_COST-unset}" "${CONVOY_RUN_TOKENS_TOTAL-unset}" "$CONVOY_RUN_DURATION_MS" > hook-usage.out' }], + pipelines: {}, + }, + })) + } catch (error) { + failure = error + } + expect(String(failure)).toContain("exited with code 1") + const values = (await readFile(join(repo, "hook-usage.out"), "utf8")).split(":") + expect(values.slice(0, 3)).toEqual(["failure", "unset", "unset"]) + expect(values[3]).toMatch(/^\d+$/) + } finally { + await rm(repo, { recursive: true, force: true }) + } + }) + test("a hosted askPermission is honoured without a TTY — the run never auto-rejects it", async () => { const repo = await cleanRepo() // A coordinated run has no TTY on the coordinator: "interactive" there diff --git a/test/usage.test.ts b/test/usage.test.ts index 9c7bb0a..0d61388 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test" import type { ProgressTokens } from "../src/progress" -import { PhaseUsage, addTokens, cloneTokens, emptyTokens, safeCost, tokensFromValue } from "../src/usage" +import { PhaseUsage, addTokens, cloneTokens, emptyTokens, safeCost, sumRunUsage, tokensFromValue } from "../src/usage" function tk(input: number, output: number): ProgressTokens { return { input, output, reasoning: 0, cacheRead: 0, cacheWrite: 0, total: input + output } @@ -99,3 +99,26 @@ describe("PhaseUsage", () => { expect(usage.totals().cost).toBe(5) }) }) + +describe("sumRunUsage", () => { + test("sums executor and advisor usage, tokens, and recorded durations", () => { + expect(sumRunUsage([ + { cost: 0.5, tokens: tk(1_000, 200), durationMs: 60_000, advisor: { cost: 0.2 } }, + { cost: 0.25, tokens: tk(400, 100), durationMs: 30_000 }, + { durationMs: 500 }, + ])).toEqual({ + cost: 0.95, + advisorCost: 0.2, + tokens: { input: 1_400, output: 300, reasoning: 0, cacheRead: 0, cacheWrite: 0, total: 1_700 }, + durationMs: 90_500, + }) + }) + + test("keeps independently recorded usage groups and ignores invalid numbers", () => { + // Advisor-only spend is still spend: the total counts it, tokens stay absent. + expect(sumRunUsage([{ advisor: { cost: 0.2 } }])).toEqual({ cost: 0.2, advisorCost: 0.2 }) + expect(sumRunUsage([{ durationMs: 10 }, { advisor: { cost: 0 } }])).toEqual({ cost: 0, durationMs: 10 }) + expect(sumRunUsage([{ cost: Number.NaN, tokens: tk(1, 1), durationMs: Number.POSITIVE_INFINITY, advisor: { cost: Number.NaN } }])).toBeUndefined() + expect(sumRunUsage([])).toBeUndefined() + }) +})