diff --git a/docs/superpowers/plans/2026-07-27-spec-6-1.md b/docs/superpowers/plans/2026-07-27-spec-6-1.md new file mode 100644 index 0000000..849de90 --- /dev/null +++ b/docs/superpowers/plans/2026-07-27-spec-6-1.md @@ -0,0 +1,1079 @@ +# SPEC-6-1 Implementation Plan — Cost-aware model tiers + cost accounting + context% + Tiers view + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a cost-aware model-tier routing layer + live $/run cost accounting + context% in the widget/Runs tab + a `/fleet` Tiers view, releasing `@getpipher/armory-fleet@0.10.0`. + +**Architecture:** A `TierRegistry` (builtins < global `tiers.json` < project `tiers.json`) feeds a pure `resolveAgentModel()` that walks a tier's `models[]` fallback chain at spawn (filtering by `contextFloor` + catalog availability). `spawnSubagent` accumulates `usage.cost.total` → `RunRecord.costTotal` + `calcContextTokens(usage)` → `RunRecord.contextTokens` per `message_end`, aborts when `costTotal > tier.costCap`. The above-editor widget + Runs tab gain `ctx%` + `$` segments; a new `/fleet` Tiers tab lists tiers + per-tier spend with inline edit actions writing `tiers.json`. + +**Tech Stack:** TypeScript (raw `.ts` via tsx, no build step), node:test + tsx for tests, pi extension API (`pi.modelRegistry`, `ctx.ui.setWidget`, `ctx.ui.custom`), typebox for tool params. + +## Global Constraints + +- Raw `.ts` via tsx at runtime — **no build step**. `pnpm typecheck` + `pnpm test:run` (`--test-timeout=30000`) green before release. +- No AI attribution in commits. Commit style: `feat(spec-6-1): …` / `fix(spec-6-1): …` / `test(spec-6-1): …` / `docs(spec-6-1): …`. +- Follow existing pure-renderer + port conventions (unit-test pure fns; panel class is term-smoke-gated, not unit-tested). +- `RunLog` widening is additive (`message.usage` gains `cost?`); audit/update existing run-log tests that assert the `usage` shape. +- Release via CI on `v*` tag (push tag → typecheck + test + npm publish + GitHub Release). `git tag` opens Vim — use `git update-ref refs/tags/vX.Y.Z ` + `git push --force origin vX.Y.Z`. +- Term smoke runs on the **published** version (pi auto-reinstalls npm packages on launch — branch code can't be smoked directly). +- Model strings are `"provider/modelId"` (e.g. `"Ollama/glm-5.2:cloud"`); `splitModel` splits on the FIRST `/`. + +--- + +## File Structure + +**New files:** +- `src/tiers/tier-registry.ts` — `Tier` type, `parseTiersFile`, `mergeTiers`, `TierRegistry` class (discovery: builtins < global < project). +- `src/tiers/builtin.ts` — `BUILTIN_TIERS` (economy/standard/frontier). +- `src/tiers/resolve.ts` — `splitModel`, `resolveAgentModel`, `ModelRegistryLike` port. +- `src/tiers/tier-store.ts` — `read`/`write` `tiers.json` (project + global), atomic + validated. +- `src/panel/tiers-rows.ts` — pure render fns for the Tiers view. +- `test/tier-registry.test.mts`, `test/resolve-model.test.mts`, `test/tier-store.test.mts`, `test/tiers-rows.test.mts`, `test/spawn-subagent-tier.test.mts`, `test/fleet-panel-tiers.test.mts`. + +**Modified files:** +- `src/registry/frontmatter.ts` — `AgentDef.tier?: string`. +- `src/engine/run-registry.ts` — `RunRecord.costTotal?`/`contextTokens?`/`tier?`. +- `src/engine/spawnSubagent.ts` — `resolveAgentModel` + fallback retry + cost/context accumulation + cap abort + `calcContextTokens` helper. +- `src/runtime/run-log.ts` — `message.usage` gains `cost?`. +- `src/panel/widget-rows.ts` — `widgetLine` adds `ctx%` + `$`; `WidgetRun` gains fields. +- `src/panel/fleet-widget.ts` — `getModelContextWindow` dep + thread `maxContext` into `WidgetRun`. +- `src/panel/runs-rows.ts` — `ctx%` + `$` columns. +- `src/panel/fleet-panel.ts` — new `tiers` View + action submenu. +- `src/index.ts` — construct `TierRegistry`, pass `pi.modelRegistry` into deps, wire tiers view + widget `getModelContextWindow`. + +--- + +### Task 1: Tier type + builtin tiers + registry + discovery + +**Files:** +- Create: `src/tiers/tier-registry.ts`, `src/tiers/builtin.ts` +- Test: `test/tier-registry.test.mts` + +**Interfaces:** +- Produces: `Tier` (`{ name, models: string[], costCap?, contextFloor? }`), `TierRegistry` (class: `get(name)`, `list()`, `usedBy(name)`), `parseTiersFile(json): Tier[]`, `mergeTiers(builtin, global, project): Tier[]`, `BUILTIN_TIERS: Tier[]`. + +- [ ] **Step 1: Write the failing test** + +```ts +// test/tier-registry.test.mts +import { test } from "node:test"; +import { strictEqual, deepStrictEqual, throws } from "node:assert"; +import { TierRegistry, parseTiersFile, mergeTiers, type Tier } from "../src/tiers/tier-registry.ts"; +import { BUILTIN_TIERS } from "../src/tiers/builtin.ts"; + +const good = (n: string, models: string[] = [`${n}-model`]): Tier => ({ name: n, models }); + +test("parseTiersFile: valid array of tiers", () => { + deepStrictEqual(parseTiersFile(`[{ "name": "x", "models": ["m1"] }]`), [{ name: "x", models: ["m1"] }]); +}); + +test("parseTiersFile: empty input → empty array", () => { + deepStrictEqual(parseTiersFile(""), []); + deepStrictEqual(parseTiersFile("[]"), []); +}); + +test("parseTiersFile: malformed JSON → throws", () => { + throws(() => parseTiersFile("{ not json"), /malformed tiers file/); +}); + +test("parseTiersFile: duplicate name → throws", () => { + throws(() => parseTiersFile(`[{ "name": "x", "models": ["m"] }, { "name": "x", "models": ["m2"] }]`), /duplicate tier name/); +}); + +test("parseTiersFile: missing name/models → throws", () => { + throws(() => parseTiersFile(`[{ "models": ["m"] }]`), /tier missing name/); + throws(() => parseTiersFile(`[{ "name": "x" }]`), /tier missing models/); + throws(() => parseTiersFile(`[{ "name": "x", "models": [] }]`), /tier 'x' has empty models/); +}); + +test("mergeTiers: builtins < global < project (project wins by name)", () => { + const merged = mergeTiers( + [good("economy"), good("standard")], + [good("standard", ["global-std"]), good("custom")], + [good("standard", ["proj-std"]), good("local")], + ); + const byName = new Map(merged.map((t) => [t.name, t])); + strictEqual(byName.get("economy")!.models[0], "economy-model", "builtin economy kept"); + strictEqual(byName.get("standard")!.models[0], "proj-std", "project wins over global + builtin"); + strictEqual(byName.get("custom")!.models[0], "custom-model", "global-only tier kept"); + strictEqual(byName.get("local")!.models[0], "local-model", "project-only tier kept"); +}); + +test("TierRegistry.get/list/usedBy", () => { + const reg = new TierRegistry({ + tiers: mergeTiers(BUILTIN_TIERS, [], []), + agents: new Map([["coder", { tier: "standard" } as any], ["oracle", { tier: "frontier" } as any]]), + }); + strictEqual(reg.get("economy")!.models[0], "Ollama/minimax-m3:cloud"); + strictEqual(reg.get("frontier")!.costCap, 5); + strictEqual(reg.get("frontier")!.contextFloor, 200000); + deepStrictEqual(reg.list().map((t) => t.name), ["economy", "standard", "frontier"]); + deepStrictEqual(reg.usedBy("standard"), ["coder"]); + deepStrictEqual(reg.usedBy("frontier"), ["oracle"]); + deepStrictEqual(reg.usedBy("economy"), []); + strictEqual(reg.get("nope"), undefined, "missing tier → undefined"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/tier-registry.test.mts` +Expected: FAIL — `Cannot find module '../src/tiers/tier-registry.ts'`. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// src/tiers/tier-registry.ts +export interface Tier { + name: string; + models: string[]; // ordered fallback chain, primary first + costCap?: number; // $ per-run; abort when run.costTotal exceeds + contextFloor?: number; // min contextWindow; skip models below it at spawn +} + +export class TierFileError extends Error { + override name = "TierFileError" as const; +} + +/** Parse a raw JSON string into validated Tier[]. Empty/blank → []. */ +export function parseTiersFile(raw: string): Tier[] { + const trimmed = raw.trim(); + if (!trimmed) return []; + let parsed: unknown; + try { parsed = JSON.parse(trimmed); } catch { throw new TierFileError("malformed tiers file (invalid JSON)"); } + if (!Array.isArray(parsed)) throw new TierFileError("tiers file must be a JSON array of tier objects"); + const seen = new Set(); + return parsed.map((t) => { + const obj = t as Record; + const name = obj.name; + const models = obj.models; + if (typeof name !== "string" || !name.trim()) throw new TierFileError("tier missing name"); + if (!Array.isArray(models) || models.length === 0) throw new TierFileError(`tier '${name}' has empty or missing models`); + if (seen.has(name)) throw new TierFileError(`duplicate tier name '${name}'`); + seen.add(name); + return { + name, models: models.map(String), + ...(typeof obj.costCap === "number" ? { costCap: obj.costCap } : {}), + ...(typeof obj.contextFloor === "number" ? { contextFloor: obj.contextFloor } : {}), + }; + }); +} + +/** Merge tiers by name: builtins < global < project (later scopes win by name). */ +export function mergeTiers(builtin: Tier[], globalTiers: Tier[], project: Tier[]): Tier[] { + const map = new Map(); + for (const t of builtin) map.set(t.name, t); + for (const t of globalTiers) map.set(t.name, t); + for (const t of project) map.set(t.name, t); + return [...map.values()]; +} + +export interface TierRegistryOpts { + tiers: Tier[]; + /** Agent defs keyed by agent name — only the `tier` field is read, for `usedBy`. */ + agents: Map; +} + +export class TierRegistry { + private readonly byName = new Map(); + private readonly agents: Map; + constructor(opts: TierRegistryOpts) { + for (const t of opts.tiers) this.byName.set(t.name, t); + this.agents = opts.agents; + } + get(name: string): Tier | undefined { return this.byName.get(name); } + list(): Tier[] { return [...this.byName.values()]; } + usedBy(name: string): string[] { + const out: string[] = []; + for (const [agentName, def] of this.agents) if (def.tier === name) out.push(agentName); + return out.sort(); + } +} +``` + +```ts +// src/tiers/builtin.ts +import type { Tier } from "./tier-registry.ts"; + +/** Shipped default tiers (Q10). Overridable via global/project tiers.json. */ +export const BUILTIN_TIERS: Tier[] = [ + { name: "economy", models: ["Ollama/minimax-m3:cloud"] }, + { name: "standard", models: ["Ollama/glm-5.2:cloud", "Ollama/minimax-m3:cloud"] }, + { name: "frontier", models: ["anthropic/claude-sonnet-4", "Ollama/glm-5.2:cloud"], costCap: 5, contextFloor: 200000 }, +]; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/tier-registry.test.mts` +Expected: PASS (8 tests). + +- [ ] **Step 5: Commit** + +```bash +git add src/tiers/tier-registry.ts src/tiers/builtin.ts test/tier-registry.test.mts +git commit -m "feat(spec-6-1): Tier type + builtin tiers + registry + discovery fns" +``` + +--- + +### Task 2: splitModel + resolveAgentModel (pure, with ModelRegistry port) + +**Files:** +- Create: `src/tiers/resolve.ts` +- Test: `test/resolve-model.test.mts` + +**Interfaces:** +- Consumes: `Tier`, `TierRegistry` (from Task 1), `AgentDef` (from `src/registry/frontmatter.ts`). +- Produces: `splitModel(model: string, parentProvider?: string): { provider: string; id: string }`, `ModelRegistryLike` (`{ find(provider, id): { contextWindow: number } | undefined }`), `resolveAgentModel(...)`. + +- [ ] **Step 1: Write the failing test** + +```ts +// test/resolve-model.test.mts +import { test } from "node:test"; +import { strictEqual, deepStrictEqual } from "node:assert"; +import { splitModel, resolveAgentModel, type ModelRegistryLike } from "../src/tiers/resolve.ts"; +import { TierRegistry } from "../src/tiers/tier-registry.ts"; +import type { AgentDef } from "../src/registry/frontmatter.ts"; + +const agent = (over: Partial = {}): AgentDef => ({ + name: "g", description: "d", rolePrompt: "r", todoSync: true, memoryHydrate: true, vision: true, + backend: "pi", sessionKey: "g", source: "builtin", filePath: "/x", ...over, +}); +const PARENT = { provider: "p", id: "m" }; +const fakeReg = (windows: Record): ModelRegistryLike => ({ + find: (provider, id) => (windows[`${provider}/${id}`] != null ? { contextWindow: windows[`${provider}/${id}`] } : undefined), +}); +const tiers = (list: any[]) => new TierRegistry({ tiers: list, agents: new Map() }); + +test("splitModel: first-slash split; colon in id preserved", () => { + deepStrictEqual(splitModel("Ollama/glm-5.2:cloud"), { provider: "Ollama", id: "glm-5.2:cloud" }); + deepStrictEqual(splitModel("anthropic/claude-sonnet-4"), { provider: "anthropic", id: "claude-sonnet-4" }); + deepStrictEqual(splitModel("bare-id", "Ollama"), { provider: "Ollama", id: "bare-id" }); +}); + +test("resolveAgentModel: opts.model beats tier beats agent.model beats parent", () => { + const reg = tiers([{ name: "t", models: ["Ollama/tm"] }]); + const mr = fakeReg({ "Ollama/tm": 128000 }); + strictEqual(resolveAgentModel(agent({ tier: "t", model: "agent-model" }), "caller-model", PARENT, reg, mr).model, "caller-model"); + strictEqual(resolveAgentModel(agent({ tier: "t", model: "agent-model" }), undefined, PARENT, reg, mr).model, "Ollama/tm", "tier beats agent.model"); + strictEqual(resolveAgentModel(agent({ model: "agent-model" }), undefined, PARENT, reg, mr).model, "agent-model", "no tier → agent.model"); + strictEqual(resolveAgentModel(agent({}), undefined, PARENT, reg, mr).model, "p/m", "no tier, no model → parent"); +}); + +test("resolveAgentModel: tier not found → error", () => { + const res = resolveAgentModel(agent({ tier: "nope" }), undefined, PARENT, tiers([]), fakeReg({})); + strictEqual((res as any).error, "tier 'nope' not found; available: "); +}); + +test("resolveAgentModel: contextFloor skips a too-small model, lands on next", () => { + const reg = tiers([{ name: "big", models: ["Ollama/small", "Ollama/big"], contextFloor: 200000 }]); + const mr = fakeReg({ "Ollama/small": 32000, "Ollama/big": 200000 }); + const res = resolveAgentModel(agent({ tier: "big" }), undefined, PARENT, reg, mr); + strictEqual(res.model, "Ollama/big", "small skipped (below floor), big chosen"); + deepStrictEqual((res as any).candidates, ["Ollama/big"]); +}); + +test("resolveAgentModel: undefined contextWindow → treated as 0 → below any floor → skipped", () => { + const reg = tiers([{ name: "big", models: ["Ollama/unknown", "Ollama/big"], contextFloor: 100000 }]); + const mr = fakeReg({ "Ollama/big": 200000 }); // unknown not in catalog at all → find returns undefined → skip + const res = resolveAgentModel(agent({ tier: "big" }), undefined, PARENT, reg, mr); + strictEqual(res.model, "Ollama/big"); +}); + +test("resolveAgentModel: all models below contextFloor → error", () => { + const reg = tiers([{ name: "big", models: ["Ollama/a", "Ollama/b"], contextFloor: 500000 }]); + const mr = fakeReg({ "Ollama/a": 128000, "Ollama/b": 200000 }); + const res = resolveAgentModel(agent({ tier: "big" }), undefined, PARENT, reg, mr); + ok((res as any).error.includes("no eligible model"), (res as any).error); + ok((res as any).error.includes("500000"), "error names the floor"); +}); +import { ok } from "node:assert"; +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/resolve-model.test.mts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// src/tiers/resolve.ts +import type { AgentDef } from "../registry/frontmatter.ts"; +import type { Tier, TierRegistry } from "./tier-registry.ts"; + +/** Narrow port over pi's ModelRegistry — only the lookup 6-1 needs. */ +export interface ModelRegistryLike { + find(provider: string, modelId: string): { contextWindow: number } | undefined; +} + +export interface ResolvedModel { + model: string; + tier?: Tier; + /** Pre-filtered eligible candidates (primary first); spawnSubagent retries these on create() rejection. */ + candidates?: string[]; +} + +export interface ResolveError { error: string; } + +/** Split "provider/modelId" on the first "/". Bare id → { parentProvider, id }. */ +export function splitModel(model: string, parentProvider = ""): { provider: string; id: string } { + const i = model.indexOf("/"); + if (i < 0) return { provider: parentProvider, id: model }; + return { provider: model.slice(0, i), id: model.slice(i + 1) }; +} + +/** Q4 precedence: optsModel > agent.tier > agent.model > parent. Q5: contextFloor + catalog filter. */ +export function resolveAgentModel( + agent: AgentDef, optsModel: string | undefined, + parentModel: { provider: string; id: string }, + tiers: TierRegistry, modelRegistry: ModelRegistryLike, +): ResolvedModel | ResolveError { + if (optsModel) return { model: optsModel }; + if (agent.tier) { + const tier = tiers.get(agent.tier); + if (!tier) return { error: `tier '${agent.tier}' not found; available: ${tiers.list().map((t) => t.name).join(", ")}` }; + const candidates: string[] = []; + for (const m of tier.models) { + const { provider, id } = splitModel(m, parentModel.provider); + const model = modelRegistry.find(provider, id); + if (!model) continue; // not in catalog → skip + if (tier.contextFloor && (model.contextWindow ?? 0) < tier.contextFloor) continue; // below floor → skip + candidates.push(m); + } + if (candidates.length === 0) { + return { error: `tier '${tier.name}': no eligible model (all missing or below contextFloor ${tier.contextFloor ?? "—"})` }; + } + return { model: candidates[0]!, tier, candidates }; + } + if (agent.model) return { model: agent.model }; + return { model: `${parentModel.provider}/${parentModel.id}` }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/resolve-model.test.mts` +Expected: PASS (6 tests). + +- [ ] **Step 5: Commit** + +```bash +git add src/tiers/resolve.ts test/resolve-model.test.mts +git commit -m "feat(spec-6-1): splitModel + resolveAgentModel (tier resolution + contextFloor filter)" +``` + +--- + +### Task 3: AgentDef.tier? frontmatter field + +**Files:** +- Modify: `src/registry/frontmatter.ts` (add `tier?` to `AgentDef` + parse it) +- Test: `test/frontmatter.test.mts` (extend) + +**Interfaces:** +- Produces: `AgentDef.tier?: string` (additive — existing agents unchanged). + +- [ ] **Step 1: Write the failing test** (append to `test/frontmatter.test.mts`) + +```ts +const WITH_TIER = `--- +name: scout +description: Recon agent +tier: standard +--- +You are a scout. +`; + +test("parses optional tier field (SPEC-6-1)", () => { + const a = parseAgentFile(WITH_TIER, "/x/.pi/agents/scout.md", "project"); + strictEqual(a.tier, "standard"); +}); + +test("tier absent → undefined (back-compat)", () => { + const a = parseAgentFile(BASE, "/x/.pi/agents/scout.md", "project"); + strictEqual(a.tier, undefined); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/frontmatter.test.mts` +Expected: FAIL — `a.tier` is undefined (field doesn't exist). + +- [ ] **Step 3: Write minimal implementation** + +In `src/registry/frontmatter.ts`: +- Add to `AgentDef` interface: `tier?: string;` (after `sessionKey`, before `source`). +- In `parseAgentFile`'s return object, add: `tier: typeof raw.tier === "string" ? raw.tier : undefined,` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/frontmatter.test.mts` +Expected: PASS (all frontmatter tests incl. the 2 new ones). + +- [ ] **Step 5: Commit** + +```bash +git add src/registry/frontmatter.ts test/frontmatter.test.mts +git commit -m "feat(spec-6-1): AgentDef.tier? frontmatter field" +``` + +--- + +### Task 4: RunRecord cost/context/tier fields + RunLog usage.cost widening + +**Files:** +- Modify: `src/engine/run-registry.ts`, `src/runtime/run-log.ts` +- Test: `test/run-registry.test.mts` (extend), `test/run-log.test.mts` (audit/update) + +**Interfaces:** +- Produces: `RunRecord.costTotal?: number`, `RunRecord.contextTokens?: number`, `RunRecord.tier?: string`; `RunLog` message event `usage` gains `cost?: { total?: number }`. + +- [ ] **Step 1: Write the failing test** (append to `test/run-registry.test.mts`) + +```ts +test("RunRecord carries costTotal/contextTokens/tier (SPEC-6-1, additive)", () => { + const r = new RunRegistry(); + r.add({ runId: "fl-c1", agent: "g", model: "m", task: "t", track: true, todoId: null, status: "running", startedAt: 1, tier: "standard", costTotal: 0, contextTokens: 0 } as any); + strictEqual(r.get("fl-c1")!.tier, "standard"); + strictEqual(r.get("fl-c1")!.costTotal, 0); + strictEqual(r.get("fl-c1")!.contextTokens, 0); + r.update("fl-c1", { costTotal: 0.01, contextTokens: 50000 } as any); + strictEqual(r.get("fl-c1")!.costTotal, 0.01); + strictEqual(r.get("fl-c1")!.contextTokens, 50000); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/run-registry.test.mts` +Expected: FAIL — `tier`/`costTotal`/`contextTokens` not on `RunRecord`. + +- [ ] **Step 3: Write minimal implementation** + +In `src/engine/run-registry.ts`, add to `RunRecord` (after `tokenTotal?`, before `session?`): +```ts + /** SPEC-6-1: cumulative $ (usage.cost.total) — live, updated on each message_end. */ + costTotal?: number; + /** SPEC-6-1: latest context tokens (calcContextTokens(usage)) — live snapshot. */ + contextTokens?: number; + /** SPEC-6-1: the tier name this run used (for Tiers-view "used by" + per-tier spend). */ + tier?: string; +``` + +In `src/runtime/run-log.ts`, widen the `message` event's `usage` type to add `cost?: { total?: number }`. (Find the `usage` shape in the `message` event type + add `cost?`.) Audit existing run-log tests: any test that deep-asserts a `message` event's `usage` object needs `cost` allowed — change `deepStrictEqual(usage, {...})` to a shape check or add `cost` to expected. Run `pnpm test:run test/run-log.test.mts` + fix failures. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/run-registry.test.mts test/run-log.test.mts` +Expected: PASS (new test + all existing run-log tests green after audit). + +- [ ] **Step 5: Commit** + +```bash +git add src/engine/run-registry.ts src/runtime/run-log.ts test/run-registry.test.mts test/run-log.test.mts +git commit -m "feat(spec-6-1): RunRecord costTotal/contextTokens/tier + RunLog usage.cost widening" +``` + +--- + +### Task 5: spawnSubagent — tier resolution + fallback + cost/context accumulation + cap abort + +**Files:** +- Modify: `src/engine/spawnSubagent.ts` +- Test: `test/spawn-subagent-tier.test.mts` (new, reuses the runlog-harness pattern) + +**Interfaces:** +- Consumes: `resolveAgentModel`, `splitModel`, `ModelRegistryLike`, `TierRegistry` (Tasks 1-2), `AgentDef.tier?` (Task 3), `RunRecord` fields (Task 4). +- Produces: `spawnSubagent` now takes `tierRegistry: TierRegistry` + `modelRegistry: ModelRegistryLike` in `SpawnOptions`; resolves the model via `resolveAgentModel`; retries `candidates[1..]` on `create()` rejection; accumulates `costTotal` + `contextTokens`; aborts on `costCap` breach. `calcContextTokens(usage)` local helper. + +- [ ] **Step 1: Write the failing test** + +```ts +// test/spawn-subagent-tier.test.mts +import { test, beforeEach, afterEach } from "node:test"; +import { strictEqual, ok } from "node:assert"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSubagent, type ChildSession, type ChildSessionFactory } from "../src/engine/spawnSubagent.ts"; +import { RunRegistry } from "../src/engine/run-registry.ts"; +import { createSingleSlotLock } from "../src/engine/concurrency-lock.ts"; +import { ArmoryTodoAdapter } from "../src/todo-sync/adapter.ts"; +import { BackendRegistry, PI_HOOK_PARITY, type Backend } from "../src/backend/port.ts"; +import { TierRegistry } from "../src/tiers/tier-registry.ts"; +import type { ModelRegistryLike } from "../src/tiers/resolve.ts"; +import type { AgentDef } from "../src/registry/frontmatter.ts"; + +let tmpDir: string, logDir: string; +beforeEach(() => { tmpDir = mkdtempSync(join(tmpdir(), "fleet-tier-")); logDir = mkdtempSync(join(tmpdir(), "fleet-tlog-")); process.env.TODO_DIR = tmpDir; }); +afterEach(() => { rmSync(tmpDir, { recursive: true, force: true }); rmSync(logDir, { recursive: true, force: true }); delete process.env.TODO_DIR; }); + +const agent = (over: Partial = {}): AgentDef => ({ name: "g", description: "d", rolePrompt: "r", todoSync: true, memoryHydrate: true, vision: true, backend: "pi", sessionKey: "g", source: "builtin", filePath: "/x", ...over }); +const PARENT = { provider: "p", id: "m" }; +const mr = (windows: Record): ModelRegistryLike => ({ find: (pr, id) => (windows[`${pr}/${id}`] != null ? { contextWindow: windows[`${pr}/${id}`] } : undefined) }); +function regWith(factory: ChildSessionFactory): BackendRegistry { const r = new BackendRegistry(); r.register({ id: "pi", factory, available: () => true, versionInfo: () => null, hookParity: PI_HOOK_PARITY }); return r; } + +function fakeChild(usage: { input: number; output: number; cost: number; totalTokens?: number }, assistantText = "done"): ChildSession { + const handlers: Array<(e: any) => void> = []; + return { + prompt: async () => { + for (const h of handlers) h({ type: "turn_start", turnIndex: 0 }); + for (const h of handlers) h({ type: "message_end", message: { role: "assistant", content: [{ type: "text", text: assistantText }], usage: { input: usage.input, output: usage.output, cacheRead: 0, cacheWrite: 0, totalTokens: usage.totalTokens ?? (usage.input + usage.output), cost: { total: usage.cost } } } }); + for (const h of handlers) h({ type: "turn_end", turnIndex: 0 }); + }, + subscribe: (h) => { handlers.push(h); return () => {}; }, abort: async () => {}, dispose: () => {}, + }; +} + +test("tier run: costTotal + contextTokens accumulated; tier recorded on RunRecord", async () => { + const factory: ChildSessionFactory = { create: async () => ({ session: fakeChild({ input: 100, output: 42, cost: 0.001 }), model: "Ollama/glm-5.2:cloud" }) }; + const tiers = new TierRegistry({ tiers: [{ name: "standard", models: ["Ollama/glm-5.2:cloud"] }], agents: new Map() }); + const runReg = new RunRegistry(); + const res = await spawnSubagent({ + agent: "g", task: "t", track: true, registry: new Map([["g", agent({ tier: "standard" })]]), + todoSync: new ArmoryTodoAdapter(), runRegistry: runReg, lock: createSingleSlotLock(), backendRegistry: regWith(factory), + parentModel: PARENT, parentCwd: tmpDir, tierRegistry: tiers, modelRegistry: mr({ "Ollama/glm-5.2:cloud": 256000 }), + } as any); + strictEqual(res.status, "completed"); + strictEqual(runReg.get(res.runId)!.tier, "standard"); + strictEqual(runReg.get(res.runId)!.costTotal, 0.001, "$ accumulated from usage.cost.total"); + strictEqual(runReg.get(res.runId)!.contextTokens, 142, "contextTokens = calcContextTokens(usage)"); +}); + +test("fallback: models[0] create() rejects → retry models[1]", async () => { + let calls = 0; + const factory: ChildSessionFactory = { create: async () => { calls++; if (calls === 1) throw new Error("provider down"); return { session: fakeChild({ input: 10, output: 5, cost: 0 }), model: "Ollama/minimax-m3:cloud" }; } }; + const tiers = new TierRegistry({ tiers: [{ name: "std", models: ["Ollama/primary", "Ollama/fallback"] }], agents: new Map() }); + const res = await spawnSubagent({ + agent: "g", task: "t", track: true, registry: new Map([["g", agent({ tier: "std" })]]), + todoSync: new ArmoryTodoAdapter(), runRegistry: new RunRegistry(), lock: createSingleSlotLock(), backendRegistry: regWith(factory), + parentModel: PARENT, parentCwd: tmpDir, tierRegistry: tiers, modelRegistry: mr({ "Ollama/primary": 128000, "Ollama/fallback": 128000 }), + } as any); + strictEqual(res.status, "completed", "fallback model succeeded"); + strictEqual(calls, 2, "create() called twice (primary rejected, fallback ok)"); +}); + +test("cap abort: costTotal > tier.costCap → aborted + budget_exceeded", async () => { + const factory: ChildSessionFactory = { create: async () => ({ session: fakeChild({ input: 100, output: 42, cost: 0.01 }), model: "anthropic/claude-sonnet-4" }) }; + const tiers = new TierRegistry({ tiers: [{ name: "frontier", models: ["anthropic/claude-sonnet-4"], costCap: 0.001 }], agents: new Map() }); + const res = await spawnSubagent({ + agent: "g", task: "t", track: true, registry: new Map([["g", agent({ tier: "frontier" })]]), + todoSync: new ArmoryTodoAdapter(), runRegistry: new RunRegistry(), lock: createSingleSlotLock(), backendRegistry: regWith(factory), + parentModel: PARENT, parentCwd: tmpDir, tierRegistry: tiers, modelRegistry: mr({ "anthropic/claude-sonnet-4": 200000 }), + } as any); + strictEqual(res.status, "aborted"); + ok(res.error?.includes("budget_exceeded"), `error mentions budget_exceeded: ${res.error}`); +}); + +test("no tier (agent.model path): cost still tracked, no cap enforcement", async () => { + const factory: ChildSessionFactory = { create: async () => ({ session: fakeChild({ input: 100, output: 42, cost: 0.5 }), model: "m" }) }; + const res = await spawnSubagent({ + agent: "g", task: "t", track: true, registry: new Map([["g", agent({ model: "m" })]]), + todoSync: new ArmoryTodoAdapter(), runRegistry: new RunRegistry(), lock: createSingleSlotLock(), backendRegistry: regWith(factory), + parentModel: PARENT, parentCwd: tmpDir, tierRegistry: new TierRegistry({ tiers: [], agents: new Map() }), modelRegistry: mr({}), + } as any); + strictEqual(res.status, "completed", "no cap → runs to completion even at $0.5"); + strictEqual(res.tokenTotal, 142); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/spawn-subagent-tier.test.mts` +Expected: FAIL — `tierRegistry`/`modelRegistry` not on `SpawnOptions`; `costTotal` not accumulated. + +- [ ] **Step 3: Write minimal implementation** + +In `src/engine/spawnSubagent.ts`: +1. Import: `import { resolveAgentModel, type ModelRegistryLike } from "../tiers/resolve.ts"; import type { TierRegistry } from "../tiers/tier-registry.ts";` +2. Add to `SpawnOptions`: `tierRegistry?: TierRegistry;` `modelRegistry?: ModelRegistryLike;` (optional — unit tests that don't pass them keep working via the `agent.model`/parent fallback). +3. Add a local helper: +```ts +function calcContextTokens(u: { totalTokens?: number; input?: number; output?: number; cacheRead?: number; cacheWrite?: number }): number { + return u.totalTokens || ((u.input ?? 0) + (u.output ?? 0) + (u.cacheRead ?? 0) + (u.cacheWrite ?? 0)); +} +``` +4. Replace the `const model = opts.model ?? agentDef.model ?? …` line with: +```ts +const resolved = resolveAgentModel(agentDef, opts.model, opts.parentModel, opts.tierRegistry ?? new TierRegistry({ tiers: [], agents: new Map() }), opts.modelRegistry ?? { find: () => undefined }); +if ("error" in resolved) { return fail(runId, startedAt, resolved.error, opts.agent); } +const model = resolved.model; +const tier = resolved.tier; +const candidates = resolved.candidates ?? [model]; +``` +5. In `runRegistry.add({ … })`, add `tier: tier?.name, costTotal: 0, contextTokens: 0`. +6. Replace the `const { session } = await backend.factory.create({ … model, … })` block with a fallback retry loop: +```ts +let session: ChildSession | undefined; +let lastErr: Error | undefined; +for (const cand of candidates) { + try { + session = (await backend.factory.create({ cwd: opts.parentCwd, model: cand, thinkingLevel: childAgent.thinkingLevel, tools, rolePrompt: childAgent.rolePrompt, skills: childAgent.skills ?? [], task: opts.task, agent: childAgent, memoryPort, visionPort })).session; + break; + } catch (e) { lastErr = e as Error; } +} +if (!session) { return await finishRun(opts, runId, startedAt, "failed", "", todoId, priorStatus, `backend create failed: ${lastErr?.message ?? "unknown"}`, agentDef.name, model, 0); } +``` +7. In the `message_end` (assistant) branch of the subscribe handler, after the existing `tokenTotal` accumulation, add: +```ts +const cost = e.message?.usage?.cost?.total ?? 0; +costTotal += cost; +contextTokens = calcContextTokens(e.message?.usage ?? {}); +opts.runRegistry.update(runId, { costTotal, contextTokens, tokenTotal }); +try { opts.runLog?.append(runId, { type: "message", role: "assistant", text, usage: { total: turnTokens, input: u?.input, output: u?.output, cacheRead: u?.cacheRead, cacheWrite: u?.cacheWrite, cost: e.message?.usage?.cost }, turnIndex: turnIdx }); } catch {} +if (tier?.costCap && costTotal > tier.costCap) { aborted = true; void session.abort(); } +``` +(Declare `let costTotal = 0; let contextTokens = 0;` alongside `let tokenTotal = 0;`.) +8. Pass `costTotal` through to `finishRun` + include `costTotal`/`contextTokens` in the `run:ended` journal event + the terminal `runRegistry.update`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/spawn-subagent-tier.test.mts` +Expected: PASS (4 tests). Also run `pnpm test:run` (full suite) to confirm no regressions in the existing spawnSubagent tests (the new `SpawnOptions` fields are optional → existing tests compile). + +- [ ] **Step 5: Commit** + +```bash +git add src/engine/spawnSubagent.ts test/spawn-subagent-tier.test.mts +git commit -m "feat(spec-6-1): spawnSubagent tier resolution + fallback + cost/context accumulation + cap abort" +``` + +--- + +### Task 6: widget-rows ctx% + $ segments + +**Files:** +- Modify: `src/panel/widget-rows.ts` +- Test: `test/widget-rows.test.mts` (extend) + +**Interfaces:** +- Produces: `WidgetRun` gains `contextTokens?`, `maxContext?`, `costTotal?`; `widgetLine` appends ` 42%` + ` $0.0123`; `toWidgetRun` projects the new `RunRecord` fields. + +- [ ] **Step 1: Write the failing test** (append to `test/widget-rows.test.mts`) + +```ts +test("widgetLine: ctx% shown when contextTokens + maxContext present (SPEC-6-1)", () => { + const w = toWidgetRun(fg({ runId: "fl-x", contextTokens: 128000, costTotal: 0.0123 } as any)); + w.maxContext = 256000; + const lines = renderWidgetLines([w], 3000); + ok(lines[0]!.includes("50%"), `ctx% shown: ${lines[0]}`); + ok(lines[0]!.includes("$0.0123"), `cost shown: ${lines[0]}`); +}); + +test("widgetLine: ctx% hidden when maxContext absent; $ hidden when costTotal 0", () => { + const w = toWidgetRun(fg({ runId: "fl-y", contextTokens: 100, costTotal: 0 } as any)); + const lines = renderWidgetLines([w], 3000); + ok(!lines[0]!.includes("%"), `no ctx% without maxContext: ${lines[0]}`); + ok(!lines[0]!.includes("$"), `no $ when costTotal 0: ${lines[0]}`); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/widget-rows.test.mts` +Expected: FAIL — `contextTokens`/`maxContext`/`costTotal` not on `WidgetRun`. + +- [ ] **Step 3: Write minimal implementation** + +In `src/panel/widget-rows.ts`: +- Add to `WidgetRun`: `contextTokens?: number; maxContext?: number; costTotal?: number;` +- In `toWidgetRun`, project: `costTotal: r.costTotal, contextTokens: r.contextTokens,` (maxContext is set by the controller — Task 7). +- In `widgetLine`, after the `tok` segment, add: +```ts +const ctx = (r.contextTokens != null && r.maxContext != null && r.maxContext > 0) ? ` ${Math.round(r.contextTokens / r.maxContext * 100)}%` : ""; +const cost = r.costTotal ? ` $${r.costTotal.toFixed(4)}` : ""; +``` +and append `${ctx}${cost}` to the returned line. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/widget-rows.test.mts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/panel/widget-rows.ts test/widget-rows.test.mts +git commit -m "feat(spec-6-1): widget-rows ctx% + $ segments" +``` + +--- + +### Task 7: fleet-widget maxContext threading + runs-rows ctx% + $ + +**Files:** +- Modify: `src/panel/fleet-widget.ts`, `src/panel/runs-rows.ts` +- Test: `test/fleet-widget.test.mts` (extend), `test/runs-rows.test.mts` (extend) + +**Interfaces:** +- Produces: `FleetWidgetDeps` gains `getModelContextWindow?: (model: string) => number | undefined`; the controller threads `maxContext` into each `WidgetRun`. `runs-rows` row fn gains `ctx%` + `$` via a `maxContext` resolver arg. + +- [ ] **Step 1: Write the failing test** (append to `test/fleet-widget.test.mts`) + +```ts +test("controller threads maxContext via getModelContextWindow (SPEC-6-1)", () => { + const rr = new RunRegistry(); + const { calls, ui } = fakeUi(); + const c = new FleetWidgetController({ + runRegistry: rr, ui, getTheme: () => ({}) as any, now: () => 1000, + setInterval: () => 1 as any, clearInterval: () => {}, + getModelContextWindow: (m) => (m === "Ollama/glm-5.2:cloud" ? 256000 : undefined), + }); + c.start(); + rr.add({ runId: "fl-1", agent: "coder", model: "Ollama/glm-5.2:cloud", task: "t", track: true, todoId: null, status: "running", startedAt: 0, contextTokens: 128000, costTotal: 0.01 } as any); + const last = calls.filter((c2) => c2.key === "fleet-active").at(-1)!; + ok(last.content![0]!.includes("50%"), `ctx% via maxContext: ${last.content![0]}`); + ok(last.content![0]!.includes("$0.01"), `cost shown: ${last.content![0]}`); + c.dispose(); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/fleet-widget.test.mts` +Expected: FAIL — `getModelContextWindow` not on `FleetWidgetDeps`. + +- [ ] **Step 3: Write minimal implementation** + +In `src/panel/fleet-widget.ts`: +- Add to `FleetWidgetDeps`: `getModelContextWindow?: (model: string) => number | undefined;` +- In `activeRuns()`, after `toWidgetRun(r)`, set `w.maxContext = this.deps.getModelContextWindow?.(r.model)` for fg runs (and similarly for bg via the bg row's model if available — defer bg maxContext to "undefined" if bg rows don't carry a model; the `ctx%` segment hides). +- Stash `this.getModelContextWindow = deps.getModelContextWindow` in the constructor. + +In `src/panel/runs-rows.ts`: +- The `runsRow` fn (or its caller) gains a `getModelContextWindow` resolver; the row appends `ctx%` (from `r.contextTokens` + resolved `maxContext`) + `$` (from `r.costTotal`) using the same format as `widgetLine`. Thread the resolver through the panel's Runs-tab render path (Task 11 wires the real `modelRegistry`). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/fleet-widget.test.mts test/runs-rows.test.mts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/panel/fleet-widget.ts src/panel/runs-rows.ts test/fleet-widget.test.mts test/runs-rows.test.mts +git commit -m "feat(spec-6-1): fleet-widget maxContext threading + runs-rows ctx%/$" +``` + +--- + +### Task 8: tiers-rows pure render fns + +**Files:** +- Create: `src/panel/tiers-rows.ts` +- Test: `test/tiers-rows.test.mts` + +**Interfaces:** +- Produces: `renderTierRow(tier, spend, usedBy, runCount): string` — pure. + +- [ ] **Step 1: Write the failing test** + +```ts +// test/tiers-rows.test.mts +import { test } from "node:test"; +import { strictEqual, ok } from "node:assert"; +import { renderTierRow } from "../src/panel/tiers-rows.ts"; +import type { Tier } from "../src/tiers/tier-registry.ts"; + +const t = (over: Partial = {}): Tier => ({ name: "standard", models: ["Ollama/glm-5.2:cloud", "Ollama/minimax-m3:cloud"], ...over }); + +test("renderTierRow: models chain arrow, spend, used-by", () => { + const row = renderTierRow(t(), 0.12, ["coder", "scout"], 3); + ok(row.includes("standard"), "name"); + ok(row.includes("Ollama/glm-5.2:cloud→Ollama/minimax-m3:cloud"), "models chain: " + row); + ok(row.includes("$0.12"), "spend"); + ok(row.includes("3 runs"), "run count"); + ok(row.includes("used by: coder, scout"), "used-by list"); +}); + +test("renderTierRow: costCap + contextFloor shown; — when absent", () => { + ok(renderTierRow(t({ costCap: 5, contextFloor: 200000 }), 0, [], 0).includes("$5"), "cap shown"); + ok(renderTierRow(t({ costCap: 5, contextFloor: 200000 }), 0, [], 0).includes("200k"), "floor shown"); + ok(renderTierRow(t(), 0, [], 0).includes("—"), "— when cap/floor absent"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/tiers-rows.test.mts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// src/panel/tiers-rows.ts +import type { Tier } from "../tiers/tier-registry.ts"; + +function fmtFloor(n: number | undefined): string { + if (n == null) return "—"; + if (n >= 1000) return `${Math.round(n / 1000)}k`; + return String(n); +} + +/** One row per tier for the /fleet Tiers view. Pure (unit-tested). */ +export function renderTierRow(tier: Tier, spend: number, usedBy: string[], runCount: number): string { + const cap = tier.costCap != null ? `$${tier.costCap}` : "—"; + const floor = fmtFloor(tier.contextFloor); + const spendStr = spend > 0 ? `$${spend.toFixed(4)}` : "$0.00"; + const used = usedBy.length ? `used by: ${usedBy.join(", ")}` : "used by: —"; + return `${tier.name} ${tier.models.join("→")} ${cap} ${floor} ${spendStr} ${runCount} runs ${used}`; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/tiers-rows.test.mts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/panel/tiers-rows.ts test/tiers-rows.test.mts +git commit -m "feat(spec-6-1): tiers-rows pure render fns" +``` + +--- + +### Task 9: tier-store read/write + +**Files:** +- Create: `src/tiers/tier-store.ts` +- Test: `test/tier-store.test.mts` + +**Interfaces:** +- Produces: `TierStore` class — `read(scope: "project" | "global"): Tier[]`, `write(scope, tiers: Tier[])`. Atomic write (temp + rename); validates via `parseTiersFile` before writing (rejects duplicates → no file change). + +- [ ] **Step 1: Write the failing test** + +```ts +// test/tier-store.test.mts +import { test, beforeEach, afterEach } from "node:test"; +import { strictEqual, ok, throws } from "node:assert"; +import { mkdtempSync, rmSync, readFileSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { TierStore } from "../src/tiers/tier-store.ts"; + +let dir: string; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "fleet-tstore-")); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + +test("read: missing file → [] (scope degrades cleanly)", () => { + const store = new TierStore({ projectPath: join(dir, "tiers.json"), globalPath: join(dir, "global-tiers.json") }); + deepStrictEqual(store.read("project"), []); +}); + +test("write+read roundtrip: project scope", () => { + const store = new TierStore({ projectPath: join(dir, "tiers.json"), globalPath: join(dir, "g.json") }); + store.write("project", [{ name: "x", models: ["m"] }]); + deepStrictEqual(store.read("project"), [{ name: "x", models: ["m"] }]); + ok(existsSync(join(dir, "tiers.json")), "file written"); +}); + +test("write rejects duplicates → no file change", () => { + const store = new TierStore({ projectPath: join(dir, "tiers.json"), globalPath: join(dir, "g.json") }); + store.write("project", [{ name: "x", models: ["m"] }]); + throws(() => store.write("project", [{ name: "x", models: ["m1"] }, { name: "x", models: ["m2"] }]), /duplicate/); + deepStrictEqual(store.read("project"), [{ name: "x", models: ["m"] }], "file unchanged after rejected write"); +}); +import { deepStrictEqual } from "node:assert"; +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/tier-store.test.mts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// src/tiers/tier-store.ts +import { readFileSync, writeFileSync, renameSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { parseTiersFile, type Tier } from "./tier-registry.ts"; + +export interface TierStoreOpts { projectPath: string; globalPath: string; } + +export class TierStore { + constructor(private readonly opts: TierStoreOpts) {} + read(scope: "project" | "global"): Tier[] { + const path = scope === "project" ? this.opts.projectPath : this.opts.globalPath; + try { return parseTiersFile(readFileSync(path, "utf8")); } catch { return []; } // missing/malformed → [] (registry skips) + } + write(scope: "project" | "global", tiers: Tier[]): void { + // Validate BEFORE writing — rejects duplicates/malformed → throws, no file change. + const json = JSON.stringify(tiers, null, 2); + parseTiersFile(json); // throws on invalid + const path = scope === "project" ? this.opts.projectPath : this.opts.globalPath; + mkdirSync(dirname(path), { recursive: true }); + const tmp = `${path}.tmp`; + writeFileSync(tmp, json, "utf8"); + renameSync(tmp, path); // atomic + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/tier-store.test.mts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/tiers/tier-store.ts test/tier-store.test.mts +git commit -m "feat(spec-6-1): tier-store read/write (atomic + validated)" +``` + +--- + +### Task 10: fleet-panel Tiers view + action submenu + +**Files:** +- Modify: `src/panel/fleet-panel.ts` +- Test: `test/fleet-panel-tiers.test.mts` (extend the panel harness from `test/panel-spec5a.test.mts` or `test/fleet-items.test.mts` — match the existing pattern) + +**Interfaces:** +- Consumes: `TierRegistry`, `TierStore`, `renderTierRow` (Tasks 1, 8, 9), the agent registry (for `usedBy`). +- Produces: a new `tiers` `View` in `FleetPanel` + action submenu (Set models / Set costCap / Set contextFloor / Add tier / Delete tier + scope toggle). + +- [ ] **Step 1: Write the failing test** + +Model on the existing panel harness (a fake `ctx.ui.custom` + `onNotify` + `Input` capture). Assert: +- The `tiers` tab renders a row per tier (via `renderTierRow`). +- Set costCap action → inline `Input` → submit `$1` → `TierStore.write("project", …)` called with the updated cap → re-render shows `$1` + the file on disk has the new value. +- Add tier → inline `Input` for name → creates a new tier with a placeholder model. +- Delete tier → removes it from the store. +- Duplicate-name Add → `onNotify("tier 'X' already exists", "error")` + no `write` call. +- Scope toggle project↔global changes which file `write` targets. + +(Use a fake `TierStore` that records `write` calls; assert on the recorded calls. The exact test code follows the `fleet-panel` harness pattern in `test/panel-spec5a.test.mts` — copy the harness setup + add the tiers-specific assertions.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test:run test/fleet-panel-tiers.test.mts` +Expected: FAIL — `tiers` view doesn't exist. + +- [ ] **Step 3: Write minimal implementation** + +In `src/panel/fleet-panel.ts`: +- Add `"tiers"` to the `View` type + the tabs array (after `"scheduled"`). +- Add a `tierRegistry: TierRegistry` + `tierStore: TierStore` to the panel deps. +- Render path for `view === "tiers"`: list `tierRegistry.list()`, each row via `renderTierRow(tier, spendFor(tier.name), tierRegistry.usedBy(tier.name), runCountFor(tier.name))` where `spendFor`/`runCountFor` sum from `runRegistry.list()` by `tier` field. +- Action submenu (on the selected tier row): `m` Set models, `c` Set costCap, `f` Set contextFloor, `a` Add tier, `d` Delete tier, `g` toggle scope (project↔global). Each opens an inline `Input` (reusing the 5b-1 resume/steer input machinery); on submit, `tierStore.write(scope, updatedTiers)` + `tierRegistry` reload + re-render. Duplicate-name Add → `onNotify` error + no write. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test:run test/fleet-panel-tiers.test.mts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/panel/fleet-panel.ts test/fleet-panel-tiers.test.mts +git commit -m "feat(spec-6-1): /fleet Tiers view + inline edit actions" +``` + +--- + +### Task 11: index.ts wiring + +**Files:** +- Modify: `src/index.ts` + +**Interfaces:** +- Consumes: `TierRegistry`, `TierStore`, `pi.modelRegistry` (the real `ModelRegistry`), the panel's new deps. + +- [ ] **Step 1: Wire the TierRegistry + ModelRegistry + Tiers view** + +In `src/index.ts` `session_start`: +1. Construct the `TierStore` with `{ projectPath: join(cwd, ".pi", "fleet", "tiers.json"), globalPath: join(home, ".pi", "agent", "fleet", "tiers.json") }`. +2. Construct the `TierRegistry` from `mergeTiers(BUILTIN_TIERS, store.read("global"), store.read("project"))` + the agent registry's `tier` fields. +3. Add `tierRegistry` + `modelRegistry: pi.modelRegistry` (narrowed to `ModelRegistryLike` via a `{ find: (p, i) => pi.modelRegistry.find(p, i) }` adapter) to `deps` (the `SubagentToolDeps` + `SpawnOptions` path). +4. Wire `getModelContextWindow: (m) => { const { provider, id } = splitModel(m, parentProvider); return pi.modelRegistry.find(provider, id)?.contextWindow; }` into the `FleetWidgetController`. +5. Pass `tierRegistry` + `tierStore` to the `FleetPanel` deps; add the `tiers` view. +6. Rebuild the `TierRegistry` when the panel writes a tier (the panel calls a `reloadTiers()` callback after `tierStore.write` → re-reads + reconstructs the registry). + +- [ ] **Step 2: Verify typecheck + full suite** + +Run: `pnpm typecheck && pnpm test:run` +Expected: typecheck clean; all tests pass (no new unit test for the wiring — it's integration-glue, covered by the term smoke + the existing index tests). + +- [ ] **Step 3: Commit** + +```bash +git add src/index.ts +git commit -m "feat(spec-6-1): wire TierRegistry + ModelRegistry + Tiers view into the extension" +``` + +--- + +### Task 12: Release + term smoke on published @0.10.0 + +**Files:** none (release + verification) + +- [ ] **Step 1: Bump version + final verification** + +```bash +# package.json: "version": "0.9.2" → "0.10.0" +pnpm typecheck && pnpm test:run # clean + all green +``` + +- [ ] **Step 2: Branch + PR + merge + tag** + +```bash +git checkout -b feat/spec-6-1-cost-tiers +git add -A && git commit -m "chore(spec-6-1): bump version 0.9.2 → 0.10.0" +git push -u origin feat/spec-6-1-cost-tiers +gh pr create --title "feat(spec-6-1): cost-aware tiers + cost accounting + context% + Tiers view (v0.10.0)" --body "..." +# merge PR → main +git checkout main && git merge --no-ff feat/spec-6-1-cost-tiers -m "Merge pull request #N ..." +git push origin main +MERGE=$(git rev-parse HEAD) +git update-ref refs/tags/v0.10.0 "$MERGE" +git push --force origin v0.10.0 +# watch CI: gh run watch (Release workflow publishes to npm + creates GitHub Release) +``` + +- [ ] **Step 3: Bump settings.json + term smoke on published** + +```bash +# ~/.pi/agent/settings.json: @getpipher/armory-fleet@0.9.2 → @0.10.0 +npm view @getpipher/armory-fleet version # confirm 0.10.0 live +``` + +Spawn a fresh pi in a temp cwd with a project `tiers.json` defining `frontier` with `costCap: 0.001` (intentionally tiny). Spawn a foreground subagent with `tier: frontier`. Verify the 6 gates from the spec §7: +1. ✅ Above-editor widget shows `ctx%` + `$` live. +2. ✅ `costTotal` crosses `0.001` → `✗ aborted` + `run aborted: budget_exceeded` notify. +3. ✅ `/fleet` → Tiers tab shows the `frontier` row + per-tier spend. +4. ✅ Tiers tab Set costCap → `$1` → re-render + file on disk updated. +5. ✅ Runs tab shows the aborted run with final `ctx%` + `$`. +6. ✅ A no-tier run shows `tok` + `ctx%` + `$` (cost tracked, no cap). + +Also verify the Ollama-Cloud `costTotal: 0` case on a no-tier run (no `$` segment, no false cap trip). Kill the session + clean up. + +- [ ] **Step 4: Update handoff pointer + complete** + +Update `~/.pi/agent/memory/-Users-rector-local-dev-getpipher-armory-fleet/handoff-pointer.md` with the v0.10.0 entry + term-smoke result. Mark the SPEC-6-1 todo done. + +--- + +## Self-Review (run before handing off) + +**1. Spec coverage:** +- Surface A (tier routing) → Tasks 1, 2, 3, 5, 11. ✅ +- Surface B (cost $) → Tasks 4, 5, 6, 7, 11. ✅ +- Surface C (context%) → Tasks 4, 5, 6, 7, 11. ✅ +- Surface D (Tiers view) → Tasks 8, 9, 10, 11. ✅ +- Q5 fallback → Task 5 (retry loop). ✅ +- Q7 cap abort → Task 5. ✅ +- Q10 builtins → Task 1. ✅ +- Term smoke → Task 12. ✅ + +**2. Placeholder scan:** no TBD/TODO/"add error handling"/"similar to Task N". Each test + impl step has real code. + +**3. Type consistency:** `Tier`/`TierRegistry`/`resolveAgentModel`/`ModelRegistryLike`/`splitModel`/`TierStore`/`renderTierRow` — names + signatures consistent across tasks. `RunRecord.costTotal`/`contextTokens`/`tier` consistent (Tasks 4, 5, 6, 7). `WidgetRun` fields consistent (Tasks 6, 7). `SpawnOptions.tierRegistry`/`modelRegistry` consistent (Tasks 5, 11). \ No newline at end of file diff --git a/docs/superpowers/specs/2026-07-27-spec-6-1-design.md b/docs/superpowers/specs/2026-07-27-spec-6-1-design.md new file mode 100644 index 0000000..7cb6d89 --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-spec-6-1-design.md @@ -0,0 +1,331 @@ +# SPEC-6-1 — Cost-aware model tiers + cost accounting + context% + Tiers view + +**Date:** 2026-07-27 +**Sub-SPEC of:** SPEC-6 (Power-user tier → v1.0) +**Package:** `@getpipher/armory-fleet` · target release `0.10.0` (minor) +**Predecessors:** SPEC-5b-4 (v0.9.1 — mid-run Steer + Stop, the retained-handle seam); v0.9.2 patch (removed stale below-editor + subagent-tool widgets) +**Pipeline step:** brainstorm (this doc) → plan → implementation + +## 1. Context + +SPEC-6-1 is the first sub-SPEC of SPEC-6 (Power-user tier → v1.0), the final SPEC. SPEC-6 bundles nine surfaces (A–I); the brainstorm decomposed it into a 4-way split mirroring the 5b pattern: + +| Sub-SPEC | Surfaces | Release | +|---|---|---| +| **6-1** (this) | A+B+C+D — cost-aware tier routing + $ accounting + context% + Tiers view | v0.10.0 | +| 6-2 | E+F — quality patterns + lifecycle hooks | v0.11.0 | +| 6-3 | G — workflows-as-code | v0.12.0 | +| 6-4 | H+I — event-bus RPC + live conversation viewer → v1.0.0 | v0.13.0 → v1.0.0 | + +6-1 is the **foundation** — cost numbers + tier-resolved model selection feed 6-2's budget gates, 6-3's phase tiers, and 6-4's observe payload. Everything downstream reads from it. + +### The data-source discovery (de-risks 6-1) + +The pi-ai `Usage` type — present in every `message_end` event — already carries everything 6-1 needs for cost + context%: + +```ts +interface Usage { + input, output, cacheRead, cacheWrite: number; // tokens (5b-2 already accumulates these as tokenTotal) + totalTokens: number; // = input+output+cacheRead+cacheWrite (pi's calculateContextTokens) + cost: { input, output, cacheRead, cacheWrite, total: number }; // ← $ already computed by pi-ai per-model +} +``` + +- **Cost $** (`usage.cost.total`) is computed by pi-ai from its own model catalog — fleet does NOT need a pricing table. 5b-2's Q9 fix deliberately switched fleet off `cost.total` (it was being misrendered as "tok"); 6-1 reads it back, honestly, as dollars. +- **Context tokens** (`usage.totalTokens`, or `input+output+cacheRead+cacheWrite` fallback) = pi's own compaction metric (`calculateContextTokens`) — the defensible "current context" definition. +- **Max-context** (`Model.contextWindow`) is available at runtime via `pi.modelRegistry.find(provider, modelId).contextWindow` — `ModelRegistry` is on the `ExtensionAPI`. + +So surfaces B (cost) + C (context%) are fully data-ready. The genuinely new design work is surface A (tier routing) + D (the Tiers view) — the tier abstraction. + +### What 6-1 closes + +- **Surface A (tier routing):** today model resolution is `opts.model ?? agentDef.model ?? parentModel` — a single string, no tier concept, no fallback chain, no context-floor gate. 6-1 adds a named tier registry + `resolveAgentModel` + spawn-time fallback. +- **Surface B (cost $):** fleet tracks `tokenTotal` (5b-2) but ignores `usage.cost.total`. 6-1 accumulates it into `RunRecord.costTotal` (live) + `RunLog run:ended` (durable) + enforces per-run `costCap` aborts. +- **Surface C (context%):** deferred from 5b-2 Q4 ("needs model max-context + a defensible current-context definition; cost-aware tier is the right home"). Both are now available (above). 6-1 surfaces context% in the live widget + Runs tab. +- **Surface D (Tiers view):** PRD §5 lists a Tiers view ("model-tier routing, cost caps, concurrency limits, worktree policy"). 6-1 delivers the tier-routing + cost-caps portion as a `/fleet` panel tab with inline edit actions (interactive-first). + +## 2. Design decisions (settled in brainstorm) + +| # | Decision | Choice | +|---|---|---| +| Q1 | Tier abstraction | **(a) Tier registry + agent references.** A named routing policy referenced by agents/phases. | +| Q2 | Tier record shape | **(c) `{ name, models: string[], costCap?, contextFloor? }`.** Ordered fallback chain + per-tier $ cap + min context window. Fleet doesn't re-own pi-ai's catalog. | +| Q3 | Registry storage + builtins | **(a) Single `tiers.json` per scope (project + global) + shipped builtins.** Merge: builtins < global < project (project wins by name). Builtins: `economy`/`standard`/`frontier`. | +| Q4 | Agent→tier reference + precedence | **(a) Additive `tier?` field on `AgentDef`.** Precedence: `opts.model` (caller override) > `agent.tier` (resolve via registry) > `agent.model` (literal) > `parentModel`. Per-phase `tier?` parallel to the existing per-phase `backend` override. | +| Q5 | Fallback trigger | **(b) Spawn failure + `contextFloor` check.** Walk `tier.models[]`; skip any not in the catalog OR whose `contextWindow < tier.contextFloor`; retry `backend.factory.create()` on rejection with the next pre-filtered candidate. | +| Q6 | Cost accounting storage | **(a) Per-run cost only.** `RunRecord.costTotal` (live) + `RunLog run:ended.costTotal` (durable). Aggregates (per-agent/tier/session) computed on-the-fly at render — no pre-aggregation store. | +| Q7 | Budget enforcement | **(b) Per-run, abort at cap.** When `run.costTotal > tier.costCap` → `aborted = true; session.abort()` → `finishRun("aborted", "budget_exceeded")`. Reuses the 5b-4 Stop abort path. | +| Q8 | Context% surface + numerator | **(b) Widget + Runs tab.** Numerator = `calcContextTokens(lastUsage)` (= `usage.totalTokens`); denominator = `modelRegistry.find(...).contextWindow`. Live in the above-editor widget + final in the Runs tab. | +| Q9 | Tiers view scope + edit | **(b) Tier list + inline edit actions.** Set models / Set costCap / Set contextFloor / Add / Delete tier via action submenu + inline `Input`; global/project scope toggle. Read-only "used by" column. No agent-file writes. | +| Q10 | Builtin tier definitions | `economy: ["Ollama/minimax-m3:cloud"]`; `standard: ["Ollama/glm-5.2:cloud", "Ollama/minimax-m3:cloud"]`; `frontier: ["anthropic/claude-sonnet-4", "Ollama/glm-5.2:cloud"], costCap: 5, contextFloor: 200000`. Editable later in `/fleet`. | + +## 3. Architecture + +The tier layer inserts at **spawn-time model resolution** + adds two **runtime accumulators** (`costTotal`, `contextTokens`) + a **cap check** per `message_end`. One new registry (tiers) + one new resolver fn + the Tiers view. The retained-handle seam (5b-4) is untouched. + +``` +spawnSubagent(opts) + ├─ resolveAgentModel(agent, opts.model, parentModel, tierRegistry, modelRegistry) + │ │ precedence: opts.model (caller) > agent.tier (resolve) > agent.model > parentModel + │ │ if tier: walk tier.models[], skip any not in catalog OR whose contextWindow < tier.contextFloor (Q5), + │ │ return first eligible candidate; if none → failRun("no eligible model in tier X") + │ └─ → { model: resolved, tier?: Tier } + ├─ runRegistry.add({…, model: resolved, tier: resolved.tier?.name, costTotal:0, contextTokens:0}) + ├─ backend.factory.create({ model: resolved.model, … }) + │ └─ on rejection: retry candidates[1], [2], … (Q5 fallback); if all reject → failRun + ├─ session.subscribe((e) => { + │ if (e.type === "message_end" && assistant) { + │ costTotal += e.message.usage.cost.total ?? 0 // NEW (Q6) + │ contextTokens = calcContextTokens(usage) // NEW (Q8) + │ runRegistry.update(runId, { costTotal, contextTokens, tokenTotal }) + │ if (tier?.costCap && costTotal > tier.costCap) // NEW (Q7) + │ { aborted = true; void session.abort(); } // → finishRun("aborted", "budget_exceeded") + │ } + │ }) + └─ finishRun → runRegistry.update(runId, { status, costTotal, contextTokens, … }) + │ + FleetWidgetController.render() (above-editor widget, existing) + └─ widgetLine adds: ` 42%` (contextTokens/maxContext) + ` $0.0123` (costTotal) // NEW (Q8) + Runs tab (runs-rows.ts) — adds `ctx%` + `$` columns // NEW (Q8) + Tiers view (new /fleet tab) — tier list + per-tier spend + inline edit actions // NEW (Q9) +``` + +**Key invariants:** +- `costTotal` / `contextTokens` are live on `RunRecord` (updated per `message_end`), parallel to `tokenTotal` (5b-2). Durable in `RunLog run:ended` (one line added). +- The tier is resolved ONCE at spawn (a run has one model — no mid-run switching). The fallback chain (Q5) walks `tier.models[]` at spawn only. +- `contextFloor` + `costCap` enforcement are tier-driven; a run with no tier (`agent.model` or `parentModel` path) has neither — backwards-compatible. + +## 4. Components + +### New: `src/tiers/tier-registry.ts` — `Tier` type + `TierRegistry` + discovery +```ts +export interface Tier { + name: string; + models: string[]; // ordered fallback chain, primary first + costCap?: number; // $ per-run; abort when run.costTotal exceeds (Q7) + contextFloor?: number; // min contextWindow; skip models below it at spawn (Q5) +} +export interface TierRegistry { + get(name: string): Tier | undefined; + list(): Tier[]; // for the Tiers view + usedBy(name: string): string[]; // agent names referencing this tier (read-only, from agent registry) +} +``` +- Discovery: builtins (`src/tiers/builtin.ts`) < global (`~/.pi/agent/fleet/tiers.json`) < project (`.pi/fleet/tiers.json`), project wins by name (Q3). Same merge pattern as the agent registry. +- Pure fns: `parseTiersFile(json): Tier[]`, `mergeTiers(builtin, global, project): Tier[]` (unit-tested, like `frontmatter.ts`). + +### New: `src/tiers/builtin.ts` — the three default tiers (Q10) +```ts +export const BUILTIN_TIERS: Tier[] = [ + { name: "economy", models: ["Ollama/minimax-m3:cloud"] }, + { name: "standard", models: ["Ollama/glm-5.2:cloud", "Ollama/minimax-m3:cloud"] }, + { name: "frontier", models: ["anthropic/claude-sonnet-4", "Ollama/glm-5.2:cloud"], costCap: 5, contextFloor: 200000 }, +]; +``` + +### New: `src/tiers/resolve.ts` — `resolveAgentModel()` (the Q4 + Q5 logic, pure fn) +```ts +export function resolveAgentModel( + agent: AgentDef, optsModel: string | undefined, + parentModel: { provider: string; id: string }, + tiers: TierRegistry, modelRegistry: ModelRegistry, +): { model: string; tier?: Tier; candidates?: string[] } | { error: string }; +``` +- Precedence (Q4): `optsModel` > `agent.tier` (resolve) > `agent.model` > `${parentModel.provider}/${parentModel.id}`. +- If a tier: walk `tier.models[]`; for each, `splitModel(m)` → `modelRegistry.find(provider, id)` — skip if not found OR `(contextWindow ?? 0) < tier.contextFloor` (Q5; undefined → 0 → below any floor); collect eligible `candidates`. If none → `{ error: "tier 'X': no eligible model…" }`. Else → `{ model: candidates[0], tier, candidates }` (the full candidate list flows to `spawnSubagent`'s retry loop). +- The actual `create()` attempt + fallback-on-rejection happens in `spawnSubagent` (the async retry lives in the engine where `backend.factory` lives; the pure resolver is testable). + +### New: `src/tiers/tier-store.ts` — read/write `tiers.json` (project + global) +- `read(scope: "project" | "global"): Tier[]`, `write(scope, tiers: Tier[])`. The write path for the Tiers view's edit actions. +- `write` is atomic (write-temp + rename) + validates via `parseTiersFile` before writing (rejects duplicates/malformed → no file change). + +### Changed: `src/registry/frontmatter.ts` — add `tier?: string` to `AgentDef` (Q4) +- Parse `raw.tier` as a string (optional). One line in `parseAgentFile`. Backward-compatible (existing agents with `model:` unchanged). + +### Changed: `src/engine/run-registry.ts` — `RunRecord` gains 3 fields +```ts +costTotal?: number; // Q6: live $ accumulated per message_end (usage.cost.total) +contextTokens?: number; // Q8: live context tokens (calcContextTokens(usage)) +tier?: string; // the tier name this run used (for Tiers-view "used by" + per-tier spend) +``` + +### Changed: `src/engine/spawnSubagent.ts` — use `resolveAgentModel` + accumulate cost/context + cap check +- Replace the `opts.model ?? agentDef.model ?? parent` line with `resolveAgentModel(...)`. +- On `message_end` (assistant): accumulate `costTotal` (`usage.cost.total ?? 0`) + set `contextTokens` (`calcContextTokens(usage)`); `runRegistry.update(runId, { costTotal, contextTokens, tokenTotal })`. +- After the update, if `tier?.costCap && costTotal > tier.costCap`: set `aborted = true`, `void session.abort()`, `finishRun` reason = `"budget_exceeded"`. +- `runRegistry.add` includes `tier: resolvedTier?.name`. +- Widen the `RunLog` message event's `usage` to carry `cost?: { total?: number }` (additive — existing `total/input/output/cacheRead/cacheWrite` unchanged). +- The fallback retry: if `backend.factory.create()` rejects for `candidates[0]`, retry `candidates[1]`, then `[2]`, …; if all reject, `failRun("backend create failed: ")`. + +### Changed: `src/panel/widget-rows.ts` — `widgetLine` adds `ctx%` + `$` +- `WidgetRun` gains `contextTokens?: number`, `maxContext?: number`, `costTotal?: number`. +- After the `tok` segment: `const ctx = (r.contextTokens && r.maxContext) ? ` ${Math.round(r.contextTokens/r.maxContext*100)}%` : "";` + `const cost = r.costTotal ? ` $${r.costTotal.toFixed(4)}` : "";`. (`maxContext` threaded into `WidgetRun` from `modelRegistry.find` at render time — see data flow.) +- `toWidgetRun` projects the new fields from `RunRecord`. + +### Changed: `src/panel/fleet-widget.ts` — thread `maxContext` into the widget render +- `FleetWidgetController` gains an optional `getModelContextWindow: (model: string) => number | undefined` dep (wired to `modelRegistry.find(splitModel(model)).contextWindow` in `index.ts`). Called once per active run per render; cached on the `WidgetRun` projection so `renderWidgetLines` stays pure. + +### Changed: `src/panel/runs-rows.ts` — Runs tab adds `ctx%` + `$` columns (Q8) +- Same segments as the widget; final context% + cost for completed runs. `maxContext` resolved from `modelRegistry` via a dep. + +### New: `src/panel/tiers-rows.ts` — pure render fns for the Tiers view (Q9) +- `renderTierRow(tier, spend, usedBy, runCount)` → `name models[0]→fallback $cap ctxFloor $spend N runs used by: a,b`. +- Unit-tested (pure-renderer convention). + +### Changed: `src/panel/fleet-panel.ts` — new `tiers` View + action submenu (Q9) +- New tab `tiers` (after `scheduled`). Action submenu: Set models / Set costCap / Set contextFloor / Add tier / Delete tier + global/project scope toggle. Inline `Input` for values (reuses the 5b-1 resume/steer input pattern). Writes to the in-scope `tiers.json` via `TierStore`. +- "Used by" column is read-only (computed from the agent registry's `tier` field). + +### Changed: `src/index.ts` — wire the `TierRegistry` + `ModelRegistry` into `deps` + the panel +- Construct `TierRegistry` at `session_start` (discovers builtins + global + project). Pass `pi.modelRegistry` (for `resolveAgentModel` + `contextWindow` lookups) into `deps`. Add the `tiers` view to the panel. Wire `getModelContextWindow` into the `FleetWidgetController`. + +### Helper: `splitModel(model: string): { provider: string; id: string }` +- Splits on the FIRST `/`; a string with no `/` → `{ provider: parentModel.provider, id: model }` (back-compat for bare model ids). Lives in `src/tiers/resolve.ts` (or a tiny `src/tiers/model-string.ts`). + +### Helper: `calcContextTokens(usage): number` +- `usage.totalTokens || (input + output + cacheRead + cacheWrite)`. Tiny local helper mirroring pi's `calculateContextTokens` — fleet doesn't import pi-ai's compaction internals. Lives in `src/engine/spawnSubagent.ts` (alongside the token accumulator). + +**No new external deps. No vendored plumbing. No build step. No journal change beyond one `run:ended` field + the `message.usage.cost?` additive widening.** + +## 5. Data flow + +### Spawn-time tier resolution + fallback (Q4 + Q5) +``` +spawnSubagent(opts) + └─ resolveAgentModel(agent, opts.model, parentModel, tierRegistry, modelRegistry) + case opts.model (caller override, e.g. subagent tool's `model` param): + → { model: opts.model } // no tier, no cap/floor (one-off pin) + case agent.tier: + tier = tierRegistry.get(agent.tier) + if (!tier) → failRun(`tier '${agent.tier}' not found; available: …`) + for (const m of tier.models): + const { provider, id } = splitModel(m) // "Ollama/glm-5.2:cloud" → {Ollama, glm-5.2:cloud} + const model = modelRegistry.find(provider, id) + if (!model) continue // not in catalog → skip + if (tier.contextFloor && (model.contextWindow ?? 0) < tier.contextFloor) continue // undefined → 0 → below any floor + candidates.push(m) // eligible + if (candidates.length === 0) + → failRun(`tier '${tier.name}': no eligible model (all missing or below contextFloor ${tier.contextFloor})`) + → { model: candidates[0], tier, candidates } // primary candidate; fallback handled below + case agent.model → { model: agent.model } // no tier + else → { model: `${parentModel.provider}/${parentModel.id}` } // no tier + └─ runRegistry.add({…, model: resolved.model, tier: resolved.tier?.name, costTotal:0, contextTokens:0}) + └─ backend.factory.create({ model: resolved.model, … }) + └─ on rejection (provider unavailable / auth error / model not found): + if (resolved.tier && candidates.length > 1) → retry create() with candidates[1], then [2], … + else → failRun(`backend create failed: ${err.message}`) + └─ … rest of spawnSubagent unchanged (subscribe, prompt, finishRun) +``` +- The fallback chain (Q5) is two-phase: **catalog/contextFloor filter** (synchronous, in `resolveAgentModel`) → **create() attempt** (async, in `spawnSubagent`); if `create()` rejects, advance to the next pre-filtered candidate. This keeps the pure resolver testable + the async retry in the engine where `backend.factory` lives. + +### Runtime cost/context accumulation + cap enforcement (Q6 + Q7 + Q8) +``` +session.subscribe((e) => { + if (e.type === "message_end" && e.message?.role === "assistant") { + // existing: tokenTotal += turnTokens (5b-2) + // NEW: + costTotal += e.message.usage?.cost?.total ?? 0 + contextTokens = calcContextTokens(e.message.usage) // usage.totalTokens || input+output+cacheRead+cacheWrite + runRegistry.update(runId, { costTotal, contextTokens, tokenTotal }) + runLog?.append(runId, { type:"message", …, usage: { …, cost: e.message.usage?.cost } }) // widen the journal entry + // Q7 cap check: + const tier = resolvedTier + if (tier?.costCap && costTotal > tier.costCap) { + aborted = true; void session.abort(); // finishRun → status:"aborted", error:"budget_exceeded" + } + } +}) +``` +- `costTotal` is cumulative across the run's turns; `contextTokens` is a latest-snapshot (overwrites — it's the current context size, not a sum). + +### Render paths +``` +FleetWidgetController.render() (existing 1s tick + store-subscribe) + └─ for each active run: maxContext = getModelContextWindow(run.model) + widgetLine(r, now, maxContext) → "▶ fl-x agent 3s 142 tok 42% $0.0123" + // ctx% only when both contextTokens + maxContext present; $ only when costTotal > 0 + +Runs tab (runs-rows.ts) — final ctx% + $ for completed runs (same segments, read from RunRecord/RunLog) + +Tiers view (fleet-panel.ts "tiers" tab) + └─ renderTierRow(tier, spend=Σ runs by tier.name, usedBy=tierRegistry.usedBy(tier.name)) + → "standard Ollama/glm-5.2:cloud→Ollama/minimax-m3:cloud — — $0.00 3 runs used by: coder,scout" + → "frontier anthropic/claude-sonnet-4→Ollama/glm-5.2:cloud $5 200k $0.12 1 runs used by: oracle" + └─ action submenu: Set models / Set costCap / Set contextFloor / Add / Delete (scope: project|global) +``` +- The `maxContext` lookup in the widget is one `modelRegistry.find` per active run per render — cheap (handful of runs, the registry is in-memory). Cached on the `WidgetRun` projection so the pure renderer stays pure. + +## 6. Error handling + +| Failure | Behavior | +|---|---| +| `agent.tier` references a non-existent tier | `failRun` with "tier 'X' not found; available: economy, standard, frontier" (actionable, lists what exists) | +| All `tier.models` missing from catalog OR below `contextFloor` | `failRun` with "tier 'X': no eligible model (all missing or below contextFloor N)" — the run never starts; the todo reverts to prior status | +| `backend.factory.create()` rejects for `models[0]` | Retry `models[1]`, then `[2]`, …; if all reject, `failRun("backend create failed: ")` | +| `costTotal` exceeds `tier.costCap` mid-run | `aborted = true; session.abort()` → `finishRun("aborted", "budget_exceeded")`; the run's todo reverts; the Runs tab shows `✗ aborted` + the $ spent | +| `usage.cost.total` absent (backend omits cost — e.g. a flat-rate provider) | `costTotal += 0`; the `$` segment hidden; the cap never trips (honest — no cost data = no cap enforcement) | +| `modelRegistry.find()` returns undefined (model not in catalog) at render | `maxContext` undefined → `ctx%` segment hidden (no division by zero); the `$` + `tok` segments still show | +| `contextTokens` absent (first turn, no `message_end` yet) | `ctx%` hidden until the first assistant message lands | +| Tiers-view edit writes a `tiers.json` with a duplicate name | `parseTiersFile` rejects duplicates → `onNotify("tier 'X' already exists in this scope", "error")`; the write is aborted (no file change) | +| `tiers.json` is malformed | `parseTiersFile` throws `TierFileError` → the registry skips that scope (builtins/global still load) + `onNotify` at session_start | +| Bg/lifecycle phase runs | Same path — `runLifecycle`'s phase spawn passes `tier` via the phase def (Q4 per-phase); cost/context/cap apply per phase child; the lifecycle todo's progress block sums phase costs | + +## 7. Testing + +### Unit tests (node:test via tsx) — the fast gate +- **`tier-registry.test.mts`** — `parseTiersFile` (valid/empty/malformed/duplicate-name rejection), `mergeTiers` (builtins < global < project, project wins by name, global wins over builtins), `TierRegistry.get`/`list`/`usedBy` (usedBy computed from a fake agent registry's `tier` fields). +- **`resolve-model.test.mts`** — `resolveAgentModel`: `opts.model` beats `agent.tier` beats `agent.model` beats parent (Q4 precedence); tier not found → error; all models below `contextFloor` → error; `contextFloor` skips a too-small model + lands on the next (Q5); `modelRegistry.find` returns undefined → skip; no tier → returns `agent.model`/parent. `splitModel` (first-`/` split; no-`/` fallback; `:`-in-id case like `glm-5.2:cloud`). +- **`spawn-subagent-tier.test.mts`** (extend the runlog harness) — a fake backend whose `create()` rejects for `models[0]` + succeeds for `models[1]` → the run uses `models[1]` (Q5 fallback); `runRegistry.get(runId).tier` set; `costTotal` accumulates from `usage.cost.total` across two `message_end`s; `contextTokens` = last `calcContextTokens`; cap breach (`costCap: 0.001` + a message with `cost.total: 0.01`) → `aborted` + `error: "budget_exceeded"`. +- **`widget-rows.test.mts`** (extend) — `widgetLine` shows `42%` when `contextTokens` + `maxContext` present; hides `ctx%` when either absent; shows `$0.0123` when `costTotal > 0`; hides `$` when `costTotal` is 0/undefined. +- **`runs-rows.test.mts`** (extend) — Runs tab row shows final `ctx%` + `$` for a completed run. +- **`tiers-rows.test.mts`** — `renderTierRow` (models chain arrow, `$cap`/`—`, `ctxFloor`/`—`, spend, run count, used-by list). +- **`frontmatter.test.mts`** (extend) — `tier?: string` parses from frontmatter; absent → undefined (back-compat). +- **`fleet-panel-tiers.test.mts`** (extend the panel harness) — Tiers tab renders tier rows; Set costCap action → inline Input → writes `tiers.json` via a fake `TierStore` → re-render shows the new cap; Add/Delete tier; duplicate-name write → error notify + no file change; scope toggle project↔global. + +### Term-driven TUI smoke (the gate, per 5b-1..5b-4 carry-forward) +Fresh temp cwd with a project `tiers.json` defining a `frontier` tier (`costCap: 0.001` — intentionally tiny). Spawn a foreground subagent with `tier: frontier` on a task that burns >$0.001 (a few turns). Verify: +1. ✅ Above-editor widget shows `ctx%` + `$` live (ticking up each turn). +2. ✅ When `costTotal` crosses `0.001` → row flips to `✗ aborted` + `run aborted: budget_exceeded` notify. +3. ✅ `/fleet` → Tiers tab shows the `frontier` row with `costCap $0.001` + the aborted run's $ spend in the per-tier total. +4. ✅ Tiers tab Set costCap action → change to `$1` → re-render reflects it; the `tiers.json` file on disk has the new value. +5. ✅ Runs tab shows the aborted run with final `ctx%` + `$`. +6. ✅ A run with no tier (`agent.model` path) shows `tok` + `ctx%` + `$` (cost still tracked) but no cap enforcement. + +**Budget tokens for the smoke:** the tiny `costCap` trips on the first real turn → one short run is enough (cheaper than 5b-4's 3-run smoke). The real-pi smoke also verifies the Ollama-Cloud `costTotal: 0` behavior is displayed cleanly (no `$` segment, no false cap trip) on a no-tier run. + +**~+22 tests on top of 311. typecheck clean. Release `@getpipher/armory-fleet@0.10.0`** (minor: new tier layer + cost/context surfaces + Tiers view; `AgentDef.tier?` + `RunRecord` fields are additive — no breaking API; the `RunLog message.usage` gains `cost?` additively). + +## 8. Scope boundaries (anti-gold-plating) + +**In 6-1:** `Tier` type + `TierRegistry` + `builtin.ts` + `resolveAgentModel` + `tier-store.ts`; `AgentDef.tier?`; `RunRecord.costTotal`/`contextTokens`/`tier`; `spawnSubagent` tier resolution + fallback + cost/context accumulation + cap abort; `widget-rows` `ctx%`+`$`; `runs-rows` `ctx%`+`$`; `tiers-rows` + the Tiers panel view + action submenu; `RunLog message.usage.cost?`; unit + term smoke. + +**Deferred (to later 6-x or explicitly out):** +- Per-agent / per-tier aggregate cost *stores* (Q6=A — computed on-the-fly; no `costByAgent` Map). +- Per-tier aggregate spawn-gating / session budgets (Q7=D — the cap is per-run only). +- Warn-at-80%-of-cap (Q7=C — the abort-at-cap is the first cut; the warn is a trivial follow-up). +- Per-agent tier assignment from the Tiers view (Q9=C — agent `.md` frontmatter is edited by hand / an agent-editor, not fleet; the Tiers view shows the assignment read-only). +- Concurrency limits + worktree policy in the Tiers view (Q9=D — SPEC-5a settings; stay in settings). +- Live conversation viewer / live-scrolling (SPEC-6-4 — Q2=a placement). +- `queue_update` "steer queued" indicator (5b-4 polish — SPEC-6-4). +- Quality gates / workflows-as-code / RPC (6-2/6-3/6-4). + +## 9. Risks + +- **`modelRegistry.find()` provider/modelId split.** Fleet's model strings are `"Ollama/glm-5.2:cloud"` — split on the FIRST `/` (`provider="Ollama"`, `id="glm-5.2:cloud"`). A model id containing `/` (rare) would mis-split. Mitigation: `splitModel` splits on first `/` only; a string with no `/` is treated as `{ provider: parent.provider, id: model }` (back-compat for bare model ids). The plan adds a test for the `:`-in-id case (`glm-5.2:cloud`). +- **`contextFloor` makes `frontier` refuse to spawn if the only available model is below the floor.** This is intentional (honest "no frontier model available" > silent downgrade) but could surprise a user without Anthropic auth whose only model is `Ollama/glm-5.2:cloud` (contextWindow unknown until queried). Mitigation: the error message names the floor + the rejected model's window; the Tiers view shows each model's `contextWindow` inline so the user sees why. **The plan verifies `modelRegistry.find("Ollama","glm-5.2:cloud")?.contextWindow` returns a real number** (not undefined) — if Ollama Cloud models don't report a window, `contextFloor` is unenforceable for them + the check degrades to "skip models with undefined contextWindow" (treat undefined as 0 → below any floor). This is the one implementation detail to probe first in the plan. +- **`usage.cost.total` for flat-rate providers (Ollama Cloud) may always be 0.** Then `costTotal` stays 0, the `$` segment hides, and the cap never trips — honest (no cost data). The `frontier` tier (metered Anthropic) is where cost/cap actually bite. The smoke uses a tiny `costCap` + a metered-model fake to verify the abort path; the real-pi smoke verifies the Ollama-Cloud `costTotal: 0` behavior is displayed cleanly (no `$` segment, no false cap trip). +- **`RunLog message.usage` widening is additive** (gains `cost?`). Existing 5b-1/5b-2 run-log tests that assert the `usage` shape need a `cost?` field added to their expected objects — the plan audits + updates them. No breaking consumer API (`replay`/rows read `usage.total` for tokens, unchanged). +- **Tiers-view writes to `tiers.json`** are a new write surface for fleet (it's been read-only on the filesystem except `.pi/fleet/` runtime artifacts). The `TierStore.write` is atomic (write-temp + rename) + validates via `parseTiersFile` before writing (rejects duplicates/malformed → no file change). The plan verifies the write doesn't corrupt a concurrent manual edit (last-write-wins is the documented contract, like git). +- **The `candidates` list must flow from `resolveAgentModel` to `spawnSubagent`'s retry loop.** The resolver returns `{ model, tier, candidates?: string[] }` so the engine can retry `candidates[1..]` on `create()` rejection. If a future refactor drops `candidates` from the return, the fallback-on-create-failure path silently breaks. Mitigation: a unit test asserts the fallback retry happens (the `spawn-subagent-tier.test.mts` fake-backend case). + +## 10. References + +- PRD §8 (SPEC-6 line — "Power-user tier → v1.0: cost-aware model tiers, real cost accounting, quality patterns, workflows-as-code, event-bus + cross-extension RPC"), §5 (interactive-first panel — the Tiers view), §6 (scope — the v1.0 success bar) +- pi-ai `Usage` type (`{ input, output, cacheRead, cacheWrite, totalTokens, cost: { total } }`): `…/pi-coding-agent/node_modules/@earendil-works/pi-ai/` (compat + compaction source) — `usage.cost.total` is $ per assistant message, computed by pi-ai per-model +- pi `ModelRegistry` (`pi.modelRegistry.find(provider, modelId).contextWindow`): `…/pi-coding-agent/dist/core/model-registry.d.ts` — `ModelRegistry` is on the `ExtensionAPI` +- pi `calculateContextTokens(usage)` (the context% numerator definition): `…/pi-coding-agent/dist/core/compaction/compaction.d.ts` — `usage.totalTokens || input+output+cacheRead+cacheWrite` +- SPEC-5b-4 (`docs/superpowers/specs/2026-07-27-spec-5b-4-design.md`) — the retained-handle seam (unchanged) + the abort path (`aborted = true; session.abort()`) the cap enforcement reuses +- SPEC-5b-2 (`docs/superpowers/specs/2026-07-26-spec-5b-2-design.md`) — `RunRecord.tokenTotal` (the 5b-2 accumulator 6-1 parallels) + the token-unit Q9 fix (the `cost.total` fleet switched off, now read back honestly as $) +- v0.9.2 patch (PR #11) — the single-widget glance surface 6-1's `ctx%`+`$` extends +- AGENTS.md LLM Backend convention: "Ollama Cloud (flat-rate, private, open-weight) as primary + OpenRouter fallback; when correctness/agentic-accuracy IS the product (forensic/financial/audited work), escalate to the proper frontier model" — the builtin tiers' intent +- getpipher conventions: `~/local-dev/getpipher/AGENTS.md` (interactive-first, EditorTheme gotcha, no AI attribution, `--test-timeout=30000` in `test:run`) \ No newline at end of file diff --git a/test/widget-lint.test.mts b/test/widget-lint.test.mts new file mode 100644 index 0000000..f54842c --- /dev/null +++ b/test/widget-lint.test.mts @@ -0,0 +1,75 @@ +// test/widget-lint.test.mts +// SPEC-5b-2 regression guard (added in the post-v0.9.2 widget-lint follow-up). +// +// The stale-widget bug class: a `ctx.ui.setWidget(key, [...])` call that sets a +// widget but never clears it (`setWidget(key, undefined)`) on completion/dispose. +// This bit fleet twice: +// - SPEC-1 (commit 26454af): the subagent tool set `setWidget("fleet", ["▶ … · running"])` +// on every turn_end and never cleared it → stale "▶ general-purpose · running" lingered +// above the editor after the run finished. Removed in v0.9.2 (PR #11). +// - SPEC-5b-2: a below-editor `fleet-view` widget duplicated the above-editor widget +// (redundant, not stale, but the same "widget proliferation" smell). Removed in v0.9.2. +// +// The architectural invariant that prevents the stale-widget class: **all `setWidget` +// calls go through `FleetWidgetController`** (src/panel/fleet-widget.ts), which owns a +// disciplined set/clear lifecycle (set on active, clear on idle + dispose). Any `setWidget` +// call elsewhere is a smell — it bypasses the lifecycle and risks never being cleared. +// +// This test enforces the invariant by scanning src/ for `.setWidget(` call sites and +// asserting they only appear in fleet-widget.ts. If a future feature needs a widget, it +// must add it to the controller (or a sibling controller with the same set/clear +// discipline), not call setWidget ad-hoc. + +import { test } from "node:test"; +import { strictEqual, fail } from "node:assert"; +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; + +const SRC = join(import.meta.dirname, "..", "src"); +const ALLOWED_FILE = join("panel", "fleet-widget.ts"); // the single sanctioned setWidget surface + +function walk(dir: string, out: string[] = []): string[] { + for (const name of readdirSync(dir)) { + const p = join(dir, name); + if (statSync(p).isDirectory()) walk(p, out); + else if (name.endsWith(".ts")) out.push(p); + } + return out; +} + +test("setWidget call sites only appear in fleet-widget.ts (stale-widget regression guard)", () => { + const offenders: string[] = []; + for (const file of walk(SRC)) { + const rel = relative(SRC, file); + if (rel === ALLOWED_FILE) continue; + const src = readFileSync(file, "utf8"); + // Match `.setWidget(` — catches ctx.ui.setWidget, this.deps.ui.setWidget, etc. + // Ignore commented-out lines (a // line that mentions setWidget is not a call). + for (const line of src.split("\n")) { + const trimmed = line.trim(); + if (trimmed.startsWith("//")) continue; + if (/\.setWidget\s*\(/.test(line)) { + offenders.push(`${rel}: ${trimmed.slice(0, 80)}`); + } + } + } + if (offenders.length > 0) { + fail( + `setWidget called outside fleet-widget.ts (the single sanctioned widget surface).\n` + + `This is the stale-widget bug class — a setWidget without a matching clear on completion.\n` + + `Add the widget to FleetWidgetController (src/panel/fleet-widget.ts) instead, which owns\n` + + `the set/clear lifecycle (set on active, clear on idle + dispose).\n` + + `Offenders:\n - ` + offenders.join("\n - "), + ); + } + strictEqual(offenders.length, 0, "no ad-hoc setWidget calls outside the controller"); +}); + +test("FleetWidgetController sets exactly one widget key (no widget proliferation)", () => { + const src = readFileSync(join(SRC, ALLOWED_FILE), "utf8"); + // The controller should declare exactly one WIDGET_KEY const + use it. + // (v0.9.2 removed the second VIEW_KEY/"fleet-view" below-editor widget.) + const keyConsts = src.match(/const\s+\w*KEY\w*\s*=\s*"[^"]+"/g) ?? []; + strictEqual(keyConsts.length, 1, `expected exactly one widget-key const, found: ${keyConsts.join(", ")}`); + strictEqual(keyConsts[0], 'const WIDGET_KEY = "fleet-active"', `the one key is fleet-active`); +}); \ No newline at end of file