diff --git a/README.md b/README.md index 0335c3f..7e3b62a 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,9 @@ The keys are the same either way: | `alwaysShow` | `false` | Keep the meter visible when idle instead of hiding it | | `idleText` | `"idle"` | Placeholder shown with `alwaysShow` when there is no data (`tok/s idle`) | | `label` | `"tok/s"` | Leading label | +| `llamaSwapUrl` | `"http://127.0.0.1:8080"` | V2: llama-swap base URL used to read live decode counts for local models | +| `llamaSwapProviders` | `[]` | V2: provider IDs served by [llama-swap](https://github.com/mostlygeek/llama-swap); their live decode counts are polled via `/upstream//slots`. Off when empty | +| `llamaSwapPollMs` | `250` | V2: how often the llama-swap slot counter is read during an execution | ## How it works @@ -187,7 +190,8 @@ generation's event bus into the same normalized events. **OpenCode V2** (`src/v2.ts`, installed via `plugins`, renders in `prompt.footer.status`): - `session.step.started` — the first step of an execution opens the assistant turn and records its start time (for TTFT). Later steps in the same tool loop keep that graph. -- `session.text.delta` / `session.reasoning.delta` — record streamed output and bucket it per second. +- `session.text.delta` / `session.reasoning.delta` / `session.tool.input.delta` — record streamed output, including tool-call arguments such as a file body, and bucket it per second. +- llama-swap models (`llamaSwapProviders`) — llama-server holds a tool call back until it is complete, so OpenCode emits no events while a local model writes one. From step start to execution end the meter polls llama-server's per-slot `n_decoded` counter through llama-swap and records the exact decoded tokens; event deltas are only used when llama-swap does not answer. - `session.tool.called` — restarts the live rate when a tool runs, keeping the graph continuous across the gap. - `session.step.ended` / `session.step.failed` — accumulate the provider's exact token usage per step, including prompt-cache reads and writes. - `session.execution.succeeded` / `.failed` / `.interrupted` — finish the turn with the summed tokens and cache hit rate, and freeze the final frame. diff --git a/src/tui.tsx b/src/tui.tsx index 6c3c4e2..1c219f8 100644 --- a/src/tui.tsx +++ b/src/tui.tsx @@ -139,7 +139,10 @@ function MeterLine(props: { return ( - {text()} + {/* Never wrap: a squeezed footer would split the line over two rows. */} + + {text()} + ) } diff --git a/src/v2.ts b/src/v2.ts index 8602572..e4c4c05 100644 --- a/src/v2.ts +++ b/src/v2.ts @@ -7,6 +7,8 @@ * by the step/execution lifecycle): * session.step.started -> turn (first step of an execution only; later steps keep the graph) * session.text/reasoning.delta-> delta (streamed output) + * session.tool.input.delta -> delta (streamed tool arguments, e.g. a file body) + * (llama-swap models) -> delta (polled /slots decode counter, step start to execution end) * session.tool.called -> segment (tool pause; the graph stays continuous) * session.step.ended/failed -> (accumulate exact provider tokens) * session.execution.* -> done (finish the turn with the summed tokens) @@ -31,17 +33,93 @@ type StepTokens = { /** Running provider-reported usage for one session's execution. */ type Usage = { output: number; input: number; cacheRead: number; cacheWrite: number } +/** Shape of one entry of llama-server's `/slots` response (only the fields used). */ +type SlotInfo = { is_processing?: boolean; id_task?: number; next_token?: Array<{ n_decoded?: number }> } + +/** Injectable side effects, so tests can fake llama-swap. */ +export type WireV2Deps = { fetch: typeof fetch; now: () => number } + /** * Wire V2 session events into the controller. Returns an unsubscribe function * that `setup` hands back to OpenCode for disposal. */ -export function wireV2(ctx: Context, controller: MeterController, onChange: () => void): () => void { + +export function wireV2( + ctx: Context, + controller: MeterController, + onChange: () => void, + deps: WireV2Deps = { fetch: (...args) => fetch(...args), now: () => Date.now() }, +): () => void { const exact = new Map() const compacting = new Set() // Sessions whose current execution already opened a meter turn. A tool loop // emits a new assistant message per step; opening again would wipe the graph. const opened = new Set() const offs: Array<() => void> = [] + const models = new Map() + // llama-server holds a tool call back until it is complete, so OpenCode + // emits no events at all while a local model writes e.g. a file body. For + // llama-swap models the meter therefore reads llama-server's live per-slot + // decode counter (proxied at /upstream//slots) for the whole + // execution and uses it instead of the event deltas while it answers. + const opts = (ctx.options ?? {}) as { llamaSwapUrl?: string; llamaSwapProviders?: string[]; llamaSwapPollMs?: number } + const swapUrl = (opts.llamaSwapUrl ?? "http://127.0.0.1:8080").replace(/\/$/, "") + const swapProviders = opts.llamaSwapProviders ?? [] + const pollMs = opts.llamaSwapPollMs ?? 250 + type Poll = { timer: ReturnType; task?: number; decoded?: number; okAt?: number } + const polls = new Map() + + const stopPoll = (sessionID: string) => { + const poll = polls.get(sessionID) + if (!poll) return + clearInterval(poll.timer) + polls.delete(sessionID) + } + + const pollOnce = async (sessionID: string, modelID: string, poll: Poll) => { + try { + const res = await deps.fetch(`${swapUrl}/upstream/${encodeURIComponent(modelID)}/slots`, { + signal: AbortSignal.timeout(Math.max(200, pollMs * 2)), + }) + if (!res.ok || polls.get(sessionID) !== poll) return + poll.okAt = deps.now() + const slots = (await res.json()) as SlotInfo[] + const slot = Array.isArray(slots) ? slots.find((x) => x.is_processing) : undefined + const decoded = slot?.next_token?.[0]?.n_decoded + if (!slot || typeof decoded !== "number" || polls.get(sessionID) !== poll) return + // The first reading only sets the baseline (the request may be under way); + // a later request on the slot (the next step of a tool loop) starts at zero. + if (poll.task === undefined) { + poll.task = slot.id_task + poll.decoded = decoded + return + } + if (poll.task !== slot.id_task || poll.decoded === undefined || decoded < poll.decoded) { + poll.task = slot.id_task + poll.decoded = 0 + } + const tokens = decoded - poll.decoded + poll.decoded = decoded + // The meter estimates tokens as UTF-8 bytes / 4, so 4 bytes per token records the exact count. + if (tokens > 0) emit({ kind: "delta", sessionID, text: "xxxx".repeat(tokens), at: deps.now() }) + } catch { + // llama-swap down or the model swapped out: the meter just shows no live rate. + } + } + + const startPoll = (sessionID: string) => { + const model = models.get(sessionID) + if (!model || !swapProviders.includes(model.providerID) || polls.has(sessionID)) return + const poll: Poll = { timer: setInterval(() => void pollOnce(sessionID, model.id, poll), pollMs) } + polls.set(sessionID, poll) + void pollOnce(sessionID, model.id, poll) + } + + /** Whether llama-swap answered recently, so its counter replaces event deltas. */ + const polled = (sessionID: string) => { + const okAt = polls.get(sessionID)?.okAt + return okAt !== undefined && deps.now() - okAt < 1000 + } const emit = (event: MeterEvent) => { if (controller.handle(event)) onChange() @@ -72,6 +150,7 @@ export function wireV2(ctx: Context, controller: MeterController, onChange: () = } const finish = (sessionID: string) => { + stopPoll(sessionID) opened.delete(sessionID) // A compaction in flight owns the session's meter. Settling here would // rescale the graph twice, but this execution's totals must not be added @@ -88,6 +167,8 @@ export function wireV2(ctx: Context, controller: MeterController, onChange: () = offs.push( ctx.data.on("session.step.started", (event) => { const sessionID = event.data.sessionID + models.set(sessionID, { providerID: event.data.model.providerID, id: event.data.model.id }) + startPoll(sessionID) if (opened.has(sessionID)) return opened.add(sessionID) emit({ @@ -102,10 +183,14 @@ export function wireV2(ctx: Context, controller: MeterController, onChange: () = const onDelta = (sessionID: string, delta: string) => { if (!delta) return if (ctx.data.session.status(sessionID) === "idle") return + if (polled(sessionID)) return emit({ kind: "delta", sessionID, text: delta, at: Date.now() }) } offs.push(ctx.data.on("session.text.delta", (event) => onDelta(event.data.sessionID, event.data.delta))) offs.push(ctx.data.on("session.reasoning.delta", (event) => onDelta(event.data.sessionID, event.data.delta))) + // Tool-call arguments (e.g. a file body for write/edit) stream as their own + // deltas; without them the meter drops to "-" during most of an agent run. + offs.push(ctx.data.on("session.tool.input.delta", (event) => onDelta(event.data.sessionID, event.data.delta))) offs.push( ctx.data.on("session.tool.called", (event) => { @@ -164,6 +249,7 @@ export function wireV2(ctx: Context, controller: MeterController, onChange: () = offs.push( ctx.data.on("session.deleted", (event) => { + stopPoll(event.data.sessionID) exact.delete(event.data.sessionID) compacting.delete(event.data.sessionID) opened.delete(event.data.sessionID) @@ -172,6 +258,7 @@ export function wireV2(ctx: Context, controller: MeterController, onChange: () = ) return () => { + for (const sessionID of [...polls.keys()]) stopPoll(sessionID) for (const off of offs) off() } } diff --git a/test/v2.test.ts b/test/v2.test.ts index 40e7ff9..c9b2940 100644 --- a/test/v2.test.ts +++ b/test/v2.test.ts @@ -6,11 +6,12 @@ import { wireV2 } from "../src/v2.ts" type Handler = (event: unknown) => void -function harness() { +function harness(options: Record = {}) { const handlers = new Map() let status: "idle" | "running" = "running" const ctx = { + options, data: { on(type: string, handler: Handler) { handlers.set(type, handler) @@ -348,8 +349,134 @@ describe("wireV2", () => { const h = harness() const controller = createMeterController(CONFIG) const dispose = wireV2(h.ctx, controller, () => {}) - assert.equal(h.handlers.size, 14) + assert.equal(h.handlers.size, 15) dispose() assert.equal(h.handlers.size, 0) }) }) + +const localStepStarted = (sessionID: string, messageID: string, started: number) => ({ + data: { sessionID, assistantMessageID: messageID, agent: "a", model: { providerID: "llamaswap", id: "local-model" }, started }, +}) + +const toolInput = (sessionID: string) => ({ data: { sessionID, assistantMessageID: "m1", id: "t1", name: "write" } }) + +/** A fake llama-swap whose slot decodes `step` more tokens on each /slots read. */ +function fakeSwap(step: number) { + let decoded = 0 + const urls: string[] = [] + const fetch = (async (url: string) => { + urls.push(url) + decoded += step + return new Response(JSON.stringify([{ id: 0, is_processing: true, id_task: 7, next_token: [{ n_decoded: decoded }] }])) + }) as unknown as typeof globalThis.fetch + return { fetch, urls } +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +describe("wireV2 tool input and llama-swap", () => { + it("counts streamed tool-call arguments", () => { + const h = harness() + const controller = createMeterController(CONFIG) + wireV2(h.ctx, controller, () => {}) + h.emit("session.tool.input.delta", { data: { sessionID: "s1", assistantMessageID: "m1", id: "t1", delta: "abcdefgh" } }) + assert.equal(controller.store.snapshot("s1", Date.now(), 4)?.tokens, 2) + }) + + it("polls llama-swap slots for a local model's whole execution", async () => { + const h = harness({ llamaSwapPollMs: 20, llamaSwapProviders: ["llamaswap"] }) + const swap = fakeSwap(10) + const controller = createMeterController(CONFIG) + const dispose = wireV2(h.ctx, controller, () => {}, { fetch: swap.fetch, now: () => Date.now() }) + h.emit("session.step.started", localStepStarted("s1", "m1", Date.now())) + await sleep(150) + const reads = swap.urls.length + const snapshot = controller.store.snapshot("s1", Date.now(), 4) + // The first read is the baseline; every later read adds exactly 10 tokens. + assert.ok(reads >= 3, `expected several polls, got ${reads}`) + assert.ok((snapshot?.tokens ?? 0) >= (reads - 2) * 10 && (snapshot?.tokens ?? 0) <= reads * 10) + assert.ok((snapshot?.rate ?? -1) > 0) + assert.equal(swap.urls[0], "http://127.0.0.1:8080/upstream/local-model/slots") + h.emit("session.execution.succeeded", executionSucceeded("s1")) + const stopped = swap.urls.length + await sleep(60) + assert.equal(swap.urls.length, stopped, "polling stops when the execution ends") + dispose() + }) + + it("prefers the slot counter over event deltas while llama-swap answers", async () => { + const h = harness({ llamaSwapPollMs: 20, llamaSwapProviders: ["llamaswap"] }) + const swap = fakeSwap(0) + const controller = createMeterController(CONFIG) + const dispose = wireV2(h.ctx, controller, () => {}, { fetch: swap.fetch, now: () => Date.now() }) + h.emit("session.step.started", localStepStarted("s1", "m1", Date.now())) + await sleep(50) + h.emit("session.text.delta", textDelta("s1", "abcdefgh")) + assert.equal(controller.store.snapshot("s1", Date.now(), 4)?.tokens ?? 0, 0) + dispose() + }) + + it("counts a new request on the slot from zero", async () => { + const h = harness({ llamaSwapPollMs: 20, llamaSwapProviders: ["llamaswap"] }) + const replies = [ + { id_task: 1, n_decoded: 500 }, + { id_task: 1, n_decoded: 520 }, + { id_task: 2, n_decoded: 15 }, + ] + let i = 0 + const fetch = (async () => { + const reply = replies[Math.min(i++, replies.length - 1)]! + return new Response(JSON.stringify([{ is_processing: true, id_task: reply.id_task, next_token: [{ n_decoded: reply.n_decoded }] }])) + }) as unknown as typeof globalThis.fetch + const controller = createMeterController(CONFIG) + const dispose = wireV2(h.ctx, controller, () => {}, { fetch, now: () => Date.now() }) + h.emit("session.step.started", localStepStarted("s1", "m1", Date.now())) + await sleep(55) + dispose() + // 500 is the baseline, +20 on the same task, then +15 for the new task. + assert.equal(controller.store.snapshot("s1", Date.now(), 4)?.tokens, 35) + }) + + it("does not poll unless llama-swap providers are configured", async () => { + const h = harness({ llamaSwapPollMs: 20 }) + const swap = fakeSwap(10) + const controller = createMeterController(CONFIG) + const dispose = wireV2(h.ctx, controller, () => {}, { fetch: swap.fetch, now: () => Date.now() }) + h.emit("session.step.started", localStepStarted("s1", "m1", Date.now())) + await sleep(60) + assert.equal(swap.urls.length, 0) + dispose() + }) + + it("does not poll for remote providers", async () => { + const h = harness({ llamaSwapPollMs: 20, llamaSwapProviders: ["llamaswap"] }) + const swap = fakeSwap(10) + const controller = createMeterController(CONFIG) + const dispose = wireV2(h.ctx, controller, () => {}, { fetch: swap.fetch, now: () => Date.now() }) + h.emit("session.step.started", stepStarted("s1", "m1", Date.now())) + await sleep(60) + assert.equal(swap.urls.length, 0) + dispose() + }) + + it("falls back to event deltas when llama-swap is down, and stops polling on dispose", async () => { + const h = harness({ llamaSwapPollMs: 20, llamaSwapProviders: ["llamaswap"] }) + let calls = 0 + const fetch = (async () => { + calls += 1 + throw new Error("ECONNREFUSED") + }) as unknown as typeof globalThis.fetch + const controller = createMeterController(CONFIG) + const dispose = wireV2(h.ctx, controller, () => {}, { fetch, now: () => Date.now() }) + h.emit("session.step.started", localStepStarted("s1", "m1", Date.now())) + await sleep(60) + h.emit("session.text.delta", textDelta("s1", "abcdefgh")) + assert.equal(controller.store.snapshot("s1", Date.now(), 4)?.tokens, 2) + dispose() + const after = calls + await sleep(60) + assert.ok(after > 0) + assert.equal(calls, after) + }) +})