diff --git a/CHANGES.md b/CHANGES.md index a2d5c51..5f7558a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,54 @@ ## Unreleased +### Fixed — pairing a phone rotated the box token and the dashboard chat plugin never re-read it (DIVE-3810), dashboard 0.4.2 + +`plugins/dashboard/server.ts` read `CONNECTORD_TOKEN` once at module scope and held it for the life +of the process. `/etc/5dive/connectord.env` is rewritten while the agent is running — pairing a phone +does it — and from that instant the plugin held a dead credential for all three control-plane calls: +`GET /server/messages/pending` (collect), `POST /server/messages/pending/ack`, and +`POST /server/messages/event` (the agent's outbound reply). Chat died in BOTH directions and stayed +dead until something restarted the agent. + +Measured on glossy-flint 2026-08-29: the env file's mtime was 17:23:20, the plugin had booted at +16:19:54.9, and the on-disk token was good (a hand `curl` with it returned 200). Five messages sent +at 17:30 never reached the transcript and all five stayed `delivered_at IS NULL` — nothing was lost, +nothing was falsely stamped. Buzz on the same box and the same claude process kept working +throughout, because it does not use this token, so the transport was never implicated. +`5dive agent restart` at 17:36:48 drained pending 5 → 0 in fifteen seconds. + +The token is now mutable and every call goes through one `authedFetch`, which on a 401/403 re-reads +`/etc/5dive/connectord.env` and retries once — but only if the token actually changed, so a +genuinely revoked credential cannot spin. `authedFetch` is the only `fetch(` and the only +`authorization` header in the file, so no call site can hold a stale credential. The ack path now +checks `res.ok`; it used to print "healed N" after a 401. + +**The failure also had no surface.** The single signal was one stderr line inside an MCP stdio +socket, written to no file on the box, while the control plane looked healthy — the messages sit in +`/pending` exactly as an ordinary undelivered queue does. `lifecycle.ts` gains an `auth` event: a +channel whose credential died mid-run is neither started, exited nor crashed, so without an event of +its own that state was recorded as `nothing`, which is what that file exists to refuse. One record +per episode, one on recovery, on disk. + +The other seven plugins take the `LifecycleEvent` union widening and its comment and nothing else — +a type-only change with no runtime behaviour, so their versions are deliberately unchanged. All +eight copies of `lifecycle.ts` stay byte-identical. + +Graded by `test/dashboard-token-rotation.test.ts`, which drives the real `server.ts` as a subprocess +against a stub control plane that 401s a stale bearer and rotates the token file under the running +process: inbound recovers with no restart, the reply tool recovers too (the mute half), the record +appears exactly once per episode, and all three call sites are asserted to route through the helper. +Deleting the reload+retry reds three of the four arms. + +Not covered: whether pairing should rotate this token at all is a control-plane question and is not +in this change; blast radius across the fleet is unmeasured (no prod database access from this +seat), and restarting an agent re-reads the file and remains the safe interim sweep. One residual is +stated in the PR: the `STATE_DIR/.env` loader copies `CONNECTORD_TOKEN` into `process.env` before the +file is read, and an env-set token is deliberately never reloaded. No live box is affected: nothing +in the provision or agent-create path writes `CONNECTORD_TOKEN` into that `.env`, the installer +writes the box token once to `/etc/5dive/connectord.env` and shelld rotates it there, so that branch +is only ever taken by a test or a deliberate off-box run — but a seat that ever acquires a non-empty +`CONNECTORD_TOKEN` in that `.env` is back in the original bug with no signal. + ### Fixed — the orphan watchdog is installed in every plugin, and its load-bearing clause could not fire (DIVE-3752), telegram 0.5.49 · buzz 0.1.2 · dashboard 0.4.1 Two defects, one of which was hiding inside the remedy for the other. diff --git a/plugins/buzz/lifecycle.ts b/plugins/buzz/lifecycle.ts index 4fa3dd3..2a0cd45 100644 --- a/plugins/buzz/lifecycle.ts +++ b/plugins/buzz/lifecycle.ts @@ -104,7 +104,11 @@ export function isBootParentAlive(pid: number): boolean { // ── the record: a channel start failure must not be encoded as `nothing` ──── -export type LifecycleEvent = 'start' | 'exit' | 'crash' +// DIVE-3810 added 'auth': a channel whose CREDENTIAL died mid-run is neither +// started, exited nor crashed — the process is healthy and every other +// surface reads healthy with it — so without an event of its own that state +// is recorded as `nothing`, which is the failure this file exists to refuse. +export type LifecycleEvent = 'start' | 'exit' | 'crash' | 'auth' /** * One line, one event, parseable and human-readable. diff --git a/plugins/dashboard/.claude-plugin/plugin.json b/plugins/dashboard/.claude-plugin/plugin.json index 07a4c0a..1992235 100644 --- a/plugins/dashboard/.claude-plugin/plugin.json +++ b/plugins/dashboard/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "dashboard", "description": "5dive dashboard chat channel \u2014 talk to this agent from the web dashboard (and mobile app). Native-push inbound via the box agent-inbox drop-dir, outbound replies via the control-plane messages API.", - "version": "0.4.1", + "version": "0.4.2", "author": { "name": "5dive", "email": "support@5dive.com" diff --git a/plugins/dashboard/lifecycle.ts b/plugins/dashboard/lifecycle.ts index 4fa3dd3..2a0cd45 100644 --- a/plugins/dashboard/lifecycle.ts +++ b/plugins/dashboard/lifecycle.ts @@ -104,7 +104,11 @@ export function isBootParentAlive(pid: number): boolean { // ── the record: a channel start failure must not be encoded as `nothing` ──── -export type LifecycleEvent = 'start' | 'exit' | 'crash' +// DIVE-3810 added 'auth': a channel whose CREDENTIAL died mid-run is neither +// started, exited nor crashed — the process is healthy and every other +// surface reads healthy with it — so without an event of its own that state +// is recorded as `nothing`, which is the failure this file exists to refuse. +export type LifecycleEvent = 'start' | 'exit' | 'crash' | 'auth' /** * One line, one event, parseable and human-readable. diff --git a/plugins/dashboard/server.ts b/plugins/dashboard/server.ts index 6f25c43..5b29384 100644 --- a/plugins/dashboard/server.ts +++ b/plugins/dashboard/server.ts @@ -30,7 +30,7 @@ import { import { readFileSync, mkdirSync, readdirSync, unlinkSync, watch, chmodSync, copyFileSync } from 'fs' import { homedir } from 'os' import { join } from 'path' -import { installLifecycle } from './lifecycle.ts' +import { installLifecycle, recordLifecycle } from './lifecycle.ts' let PLUGIN_VERSION = '?' try { @@ -70,17 +70,41 @@ const OUTBOX_DIR = process.env.DASHBOARD_OUTBOX ?? '/home/claude/chat-downloads' // The box's connectord token authenticates outbound replies to the control // plane. Standard location is /etc/5dive/connectord.env (root:claude 640; // agent users are in the claude group). Env/.env override for tests. +// +// DIVE-3810: this file is REWRITTEN UNDER US while the agent runs — pairing a +// phone rotates the box token (shelld's /shell/rotate-token does the line +// surgery). A token read once at module scope therefore outlives the rotation +// that invalidates it, and from that instant the channel is deaf AND mute: all +// three calls below carry a dead credential. So `TOKEN` is mutable and gets +// re-read on rejection. shelld itself already treats it this way +// (`let TOKEN` + rotate-in-place); this plugin was the reader that did not. +const TOKEN_FILE = process.env.CONNECTORD_ENV_FILE ?? '/etc/5dive/connectord.env' function loadConnectordToken(): string { + // An explicit env override stays authoritative and is never reloaded: it is + // set by a test or an off-box run, and nothing rotates it. + // + // RESIDUAL, stated rather than fixed: the ENV_FILE loader above copies + // CONNECTORD_TOKEN out of ~/.claude/channels/dashboard/.env into process.env + // BEFORE this runs, so a token that arrives that way is read here as an + // override and is never reloaded — that seat is back in the DIVE-3810 bug + // with no signal. No live box is affected today: pairing rotates the FILE, + // and nothing in the provision or agent-create path writes CONNECTORD_TOKEN + // into that .env at all (agent-create passes the dashboard channel an EMPTY + // token on purpose, DIVE-841) — the box token is written once to + // /etc/5dive/connectord.env by the installer and rotated there by shelld, so + // this branch is only ever taken by a test or a deliberate off-box run. + // Fixing it means deciding that the .env copy is rotatable too, which is a + // different question from this one. if (process.env.CONNECTORD_TOKEN) return process.env.CONNECTORD_TOKEN try { - for (const line of readFileSync('/etc/5dive/connectord.env', 'utf8').split('\n')) { + for (const line of readFileSync(TOKEN_FILE, 'utf8').split('\n')) { const m = line.match(/^CONNECTORD_TOKEN=(.+)$/) if (m) return m[1].trim() } } catch {} return '' } -const TOKEN = loadConnectordToken() +let TOKEN = loadConnectordToken() if (!TOKEN) { process.stderr.write( `dashboard channel: connectord token not found\n` + @@ -97,6 +121,78 @@ if (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(AGENT)) { process.exit(1) } +// --- DIVE-3810: the credential is mutable, and its failure has to be visible -- +// +// Pairing a phone rewrites /etc/5dive/connectord.env while this process runs. +// Every surface a triager would check then says "healthy" — the process is +// alive, the MCP socket is ESTAB with empty queues, the control plane's +// /pending holds the messages with their text intact and delivered_at NULL, +// and the OTHER channel on the same agent keeps working because buzz does not +// use this token. The only signal was one stderr line that goes down the stdio +// socket into the harness and is written to no file on the box. +// +// So: reload on rejection (the cheapest correct fix — no watch, no timer, and +// it costs exactly one extra request on the request that was going to fail +// anyway), and write the state CHANGE to lifecycle.log, which is a file on the +// box that a human or an agent can read after the fact. +let authFailing = false + +/** Re-read the token from disk. True only if it actually CHANGED. */ +function reloadToken(): boolean { + const next = loadConnectordToken() + if (!next || next === TOKEN) return false + TOKEN = next + return true +} + +function recordAuth(reason: string): void { + recordLifecycle(STATE_DIR, 'auth', 'dashboard', reason) +} + +/** + * Every control-plane call goes through here so no call site can hold a stale + * credential — a fix applied at one of the three would leave the channel half + * deaf. `what` names the call in the record. + * + * On 401/403 the token is re-read; if (and only if) it changed, the request is + * retried ONCE with the new one. A rejection that survives a reload is a real + * rejection and is returned to the caller unchanged — this must not turn an + * auth failure into a retry loop. + */ +async function authedFetch(url: string, what: string, init: RequestInit = {}): Promise { + const send = () => + fetch(url, { + ...init, + headers: { ...((init.headers as Record) ?? {}), authorization: `Bearer ${TOKEN}` }, + }) + let res = await send() + if (res.status !== 401 && res.status !== 403) { + if (authFailing) { + authFailing = false + recordAuth(`credential accepted again on ${what} (${res.status})`) + } + return res + } + if (reloadToken()) { + // Drain the rejected body so the retry is not racing a live stream. + void res.text().catch(() => '') + res = await send() + if (res.status !== 401 && res.status !== 403) { + authFailing = false + recordAuth(`token rotated on disk (${TOKEN_FILE}); reloaded and retried ${what} ok`) + return res + } + } + if (!authFailing) { + authFailing = true + recordAuth( + `${what} rejected ${res.status} and reloading ${TOKEN_FILE} did not fix it — ` + + `dashboard chat is deaf and mute until this clears`, + ) + } + return res +} + const mcp = new Server( { name: 'dashboard', version: '1.0.0' }, { @@ -234,9 +330,10 @@ async function drainPending(): Promise { async function drainPendingOnce(): Promise { let items: Array<{ id: number; text: string; from?: string; chat_id?: string; ts?: string; image_path?: string }> try { - const res = await fetch(`${API_BASE}/server/messages/pending?agent=${encodeURIComponent(AGENT)}`, { - headers: { authorization: `Bearer ${TOKEN}` }, - }) + const res = await authedFetch( + `${API_BASE}/server/messages/pending?agent=${encodeURIComponent(AGENT)}`, + 'pending fetch', + ) if (!res.ok) throw new Error(`${res.status}`) items = ((await res.json()) as { pending?: typeof items }).pending ?? [] } catch (err) { @@ -270,11 +367,12 @@ async function drainPendingOnce(): Promise { } if (acked.length === 0) return try { - await fetch(`${API_BASE}/server/messages/pending/ack`, { + const ack = await authedFetch(`${API_BASE}/server/messages/pending/ack`, 'pending ack', { method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${TOKEN}` }, + headers: { 'content-type': 'application/json' }, body: JSON.stringify({ agent: AGENT, ids: acked }), }) + if (!ack.ok) throw new Error(`${ack.status}`) process.stderr.write(`dashboard channel: healed ${acked.length} undelivered message(s)\n`) } catch (err) { process.stderr.write(`dashboard channel: pending ack failed (will redeliver next boot): ${err}\n`) @@ -334,11 +432,10 @@ mcp.setRequestHandler(CallToolRequestSchema, async req => { } }) - const res = await fetch(`${API_BASE}/server/messages/event`, { + const res = await authedFetch(`${API_BASE}/server/messages/event`, 'outbound reply', { method: 'POST', headers: { 'content-type': 'application/json', - authorization: `Bearer ${TOKEN}`, }, body: JSON.stringify({ agent: AGENT, diff --git a/plugins/telegram-agy/lifecycle.ts b/plugins/telegram-agy/lifecycle.ts index 4fa3dd3..2a0cd45 100644 --- a/plugins/telegram-agy/lifecycle.ts +++ b/plugins/telegram-agy/lifecycle.ts @@ -104,7 +104,11 @@ export function isBootParentAlive(pid: number): boolean { // ── the record: a channel start failure must not be encoded as `nothing` ──── -export type LifecycleEvent = 'start' | 'exit' | 'crash' +// DIVE-3810 added 'auth': a channel whose CREDENTIAL died mid-run is neither +// started, exited nor crashed — the process is healthy and every other +// surface reads healthy with it — so without an event of its own that state +// is recorded as `nothing`, which is the failure this file exists to refuse. +export type LifecycleEvent = 'start' | 'exit' | 'crash' | 'auth' /** * One line, one event, parseable and human-readable. diff --git a/plugins/telegram-codex/lifecycle.ts b/plugins/telegram-codex/lifecycle.ts index 4fa3dd3..2a0cd45 100644 --- a/plugins/telegram-codex/lifecycle.ts +++ b/plugins/telegram-codex/lifecycle.ts @@ -104,7 +104,11 @@ export function isBootParentAlive(pid: number): boolean { // ── the record: a channel start failure must not be encoded as `nothing` ──── -export type LifecycleEvent = 'start' | 'exit' | 'crash' +// DIVE-3810 added 'auth': a channel whose CREDENTIAL died mid-run is neither +// started, exited nor crashed — the process is healthy and every other +// surface reads healthy with it — so without an event of its own that state +// is recorded as `nothing`, which is the failure this file exists to refuse. +export type LifecycleEvent = 'start' | 'exit' | 'crash' | 'auth' /** * One line, one event, parseable and human-readable. diff --git a/plugins/telegram-grok/lifecycle.ts b/plugins/telegram-grok/lifecycle.ts index 4fa3dd3..2a0cd45 100644 --- a/plugins/telegram-grok/lifecycle.ts +++ b/plugins/telegram-grok/lifecycle.ts @@ -104,7 +104,11 @@ export function isBootParentAlive(pid: number): boolean { // ── the record: a channel start failure must not be encoded as `nothing` ──── -export type LifecycleEvent = 'start' | 'exit' | 'crash' +// DIVE-3810 added 'auth': a channel whose CREDENTIAL died mid-run is neither +// started, exited nor crashed — the process is healthy and every other +// surface reads healthy with it — so without an event of its own that state +// is recorded as `nothing`, which is the failure this file exists to refuse. +export type LifecycleEvent = 'start' | 'exit' | 'crash' | 'auth' /** * One line, one event, parseable and human-readable. diff --git a/plugins/telegram-opencode/lifecycle.ts b/plugins/telegram-opencode/lifecycle.ts index 4fa3dd3..2a0cd45 100644 --- a/plugins/telegram-opencode/lifecycle.ts +++ b/plugins/telegram-opencode/lifecycle.ts @@ -104,7 +104,11 @@ export function isBootParentAlive(pid: number): boolean { // ── the record: a channel start failure must not be encoded as `nothing` ──── -export type LifecycleEvent = 'start' | 'exit' | 'crash' +// DIVE-3810 added 'auth': a channel whose CREDENTIAL died mid-run is neither +// started, exited nor crashed — the process is healthy and every other +// surface reads healthy with it — so without an event of its own that state +// is recorded as `nothing`, which is the failure this file exists to refuse. +export type LifecycleEvent = 'start' | 'exit' | 'crash' | 'auth' /** * One line, one event, parseable and human-readable. diff --git a/plugins/telegram-pi/lifecycle.ts b/plugins/telegram-pi/lifecycle.ts index 4fa3dd3..2a0cd45 100644 --- a/plugins/telegram-pi/lifecycle.ts +++ b/plugins/telegram-pi/lifecycle.ts @@ -104,7 +104,11 @@ export function isBootParentAlive(pid: number): boolean { // ── the record: a channel start failure must not be encoded as `nothing` ──── -export type LifecycleEvent = 'start' | 'exit' | 'crash' +// DIVE-3810 added 'auth': a channel whose CREDENTIAL died mid-run is neither +// started, exited nor crashed — the process is healthy and every other +// surface reads healthy with it — so without an event of its own that state +// is recorded as `nothing`, which is the failure this file exists to refuse. +export type LifecycleEvent = 'start' | 'exit' | 'crash' | 'auth' /** * One line, one event, parseable and human-readable. diff --git a/plugins/telegram/lifecycle.ts b/plugins/telegram/lifecycle.ts index 4fa3dd3..2a0cd45 100644 --- a/plugins/telegram/lifecycle.ts +++ b/plugins/telegram/lifecycle.ts @@ -104,7 +104,11 @@ export function isBootParentAlive(pid: number): boolean { // ── the record: a channel start failure must not be encoded as `nothing` ──── -export type LifecycleEvent = 'start' | 'exit' | 'crash' +// DIVE-3810 added 'auth': a channel whose CREDENTIAL died mid-run is neither +// started, exited nor crashed — the process is healthy and every other +// surface reads healthy with it — so without an event of its own that state +// is recorded as `nothing`, which is the failure this file exists to refuse. +export type LifecycleEvent = 'start' | 'exit' | 'crash' | 'auth' /** * One line, one event, parseable and human-readable. diff --git a/test/dashboard-token-rotation.test.ts b/test/dashboard-token-rotation.test.ts new file mode 100644 index 0000000..8e6538c --- /dev/null +++ b/test/dashboard-token-rotation.test.ts @@ -0,0 +1,312 @@ +// DIVE-3810: pairing a phone rotates the box's connectord token while the agent +// is running, and the dashboard channel held the one it read at boot. +// +// This drives the REAL plugins/dashboard/server.ts as a subprocess against a +// stub control plane that rejects a stale bearer, exactly as the live one does, +// and rotates the token FILE underneath the running process — the thing +// shelld's /shell/rotate-token does. A static assertion that a reload exists +// cannot tell you the channel recovers; only a running process that was 401ing +// and then delivers can. +// +// Both directions are exercised on purpose. The row's defect is one cause with +// two symptoms — the channel goes deaf (collect) AND mute (the agent's reply) — +// and a fix applied to the collect path alone would still leave the customer +// talking to a wall. + +import { describe, test, expect } from 'bun:test' +import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const SERVER = join(import.meta.dir, '..', 'plugins', 'dashboard', 'server.ts') +// server.ts gives the harness a 5s head start before its first drain and before +// the watchers install; everything here waits that out. +const BOOT_MS = 5_000 +const OLD_TOKEN = 'tok-old-abcdefghijkl' +const NEW_TOKEN = 'tok-new-mnopqrstuvwx' + +type Harness = { + dir: string + pendingHits: number + rejected: number + events: Array<{ body: string }> + delivered: string[] + stop: () => void + nudge: () => void + enqueue: (msgs: Array<{ id: number; text: string }>) => void + rotate: (next: string) => void + reply: (text: string) => void + lifecycleLines: () => string[] + waitFor: (pred: () => boolean, ms: number) => Promise +} + +async function start(pending: Array<{ id: number; text: string }>): Promise { + const dir = mkdtempSync(join(tmpdir(), 'token-rotation-')) + const envFile = join(dir, 'connectord.env') + writeFileSync(envFile, `CONNECTORD_TOKEN=${OLD_TOKEN}\nOTHER=keep-me\n`) + + let accepted = OLD_TOKEN + let queue = [...pending] + const delivered: string[] = [] + const events: Array<{ body: string }> = [] + const h = { pendingHits: 0, rejected: 0 } + + const api = Bun.serve({ + port: 0, + async fetch(req) { + const url = new URL(req.url) + // The control plane's own behaviour: a stale bearer is a 401, on every + // route, whatever the payload. + if (req.headers.get('authorization') !== `Bearer ${accepted}`) { + h.rejected++ + return new Response('unauthorized', { status: 401 }) + } + if (url.pathname === '/server/messages/pending') { + h.pendingHits++ + return Response.json({ pending: queue }) + } + if (url.pathname === '/server/messages/pending/ack') { + const body = (await req.json()) as { ids: number[] } + queue = queue.filter(m => !body.ids.includes(m.id)) + return Response.json({ ok: true }) + } + if (url.pathname === '/server/messages/event') { + const body = (await req.json()) as { body: string } + events.push({ body: body.body }) + return Response.json({ id: events.length }) + } + return new Response('not found', { status: 404 }) + }, + }) + + const proc = Bun.spawn(['bun', SERVER], { + env: { + ...process.env, + DASHBOARD_STATE_DIR: dir, + DASHBOARD_API_BASE: `http://127.0.0.1:${api.port}`, + // Deliberately NOT CONNECTORD_TOKEN: an explicit env override is + // authoritative and never reloaded, so setting it here would test the one + // path the rotation cannot reach. + CONNECTORD_ENV_FILE: envFile, + CONNECTORD_TOKEN: undefined as unknown as string, + USER: 'agent-dev', + }, + stdin: 'pipe', + stdout: 'pipe', + stderr: 'pipe', + }) + + proc.stdin.write( + JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'token-rotation-test', version: '0' }, + }, + }) + '\n' + ) + proc.stdin.flush() + + void (async () => { + const reader = proc.stdout.getReader() + const dec = new TextDecoder() + let buf = '' + for (;;) { + const { done, value } = await reader.read() + if (done) return + buf += dec.decode(value, { stream: true }) + const lines = buf.split('\n') + buf = lines.pop() ?? '' + for (const line of lines) { + if (!line.trim()) continue + try { + const msg = JSON.parse(line) + if (msg.method === 'notifications/claude/channel') { + delivered.push(String(msg.params?.content ?? '')) + } + if (msg.id === 1) { + proc.stdin.write( + JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n' + ) + proc.stdin.flush() + } + } catch {} + } + } + })() + + const nudgeDir = join(dir, 'collect-now') + let replyId = 100 + return { + dir, + get pendingHits() { return h.pendingHits }, + get rejected() { return h.rejected }, + events, + delivered, + stop: () => { + proc.kill() + api.stop(true) + rmSync(dir, { recursive: true, force: true }) + }, + enqueue: msgs => { queue = [...queue, ...msgs] }, + nudge: () => { + mkdirSync(nudgeDir, { recursive: true }) + writeFileSync(join(nudgeDir, 'nudge'), '') + }, + // Line surgery over the file, then flip the control plane — byte-for-byte + // the shape of shelld's rotate-token (it keeps the other lines). + rotate: next => { + const kept = readFileSync(envFile, 'utf8') + .split('\n') + .filter(l => l && !l.startsWith('CONNECTORD_TOKEN=')) + writeFileSync(envFile, [`CONNECTORD_TOKEN=${next}`, ...kept].join('\n') + '\n') + accepted = next + }, + reply: text => { + proc.stdin.write( + JSON.stringify({ + jsonrpc: '2.0', + id: replyId++, + method: 'tools/call', + params: { name: 'reply', arguments: { chat_id: 'dashboard', text } }, + }) + '\n' + ) + proc.stdin.flush() + }, + lifecycleLines: () => { + const p = join(dir, 'lifecycle.log') + if (!existsSync(p)) return [] + return readFileSync(p, 'utf8').split('\n').filter(Boolean) + }, + waitFor: async (pred, ms) => { + const deadline = Date.now() + ms + while (Date.now() < deadline) { + if (pred()) return true + await Bun.sleep(25) + } + return pred() + }, + } as Harness +} + +describe('dashboard token rotation (DIVE-3810)', () => { + test('a token rotated under a running plugin is picked up — inbound recovers with no restart', async () => { + const h = await start([{ id: 1, text: 'before the rotation' }]) + try { + expect(await h.waitFor(() => h.delivered.includes('before the rotation'), BOOT_MS + 8_000)).toBe(true) + + // Pairing a phone: connectord.env is rewritten while this process runs. + h.rotate(NEW_TOKEN) + h.enqueue([{ id: 2, text: 'after the rotation' }]) + h.nudge() + + // On the shipped code this never arrives: the plugin 401s every collect + // until someone restarts the agent. + expect(await h.waitFor(() => h.delivered.includes('after the rotation'), 15_000)).toBe(true) + // And it was actually rejected first — otherwise this passes for the + // trivial reason that the stale credential still worked. + expect(h.rejected).toBeGreaterThan(0) + } finally { + h.stop() + } + }, 40_000) + + test('the agent can still SPEAK after the rotation — the mute half of the same defect', async () => { + const h = await start([]) + try { + await h.waitFor(() => h.pendingHits >= 1, BOOT_MS + 8_000) + h.rotate(NEW_TOKEN) + h.reply('are you still there?') + expect(await h.waitFor(() => h.events.length >= 1, 15_000)).toBe(true) + expect(h.events[0].body).toBe('are you still there?') + } finally { + h.stop() + } + }, 40_000) + + test('the failure reaches a file on the box, and one episode writes one record', async () => { + // The row's second requirement: a stderr line that goes down the stdio + // socket into the harness is written nowhere and is not a signal. A + // credential that is dead for good must leave a readable trace, and a + // 5-minute poll against a dead credential must not fill the log with it. + // + // The message is enqueued only AFTER the credential is dead, so its arrival + // can only come from the reload. Enqueuing it before boot makes the final + // assertion pass on the boot drain and grades nothing. + const h = await start([]) + try { + expect(await h.waitFor(() => h.pendingHits >= 1, BOOT_MS + 8_000)).toBe(true) + + // Rotate the CONTROL PLANE only, leaving a stale token on disk: a reload + // cannot fix this one. That is the genuinely-broken case, and it is the + // one that has to be legible after the fact. + h.rotate(NEW_TOKEN) + writeFileSync(join(h.dir, 'connectord.env'), `CONNECTORD_TOKEN=${OLD_TOKEN}\n`) + h.enqueue([{ id: 1, text: 'arrives only after the reload' }]) + const before = h.rejected + h.nudge() + expect(await h.waitFor(() => h.rejected > before, 10_000)).toBe(true) + expect(await h.waitFor(() => h.lifecycleLines().some(l => l.includes('\tauth\t')), 10_000)).toBe(true) + expect(h.delivered).not.toContain('arrives only after the reload') + + const auth = h.lifecycleLines().filter(l => l.includes('\tauth\t')) + expect(auth.length).toBe(1) + expect(auth[0]).toContain('dashboard') + expect(auth[0]).toMatch(/deaf and mute/) + + // A second nudge against the same dead credential is the same episode, + // not a second record — otherwise the 5-minute poll turns the log into + // one line per poll for as long as the box lives. + h.nudge() + await Bun.sleep(2_000) + expect(h.lifecycleLines().filter(l => l.includes('\tauth\t')).length).toBe(1) + + // And when the on-disk token catches up, the channel recovers with no + // restart and the recovery is recorded too — an episode that only ever + // opens cannot be read as closed. + writeFileSync(join(h.dir, 'connectord.env'), `CONNECTORD_TOKEN=${NEW_TOKEN}\n`) + h.nudge() + expect(await h.waitFor(() => h.delivered.includes('arrives only after the reload'), 15_000)).toBe(true) + const auth2 = h.lifecycleLines().filter(l => l.includes('\tauth\t')) + expect(auth2.length).toBe(2) + expect(auth2[1]).toMatch(/rotated on disk/) + } finally { + h.stop() + } + }, 60_000) + + test('ALL THREE control-plane calls go through the reloading path, not just the one driven above', () => { + // Arming one call site is not arming the caller: collect, ack and the + // outbound reply each carry the credential, and a fix applied to the one a + // test happens to drive leaves the channel half broken. Assert it of the + // source, because no single scenario reaches all three. + const src = readFileSync(SERVER, 'utf8') + for (const route of [ + '/server/messages/pending?agent=', + '/server/messages/pending/ack', + '/server/messages/event', + ]) { + // Only the occurrences that are actually a URL — every real call builds + // it as `${API_BASE}`. Matching the first mention instead would + // grade the file header, which names two of these routes in prose. + const sites: number[] = [] + for (let i = src.indexOf(route); i > -1; i = src.indexOf(route, i + 1)) { + if (src.slice(i - '${API_BASE}'.length, i) === '${API_BASE}') sites.push(i) + } + expect(sites.length).toBeGreaterThan(0) + for (const at of sites) { + // The call that owns this route must be an authedFetch. Look back from + // the route to the nearest fetch-ish call, and check which one it is. + const before = src.slice(Math.max(0, at - 400), at) + expect(before.lastIndexOf('authedFetch(')).toBeGreaterThan(before.lastIndexOf('await fetch(')) + } + } + // And the bearer header is built in exactly ONE place — the helper. + expect(src.split('authorization: `Bearer ${TOKEN}`').length - 1).toBe(1) + // The token must stay mutable: a `const` here is the whole defect. + expect(src).toMatch(/\nlet TOKEN = loadConnectordToken\(\)/) + }) +})