From 03abeb3bdc50f3619cec6083ebdefa56d3ca0ab0 Mon Sep 17 00:00:00 2001 From: lodar Date: Fri, 28 Aug 2026 06:08:43 +0000 Subject: [PATCH] DIVE-3786: stop reporting an untested mechanism, and make the re-arm kick observe its own submit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit detectStallCause() tested two pane patterns and reported everything else as "listen loop wedged / agent is idle outside wait_for_message and won't re-arm". It reads none of those things — the string was the else-branch, shown to a paying customer as a diagnosis. It now says "not responding — cause unknown" and prints the last lines of the agent's screen. kickListenLoop() typed with `send-keys -l` (which APPENDS) and fired Enter with the result discarded, under a comment noting the TUI drops Enters. A dropped Enter stranded text in the composer and every later kick concatenated onto it — the recovery path using the delivery path that broke, which is why the customer saw the alert constantly. It now clears with C-u, submits separately, and verifies by comparing the pane before and after; one retry, then it sets rearmSubmitFailed. That flag is the only wedged state we actually measure, so it is the only one the alert names ("stuck at the input prompt"). The pane at alert time is written to last-stall-pane.txt next to the existing stamp, so the next occurrence is diagnosable on the box. Co-Authored-By: Claude Opus 5 --- plugins/telegram-agy/server.ts | 114 ++++++++++++++++++++--- plugins/telegram-codex/server.ts | 114 ++++++++++++++++++++--- plugins/telegram-grok/server.ts | 114 ++++++++++++++++++++--- test/dive3786-honest-stall-cause.test.ts | 98 +++++++++++++++++++ 4 files changed, 407 insertions(+), 33 deletions(-) create mode 100644 test/dive3786-honest-stall-cause.test.ts diff --git a/plugins/telegram-agy/server.ts b/plugins/telegram-agy/server.ts index c6c5404..a4dddc8 100644 --- a/plugins/telegram-agy/server.ts +++ b/plugins/telegram-agy/server.ts @@ -2661,6 +2661,15 @@ const LAST_STALL_ALERT_FILE = join(STATE_DIR, 'last-stall-alert.stamp') // recovers (a waiter re-parks). Persisted to a stamp so a server bounce // mid-stall doesn't re-ping. let stallAlerted = existsSync(LAST_STALL_ALERT_FILE) +// The pane as it looked when we alerted. Written next to the stamp so the NEXT +// occurrence is diagnosable from the box itself instead of from a forwarded +// screenshot (DIVE-3786: the customer report we had was a photo of a chat). +const LAST_STALL_PANE_FILE = join(STATE_DIR, 'last-stall-pane.txt') +let _lastPane = '' +// Set by kickListenLoop() when it types a re-arm prompt and then CANNOT observe +// its own Enter landing. This is the one "wedged" state we actually measure, as +// opposed to inferring it from a pane scrape that matched none of our patterns. +let rearmSubmitFailed = false function capitalize(s: string): string { return s.charAt(0).toUpperCase() + s.slice(1) } @@ -2674,6 +2683,7 @@ function detectStallCause(): { cause: string; detail: string } { const pane: string = cp.execFileSync('tmux', ['capture-pane', '-t', `agent-${name}`, '-p'], { timeout: 5_000, encoding: 'utf8' }) + _lastPane = pane const tail = pane.slice(-4000) // Model/credit quota exhausted (e.g. Antigravity "Individual quota reached"). if (/quota reached|out of (?:credits|quota)|usage limit|rate limit exceeded/i.test(tail)) { @@ -2684,12 +2694,34 @@ function detectStallCause(): { cause: string; detail: string } { if (/\b(sign in|log ?in|authenticate|re-?authenticate|oauth|enter your api key)\b/i.test(tail)) { return { cause: 'auth expired — sitting at a login screen', detail: 're-run `5dive agent auth …` for this agent' } } - return { cause: 'listen loop wedged', detail: 'agent is idle outside wait_for_message and won’t re-arm' } + // Our own re-arm keystrokes went in and we could not see the submit land. + // Unlike everything below this line, that IS an observation. + if (rearmSubmitFailed) { + return { + cause: 'stuck at the input prompt', + detail: 'the agent is idle and our re-arm keystrokes are not being submitted — try /restart', + } + } + // Nothing matched. This branch means exactly "it stopped answering and none + // of the patterns above fired" — so say that, and show the owner what is + // actually on the screen. Do NOT name a mechanism we never tested: the old + // text here claimed the listen loop was wedged and that the agent would not + // re-arm, neither of which this function looks at (DIVE-3786). + return { cause: 'not responding — cause unknown', detail: paneTailSummary(tail) } } catch { return { cause: 'not responding', detail: 'could not read the agent pane' } } } +// The last few non-empty screen lines, for an alert that cannot name a cause. +// Trimmed hard: this goes in a Telegram message, not a log. +function paneTailSummary(tail: string): string { + const lines = tail.split('\n').map((l) => l.replace(/\s+$/, '')).filter((l) => l.trim() !== '') + const last = lines.slice(-3).map((l) => (l.length > 100 ? l.slice(0, 99) + '…' : l)) + if (last.length === 0) return 'the agent screen is blank' + return 'Last lines on its screen:\n' + last.map((l) => ' ' + l).join('\n') +} + // detectStallCause does a synchronous pane capture; cache it briefly so a burst // of inbound messages (each auto-replied while stalled) doesn't fire one tmux // capture per message. 5s TTL keeps the reported cause effectively live. @@ -2721,29 +2753,89 @@ function sendStallAlert(): void { } stallAlerted = true try { writeFileSync(LAST_STALL_ALERT_FILE, String(Date.now())) } catch {} + // Keep the evidence on the box. Without this the only record of what the + // agent was sitting on is whatever the owner happens to screenshot. + try { + writeFileSync(LAST_STALL_PANE_FILE, + `# ${new Date().toISOString()} ${name} — ${cause}: ${detail}\n${_lastPane}`) + } catch {} process.stderr.write(`telegram-agy: stall alert sent (${cause}) to ${owners.length} owner(s)\n`) } // Loop recovered — drop the dedup flag so a future stall alerts again. function clearStallAlert(): void { + rearmSubmitFailed = false if (!stallAlerted) return stallAlerted = false try { if (existsSync(LAST_STALL_ALERT_FILE)) unlinkSync(LAST_STALL_ALERT_FILE) } catch {} } +// ── Re-arm kick ───────────────────────────────────────────────────────────── +// Typing into a live TUI has two silent failure modes, and the old version of +// this function had both (DIVE-3786): +// +// 1. `send-keys -l` APPENDS. If a previous delivery's Enter was dropped, its +// text is still sitting in the composer, and every later kick concatenates +// onto that stranded line instead of replacing it. Since the recovery path +// IS the delivery path, the failure repairs itself into a worse one. +// 2. The Enter was fire-and-forget — the callback discarded the result, right +// under a comment saying the TUI "occasionally drops an Enter". +// +// So now: clear the composer first, submit as its own call, and CHECK. A landed +// Enter clears the composer and starts a turn, so the pane must change; byte- +// identical panes before and after mean the keystroke went nowhere. +const REARM_TYPE_SETTLE_MS = 400 +const REARM_SUBMIT_SETTLE_MS = 700 + +function capturePaneSync(name: string): string | null { + try { + const cp = require('child_process') + return cp.execFileSync('tmux', ['capture-pane', '-t', `agent-${name}`, '-p'], + { timeout: 5_000, encoding: 'utf8' }) as string + } catch { return null } +} + +function sendKeysAsync(name: string, args: string[]): Promise { + return new Promise((resolve) => { + const cp = require('child_process') + cp.execFile('tmux', ['send-keys', '-t', `agent-${name}`, ...args], + { timeout: 5_000 }, (err: any) => { + if (err) process.stderr.write(`telegram-agy: rearm send-keys failed: ${err.message}\n`) + resolve(!err) + }) + }) +} + +const rearmSleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +// One clear-type-submit-verify attempt. Returns true only if the submit was +// OBSERVED to change the pane; a pane we cannot read counts as unconfirmed. +async function kickListenLoopOnce(name: string): Promise { + // C-u kills the composer line. Verified against a live codex TUI on 2026-08-28: + // typing a probe string then sending C-u returns the composer to its + // placeholder hint, i.e. genuinely empty. + if (!(await sendKeysAsync(name, ['C-u']))) return false + if (!(await sendKeysAsync(name, ['-l', REARM_KICK_TEXT]))) return false + await rearmSleep(REARM_TYPE_SETTLE_MS) + const typed = capturePaneSync(name) + if (!(await sendKeysAsync(name, ['Enter']))) return false + await rearmSleep(REARM_SUBMIT_SETTLE_MS) + const after = capturePaneSync(name) + if (typed === null || after === null) return false + return after !== typed +} + function kickListenLoop(): void { const name = agentName() if (name === 'unknown') return - const cp = require('child_process') - // Type the prompt as a literal line, then submit. Two send-keys calls because - // the TUI occasionally drops an Enter folded into the same call. - cp.execFile('tmux', ['send-keys', '-t', `agent-${name}`, '-l', REARM_KICK_TEXT], - { timeout: 5_000 }, (err: any) => { - if (err) { process.stderr.write(`telegram-agy: rearm send-keys failed: ${err.message}\n`); return } - setTimeout(() => { - cp.execFile('tmux', ['send-keys', '-t', `agent-${name}`, 'Enter'], { timeout: 5_000 }, () => {}) - }, 400) - }) + void (async () => { + if (await kickListenLoopOnce(name)) { rearmSubmitFailed = false; return } + // A busy TUI can swallow one Enter; a wedged one swallows every Enter. + process.stderr.write('telegram-agy: rearm submit not observed, retrying once\n') + if (await kickListenLoopOnce(name)) { rearmSubmitFailed = false; return } + rearmSubmitFailed = true + process.stderr.write('telegram-agy: rearm submit still not observed — agent is not accepting input\n') + })() } // Newest agent-turn mtime (ms) — the "still doing real work" signal used by the diff --git a/plugins/telegram-codex/server.ts b/plugins/telegram-codex/server.ts index d82c5eb..181afdd 100644 --- a/plugins/telegram-codex/server.ts +++ b/plugins/telegram-codex/server.ts @@ -2864,6 +2864,15 @@ const LAST_STALL_ALERT_FILE = join(STATE_DIR, 'last-stall-alert.stamp') // recovers (a waiter re-parks). Persisted to a stamp so a server bounce // mid-stall doesn't re-ping. let stallAlerted = existsSync(LAST_STALL_ALERT_FILE) +// The pane as it looked when we alerted. Written next to the stamp so the NEXT +// occurrence is diagnosable from the box itself instead of from a forwarded +// screenshot (DIVE-3786: the customer report we had was a photo of a chat). +const LAST_STALL_PANE_FILE = join(STATE_DIR, 'last-stall-pane.txt') +let _lastPane = '' +// Set by kickListenLoop() when it types a re-arm prompt and then CANNOT observe +// its own Enter landing. This is the one "wedged" state we actually measure, as +// opposed to inferring it from a pane scrape that matched none of our patterns. +let rearmSubmitFailed = false function capitalize(s: string): string { return s.charAt(0).toUpperCase() + s.slice(1) } @@ -2877,6 +2886,7 @@ function detectStallCause(): { cause: string; detail: string } { const pane: string = cp.execFileSync('tmux', ['capture-pane', '-t', `agent-${name}`, '-p'], { timeout: 5_000, encoding: 'utf8' }) + _lastPane = pane const tail = pane.slice(-4000) // Model/credit quota exhausted (e.g. Antigravity "Individual quota reached"). if (/quota reached|out of (?:credits|quota)|usage limit|rate limit exceeded/i.test(tail)) { @@ -2887,12 +2897,34 @@ function detectStallCause(): { cause: string; detail: string } { if (/\b(sign in|log ?in|authenticate|re-?authenticate|oauth|enter your api key)\b/i.test(tail)) { return { cause: 'auth expired — sitting at a login screen', detail: 're-run `5dive agent auth …` for this agent' } } - return { cause: 'listen loop wedged', detail: 'agent is idle outside wait_for_message and won’t re-arm' } + // Our own re-arm keystrokes went in and we could not see the submit land. + // Unlike everything below this line, that IS an observation. + if (rearmSubmitFailed) { + return { + cause: 'stuck at the input prompt', + detail: 'the agent is idle and our re-arm keystrokes are not being submitted — try /restart', + } + } + // Nothing matched. This branch means exactly "it stopped answering and none + // of the patterns above fired" — so say that, and show the owner what is + // actually on the screen. Do NOT name a mechanism we never tested: the old + // text here claimed the listen loop was wedged and that the agent would not + // re-arm, neither of which this function looks at (DIVE-3786). + return { cause: 'not responding — cause unknown', detail: paneTailSummary(tail) } } catch { return { cause: 'not responding', detail: 'could not read the agent pane' } } } +// The last few non-empty screen lines, for an alert that cannot name a cause. +// Trimmed hard: this goes in a Telegram message, not a log. +function paneTailSummary(tail: string): string { + const lines = tail.split('\n').map((l) => l.replace(/\s+$/, '')).filter((l) => l.trim() !== '') + const last = lines.slice(-3).map((l) => (l.length > 100 ? l.slice(0, 99) + '…' : l)) + if (last.length === 0) return 'the agent screen is blank' + return 'Last lines on its screen:\n' + last.map((l) => ' ' + l).join('\n') +} + // detectStallCause does a synchronous pane capture; cache it briefly so a burst // of inbound messages (each auto-replied while stalled) doesn't fire one tmux // capture per message. 5s TTL keeps the reported cause effectively live. @@ -2924,29 +2956,89 @@ function sendStallAlert(): void { } stallAlerted = true try { writeFileSync(LAST_STALL_ALERT_FILE, String(Date.now())) } catch {} + // Keep the evidence on the box. Without this the only record of what the + // agent was sitting on is whatever the owner happens to screenshot. + try { + writeFileSync(LAST_STALL_PANE_FILE, + `# ${new Date().toISOString()} ${name} — ${cause}: ${detail}\n${_lastPane}`) + } catch {} process.stderr.write(`telegram-codex: stall alert sent (${cause}) to ${owners.length} owner(s)\n`) } // Loop recovered — drop the dedup flag so a future stall alerts again. function clearStallAlert(): void { + rearmSubmitFailed = false if (!stallAlerted) return stallAlerted = false try { if (existsSync(LAST_STALL_ALERT_FILE)) unlinkSync(LAST_STALL_ALERT_FILE) } catch {} } +// ── Re-arm kick ───────────────────────────────────────────────────────────── +// Typing into a live TUI has two silent failure modes, and the old version of +// this function had both (DIVE-3786): +// +// 1. `send-keys -l` APPENDS. If a previous delivery's Enter was dropped, its +// text is still sitting in the composer, and every later kick concatenates +// onto that stranded line instead of replacing it. Since the recovery path +// IS the delivery path, the failure repairs itself into a worse one. +// 2. The Enter was fire-and-forget — the callback discarded the result, right +// under a comment saying the TUI "occasionally drops an Enter". +// +// So now: clear the composer first, submit as its own call, and CHECK. A landed +// Enter clears the composer and starts a turn, so the pane must change; byte- +// identical panes before and after mean the keystroke went nowhere. +const REARM_TYPE_SETTLE_MS = 400 +const REARM_SUBMIT_SETTLE_MS = 700 + +function capturePaneSync(name: string): string | null { + try { + const cp = require('child_process') + return cp.execFileSync('tmux', ['capture-pane', '-t', `agent-${name}`, '-p'], + { timeout: 5_000, encoding: 'utf8' }) as string + } catch { return null } +} + +function sendKeysAsync(name: string, args: string[]): Promise { + return new Promise((resolve) => { + const cp = require('child_process') + cp.execFile('tmux', ['send-keys', '-t', `agent-${name}`, ...args], + { timeout: 5_000 }, (err: any) => { + if (err) process.stderr.write(`telegram-codex: rearm send-keys failed: ${err.message}\n`) + resolve(!err) + }) + }) +} + +const rearmSleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +// One clear-type-submit-verify attempt. Returns true only if the submit was +// OBSERVED to change the pane; a pane we cannot read counts as unconfirmed. +async function kickListenLoopOnce(name: string): Promise { + // C-u kills the composer line. Verified against a live codex TUI on 2026-08-28: + // typing a probe string then sending C-u returns the composer to its + // placeholder hint, i.e. genuinely empty. + if (!(await sendKeysAsync(name, ['C-u']))) return false + if (!(await sendKeysAsync(name, ['-l', REARM_KICK_TEXT]))) return false + await rearmSleep(REARM_TYPE_SETTLE_MS) + const typed = capturePaneSync(name) + if (!(await sendKeysAsync(name, ['Enter']))) return false + await rearmSleep(REARM_SUBMIT_SETTLE_MS) + const after = capturePaneSync(name) + if (typed === null || after === null) return false + return after !== typed +} + function kickListenLoop(): void { const name = agentName() if (name === 'unknown') return - const cp = require('child_process') - // Type the prompt as a literal line, then submit. Two send-keys calls because - // codex's TUI occasionally drops an Enter folded into the same call. - cp.execFile('tmux', ['send-keys', '-t', `agent-${name}`, '-l', REARM_KICK_TEXT], - { timeout: 5_000 }, (err: any) => { - if (err) { process.stderr.write(`telegram-codex: rearm send-keys failed: ${err.message}\n`); return } - setTimeout(() => { - cp.execFile('tmux', ['send-keys', '-t', `agent-${name}`, 'Enter'], { timeout: 5_000 }, () => {}) - }, 400) - }) + void (async () => { + if (await kickListenLoopOnce(name)) { rearmSubmitFailed = false; return } + // A busy TUI can swallow one Enter; a wedged one swallows every Enter. + process.stderr.write('telegram-codex: rearm submit not observed, retrying once\n') + if (await kickListenLoopOnce(name)) { rearmSubmitFailed = false; return } + rearmSubmitFailed = true + process.stderr.write('telegram-codex: rearm submit still not observed — agent is not accepting input\n') + })() } // Newest agent-turn mtime (ms) — the "still doing real work" signal used by the diff --git a/plugins/telegram-grok/server.ts b/plugins/telegram-grok/server.ts index 24247be..2330cb0 100755 --- a/plugins/telegram-grok/server.ts +++ b/plugins/telegram-grok/server.ts @@ -2691,6 +2691,15 @@ const LAST_STALL_ALERT_FILE = join(STATE_DIR, 'last-stall-alert.stamp') // recovers (a waiter re-parks). Persisted to a stamp so a server bounce // mid-stall doesn't re-ping. let stallAlerted = existsSync(LAST_STALL_ALERT_FILE) +// The pane as it looked when we alerted. Written next to the stamp so the NEXT +// occurrence is diagnosable from the box itself instead of from a forwarded +// screenshot (DIVE-3786: the customer report we had was a photo of a chat). +const LAST_STALL_PANE_FILE = join(STATE_DIR, 'last-stall-pane.txt') +let _lastPane = '' +// Set by kickListenLoop() when it types a re-arm prompt and then CANNOT observe +// its own Enter landing. This is the one "wedged" state we actually measure, as +// opposed to inferring it from a pane scrape that matched none of our patterns. +let rearmSubmitFailed = false function capitalize(s: string): string { return s.charAt(0).toUpperCase() + s.slice(1) } @@ -2704,6 +2713,7 @@ function detectStallCause(): { cause: string; detail: string } { const pane: string = cp.execFileSync('tmux', ['capture-pane', '-t', `agent-${name}`, '-p'], { timeout: 5_000, encoding: 'utf8' }) + _lastPane = pane const tail = pane.slice(-4000) // Model/credit quota exhausted (e.g. Antigravity "Individual quota reached"). if (/quota reached|out of (?:credits|quota)|usage limit|rate limit exceeded/i.test(tail)) { @@ -2714,12 +2724,34 @@ function detectStallCause(): { cause: string; detail: string } { if (/\b(sign in|log ?in|authenticate|re-?authenticate|oauth|enter your api key)\b/i.test(tail)) { return { cause: 'auth expired — sitting at a login screen', detail: 're-run `5dive agent auth …` for this agent' } } - return { cause: 'listen loop wedged', detail: 'agent is idle outside wait_for_message and won’t re-arm' } + // Our own re-arm keystrokes went in and we could not see the submit land. + // Unlike everything below this line, that IS an observation. + if (rearmSubmitFailed) { + return { + cause: 'stuck at the input prompt', + detail: 'the agent is idle and our re-arm keystrokes are not being submitted — try /restart', + } + } + // Nothing matched. This branch means exactly "it stopped answering and none + // of the patterns above fired" — so say that, and show the owner what is + // actually on the screen. Do NOT name a mechanism we never tested: the old + // text here claimed the listen loop was wedged and that the agent would not + // re-arm, neither of which this function looks at (DIVE-3786). + return { cause: 'not responding — cause unknown', detail: paneTailSummary(tail) } } catch { return { cause: 'not responding', detail: 'could not read the agent pane' } } } +// The last few non-empty screen lines, for an alert that cannot name a cause. +// Trimmed hard: this goes in a Telegram message, not a log. +function paneTailSummary(tail: string): string { + const lines = tail.split('\n').map((l) => l.replace(/\s+$/, '')).filter((l) => l.trim() !== '') + const last = lines.slice(-3).map((l) => (l.length > 100 ? l.slice(0, 99) + '…' : l)) + if (last.length === 0) return 'the agent screen is blank' + return 'Last lines on its screen:\n' + last.map((l) => ' ' + l).join('\n') +} + // detectStallCause does a synchronous pane capture; cache it briefly so a burst // of inbound messages (each auto-replied while stalled) doesn't fire one tmux // capture per message. 5s TTL keeps the reported cause effectively live. @@ -2751,29 +2783,89 @@ function sendStallAlert(): void { } stallAlerted = true try { writeFileSync(LAST_STALL_ALERT_FILE, String(Date.now())) } catch {} + // Keep the evidence on the box. Without this the only record of what the + // agent was sitting on is whatever the owner happens to screenshot. + try { + writeFileSync(LAST_STALL_PANE_FILE, + `# ${new Date().toISOString()} ${name} — ${cause}: ${detail}\n${_lastPane}`) + } catch {} process.stderr.write(`telegram-grok: stall alert sent (${cause}) to ${owners.length} owner(s)\n`) } // Loop recovered — drop the dedup flag so a future stall alerts again. function clearStallAlert(): void { + rearmSubmitFailed = false if (!stallAlerted) return stallAlerted = false try { if (existsSync(LAST_STALL_ALERT_FILE)) unlinkSync(LAST_STALL_ALERT_FILE) } catch {} } +// ── Re-arm kick ───────────────────────────────────────────────────────────── +// Typing into a live TUI has two silent failure modes, and the old version of +// this function had both (DIVE-3786): +// +// 1. `send-keys -l` APPENDS. If a previous delivery's Enter was dropped, its +// text is still sitting in the composer, and every later kick concatenates +// onto that stranded line instead of replacing it. Since the recovery path +// IS the delivery path, the failure repairs itself into a worse one. +// 2. The Enter was fire-and-forget — the callback discarded the result, right +// under a comment saying the TUI "occasionally drops an Enter". +// +// So now: clear the composer first, submit as its own call, and CHECK. A landed +// Enter clears the composer and starts a turn, so the pane must change; byte- +// identical panes before and after mean the keystroke went nowhere. +const REARM_TYPE_SETTLE_MS = 400 +const REARM_SUBMIT_SETTLE_MS = 700 + +function capturePaneSync(name: string): string | null { + try { + const cp = require('child_process') + return cp.execFileSync('tmux', ['capture-pane', '-t', `agent-${name}`, '-p'], + { timeout: 5_000, encoding: 'utf8' }) as string + } catch { return null } +} + +function sendKeysAsync(name: string, args: string[]): Promise { + return new Promise((resolve) => { + const cp = require('child_process') + cp.execFile('tmux', ['send-keys', '-t', `agent-${name}`, ...args], + { timeout: 5_000 }, (err: any) => { + if (err) process.stderr.write(`telegram-grok: rearm send-keys failed: ${err.message}\n`) + resolve(!err) + }) + }) +} + +const rearmSleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +// One clear-type-submit-verify attempt. Returns true only if the submit was +// OBSERVED to change the pane; a pane we cannot read counts as unconfirmed. +async function kickListenLoopOnce(name: string): Promise { + // C-u kills the composer line. Verified against a live codex TUI on 2026-08-28: + // typing a probe string then sending C-u returns the composer to its + // placeholder hint, i.e. genuinely empty. + if (!(await sendKeysAsync(name, ['C-u']))) return false + if (!(await sendKeysAsync(name, ['-l', REARM_KICK_TEXT]))) return false + await rearmSleep(REARM_TYPE_SETTLE_MS) + const typed = capturePaneSync(name) + if (!(await sendKeysAsync(name, ['Enter']))) return false + await rearmSleep(REARM_SUBMIT_SETTLE_MS) + const after = capturePaneSync(name) + if (typed === null || after === null) return false + return after !== typed +} + function kickListenLoop(): void { const name = agentName() if (name === 'unknown') return - const cp = require('child_process') - // Type the prompt as a literal line, then submit. Two send-keys calls because - // the TUI occasionally drops an Enter folded into the same call. - cp.execFile('tmux', ['send-keys', '-t', `agent-${name}`, '-l', REARM_KICK_TEXT], - { timeout: 5_000 }, (err: any) => { - if (err) { process.stderr.write(`telegram-grok: rearm send-keys failed: ${err.message}\n`); return } - setTimeout(() => { - cp.execFile('tmux', ['send-keys', '-t', `agent-${name}`, 'Enter'], { timeout: 5_000 }, () => {}) - }, 400) - }) + void (async () => { + if (await kickListenLoopOnce(name)) { rearmSubmitFailed = false; return } + // A busy TUI can swallow one Enter; a wedged one swallows every Enter. + process.stderr.write('telegram-grok: rearm submit not observed, retrying once\n') + if (await kickListenLoopOnce(name)) { rearmSubmitFailed = false; return } + rearmSubmitFailed = true + process.stderr.write('telegram-grok: rearm submit still not observed — agent is not accepting input\n') + })() } // Newest agent-turn mtime (ms) — the "still doing real work" signal used by the diff --git a/test/dive3786-honest-stall-cause.test.ts b/test/dive3786-honest-stall-cause.test.ts new file mode 100644 index 0000000..70cbb06 --- /dev/null +++ b/test/dive3786-honest-stall-cause.test.ts @@ -0,0 +1,98 @@ +// DIVE-3786 — the stall alert must not name a mechanism it never tested, and +// the re-arm kick must observe its own submit. +// +// Two shipped defects, both customer-visible on every codex/grok/antigravity +// box: +// +// 1. detectStallCause() tested exactly two pane patterns and reported +// everything else as "listen loop wedged / agent is idle outside +// wait_for_message and won't re-arm". It reads neither the listen loop nor +// wait_for_message nor the re-arm state — the string was the else-branch, +// shown to a paying customer as a diagnosis. +// 2. kickListenLoop() typed with `send-keys -l` (which APPENDS to whatever is +// already in the composer) and then fired Enter with its result discarded. +// A dropped Enter therefore stranded text in the composer, and every later +// kick concatenated onto it — the recovery path breaking itself, which is +// why the customer saw the alert constantly. +// +// Static, like rearm-loop-regression.test.ts: importing a server long-polls +// Telegram. These guards fail if a regeneration or later edit restores either +// shape. + +import { describe, test, expect } from 'bun:test' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +const PLUGINS = join(import.meta.dir, '..', 'plugins') +const FORKS = ['telegram-codex', 'telegram-grok', 'telegram-agy'] as const + +const src = (fork: string) => readFileSync(join(PLUGINS, fork, 'server.ts'), 'utf8') + +describe.each(FORKS)('%s stall reporting', (fork) => { + test('never reports "listen loop wedged" — it was the unmatched-pattern branch', () => { + expect(src(fork)).not.toMatch(/listen loop wedged/i) + }) + + test('never claims the agent is outside wait_for_message from a pane scrape', () => { + // The pane capture cannot see wait_for_message state at all. + const detect = src(fork).split('function detectStallCause')[1]?.split('\nfunction ')[0] ?? '' + expect(detect).not.toMatch(/wait_for_message/) + }) + + test('the unmatched branch admits ignorance and shows the pane instead', () => { + const s = src(fork) + expect(s).toMatch(/cause: 'not responding — cause unknown', detail: paneTailSummary\(tail\)/) + expect(s).toMatch(/function paneTailSummary/) + }) + + test('the one wedged cause it does name is one it measured', () => { + // rearmSubmitFailed is set only after kickListenLoop typed a prompt and + // could not observe the submit landing — an observation, not an inference. + const s = src(fork) + expect(s).toMatch(/if \(rearmSubmitFailed\) \{/) + expect(s).toMatch(/cause: 'stuck at the input prompt'/) + }) + + test('keeps the pane that triggered the alert on the box', () => { + const s = src(fork) + expect(s).toMatch(/LAST_STALL_PANE_FILE/) + expect(s).toMatch(/writeFileSync\(LAST_STALL_PANE_FILE/) + }) +}) + +describe.each(FORKS)('%s re-arm kick', (fork) => { + const kick = (fork: string) => { + const s = src(fork) + const i = s.indexOf('async function kickListenLoopOnce') + expect(i).toBeGreaterThan(-1) + return s.slice(i, s.indexOf('\n// ', s.indexOf('function kickListenLoop(): void'))) + } + + test('clears the composer before typing, because send-keys -l appends', () => { + const k = kick(fork) + const clear = k.indexOf("'C-u'") + const type = k.indexOf('REARM_KICK_TEXT') + expect(clear).toBeGreaterThan(-1) + expect(type).toBeGreaterThan(clear) + }) + + test('submits in its own send-keys call, after the literal text', () => { + const k = kick(fork) + expect(k.indexOf("'Enter'")).toBeGreaterThan(k.indexOf('REARM_KICK_TEXT')) + }) + + test('verifies the submit landed instead of discarding the result', () => { + const k = kick(fork) + // A landed Enter clears the composer and starts a turn, so the pane changes. + expect(k).toMatch(/return after !== typed/) + // The old code ended the Enter call with an empty callback `() => {}`. + expect(k).not.toMatch(/'Enter'\][^\n]*\(\) => \{\}/) + }) + + test('retries once, then stops claiming success', () => { + const s = src(fork) + const k = s.slice(s.indexOf('function kickListenLoop(): void')) + expect(k).toMatch(/retrying once/) + expect(k).toMatch(/rearmSubmitFailed = true/) + }) +})