diff --git a/packages/advisor/src/board-watcher.test.ts b/packages/advisor/src/board-watcher.test.ts new file mode 100644 index 0000000..1ef7c0e --- /dev/null +++ b/packages/advisor/src/board-watcher.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { buildBoardLedger } from "./board.js"; +import { defaultBoardWatchConfig, defaultBoardWatchState, normalizeBoardWatchConfig, runBoardWatch } from "./board-watcher.js"; + +function riskyLedger(turn = 1) { + return buildBoardLedger([ + { type: "session", id: "watch-session" }, + { type: "turn", turn }, + { type: "file_changed", path: "src/change.ts", turn }, + ]); +} + +describe("Board watcher", () => { + it("defaults to deterministic shadow mode", () => { + expect(defaultBoardWatchConfig()).toEqual({ mode: "shadow", cooldownTurns: 3, maxInterventions: 4 }); + expect(normalizeBoardWatchConfig({ mode: "intervene", cooldownTurns: 999, maxInterventions: -1 })).toEqual({ mode: "intervene", cooldownTurns: 100, maxInterventions: 0 }); + }); + + it("records material risks without calling a model or queuing advice in shadow mode", () => { + const result = runBoardWatch(defaultBoardWatchConfig(), defaultBoardWatchState(), riskyLedger(), 1); + expect(result.decision.action).toBe("would_whisper"); + expect(result.advice).toBeUndefined(); + expect(result.state.runs).toBe(1); + expect(result.state.interventions).toBe(0); + }); + + it("queues one bounded non-binding intervention and deduplicates it", () => { + const config = { ...defaultBoardWatchConfig(), mode: "intervene" as const, cooldownTurns: 0 }; + const first = runBoardWatch(config, defaultBoardWatchState(), riskyLedger(), 1); + expect(first.advice).toMatchObject({ details: { nonBinding: true, readOnly: true } }); + expect(first.advice?.content.length).toBeLessThanOrEqual(1800); + const duplicate = runBoardWatch(config, first.state, riskyLedger(), 2); + expect(duplicate.advice).toBeUndefined(); + expect(duplicate.skipped).toBe("duplicate"); + expect(duplicate.state.suppressed).toBe(1); + }); + + it("enforces cooldown for a changed risk fingerprint", () => { + const config = { ...defaultBoardWatchConfig(), mode: "intervene" as const, cooldownTurns: 3 }; + const first = runBoardWatch(config, defaultBoardWatchState(), riskyLedger(1), 1); + const changed = runBoardWatch(config, first.state, riskyLedger(2), 2); + expect(changed.advice).toBeUndefined(); + expect(changed.skipped).toBe("cooldown"); + const consecutive = runBoardWatch(config, changed.state, riskyLedger(3), 3); + expect(consecutive.advice).toBeUndefined(); + expect(consecutive.skipped).toBe("cooldown"); + const afterCooldown = runBoardWatch(config, consecutive.state, riskyLedger(4), 4); + expect(afterCooldown.advice).toBeDefined(); + }); +}); diff --git a/packages/advisor/src/board-watcher.ts b/packages/advisor/src/board-watcher.ts new file mode 100644 index 0000000..cbb7234 --- /dev/null +++ b/packages/advisor/src/board-watcher.ts @@ -0,0 +1,142 @@ +import { createHash } from "node:crypto"; +import { decideBoardAction, type BoardDecision, type BoardLedger } from "./board.js"; + +export type BoardWatchMode = "off" | "shadow" | "intervene"; + +export interface BoardWatchConfig { + mode: BoardWatchMode; + cooldownTurns: number; + maxInterventions: number; +} + +export interface BoardWatchState { + runs: number; + interventions: number; + suppressed: number; + lastAt?: string; + lastTurn?: number; + lastInterventionTurn?: number; + lastRiskFingerprint?: string; + lastDecision?: BoardDecision; +} + +export interface BoardWatchAdvice { + fingerprint: string; + content: string; + details: { + kind: "board-watch"; + decision: "would_whisper"; + severity: "note" | "important" | "blocker"; + riskIds: string[]; + nonBinding: true; + readOnly: true; + }; +} + +export interface BoardWatchResult { + state: BoardWatchState; + decision: BoardDecision; + advice?: BoardWatchAdvice; + riskFingerprint?: string; + skipped?: "off" | "silent" | "ledger_update" | "duplicate" | "cooldown" | "limit"; +} + +export function defaultBoardWatchConfig(): BoardWatchConfig { + return { mode: "shadow", cooldownTurns: 3, maxInterventions: 4 }; +} + +export function normalizeBoardWatchConfig(raw: unknown): BoardWatchConfig { + const defaults = defaultBoardWatchConfig(); + if (!raw || typeof raw !== "object") return defaults; + const record = raw as Record; + const bounded = (value: unknown, fallback: number, min: number, max: number) => { + const number = Number(value); + return Number.isFinite(number) ? Math.min(max, Math.max(min, Math.floor(number))) : fallback; + }; + return { + mode: record.mode === "off" || record.mode === "intervene" ? record.mode : "shadow", + cooldownTurns: bounded(record.cooldownTurns, defaults.cooldownTurns, 0, 100), + maxInterventions: bounded(record.maxInterventions, defaults.maxInterventions, 0, 32), + }; +} + +export function defaultBoardWatchState(): BoardWatchState { + return { runs: 0, interventions: 0, suppressed: 0 }; +} + +export function normalizeBoardWatchState(raw: unknown): BoardWatchState { + if (!raw || typeof raw !== "object") return defaultBoardWatchState(); + const record = raw as Record; + const count = (value: unknown) => Number.isFinite(Number(value)) && Number(value) > 0 ? Math.floor(Number(value)) : 0; + return { + runs: count(record.runs), + interventions: count(record.interventions), + suppressed: count(record.suppressed), + lastAt: typeof record.lastAt === "string" ? record.lastAt : undefined, + lastTurn: Number.isFinite(Number(record.lastTurn)) ? Math.max(0, Math.floor(Number(record.lastTurn))) : undefined, + lastInterventionTurn: Number.isFinite(Number(record.lastInterventionTurn)) ? Math.max(0, Math.floor(Number(record.lastInterventionTurn))) : undefined, + lastRiskFingerprint: typeof record.lastRiskFingerprint === "string" ? record.lastRiskFingerprint : undefined, + lastDecision: record.lastDecision as BoardDecision | undefined, + }; +} + +function fingerprint(ledger: BoardLedger, decision: BoardDecision): string { + return createHash("sha256") + .update(JSON.stringify({ risks: ledger.risks.map((risk) => ({ id: risk.id, type: risk.type, severity: risk.severity })).sort((a, b) => a.id.localeCompare(b.id)), decision })) + .digest("hex") + .slice(0, 16); +} + +function adviceText(decision: Extract, ledger: BoardLedger): string { + const risks = ledger.risks.filter((risk) => decision.riskIds.includes(risk.id)).slice(0, 3); + const pointers = risks.flatMap((risk) => risk.evidencePointers).slice(0, 5); + return [ + "Advisory Board suggestion (read-only, non-binding): pause and consider the following before continuing.", + `Severity: ${decision.severity}.`, + `Reason: ${decision.reason}`, + pointers.length ? `Evidence: ${pointers.join(", ")}` : "Evidence: compact Board ledger only.", + "The main model may accept, ignore, or ask for clarification; no action was taken automatically.", + ].join("\n").slice(0, 1800); +} + +export function runBoardWatch(config: BoardWatchConfig, previous: BoardWatchState, ledger: BoardLedger, turn: number, now = new Date().toISOString()): BoardWatchResult { + const prior = normalizeBoardWatchState(previous); + const state: BoardWatchState = { + ...prior, + runs: prior.runs + 1, + lastAt: now, + lastTurn: turn, + }; + const decision = decideBoardAction(ledger); + state.lastDecision = decision; + if (config.mode === "off") return { state, decision, skipped: "off" }; + if (decision.action !== "would_whisper") return { state, decision, skipped: decision.action }; + + const id = fingerprint(ledger, decision); + if (prior.lastRiskFingerprint === id) { + state.suppressed += 1; + return { state, decision, riskFingerprint: id, skipped: "duplicate" }; + } + if (prior.lastInterventionTurn !== undefined && turn - prior.lastInterventionTurn < config.cooldownTurns) { + state.suppressed += 1; + return { state, decision, riskFingerprint: id, skipped: "cooldown" }; + } + if (prior.interventions >= config.maxInterventions) { + state.suppressed += 1; + return { state, decision, riskFingerprint: id, skipped: "limit" }; + } + if (config.mode !== "intervene") return { state, decision, riskFingerprint: id }; + state.lastRiskFingerprint = id; + state.lastInterventionTurn = turn; + state.interventions += 1; + return { + state, + decision, + riskFingerprint: id, + advice: { + fingerprint: id, + content: adviceText(decision, ledger), + details: { kind: "board-watch", decision: "would_whisper", severity: decision.severity, riskIds: decision.riskIds.slice(0, 8), nonBinding: true, readOnly: true }, + }, + }; +} diff --git a/packages/advisor/src/completions.test.ts b/packages/advisor/src/completions.test.ts index 942c97b..9c2cead 100644 --- a/packages/advisor/src/completions.test.ts +++ b/packages/advisor/src/completions.test.ts @@ -27,8 +27,9 @@ describe("advisor completions", () => { expect(advisorArgumentCompletions("model list ")?.map((i) => i.value)).toEqual(["advisor", "specialist", "head"]); }); - it("offers explicit Board roles without shadow or discovery controls", () => { - expect(advisorArgumentCompletions("board ")?.map((i) => i.value)).toEqual(["specialist", "head"]); + it("offers explicit Board controls and roles", () => { + expect(advisorArgumentCompletions("board ")?.map((i) => i.value)).toEqual(["watch", "specialist", "head"]); + expect(advisorArgumentCompletions("board watch ")?.map((i) => i.value)).toEqual(["status", "off", "shadow", "intervene"]); expect(advisorArgumentCompletions("review ")).toBeNull(); expect(advisorArgumentCompletions("profile ")).toBeNull(); }); diff --git a/packages/advisor/src/completions.ts b/packages/advisor/src/completions.ts index 8c09eb7..0c4d8d5 100644 --- a/packages/advisor/src/completions.ts +++ b/packages/advisor/src/completions.ts @@ -46,7 +46,8 @@ const advisorTopLevel: Array<[string, string?]> = [ const advisorNested: Record> = { model: [["list", "inspect available role candidates"], ["advisor"], ["specialist"], ["head"], ["null"]], "model list": [["advisor"], ["specialist"], ["head"]], - board: [["specialist"], ["head"]], + board: [["watch"], ["specialist"], ["head"]], + "board watch": [["status"], ["off"], ["shadow"], ["intervene"]], }; const piRogueTopLevel: Array<[string, string?]> = [ diff --git a/packages/advisor/src/extension.test.ts b/packages/advisor/src/extension.test.ts index 5667c64..837ca15 100644 --- a/packages/advisor/src/extension.test.ts +++ b/packages/advisor/src/extension.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from "vitest"; -import { readFileSync } from "node:fs"; +import { readFileSync, unlinkSync, writeFileSync } from "node:fs"; import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { completeSimple } from "@earendil-works/pi-ai/compat"; -import { advisorModelInspectionText, inspectAdvisorModels, advisorSessionStatePath, normalizeAdvisorConfig, rankAvailableAdvisorModels, registerAdvisor, resolveModelCandidates, type AdvisorConfig } from "./extension.js"; +import { advisorBoardWatchConfigPath, advisorModelInspectionText, inspectAdvisorModels, advisorSessionStatePath, normalizeAdvisorConfig, rankAvailableAdvisorModels, registerAdvisor, resolveModelCandidates, type AdvisorConfig } from "./extension.js"; vi.mock("@earendil-works/pi-ai/compat", async () => { const actual = await vi.importActual("@earendil-works/pi-ai/compat"); @@ -240,6 +240,47 @@ describe("Advisor PR1 lifecycle", () => { ]); }); + it("queues a non-binding next-turn message only when intervention mode is explicit", () => { + const handlers = new Map void>(); + const sendMessage = vi.fn(); + const pi = { + on: (event: string, handler: (event: unknown, ctx: any) => void) => { handlers.set(event, handler); }, + registerMessageRenderer: vi.fn(), + registerTool: vi.fn(), + registerCommand: vi.fn(), + sendMessage, + } as unknown as ExtensionAPI; + writeFileSync(advisorBoardWatchConfigPath(), JSON.stringify({ mode: "intervene", cooldownTurns: 0, maxInterventions: 4 })); + try { + registerAdvisor(pi); + const ctx = { session: { id: `advisor-watch-${Date.now()}-${Math.random()}` }, cwd: process.cwd(), ui: { setStatus: vi.fn() } }; + handlers.get("session_start")?.({}, ctx); + handlers.get("turn_end")?.({ turnIndex: 0, toolResults: [{ toolName: "edit", input: { path: "packages/advisor/src/watched.ts" }, status: "success" }] }, ctx); + expect(sendMessage).toHaveBeenCalledWith(expect.objectContaining({ customType: "advisor:board", details: expect.objectContaining({ nonBinding: true, readOnly: true }) }), { triggerTurn: false, deliverAs: "nextTurn" }); + expect(vi.mocked(completeSimple)).not.toHaveBeenCalled(); + } finally { + unlinkSync(advisorBoardWatchConfigPath()); + } + }); + + it("deduplicates repeated lifecycle events while preserving same-turn failures", () => { + const handlers = new Map void>(); + const pi = { + on: (event: string, handler: (event: unknown, ctx: any) => void) => { handlers.set(event, handler); }, + registerMessageRenderer: vi.fn(), + registerTool: vi.fn(), + registerCommand: vi.fn(), + } as unknown as ExtensionAPI; + registerAdvisor(pi); + const ctx = { session: { id: `advisor-dedupe-${Date.now()}-${Math.random()}` }, cwd: process.cwd(), ui: { setStatus: vi.fn() } }; + const failure = { toolName: "bash", input: { command: "npm test" }, status: "error", error: "failure" }; + const event = { turnIndex: 0, toolResults: [failure, failure, failure] }; + handlers.get("turn_end")?.(event, ctx); + handlers.get("agent_end")?.(event, ctx); + const state = JSON.parse(readFileSync(advisorSessionStatePath(ctx), "utf8")); + expect(state.boardEvents.filter((item: any) => item.type === "tool_failure")).toHaveLength(3); + }); + it("increments turns on turn_end only, keeping cooldown turn accounting stable", () => { const handlers = new Map void>(); const pi = { diff --git a/packages/advisor/src/extension.ts b/packages/advisor/src/extension.ts index 821f143..7b515c6 100644 --- a/packages/advisor/src/extension.ts +++ b/packages/advisor/src/extension.ts @@ -23,6 +23,7 @@ import { type SpecialistCallState, } from "./board-specialist.js"; import { loadBoardRoleBody, loadBoardRoleCatalog } from "./board-roles.js"; +import { defaultBoardWatchConfig, normalizeBoardWatchConfig, normalizeBoardWatchState, runBoardWatch, type BoardWatchConfig, type BoardWatchState } from "./board-watcher.js"; // ── Explicit-only configuration ───────────────────────────────────────── @@ -71,7 +72,12 @@ const DEFAULT_CONFIG: AdvisorConfig = { }; const CONFIG_PATH = featureFile("advisor", "config.json"); +const BOARD_WATCH_CONFIG_PATH = featureFile("advisor", "board-watch.json"); const LEGACY_STATE_PATH = featureFile("advisor", "state.json"); + +export function advisorBoardWatchConfigPath(): string { + return BOARD_WATCH_CONFIG_PATH; +} const CACHE_PATH = featureFile("advisor", "cache.json"); const DEFAULT_DIAGNOSTICS_PATH = featureFile("advisor", "diagnostics.jsonl"); const SESSION_STATE_PROP = "__piRogueAdvisorStatePath"; @@ -144,6 +150,7 @@ interface SessionState { lastNote?: string; lastDenied?: string; }; + boardWatch: BoardWatchState; } type BoardLifecycleEvent = Extract; @@ -239,6 +246,14 @@ function saveConfig(c: AdvisorConfig) { writeJson(CONFIG_PATH, c); } +function loadBoardWatchConfig(): BoardWatchConfig { + return normalizeBoardWatchConfig(readJson(BOARD_WATCH_CONFIG_PATH, defaultBoardWatchConfig())); +} + +function saveBoardWatchConfig(config: BoardWatchConfig): void { + writeJson(BOARD_WATCH_CONFIG_PATH, normalizeBoardWatchConfig(config)); +} + function advisorSessionDir(ctxOrKey?: any): string { const root = join(featureDir("advisor"), "sessions"); if (typeof ctxOrKey === "string") return join(root, safeSessionKey(ctxOrKey)); @@ -384,6 +399,7 @@ function loadStateFromPath(path: string): SessionState { cacheHits: raw.cacheHits ?? 0, evidenceLedger: normalizeEvidenceLedger(raw.evidenceLedger), boardEvents: normalizeBoardEvents(raw.boardEvents), + boardWatch: normalizeBoardWatchState(raw.boardWatch), workflow: normalizeWorkflowState(raw.workflow), rateLimit: normalizeRateLimitState(raw.rateLimit), advisorLoop: raw.advisorLoop && typeof raw.advisorLoop === "object" ? { @@ -1044,8 +1060,9 @@ const STRUCTURED_FAILING_TEST_RE = /(?:\bTests?\s+.*?\bfailed\s+\([1-9]\d*\)|\bT const HUMAN_TEST_SUMMARY_RE = /(?:\bTests?\s+\d+\s+(?:passed|failed)\s+\(\d+\)|\bTest Files\s+\d+\s+(?:passed|failed)\s+\(\d+\))/i; type AdvisorHintDetails = { - kind?: "handoff" | "answer"; - decision?: "continue" | "review" | "defer"; + kind?: "handoff" | "answer" | "board-watch" | "board-head"; + decision?: "continue" | "review" | "defer" | "would_whisper"; + severity?: "note" | "important" | "blocker"; reason?: string; summary?: string; actions?: unknown; @@ -1215,6 +1232,15 @@ function renderAdvisorHint(message: any, options: { expanded?: boolean }, theme: const sourceColor = customType === "advisor:llm" ? "success" : customType === "advisor:model" ? "accent" : "muted"; const source = theme.bold(theme.fg(sourceColor, `[${customType}]`)); + if (details.kind === "board-watch" || details.kind === "board-head") { + const body = sanitizeAdvisorText(contentText(message?.content) || "No Board advice."); + const box = new Box(1, 1, (s: string) => theme.bg("customMessageBg", s)); + const label = details.kind === "board-head" ? "Head-of-Board advice" : "Board suggestion"; + box.addChild(new Text(`${theme.bold(theme.fg("accent", "⚠"))} ${source} ${theme.bold(theme.fg("accent", label))}`, 0, 0)); + box.addChild(new Text(theme.fg("dim", `${body}\n(non-binding, read-only; the main model may ignore it)`), 0, 0)); + return box; + } + if (details.kind === "answer") { const body = sanitizeAdvisorText(contentText(message?.content) || details.summary || "No advisor response."); const box = new Box(1, 1, (s: string) => theme.bg("customMessageBg", s)); @@ -1223,7 +1249,7 @@ function renderAdvisorHint(message: any, options: { expanded?: boolean }, theme: return box; } - const decision = details.decision ?? "defer"; + const decision = details.decision === "would_whisper" ? "defer" : details.decision ?? "defer"; const decisionColor = decision === "review" ? "accent" : decision === "continue" ? "muted" : "dim"; const verdict = theme.bold(theme.fg(decisionColor, decision)); const glyph = decision === "review" ? "↗" : decision === "defer" ? "…" : "·"; @@ -1818,7 +1844,22 @@ function collectLifecycleBoardEvents(state: SessionState, toolResults: unknown[] state.boardEvents = [...state.boardEvents, ...events].slice(-MAX_BOARD_EVENTS); } -function collectLifecycleEvidence(event: unknown, ctx: any, agentEnd: boolean): void { +function recordBoardWatchIfEnabled(pi: ExtensionAPI, ctx: any, state: SessionState): void { + const config = loadBoardWatchConfig(); + if (config.mode === "off") return; + const result = runBoardWatch(config, state.boardWatch, currentBoardLedger(ctx, state), state.turns); + state.boardWatch = result.state; + if (result.advice && typeof pi.sendMessage === "function") { + pi.sendMessage({ + customType: "advisor:board", + content: result.advice.content, + display: true, + details: result.advice.details, + }, { triggerTurn: false, deliverAs: "nextTurn" }); + } +} + +function collectLifecycleEvidence(event: unknown, ctx: any, agentEnd: boolean, pi: ExtensionAPI): void { const state = loadState(ctx); const record = event && typeof event === "object" ? event as Record : {}; const turnIndex = Number(record.turnIndex); @@ -1828,7 +1869,10 @@ function collectLifecycleEvidence(event: unknown, ctx: any, agentEnd: boolean): if (text) state.notes = [...state.notes, text].slice(-MAX_NOTES); const toolResults = Array.isArray(record.toolResults) ? record.toolResults : []; const timestamp = new Date().toISOString(); - collectLifecycleBoardEvents(state, toolResults, state.turns, timestamp); + // turn_end is the authoritative tool-result boundary. agent_end repeats the + // same results for the completed run, so collecting there would double-count + // retries and inflate repeated-failure risks. + if (!agentEnd) collectLifecycleBoardEvents(state, toolResults, state.turns, timestamp); state.files = [...new Set([ ...state.files, ...state.boardEvents.filter((entry) => entry.type === "file_changed").map((entry) => entry.path), @@ -1839,6 +1883,7 @@ function collectLifecycleEvidence(event: unknown, ctx: any, agentEnd: boolean): if (summary && /error|fail|exception/i.test(summary)) state.errors = [...state.errors, summary].slice(-MAX_ERRORS); } if (agentEnd && text) state.lastTask = text.slice(0, 500); + if (!agentEnd) recordBoardWatchIfEnabled(pi, ctx, state); saveState(state); syncCloseoutFacts(ctx, state); } @@ -1850,7 +1895,7 @@ export function registerAdvisor(pi: ExtensionAPI): void { if (p.__piRogueAdvisorRegistered) return; p.__piRogueAdvisorRegistered = true; - for (const customType of ["advisor:model", "advisor:rules", "advisor:llm"] as const) { + for (const customType of ["advisor:model", "advisor:rules", "advisor:llm", "advisor:board"] as const) { pi.registerMessageRenderer(customType, renderAdvisorHint); } @@ -1863,10 +1908,10 @@ export function registerAdvisor(pi: ExtensionAPI): void { saveState(loadState(ctx)); }); pi.on("turn_end", (event, ctx) => { - collectLifecycleEvidence(event, ctx, false); + collectLifecycleEvidence(event, ctx, false, pi); }); pi.on("agent_end", (event, ctx) => { - collectLifecycleEvidence(event, ctx, true); + collectLifecycleEvidence(event, ctx, true, pi); }); pi.on("agent_settled", (_event, ctx) => { syncCloseoutFacts(ctx, loadState(ctx)); @@ -2025,6 +2070,26 @@ export function registerAdvisor(pi: ExtensionAPI): void { ctx.ui.notify("Usage: board specialist status|suggest|ask ", "error"); return; } + if (area === "watch") { + const action = String(parts[2] ?? "status").toLowerCase(); + const current = loadBoardWatchConfig(); + if (action === "status") { + ctx.ui.notify([ + `Board watcher: ${current.mode}`, + `Runs: ${state.boardWatch.runs}; interventions: ${state.boardWatch.interventions}; suppressed: ${state.boardWatch.suppressed}`, + `Cooldown: ${current.cooldownTurns} turn(s); max interventions: ${current.maxInterventions}`, + "Watcher is deterministic and read-only; interventions are non-binding next-turn advice.", + ].join("\n"), "info"); + return; + } + if (action === "off" || action === "shadow" || action === "intervene") { + saveBoardWatchConfig({ ...current, mode: action }); + ctx.ui.notify(`Board watcher mode set to ${action}.`, "info"); + return; + } + ctx.ui.notify("Usage: board watch status|off|shadow|intervene", "error"); + return; + } if (area === "head") { const action = String(parts[2] ?? "status").toLowerCase(); if (action === "status") { @@ -2043,7 +2108,7 @@ export function registerAdvisor(pi: ExtensionAPI): void { ctx.ui.notify("Usage: board head status|ask ", "error"); return; } - ctx.ui.notify("Usage: board specialist status|suggest|ask or board head status|ask", "error"); + ctx.ui.notify("Usage: board watch status|off|shadow|intervene; specialist status|suggest|ask; or head status|ask", "error"); return; }