Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions packages/advisor/src/board-watcher.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { describe, expect, it } from "vitest";
import { buildBoardLedger } from "./board.js";
import { defaultBoardWatchConfig, defaultBoardWatchState, normalizeBoardWatchConfig, runBoardWatch } from "./board-watcher.js";

function riskyLedger(turn = 1) {
return buildBoardLedger([
{ type: "session", id: "watch-session" },
{ type: "turn", turn },
{ type: "file_changed", path: "src/change.ts", turn },
]);
}

describe("Board watcher", () => {
it("defaults to deterministic shadow mode", () => {
expect(defaultBoardWatchConfig()).toEqual({ mode: "shadow", cooldownTurns: 3, maxInterventions: 4 });
expect(normalizeBoardWatchConfig({ mode: "intervene", cooldownTurns: 999, maxInterventions: -1 })).toEqual({ mode: "intervene", cooldownTurns: 100, maxInterventions: 0 });
});

it("records material risks without calling a model or queuing advice in shadow mode", () => {
const result = runBoardWatch(defaultBoardWatchConfig(), defaultBoardWatchState(), riskyLedger(), 1);
expect(result.decision.action).toBe("would_whisper");
expect(result.advice).toBeUndefined();
expect(result.state.runs).toBe(1);
expect(result.state.interventions).toBe(0);
});

it("queues one bounded non-binding intervention and deduplicates it", () => {
const config = { ...defaultBoardWatchConfig(), mode: "intervene" as const, cooldownTurns: 0 };
const first = runBoardWatch(config, defaultBoardWatchState(), riskyLedger(), 1);
expect(first.advice).toMatchObject({ details: { nonBinding: true, readOnly: true } });
expect(first.advice?.content.length).toBeLessThanOrEqual(1800);
const duplicate = runBoardWatch(config, first.state, riskyLedger(), 2);
expect(duplicate.advice).toBeUndefined();
expect(duplicate.skipped).toBe("duplicate");
expect(duplicate.state.suppressed).toBe(1);
});

it("enforces cooldown for a changed risk fingerprint", () => {
const config = { ...defaultBoardWatchConfig(), mode: "intervene" as const, cooldownTurns: 3 };
const first = runBoardWatch(config, defaultBoardWatchState(), riskyLedger(1), 1);
const changed = runBoardWatch(config, first.state, riskyLedger(2), 2);
expect(changed.advice).toBeUndefined();
expect(changed.skipped).toBe("cooldown");
const consecutive = runBoardWatch(config, changed.state, riskyLedger(3), 3);
expect(consecutive.advice).toBeUndefined();
expect(consecutive.skipped).toBe("cooldown");
const afterCooldown = runBoardWatch(config, consecutive.state, riskyLedger(4), 4);
expect(afterCooldown.advice).toBeDefined();
});
});
142 changes: 142 additions & 0 deletions packages/advisor/src/board-watcher.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
import { createHash } from "node:crypto";
import { decideBoardAction, type BoardDecision, type BoardLedger } from "./board.js";

export type BoardWatchMode = "off" | "shadow" | "intervene";

export interface BoardWatchConfig {
mode: BoardWatchMode;
cooldownTurns: number;
maxInterventions: number;
}

export interface BoardWatchState {
runs: number;
interventions: number;
suppressed: number;
lastAt?: string;
lastTurn?: number;
lastInterventionTurn?: number;
lastRiskFingerprint?: string;
lastDecision?: BoardDecision;
}

export interface BoardWatchAdvice {
fingerprint: string;
content: string;
details: {
kind: "board-watch";
decision: "would_whisper";
severity: "note" | "important" | "blocker";
riskIds: string[];
nonBinding: true;
readOnly: true;
};
}

export interface BoardWatchResult {
state: BoardWatchState;
decision: BoardDecision;
advice?: BoardWatchAdvice;
riskFingerprint?: string;
skipped?: "off" | "silent" | "ledger_update" | "duplicate" | "cooldown" | "limit";
}

export function defaultBoardWatchConfig(): BoardWatchConfig {
return { mode: "shadow", cooldownTurns: 3, maxInterventions: 4 };
}

export function normalizeBoardWatchConfig(raw: unknown): BoardWatchConfig {
const defaults = defaultBoardWatchConfig();
if (!raw || typeof raw !== "object") return defaults;
const record = raw as Record<string, unknown>;
const bounded = (value: unknown, fallback: number, min: number, max: number) => {
const number = Number(value);
return Number.isFinite(number) ? Math.min(max, Math.max(min, Math.floor(number))) : fallback;
};
return {
mode: record.mode === "off" || record.mode === "intervene" ? record.mode : "shadow",
cooldownTurns: bounded(record.cooldownTurns, defaults.cooldownTurns, 0, 100),
maxInterventions: bounded(record.maxInterventions, defaults.maxInterventions, 0, 32),
};
}

export function defaultBoardWatchState(): BoardWatchState {
return { runs: 0, interventions: 0, suppressed: 0 };
}

export function normalizeBoardWatchState(raw: unknown): BoardWatchState {
if (!raw || typeof raw !== "object") return defaultBoardWatchState();
const record = raw as Record<string, unknown>;
const count = (value: unknown) => Number.isFinite(Number(value)) && Number(value) > 0 ? Math.floor(Number(value)) : 0;
return {
runs: count(record.runs),
interventions: count(record.interventions),
suppressed: count(record.suppressed),
lastAt: typeof record.lastAt === "string" ? record.lastAt : undefined,
lastTurn: Number.isFinite(Number(record.lastTurn)) ? Math.max(0, Math.floor(Number(record.lastTurn))) : undefined,
lastInterventionTurn: Number.isFinite(Number(record.lastInterventionTurn)) ? Math.max(0, Math.floor(Number(record.lastInterventionTurn))) : undefined,
lastRiskFingerprint: typeof record.lastRiskFingerprint === "string" ? record.lastRiskFingerprint : undefined,
lastDecision: record.lastDecision as BoardDecision | undefined,
};
}

function fingerprint(ledger: BoardLedger, decision: BoardDecision): string {
return createHash("sha256")
.update(JSON.stringify({ risks: ledger.risks.map((risk) => ({ id: risk.id, type: risk.type, severity: risk.severity })).sort((a, b) => a.id.localeCompare(b.id)), decision }))
.digest("hex")
.slice(0, 16);
}

function adviceText(decision: Extract<BoardDecision, { action: "would_whisper" }>, ledger: BoardLedger): string {
const risks = ledger.risks.filter((risk) => decision.riskIds.includes(risk.id)).slice(0, 3);
const pointers = risks.flatMap((risk) => risk.evidencePointers).slice(0, 5);
return [
"Advisory Board suggestion (read-only, non-binding): pause and consider the following before continuing.",
`Severity: ${decision.severity}.`,
`Reason: ${decision.reason}`,
pointers.length ? `Evidence: ${pointers.join(", ")}` : "Evidence: compact Board ledger only.",
"The main model may accept, ignore, or ask for clarification; no action was taken automatically.",
].join("\n").slice(0, 1800);
}

export function runBoardWatch(config: BoardWatchConfig, previous: BoardWatchState, ledger: BoardLedger, turn: number, now = new Date().toISOString()): BoardWatchResult {
const prior = normalizeBoardWatchState(previous);
const state: BoardWatchState = {
...prior,
runs: prior.runs + 1,
lastAt: now,
lastTurn: turn,
};
const decision = decideBoardAction(ledger);
state.lastDecision = decision;
if (config.mode === "off") return { state, decision, skipped: "off" };
if (decision.action !== "would_whisper") return { state, decision, skipped: decision.action };

const id = fingerprint(ledger, decision);
if (prior.lastRiskFingerprint === id) {
state.suppressed += 1;
return { state, decision, riskFingerprint: id, skipped: "duplicate" };
}
if (prior.lastInterventionTurn !== undefined && turn - prior.lastInterventionTurn < config.cooldownTurns) {
state.suppressed += 1;
return { state, decision, riskFingerprint: id, skipped: "cooldown" };
}
if (prior.interventions >= config.maxInterventions) {
state.suppressed += 1;
return { state, decision, riskFingerprint: id, skipped: "limit" };
}
if (config.mode !== "intervene") return { state, decision, riskFingerprint: id };
state.lastRiskFingerprint = id;
state.lastInterventionTurn = turn;
state.interventions += 1;
return {
state,
decision,
riskFingerprint: id,
advice: {
fingerprint: id,
content: adviceText(decision, ledger),
details: { kind: "board-watch", decision: "would_whisper", severity: decision.severity, riskIds: decision.riskIds.slice(0, 8), nonBinding: true, readOnly: true },
},
};
}
5 changes: 3 additions & 2 deletions packages/advisor/src/completions.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ describe("advisor completions", () => {
expect(advisorArgumentCompletions("model list ")?.map((i) => i.value)).toEqual(["advisor", "specialist", "head"]);
});

it("offers explicit Board roles without shadow or discovery controls", () => {
expect(advisorArgumentCompletions("board ")?.map((i) => i.value)).toEqual(["specialist", "head"]);
it("offers explicit Board controls and roles", () => {
expect(advisorArgumentCompletions("board ")?.map((i) => i.value)).toEqual(["watch", "specialist", "head"]);
expect(advisorArgumentCompletions("board watch ")?.map((i) => i.value)).toEqual(["status", "off", "shadow", "intervene"]);
expect(advisorArgumentCompletions("review ")).toBeNull();
expect(advisorArgumentCompletions("profile ")).toBeNull();
});
Expand Down
3 changes: 2 additions & 1 deletion packages/advisor/src/completions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ const advisorTopLevel: Array<[string, string?]> = [
const advisorNested: Record<string, Array<[string, string?]>> = {
model: [["list", "inspect available role candidates"], ["advisor"], ["specialist"], ["head"], ["null"]],
"model list": [["advisor"], ["specialist"], ["head"]],
board: [["specialist"], ["head"]],
board: [["watch"], ["specialist"], ["head"]],
"board watch": [["status"], ["off"], ["shadow"], ["intervene"]],
};

const piRogueTopLevel: Array<[string, string?]> = [
Expand Down
45 changes: 43 additions & 2 deletions packages/advisor/src/extension.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { describe, expect, it, vi } from "vitest";
import { readFileSync } from "node:fs";
import { readFileSync, unlinkSync, writeFileSync } from "node:fs";
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { completeSimple } from "@earendil-works/pi-ai/compat";
import { advisorModelInspectionText, inspectAdvisorModels, advisorSessionStatePath, normalizeAdvisorConfig, rankAvailableAdvisorModels, registerAdvisor, resolveModelCandidates, type AdvisorConfig } from "./extension.js";
import { advisorBoardWatchConfigPath, advisorModelInspectionText, inspectAdvisorModels, advisorSessionStatePath, normalizeAdvisorConfig, rankAvailableAdvisorModels, registerAdvisor, resolveModelCandidates, type AdvisorConfig } from "./extension.js";

vi.mock("@earendil-works/pi-ai/compat", async () => {
const actual = await vi.importActual<typeof import("@earendil-works/pi-ai/compat")>("@earendil-works/pi-ai/compat");
Expand Down Expand Up @@ -240,6 +240,47 @@ describe("Advisor PR1 lifecycle", () => {
]);
});

it("queues a non-binding next-turn message only when intervention mode is explicit", () => {
const handlers = new Map<string, (event: unknown, ctx: any) => void>();
const sendMessage = vi.fn();
const pi = {
on: (event: string, handler: (event: unknown, ctx: any) => void) => { handlers.set(event, handler); },
registerMessageRenderer: vi.fn(),
registerTool: vi.fn(),
registerCommand: vi.fn(),
sendMessage,
} as unknown as ExtensionAPI;
writeFileSync(advisorBoardWatchConfigPath(), JSON.stringify({ mode: "intervene", cooldownTurns: 0, maxInterventions: 4 }));
try {
registerAdvisor(pi);
const ctx = { session: { id: `advisor-watch-${Date.now()}-${Math.random()}` }, cwd: process.cwd(), ui: { setStatus: vi.fn() } };
handlers.get("session_start")?.({}, ctx);
handlers.get("turn_end")?.({ turnIndex: 0, toolResults: [{ toolName: "edit", input: { path: "packages/advisor/src/watched.ts" }, status: "success" }] }, ctx);
expect(sendMessage).toHaveBeenCalledWith(expect.objectContaining({ customType: "advisor:board", details: expect.objectContaining({ nonBinding: true, readOnly: true }) }), { triggerTurn: false, deliverAs: "nextTurn" });
expect(vi.mocked(completeSimple)).not.toHaveBeenCalled();
} finally {
unlinkSync(advisorBoardWatchConfigPath());
}
});

it("deduplicates repeated lifecycle events while preserving same-turn failures", () => {
const handlers = new Map<string, (event: unknown, ctx: any) => void>();
const pi = {
on: (event: string, handler: (event: unknown, ctx: any) => void) => { handlers.set(event, handler); },
registerMessageRenderer: vi.fn(),
registerTool: vi.fn(),
registerCommand: vi.fn(),
} as unknown as ExtensionAPI;
registerAdvisor(pi);
const ctx = { session: { id: `advisor-dedupe-${Date.now()}-${Math.random()}` }, cwd: process.cwd(), ui: { setStatus: vi.fn() } };
const failure = { toolName: "bash", input: { command: "npm test" }, status: "error", error: "failure" };
const event = { turnIndex: 0, toolResults: [failure, failure, failure] };
handlers.get("turn_end")?.(event, ctx);
handlers.get("agent_end")?.(event, ctx);
const state = JSON.parse(readFileSync(advisorSessionStatePath(ctx), "utf8"));
expect(state.boardEvents.filter((item: any) => item.type === "tool_failure")).toHaveLength(3);
});

it("increments turns on turn_end only, keeping cooldown turn accounting stable", () => {
const handlers = new Map<string, (event: unknown, ctx: any) => void>();
const pi = {
Expand Down
Loading
Loading