diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..ae009f0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: test + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + # `node --test` type-strips test/*.test.ts natively; that needs + # Node >=22.18 (or 24+). No package.json "engines" field is pinned, + # so this just satisfies the runtime's own requirement. + node-version: "22" + - run: npm install + - run: npm test diff --git a/test/conformance.test.ts b/test/conformance.test.ts new file mode 100644 index 0000000..9eb3465 --- /dev/null +++ b/test/conformance.test.ts @@ -0,0 +1,291 @@ +// ACP plugin conformance adapter for prime-agent-acp-plugin. +// +// Drives the REAL plugin entry point (../index.ts) against a fake gateway, +// against the shared corpus vendored byte-identically at +// test/fixtures/plugin-corpus.json (canonical home: +// davidcrowe/gatewaystack-connect:conformance/plugin-corpus.json, tracking +// davidcrowe/gatewaystack-connect#1344). Do not hand-edit the fixture; re-copy +// it from the canonical source and update CORPUS_FINGERPRINT below together. +// +// Invocation follows test/decisions.test.ts's existing convention: a fake +// `pi` object that records registered event handlers, and a fake +// ExtensionContext, driven directly (no real @earendil-works/pi-coding-agent +// runtime needed — the plugin only imports its types, which are erased). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createServer, type Server } from "node:http"; +import { createHash } from "node:crypto"; +import { readFileSync, mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import acp from "../index.ts"; + +const CORPUS_PATH = fileURLToPath(new URL("./fixtures/plugin-corpus.json", import.meta.url)); +const CORPUS_FINGERPRINT = "aa186d3fb3e7d18c"; +const PLUGIN_NAME = "prime-agent-acp-plugin"; + +// Canonical tool-name mapping for prime-agent: NONE. index.ts's checkPayload() +// forwards event.toolName straight through as tool_name on both +// /govern/tool-use and /govern/tool-output (see index.ts:311 / :444) — there +// is no rename table. So "the adapter's declared canonical mapping" is the +// identity function. +const CANONICAL_TOOL_NAME = (nativeToolName: string): string => nativeToolName; + +// Populated only for a capability the corpus lists "supported" that this +// adapter actually observed failing on origin/main. Each entry here MUST be +// asserted to fail below; the final test asserts this list is exactly what +// was recorded (an empty list here means everything the corpus called +// "supported" for this plugin genuinely passed). +const EXPECTED_DIVERGENCES: Array<{ id: string; issue: string; evidence: string }> = []; + +type Handler = (event: any, ctx: any) => Promise | any; + +/** Minimal prime-agent-shaped API that captures registered handlers (copied style: test/decisions.test.ts). */ +function fakePi() { + const handlers: Record = {}; + return { + handlers, + on: (event: string, fn: Handler) => { + handlers[event] = fn; + }, + }; +} + +/** + * Fake ExtensionContext. `notifies` records every ctx.ui.notify() call — the + * person-visible channel tell() (index.ts:127) writes to whenever + * ctx.hasUI is true, which is the default here and in the plugin's own + * TUI/RPC/attended posture. + */ +function fakeCtx({ hasUI = true, sessionId = "conf-sess" } = {}) { + const notifies: Array<{ msg: string; level: string }> = []; + return { + notifies, + hasUI, + cwd: "/tmp", + signal: new AbortController().signal, + sessionManager: { getSessionId: () => sessionId }, + ui: { + notify: (msg: string, level: string) => notifies.push({ msg, level }), + confirm: async () => true, + }, + }; +} + +/** One-route stub gateway; respond(url, body) decides per request. Records every request. */ +function stubGateway( + respond: (url: string, body: any) => { status?: number; json?: any }, +): Promise<{ server: Server; base: string; requests: Array<{ method: string; path: string; body: any }> }> { + const requests: Array<{ method: string; path: string; body: any }> = []; + const server = createServer((req, res) => { + let raw = ""; + req.on("data", (c) => { + raw += c; + }); + req.on("end", () => { + const body = JSON.parse(raw || "{}"); + requests.push({ method: req.method ?? "", path: req.url ?? "", body }); + const { status = 200, json = {} } = respond(req.url ?? "", body); + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify(json)); + }); + }); + return new Promise((resolve) => + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + resolve({ server, base: `http://127.0.0.1:${port}`, requests }); + }), + ); +} + +/** + * Isolate HOME/state to a fresh temp dir per case (so ~/.acp/credentials or + * lapse.log on the developer's machine can't influence a run), set the + * dummy credential the plugin requires to install its handlers, clear any + * inherited ACP_SHADOW, then apply the case's own env. Returns a restore fn. + */ +function isolateEnv(caseEnv: Record = {}): () => void { + const saved = { ...process.env }; + delete process.env.ACP_SHADOW; + delete process.env.ACP_AGENT_TIER; + delete process.env.ACP_GOVERN_BASE; + delete process.env.ACP_API_BASE; + process.env.HOME = mkdtempSync(join(tmpdir(), "prime-agent-conf-")); + process.env.ACP_BEARER_TOKEN = "acpconf-dummy-token-never-real"; + for (const [k, v] of Object.entries(caseEnv)) process.env[k] = v; + return () => { + process.env = saved; + }; +} + +function expectedDivergence(id: string) { + return EXPECTED_DIVERGENCES.find((d) => d.id === id); +} + +/** Run `assertion`; if `id` is a declared divergence, require it to FAIL instead. */ +function checkCase(id: string, assertion: () => void): void { + const div = expectedDivergence(id); + if (!div) { + assertion(); + return; + } + assert.throws( + assertion, + undefined, + `case "${id}" is listed in EXPECTED_DIVERGENCES (${div.issue}) but now PASSES — remove the entry`, + ); +} + +// --- fingerprint gate: refuse to run against a corpus that has drifted --- + +test("vendored corpus matches the pinned fingerprint", () => { + const raw = readFileSync(CORPUS_PATH); + const digest = createHash("sha256").update(raw).digest("hex").slice(0, 16); + assert.equal( + digest, + CORPUS_FINGERPRINT, + "test/fixtures/plugin-corpus.json has drifted from the canonical corpus " + + "(davidcrowe/gatewaystack-connect:conformance/plugin-corpus.json) — re-copy it, don't hand-edit", + ); +}); + +const corpus = JSON.parse(readFileSync(CORPUS_PATH, "utf8")); +const myRows: Array<{ plugin: string; capability: string; status: string }> = corpus.harnesses.filter( + (h: any) => h.plugin === PLUGIN_NAME, +); +const supportedCapabilities = new Set(myRows.filter((r) => r.status === "supported").map((r) => r.capability)); + +function caseById(id: string): any { + const c = corpus.cases.find((x: any) => x.id === id); + assert.ok(c, `corpus is missing case "${id}"`); + return c; +} + +test("corpus declares both notice and post-tool as supported for prime-agent-acp-plugin", () => { + assert.deepEqual([...supportedCapabilities].sort(), ["notice", "post-tool"]); +}); + +// --- notice capability (issue #1334) --- +// Channel captured: ctx.ui.notify() — the fake ExtensionContext's spy on the +// call tell() (index.ts:127) makes when ctx.hasUI is true, reached from the +// tool_result handler's shadow-notice branch at index.ts:465-469. + +test('notice-shown: ACP_SHADOW unset -> gateway notice reaches ctx.ui.notify() ("notify" channel)', async () => { + assert.ok(supportedCapabilities.has("notice")); + const c = caseById("notice-shown"); + const restore = isolateEnv(c.env); + const { server, base, requests } = await stubGateway((url) => + url.includes("/govern/tool-output") ? { json: c.gatewayReply } : { json: { decision: "allow" } }, + ); + process.env.ACP_GOVERN_BASE = base; + const pi = fakePi(); + acp(pi as any); + const ctx = fakeCtx(); + await pi.handlers.tool_result( + { type: "tool_result", toolName: "shell", toolCallId: "conf-1", input: {}, content: [{ type: "text", text: "ok" }], isError: false }, + ctx, + ); + server.close(); + restore(); + + const req = requests.find((r) => r.path.includes("/govern/tool-output")); + assert.ok(req, "expected the plugin to POST /govern/tool-output"); + const personSees = ctx.notifies.some((n) => n.msg.includes(c.expect.contains)); + checkCase("notice-shown", () => { + assert.equal(personSees, c.expect.personSees, `expected ctx.ui.notify() to ${c.expect.personSees ? "" : "NOT "}carry "${c.expect.contains}"`); + }); +}); + +test("notice-shadow-off: ACP_SHADOW=off -> marker absent from ctx.ui.notify() AND stderr", async () => { + assert.ok(supportedCapabilities.has("notice")); + const c = caseById("notice-shadow-off"); + const restore = isolateEnv(c.env); + const { server, base } = await stubGateway((url) => + url.includes("/govern/tool-output") ? { json: c.gatewayReply } : { json: { decision: "allow" } }, + ); + process.env.ACP_GOVERN_BASE = base; + const pi = fakePi(); + acp(pi as any); + // Also watch stderr — tell() falls back to console.error when ctx.hasUI is + // false or ctx.ui.notify throws; the corpus requires the marker absent from + // EVERY channel, so this test spies both. + const ctx = fakeCtx(); + const originalErrorWrite = process.stderr.write.bind(process.stderr); + let stderrCaptured = ""; + (process.stderr.write as any) = (chunk: any, ...rest: any[]) => { + stderrCaptured += String(chunk); + return originalErrorWrite(chunk, ...(rest as [])); + }; + try { + await pi.handlers.tool_result( + { type: "tool_result", toolName: "shell", toolCallId: "conf-2", input: {}, content: [{ type: "text", text: "ok" }], isError: false }, + ctx, + ); + } finally { + process.stderr.write = originalErrorWrite; + server.close(); + restore(); + } + + const notifySaw = ctx.notifies.some((n) => n.msg.includes(c.expect.contains)); + const stderrSaw = stderrCaptured.includes(c.expect.contains); + checkCase("notice-shadow-off", () => { + assert.equal(notifySaw, false, "ACP_SHADOW=off must suppress the notify() channel"); + assert.equal(stderrSaw, false, "ACP_SHADOW=off must suppress the stderr channel"); + }); +}); + +// --- post-tool capability (issue #1344) --- +// prime-agent's native post-tool hook is the `tool_result` pi event; its own +// field names are toolName / input / content (an array of {type, text} +// blocks). Built here from the corpus's harness-neutral case.call. + +test("post-tool-fields: native tool_result payload -> POST /govern/tool-output carries required fields", async () => { + assert.ok(supportedCapabilities.has("post-tool")); + const c = caseById("post-tool-fields"); + const restore = isolateEnv(c.env); + const { server, base, requests } = await stubGateway((url) => + url.includes("/govern/tool-output") ? { json: c.gatewayReply } : { json: { decision: "allow" } }, + ); + process.env.ACP_GOVERN_BASE = base; + const pi = fakePi(); + acp(pi as any); + const nativeToolName = c.call.tool; // "shell" — prime-agent forwards this verbatim, see CANONICAL_TOOL_NAME above + const ctx = fakeCtx({ sessionId: c.call.sessionId }); + await pi.handlers.tool_result( + { + type: "tool_result", + toolName: nativeToolName, + toolCallId: "conf-post-tool-1", + input: { command: c.call.command }, + content: [{ type: "text", text: c.call.output }], + isError: false, + }, + ctx, + ); + server.close(); + restore(); + + const req = requests.find((r) => r.path.includes("/govern/tool-output")); + assert.ok(req, "expected a POST to /govern/tool-output"); + const marker = corpus.marker as string; + + checkCase("post-tool-fields", () => { + assert.equal(req!.method, "POST"); + assert.equal(req!.body.hook_event_name, "PostToolUse"); + assert.equal(req!.body.tool_name, CANONICAL_TOOL_NAME(nativeToolName)); + assert.match(JSON.stringify(req!.body.tool_input), new RegExp(marker), "tool_input must carry the marker"); + assert.match(JSON.stringify(req!.body.tool_output), new RegExp(marker), "tool_output must carry the marker"); + assert.equal(typeof req!.body.session_id, "string"); + assert.ok((req!.body.session_id as string).length > 0, "session_id must be non-empty"); + }); +}); + +test("EXPECTED_DIVERGENCES is exactly what this adapter observed on origin/main", () => { + // prime-agent-acp-plugin passed every supported case when this adapter was + // written — see the report accompanying this commit for the break-proof + // evidence that the harness genuinely detects a regression here. + assert.deepEqual(EXPECTED_DIVERGENCES, []); +}); diff --git a/test/fixtures/plugin-corpus.json b/test/fixtures/plugin-corpus.json new file mode 100644 index 0000000..8f0d2de --- /dev/null +++ b/test/fixtures/plugin-corpus.json @@ -0,0 +1,90 @@ +{ + "corpus": "acp-plugin-conformance", + "version": 1, + "canonicalHome": "davidcrowe/gatewaystack-connect:conformance/plugin-corpus.json", + "tracking": "davidcrowe/gatewaystack-connect#1344 (L1 shared conformance corpus, build step 1)", + "purpose": "One table of plugin capabilities, run against EVERY ACP harness plugin through a thin per-plugin adapter. The adapter drives the plugin's real entry point against a fake gateway on 127.0.0.1 (or a stubbed fetch/urlopen where the repo already does that) and asserts on what the person would see or on what the plugin sent. Unit tests stayed green while four plugins dropped every gateway notice (#1334); this corpus is the seam test that would have caught it.", + "vendoring": "Each plugin vendors a byte-identical copy at test/fixtures/plugin-corpus.json (tests/fixtures/ for pytest repos). Its adapter pins the fingerprint below and fails when the copy differs. To change the corpus: edit this file, recompute the fingerprint, and update every plugin's copy and pin in the same change.", + "fingerprint": "sha256 of the raw file bytes, first 16 hex characters. Byte hashing (not a hash of a re-serialised object) so Node and Python compute the same value without agreeing on JSON serialisation.", + "divergences": "A plugin that fails a supported capability records it in its adapter's EXPECTED_DIVERGENCES list with the issue number. The adapter asserts the list exactly: a NEW failure fails CI, and a FIX also fails CI until the entry is removed. Neither can be quietly forgotten. A divergence is not a not-possible row: not-possible means the harness gives the plugin no way to do it.", + "marker": "ACPCONF7F3A", + "capabilities": { + "notice": { + "contract": "When the gateway's reply to POST /govern/tool-output contains notice: \"\", that text reaches the person-visible channel of the harness (stdout systemMessage, stderr, a toast, a UI notify call, a logger the harness shows, or an editor protocol message). ACP_SHADOW=off silences it on the client side.", + "adapterMust": "Answer every other gateway path with an allow verdict. Isolate HOME (or the plugin's state dir) to a temp directory so first-per-session markers from the developer's machine cannot suppress the notice." + }, + "post-tool": { + "contract": "Given the harness's NATIVE post-tool payload, the plugin's outgoing POST /govern/tool-output request body carries tool_name, tool_input, tool_output, session_id and hook_event_name \"PostToolUse\".", + "adapterMust": "Build the payload in the harness's own native shape (its own field names and tool name) from the canonical call below. Assert tool_name equals the native tool name the adapter fed in, or the plugin's documented canonical mapping of it (the adapter names that mapping explicitly)." + } + }, + "cases": [ + { + "id": "notice-shown", + "capability": "notice", + "env": {}, + "gatewayReply": { "decision": "allow", "notice": "ACPCONF7F3A shadow mode: this call would have been held for review" }, + "expect": { "personSees": true, "contains": "ACPCONF7F3A" }, + "issue": "#1334", + "why": "Codex, OpenCode, Hermes and fx dropped every notice while their unit tests stayed green." + }, + { + "id": "notice-shadow-off", + "capability": "notice", + "env": { "ACP_SHADOW": "off" }, + "gatewayReply": { "decision": "allow", "notice": "ACPCONF7F3A shadow mode: this call would have been held for review" }, + "expect": { "personSees": false, "contains": "ACPCONF7F3A" }, + "issue": "#1334", + "why": "ACP_SHADOW=off is the client-side belt to the server's own shadow switch." + }, + { + "id": "post-tool-fields", + "capability": "post-tool", + "env": {}, + "call": { + "tool": "shell", + "command": "echo ACPCONF7F3A", + "output": "ACPCONF7F3A\n", + "sessionId": "acpconf-session-0001" + }, + "gatewayReply": { "decision": "allow" }, + "expect": { + "method": "POST", + "path": "/govern/tool-output", + "hook_event_name": "PostToolUse", + "tool_name": "equals the native tool name fed in, or the adapter's declared canonical mapping", + "tool_input": "a JSON object whose serialisation contains the marker", + "tool_output": "a value whose serialisation contains the marker", + "session_id": "a non-empty string" + }, + "issue": "#1344", + "why": "fx's post-tool call sends no tool_name or tool_input, so the gateway cannot scan or attribute the output." + } + ], + "harnesses": [ + { "plugin": "claude-code-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "claude-code-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "codex-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "codex-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "opencode-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "opencode-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "hermes-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "hermes-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "pi-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "pi-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "grok-build-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "grok-build-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "antigravity-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "antigravity-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "openclaw-acp-plugin", "capability": "notice", "status": "not-possible", "reason": "OpenClaw exposes no after-tool hook to plugins, so the plugin never calls /govern/tool-output and has no reply to surface. Revisit when OpenClaw ships one." }, + { "plugin": "openclaw-acp-plugin", "capability": "post-tool", "status": "not-possible", "reason": "OpenClaw exposes no after-tool hook to plugins, so there is no native post-tool payload to forward. Revisit when OpenClaw ships one." }, + { "plugin": "dsh-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "dsh-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "fx-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "fx-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "muse-code-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "muse-code-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "prime-agent-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "prime-agent-acp-plugin", "capability": "post-tool", "status": "supported" } + ] +}