From c652350439fc107c6cd32cece755d72c02d027a6 Mon Sep 17 00:00:00 2001 From: David Crowe Date: Wed, 23 Sep 2026 11:41:21 -0700 Subject: [PATCH] Add ACP plugin conformance adapter (notice + post-tool) Vendors the shared corpus (gatewaystack-connect#1344 step 1) and adds test/conformance.test.ts, which drives the real extension entry point against a fake gateway to check that gateway notices reach the operator (ctx.ui.notify / stderr, honoring ACP_SHADOW=off) and that the outgoing POST /govern/tool-output body carries tool_name, tool_input, tool_output, session_id, and hook_event_name "PostToolUse". Both capabilities pass against origin/main; EXPECTED_DIVERGENCES is empty. Also adds a GitHub Actions workflow so the existing suite and the new adapter run on pull requests and pushes to main. --- .github/workflows/test.yml | 23 +++ test/conformance.test.ts | 300 +++++++++++++++++++++++++++++++ test/fixtures/plugin-corpus.json | 90 ++++++++++ 3 files changed, 413 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 test/conformance.test.ts create mode 100644 test/fixtures/plugin-corpus.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..8a36de1 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: test + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + # node --test on .ts files needs unflagged type stripping + # (stable from Node 22.18 / 24). + node-version: "24" + - run: npm install + - run: npm test + - run: node --test test/conformance.test.ts diff --git a/test/conformance.test.ts b/test/conformance.test.ts new file mode 100644 index 0000000..c57113e --- /dev/null +++ b/test/conformance.test.ts @@ -0,0 +1,300 @@ +// ACP plugin conformance adapter for pi-acp-plugin (gatewaystack-connect#1344, +// step 1). Drives the REAL plugin entry point (imported from ../index.ts, not +// reimplemented) against a fake gateway, using the shared corpus vendored at +// test/fixtures/plugin-corpus.json. See that corpus's own "purpose" field for +// the seam this closes: unit tests stayed green while four plugins dropped +// every gateway notice (#1334). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createServer, type Server } from "node:http"; +import { readFileSync, mkdtempSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import acp from "../index.ts"; + +type Handler = (event: any, ctx: any) => Promise | any; + +// --- Corpus loading + fingerprint pin ------------------------------------- + +const CORPUS_PATH = join(import.meta.dirname, "fixtures", "plugin-corpus.json"); +const PINNED_FINGERPRINT = "aa186d3fb3e7d18c"; +const PLUGIN_NAME = "pi-acp-plugin"; +const MARKER = "ACPCONF7F3A"; + +const rawCorpus = readFileSync(CORPUS_PATH); +const actualFingerprint = createHash("sha256").update(rawCorpus).digest("hex").slice(0, 16); + +test("vendored corpus matches the pinned fingerprint", () => { + assert.equal( + actualFingerprint, + PINNED_FINGERPRINT, + "test/fixtures/plugin-corpus.json has drifted from the canonical copy at " + + "gatewaystack-connect:conformance/plugin-corpus.json — re-copy it with `cp` " + + "(never retype) and update PINNED_FINGERPRINT in the same change.", + ); +}); + +const corpus = JSON.parse(rawCorpus.toString("utf8")); +assert.equal(corpus.marker, MARKER, "corpus marker constant drifted from this adapter's copy"); + +const rows = corpus.harnesses.filter((h: any) => h.plugin === PLUGIN_NAME); +test("corpus declares both capabilities supported for pi-acp-plugin", () => { + assert.deepEqual( + rows.map((r: any) => [r.capability, r.status]).sort(), + [ + ["notice", "supported"], + ["post-tool", "supported"], + ], + ); +}); + +function caseFor(id: string) { + const c = corpus.cases.find((c: any) => c.id === id); + assert.ok(c, `corpus is missing case ${id}`); + return c; +} + +/** + * Canonical tool-name mapping (documented, per the corpus's "post-tool" + * adapterMust): the corpus's generic call.tool "shell" maps onto pi's own + * native tool name for its bash tool, "bash" — pi ships four built-in tools + * (bash, read, write, edit; see index.ts's module doc and README.md) and the + * shell/echo case in the corpus is pi's "bash" tool. + */ +const NATIVE_TOOL_NAME_FOR_SHELL = "bash"; + +/** + * Known, currently-observed failures against origin/main. Each entry must be + * accompanied by a case that is asserted to FAIL (see runCase below) — a + * fix removes the entry, a new regression here fails CI until it's added + * back with evidence, and this array is checked to be exactly this by the + * test at the bottom of the file. + */ +const EXPECTED_DIVERGENCES: Array<{ id: string; issue: string; evidence: string }> = []; + +const observedDivergences: string[] = []; + +/** Runs `fn`; if `id` is a declared divergence, asserts `fn` throws/rejects instead of passing. */ +async function runCase(id: string, fn: () => Promise | void): Promise { + const declared = EXPECTED_DIVERGENCES.find((d) => d.id === id); + if (!declared) { + await fn(); + return; + } + observedDivergences.push(id); + await assert.rejects( + async () => { + await fn(); + }, + undefined, + `case ${id} is listed in EXPECTED_DIVERGENCES (${declared.issue}) but its assertions passed — remove the entry, it's fixed`, + ); +} + +// --- Fake pi host ---------------------------------------------------------- + +function fakePi() { + const handlers: Record = {}; + return { handlers, on: (event: string, fn: Handler) => { handlers[event] = fn; } }; +} + +function fakeCtx(opts: { hasUI: boolean; sessionId?: string }) { + const notes: Array<{ msg: string; level: string }> = []; + return { + notes, + hasUI: opts.hasUI, + cwd: "/tmp", + signal: new AbortController().signal, + sessionManager: { getSessionId: () => opts.sessionId ?? "acpconf-session-0001" }, + ui: { + notify: (msg: string, level: string) => notes.push({ msg, level }), + confirm: async () => true, + }, + }; +} + +/** Fake gateway: /govern/tool-output answers with `gatewayReply`; every other path allows. Records every request. */ +function stubGateway(gatewayReply: unknown): Promise<{ + server: Server; + base: string; + requests: Array<{ method: string; path: string; body: any }>; +}> { + const requests: Array<{ method: string; path: string; body: any }> = []; + const server = createServer((req, res) => { + let raw = ""; + req.on("data", (c) => { raw += c; }); + req.on("end", () => { + const path = req.url ?? ""; + const body = raw ? JSON.parse(raw) : {}; + requests.push({ method: req.method ?? "", path, body }); + const json = path === "/govern/tool-output" ? gatewayReply : { decision: "allow" }; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(json)); + }); + }); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + resolve({ server, base: `http://127.0.0.1:${port}`, requests }); + }); + }); +} + +/** + * Mounts the real plugin: fresh HOME (per adapterMust, so no developer-machine + * state — e.g. a warned-once flag or lapse.log — can suppress a notice), + * dummy credential, base pointed at the fake gateway, case.env applied, and + * ACP_SHADOW cleared unless the case sets it. + */ +function mount(base: string, caseEnv: Record = {}): Record { + const homeDir = mkdtempSync(join(tmpdir(), "acpconf-pi-home-")); + process.env.ACP_BEARER_TOKEN = "acpconf-dummy-credential-not-real"; + process.env.ACP_GOVERN_BASE = base; + process.env.HOME = homeDir; + delete process.env.ACP_API_BASE; + delete process.env.ACP_AGENT_TIER; + delete process.env.ACP_SHADOW; + for (const [k, v] of Object.entries(caseEnv)) process.env[k] = v; + const pi = fakePi(); + acp(pi as any); + return pi.handlers; +} + +const toolResultEvent = (overrides: Partial = {}) => ({ + type: "tool_result", + toolName: "bash", + toolCallId: "acpconf-call-1", + input: {}, + content: [{ type: "text", text: "irrelevant for the notice cases" }], + isError: false, + ...overrides, +}); + +// --- notice-shown: attended channel (ctx.ui.notify) ------------------------- + +test("notice-shown: gateway notice reaches the operator via ctx.ui.notify (attended)", async () => { + await runCase("notice-shown", async () => { + const c = caseFor("notice-shown"); + const { server, base } = await stubGateway(c.gatewayReply); + try { + const handlers = mount(base, c.env); + const ctx = fakeCtx({ hasUI: true }); + await handlers.tool_result(toolResultEvent(), ctx); + const seen = ctx.notes.some((n) => n.msg.includes(MARKER)); + assert.equal(seen, c.expect.personSees, `ctx.ui.notify notes: ${JSON.stringify(ctx.notes)}`); + } finally { + server.close(); + } + }); +}); + +// --- notice-shown: unattended channel (stderr via console.error) ----------- + +test("notice-shown: gateway notice reaches the operator via stderr (unattended, no UI)", async () => { + await runCase("notice-shown", async () => { + const c = caseFor("notice-shown"); + const { server, base } = await stubGateway(c.gatewayReply); + const originalError = console.error; + const stderrLines: string[] = []; + console.error = (...args: unknown[]) => { stderrLines.push(args.map(String).join(" ")); }; + try { + const handlers = mount(base, c.env); + const ctx = fakeCtx({ hasUI: false }); + await handlers.tool_result(toolResultEvent(), ctx); + const seen = stderrLines.some((l) => l.includes(MARKER)); + assert.equal(seen, c.expect.personSees, `stderr lines: ${JSON.stringify(stderrLines)}`); + } finally { + console.error = originalError; + server.close(); + } + }); +}); + +// --- notice-shadow-off: both channels, marker must be ABSENT everywhere ---- + +test("notice-shadow-off: ACP_SHADOW=off silences ctx.ui.notify (attended)", async () => { + await runCase("notice-shadow-off", async () => { + const c = caseFor("notice-shadow-off"); + const { server, base } = await stubGateway(c.gatewayReply); + try { + const handlers = mount(base, c.env); + const ctx = fakeCtx({ hasUI: true }); + await handlers.tool_result(toolResultEvent(), ctx); + const seen = ctx.notes.some((n) => n.msg.includes(MARKER)); + assert.equal(seen, c.expect.personSees, `ctx.ui.notify notes: ${JSON.stringify(ctx.notes)}`); + } finally { + server.close(); + } + }); +}); + +test("notice-shadow-off: ACP_SHADOW=off silences stderr (unattended, no UI)", async () => { + await runCase("notice-shadow-off", async () => { + const c = caseFor("notice-shadow-off"); + const { server, base } = await stubGateway(c.gatewayReply); + const originalError = console.error; + const stderrLines: string[] = []; + console.error = (...args: unknown[]) => { stderrLines.push(args.map(String).join(" ")); }; + try { + const handlers = mount(base, c.env); + const ctx = fakeCtx({ hasUI: false }); + await handlers.tool_result(toolResultEvent(), ctx); + const seen = stderrLines.some((l) => l.includes(MARKER)); + assert.equal(seen, c.expect.personSees, `stderr lines: ${JSON.stringify(stderrLines)}`); + } finally { + console.error = originalError; + server.close(); + } + }); +}); + +// --- post-tool-fields -------------------------------------------------- + +test("post-tool-fields: POST /govern/tool-output carries the required native fields", async () => { + await runCase("post-tool-fields", async () => { + const c = caseFor("post-tool-fields"); + const { server, base, requests } = await stubGateway(c.gatewayReply); + try { + const handlers = mount(base, c.env); + // pi's own native tool_result shape for a shell/bash call: toolName is + // pi's own name for the tool, input/content are pi's own field names. + const event = toolResultEvent({ + toolName: NATIVE_TOOL_NAME_FOR_SHELL, + toolCallId: "acpconf-call-post-tool", + input: { command: c.call.command }, + content: [{ type: "text", text: c.call.output }], + }); + const ctx = fakeCtx({ hasUI: true, sessionId: c.call.sessionId }); + await handlers.tool_result(event, ctx); + + const req = requests.find((r) => r.path === "/govern/tool-output"); + assert.ok(req, `no request recorded to /govern/tool-output; saw: ${JSON.stringify(requests.map((r) => r.path))}`); + assert.equal(req!.method, "POST"); + assert.equal(req!.body.hook_event_name, "PostToolUse"); + assert.equal( + req!.body.tool_name, + NATIVE_TOOL_NAME_FOR_SHELL, + "tool_name must equal the native tool name fed in, or the adapter's declared canonical mapping " + + `(NATIVE_TOOL_NAME_FOR_SHELL = ${JSON.stringify(NATIVE_TOOL_NAME_FOR_SHELL)})`, + ); + assert.ok(JSON.stringify(req!.body.tool_input).includes(MARKER), `tool_input missing marker: ${JSON.stringify(req!.body.tool_input)}`); + assert.ok(JSON.stringify(req!.body.tool_output).includes(MARKER), `tool_output missing marker: ${JSON.stringify(req!.body.tool_output)}`); + assert.equal(typeof req!.body.session_id, "string"); + assert.ok(req!.body.session_id.length > 0, "session_id must be a non-empty string"); + assert.equal(req!.body.session_id, c.call.sessionId); + } finally { + server.close(); + } + }); +}); + +// --- EXPECTED_DIVERGENCES bookkeeping --------------------------------------- + +test("EXPECTED_DIVERGENCES is exactly what this run observed", () => { + assert.deepEqual( + observedDivergences.sort(), + EXPECTED_DIVERGENCES.map((d) => d.id).sort(), + ); +}); diff --git a/test/fixtures/plugin-corpus.json b/test/fixtures/plugin-corpus.json new file mode 100644 index 0000000..8f0d2de --- /dev/null +++ b/test/fixtures/plugin-corpus.json @@ -0,0 +1,90 @@ +{ + "corpus": "acp-plugin-conformance", + "version": 1, + "canonicalHome": "davidcrowe/gatewaystack-connect:conformance/plugin-corpus.json", + "tracking": "davidcrowe/gatewaystack-connect#1344 (L1 shared conformance corpus, build step 1)", + "purpose": "One table of plugin capabilities, run against EVERY ACP harness plugin through a thin per-plugin adapter. The adapter drives the plugin's real entry point against a fake gateway on 127.0.0.1 (or a stubbed fetch/urlopen where the repo already does that) and asserts on what the person would see or on what the plugin sent. Unit tests stayed green while four plugins dropped every gateway notice (#1334); this corpus is the seam test that would have caught it.", + "vendoring": "Each plugin vendors a byte-identical copy at test/fixtures/plugin-corpus.json (tests/fixtures/ for pytest repos). Its adapter pins the fingerprint below and fails when the copy differs. To change the corpus: edit this file, recompute the fingerprint, and update every plugin's copy and pin in the same change.", + "fingerprint": "sha256 of the raw file bytes, first 16 hex characters. Byte hashing (not a hash of a re-serialised object) so Node and Python compute the same value without agreeing on JSON serialisation.", + "divergences": "A plugin that fails a supported capability records it in its adapter's EXPECTED_DIVERGENCES list with the issue number. The adapter asserts the list exactly: a NEW failure fails CI, and a FIX also fails CI until the entry is removed. Neither can be quietly forgotten. A divergence is not a not-possible row: not-possible means the harness gives the plugin no way to do it.", + "marker": "ACPCONF7F3A", + "capabilities": { + "notice": { + "contract": "When the gateway's reply to POST /govern/tool-output contains notice: \"\", that text reaches the person-visible channel of the harness (stdout systemMessage, stderr, a toast, a UI notify call, a logger the harness shows, or an editor protocol message). ACP_SHADOW=off silences it on the client side.", + "adapterMust": "Answer every other gateway path with an allow verdict. Isolate HOME (or the plugin's state dir) to a temp directory so first-per-session markers from the developer's machine cannot suppress the notice." + }, + "post-tool": { + "contract": "Given the harness's NATIVE post-tool payload, the plugin's outgoing POST /govern/tool-output request body carries tool_name, tool_input, tool_output, session_id and hook_event_name \"PostToolUse\".", + "adapterMust": "Build the payload in the harness's own native shape (its own field names and tool name) from the canonical call below. Assert tool_name equals the native tool name the adapter fed in, or the plugin's documented canonical mapping of it (the adapter names that mapping explicitly)." + } + }, + "cases": [ + { + "id": "notice-shown", + "capability": "notice", + "env": {}, + "gatewayReply": { "decision": "allow", "notice": "ACPCONF7F3A shadow mode: this call would have been held for review" }, + "expect": { "personSees": true, "contains": "ACPCONF7F3A" }, + "issue": "#1334", + "why": "Codex, OpenCode, Hermes and fx dropped every notice while their unit tests stayed green." + }, + { + "id": "notice-shadow-off", + "capability": "notice", + "env": { "ACP_SHADOW": "off" }, + "gatewayReply": { "decision": "allow", "notice": "ACPCONF7F3A shadow mode: this call would have been held for review" }, + "expect": { "personSees": false, "contains": "ACPCONF7F3A" }, + "issue": "#1334", + "why": "ACP_SHADOW=off is the client-side belt to the server's own shadow switch." + }, + { + "id": "post-tool-fields", + "capability": "post-tool", + "env": {}, + "call": { + "tool": "shell", + "command": "echo ACPCONF7F3A", + "output": "ACPCONF7F3A\n", + "sessionId": "acpconf-session-0001" + }, + "gatewayReply": { "decision": "allow" }, + "expect": { + "method": "POST", + "path": "/govern/tool-output", + "hook_event_name": "PostToolUse", + "tool_name": "equals the native tool name fed in, or the adapter's declared canonical mapping", + "tool_input": "a JSON object whose serialisation contains the marker", + "tool_output": "a value whose serialisation contains the marker", + "session_id": "a non-empty string" + }, + "issue": "#1344", + "why": "fx's post-tool call sends no tool_name or tool_input, so the gateway cannot scan or attribute the output." + } + ], + "harnesses": [ + { "plugin": "claude-code-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "claude-code-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "codex-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "codex-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "opencode-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "opencode-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "hermes-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "hermes-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "pi-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "pi-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "grok-build-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "grok-build-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "antigravity-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "antigravity-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "openclaw-acp-plugin", "capability": "notice", "status": "not-possible", "reason": "OpenClaw exposes no after-tool hook to plugins, so the plugin never calls /govern/tool-output and has no reply to surface. Revisit when OpenClaw ships one." }, + { "plugin": "openclaw-acp-plugin", "capability": "post-tool", "status": "not-possible", "reason": "OpenClaw exposes no after-tool hook to plugins, so there is no native post-tool payload to forward. Revisit when OpenClaw ships one." }, + { "plugin": "dsh-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "dsh-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "fx-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "fx-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "muse-code-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "muse-code-acp-plugin", "capability": "post-tool", "status": "supported" }, + { "plugin": "prime-agent-acp-plugin", "capability": "notice", "status": "supported" }, + { "plugin": "prime-agent-acp-plugin", "capability": "post-tool", "status": "supported" } + ] +}