Shared plugin conformance suite (gatewaystack-connect#1344 step 1)

.github/workflows/test.yml

@@ -0,0 +1,20 @@
+name: test
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+      - run: npm ci
+      - run: npm test

test/conformance.test.mjs

@@ -0,0 +1,128 @@
+// ACP plugin conformance adapter for openclaw-acp-plugin.
+// Corpus: davidcrowe/gatewaystack-connect:conformance/plugin-corpus.json
+// Tracking: davidcrowe/gatewaystack-connect#1344
+//
+// Both corpus capabilities ("notice" and "post-tool") are declared
+// "not-possible" for this plugin: OpenClaw's plugin API exposes only
+// before_tool_call — there is no after-tool/post-tool hook for a plugin to
+// register, so the plugin never calls POST /govern/tool-output on the way
+// out and has no native post-tool payload to build or reply to surface.
+// Because nothing is "supported" here, there is nothing to drive against a
+// fake gateway (no notice-shown / notice-shadow-off / post-tool-fields
+// cases apply) and no EXPECTED_DIVERGENCES list — a divergence can only
+// exist for a supported capability, and none exists here.
+//
+// What this file actually checks is the STATED FACT behind both
+// not-possible rows: that the plugin's register() call, exercised against
+// a fake OpenClaw host, registers before_tool_call and nothing that looks
+// like an after-tool/post-tool hook. If OpenClaw ever ships an after-tool
+// hook and this plugin starts registering it, this test fails loudly and
+// both corpus rows above have gone stale.
+//
+// Run with: npm test (builds src/ first, then node --test test/)
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CORPUS_PATH = join(__dirname, "fixtures", "plugin-corpus.json");
+const PINNED_FINGERPRINT = "aa186d3fb3e7d18c";
+const PLUGIN_ID = "openclaw-acp-plugin";
+
+// A hook name that behaves like an after-tool/post-tool hook, even one
+// OpenClaw might name differently than we expect today.
+const AFTER_TOOL_HOOK_PATTERN = /after.?tool|post.?tool|tool.?(result|output|complete|end)/i;
+
+function loadCorpus() {
+  const raw = readFileSync(CORPUS_PATH);
+  const fingerprint = createHash("sha256").update(raw).digest("hex").slice(0, 16);
+  assert.equal(
+    fingerprint,
+    PINNED_FINGERPRINT,
+    `vendored plugin-corpus.json at ${CORPUS_PATH} does not match the pinned fingerprint ` +
+      `(${PINNED_FINGERPRINT}) — got ${fingerprint}. Re-vendor a byte-identical copy from ` +
+      `davidcrowe/gatewaystack-connect:conformance/plugin-corpus.json and update the pin ` +
+      `together if the corpus changed intentionally.`
+  );
+  return JSON.parse(raw.toString("utf8"));
+}
+
+function corpusRow(corpus, capability) {
+  const row = corpus.harnesses.find(
+    (h) => h.plugin === PLUGIN_ID && h.capability === capability
+  );
+  assert.ok(row, `corpus has no ${PLUGIN_ID}/${capability} row`);
+  return row;
+}
+
+// dist/plugin.js is CommonJS (`exports.default = ...`); Node's ESM interop
+// surfaces that as `module.default.default` rather than `module.default`
+// (see test/fail-posture.test.mjs for the same pattern).
+const pluginModule = await import("../dist/plugin.js");
+const plugin = pluginModule.default.default ?? pluginModule.default;
+
+// Registers the plugin against a fake OpenClaw host that records every
+// hook name it's asked to register, then hands back what was recorded.
+function registerAgainstFakeHost() {
+  const registeredHooks = [];
+  const fakeApi = {
+    on(hook, _handler, _opts) {
+      registeredHooks.push(hook);
+    },
+  };
+  plugin.register(fakeApi);
+  return registeredHooks;
+}
+
+test("vendored corpus fingerprint matches the pinned value", () => {
+  loadCorpus();
+});
+
+test("openclaw-acp-plugin/notice is declared not-possible with a non-empty reason", () => {
+  const corpus = loadCorpus();
+  const row = corpusRow(corpus, "notice");
+  assert.equal(row.status, "not-possible");
+  assert.equal(typeof row.reason, "string");
+  assert.ok(row.reason.trim().length > 0, "reason must be non-empty");
+});
+
+test("openclaw-acp-plugin/post-tool is declared not-possible with a non-empty reason", () => {
+  const corpus = loadCorpus();
+  const row = corpusRow(corpus, "post-tool");
+  assert.equal(row.status, "not-possible");
+  assert.equal(typeof row.reason, "string");
+  assert.ok(row.reason.trim().length > 0, "reason must be non-empty");
+});
+
+// The concrete "stated fact" check backing BOTH not-possible rows above:
+// both reasons rest on the same underlying fact (OpenClaw exposes no
+// after-tool hook to plugins), so one shared registration check is enough
+// to back both assertions rather than duplicating it per capability.
+test("register() only registers before_tool_call — no after-tool/post-tool hook exists to go stale", () => {
+  const registeredHooks = registerAgainstFakeHost();
+
+  // Proves the fake harness actually exercised the real registration path
+  // (not a no-op registration whose absence of post-tool hooks would be
+  // vacuously true).
+  assert.ok(
+    registeredHooks.includes("before_tool_call"),
+    `expected the plugin to register "before_tool_call"; got [${registeredHooks.join(", ")}]`
+  );
+
+  // The concrete claim behind both not-possible rows: nothing registered
+  // looks like an after-tool/post-tool hook. Checked both against the
+  // literal hook name this plugin's src/plugin.ts knows about today, and
+  // defensively via a naming-convention regex so a differently-spelled new
+  // hook still trips this.
+  for (const hook of registeredHooks) {
+    assert.notEqual(hook, "after_tool_call", `plugin registered "${hook}" — the notice/post-tool corpus rows are now stale`);
+    assert.doesNotMatch(
+      hook,
+      AFTER_TOOL_HOOK_PATTERN,
+      `registered hook "${hook}" looks like an after-tool/post-tool hook — the notice/post-tool corpus rows are now stale`
+    );
+  }
+});

test/fixtures/plugin-corpus.json

@@ -0,0 +1,90 @@
+{
+  "corpus": "acp-plugin-conformance",
+  "version": 1,
+  "canonicalHome": "davidcrowe/gatewaystack-connect:conformance/plugin-corpus.json",
+  "tracking": "davidcrowe/gatewaystack-connect#1344 (L1 shared conformance corpus, build step 1)",
+  "purpose": "One table of plugin capabilities, run against EVERY ACP harness plugin through a thin per-plugin adapter. The adapter drives the plugin's real entry point against a fake gateway on 127.0.0.1 (or a stubbed fetch/urlopen where the repo already does that) and asserts on what the person would see or on what the plugin sent. Unit tests stayed green while four plugins dropped every gateway notice (#1334); this corpus is the seam test that would have caught it.",
+  "vendoring": "Each plugin vendors a byte-identical copy at test/fixtures/plugin-corpus.json (tests/fixtures/ for pytest repos). Its adapter pins the fingerprint below and fails when the copy differs. To change the corpus: edit this file, recompute the fingerprint, and update every plugin's copy and pin in the same change.",
+  "fingerprint": "sha256 of the raw file bytes, first 16 hex characters. Byte hashing (not a hash of a re-serialised object) so Node and Python compute the same value without agreeing on JSON serialisation.",
+  "divergences": "A plugin that fails a supported capability records it in its adapter's EXPECTED_DIVERGENCES list with the issue number. The adapter asserts the list exactly: a NEW failure fails CI, and a FIX also fails CI until the entry is removed. Neither can be quietly forgotten. A divergence is not a not-possible row: not-possible means the harness gives the plugin no way to do it.",
+  "marker": "ACPCONF7F3A",
+  "capabilities": {
+    "notice": {
+      "contract": "When the gateway's reply to POST /govern/tool-output contains notice: \"<text>\", that text reaches the person-visible channel of the harness (stdout systemMessage, stderr, a toast, a UI notify call, a logger the harness shows, or an editor protocol message). ACP_SHADOW=off silences it on the client side.",
+      "adapterMust": "Answer every other gateway path with an allow verdict. Isolate HOME (or the plugin's state dir) to a temp directory so first-per-session markers from the developer's machine cannot suppress the notice."
+    },
+    "post-tool": {
+      "contract": "Given the harness's NATIVE post-tool payload, the plugin's outgoing POST /govern/tool-output request body carries tool_name, tool_input, tool_output, session_id and hook_event_name \"PostToolUse\".",
+      "adapterMust": "Build the payload in the harness's own native shape (its own field names and tool name) from the canonical call below. Assert tool_name equals the native tool name the adapter fed in, or the plugin's documented canonical mapping of it (the adapter names that mapping explicitly)."
+    }
+  },
+  "cases": [
+    {
+      "id": "notice-shown",
+      "capability": "notice",
+      "env": {},
+      "gatewayReply": { "decision": "allow", "notice": "ACPCONF7F3A shadow mode: this call would have been held for review" },
+      "expect": { "personSees": true, "contains": "ACPCONF7F3A" },
+      "issue": "#1334",
+      "why": "Codex, OpenCode, Hermes and fx dropped every notice while their unit tests stayed green."
+    },
+    {
+      "id": "notice-shadow-off",
+      "capability": "notice",
+      "env": { "ACP_SHADOW": "off" },
+      "gatewayReply": { "decision": "allow", "notice": "ACPCONF7F3A shadow mode: this call would have been held for review" },
+      "expect": { "personSees": false, "contains": "ACPCONF7F3A" },
+      "issue": "#1334",
+      "why": "ACP_SHADOW=off is the client-side belt to the server's own shadow switch."
+    },
+    {
+      "id": "post-tool-fields",
+      "capability": "post-tool",
+      "env": {},
+      "call": {
+        "tool": "shell",
+        "command": "echo ACPCONF7F3A",
+        "output": "ACPCONF7F3A\n",
+        "sessionId": "acpconf-session-0001"
+      },
+      "gatewayReply": { "decision": "allow" },
+      "expect": {
+        "method": "POST",
+        "path": "/govern/tool-output",
+        "hook_event_name": "PostToolUse",
+        "tool_name": "equals the native tool name fed in, or the adapter's declared canonical mapping",
+        "tool_input": "a JSON object whose serialisation contains the marker",
+        "tool_output": "a value whose serialisation contains the marker",
+        "session_id": "a non-empty string"
+      },
+      "issue": "#1344",
+      "why": "fx's post-tool call sends no tool_name or tool_input, so the gateway cannot scan or attribute the output."
+    }
+  ],
+  "harnesses": [
+    { "plugin": "claude-code-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "claude-code-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "codex-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "codex-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "opencode-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "opencode-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "hermes-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "hermes-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "pi-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "pi-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "grok-build-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "grok-build-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "antigravity-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "antigravity-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "openclaw-acp-plugin", "capability": "notice", "status": "not-possible", "reason": "OpenClaw exposes no after-tool hook to plugins, so the plugin never calls /govern/tool-output and has no reply to surface. Revisit when OpenClaw ships one." },
+    { "plugin": "openclaw-acp-plugin", "capability": "post-tool", "status": "not-possible", "reason": "OpenClaw exposes no after-tool hook to plugins, so there is no native post-tool payload to forward. Revisit when OpenClaw ships one." },
+    { "plugin": "dsh-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "dsh-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "fx-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "fx-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "muse-code-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "muse-code-acp-plugin", "capability": "post-tool", "status": "supported" },
+    { "plugin": "prime-agent-acp-plugin", "capability": "notice", "status": "supported" },
+    { "plugin": "prime-agent-acp-plugin", "capability": "post-tool", "status": "supported" }
+  ]
+}