test(qa): add credential-free native VNC runtime matrix

qa/README.md

@@ -0,0 +1,83 @@
+# Coven Cave QA
+
+This directory contains opt-in quality harnesses that are intentionally kept
+separate from the application and its existing CI. The native VNC harness runs
+the real Linux Tauri window inside an isolated desktop, exposes that desktop to
+VNC automation, and serves the same screen through noVNC for a human observer.
+
+## Native VNC harness
+
+The harness creates a temporary home directory and XDG profile, starts a real
+Coven daemon, opens Cave as a centered `1180x760` window on a `1440x900`
+Openbox desktop, and binds VNC and noVNC to loopback only. The desktop includes
+a wallpaper, Applications menu, taskbar, and clock, so app activity stays
+visible in context instead of filling the screen.
+
+Install the Linux dependencies:
+
+```text
+build-essential dbus-x11 feh ffmpeg imagemagick
+libayatana-appindicator3-dev libgtk-3-dev libjavascriptcoregtk-4.1-dev
+librsvg2-dev libsoup-3.0-dev libssl-dev libwebkit2gtk-4.1-dev libxdo-dev
+lxpanel novnc openbox pkg-config websockify x11-utils x11vnc xauth xdotool xvfb
+```
+
+Install a compatible Coven CLI, then start the harness in the foreground:
+
+```bash
+bun qa/native-vnc/cli/start.ts
+```
+
+The launcher prints the loopback VNC endpoint, noVNC URL, app URL, and artifact
+directory. Open the viewer in the operating system's default browser:
+
+```bash
+bun qa/native-vnc/cli/view.ts
+```
+
+From another terminal, verify connectivity, input, daemon health, and a real
+screenshot:
+
+```bash
+bun qa/native-vnc/cli/smoke.ts
+```
+
+The principal overrides are `CAVE_VNC_DISPLAY`, `CAVE_VNC_PORT`,
+`CAVE_VNC_VIEWER_PORT`, `CAVE_VNC_APP_PORT`, `CAVE_VNC_GEOMETRY`,
+`CAVE_VNC_WINDOW_SIZE`, `CAVE_VNC_COVEN_BIN`, `CAVE_VNC_NOVNC_ROOT`, and
+`CAVE_VNC_ARTIFACT_DIR`.
+
+## Credential-free runtime matrix
+
+Set `CAVE_VNC_FAKE_RUNTIMES=1` before starting the harness to replace only the
+credential-bearing vendor processes with deterministic local doubles. The
+Tauri shell, Next server, Coven daemon, application routes, persistence, VNC
+transport, and desktop remain real.
+
+```bash
+CAVE_VNC_FAKE_RUNTIMES=1 bun qa/native-vnc/cli/start.ts
+bun qa/native-vnc/cli/record-scenarios.ts
+```
+
+The runtime contract is deliberately asymmetric:
+
+| Runtime | Inventory | Summon flow | Chat contract |
+| --- | --- | --- | --- |
+| Codex | Supported | Supported | Generic stream |
+| Claude Code | Supported | Supported | Generic stream |
+| Copilot | Supported | Supported | Copilot JSONL |
+| Hermes | Supported | Supported | One-shot manifest |
+| OpenCode | Supported | Not yet offered | One-shot manifest |
+| OpenClaw | Supported | Dedicated agent path | Local OpenClaw bridge |
+| External manifests | Visible | Not trusted | Rejected before spawn |
+
+The 11 scenarios cover first launch, familiar creation, daemon recovery,
+runtime inventory, summon choices, chat round trips, resume fallback,
+permissions and directory scope, process failure UX, trust boundaries, and
+OpenClaw limitations. Results are written to `artifacts/openvnc-qa/`, including
+`scenario-manifest.json` and optional recordings. Generated media is evidence,
+not source: keep it out of Git and link externally hosted previews from the PR.
+
+For the design and process model, see [architecture](native-vnc/docs/architecture.md).
+For extending coverage, see [adding scenarios](native-vnc/docs/adding-scenarios.md).
+For the hardening roadmap, see [future steps](native-vnc/docs/future-steps.md).

qa/native-vnc/bun-env.d.ts

@@ -0,0 +1,66 @@
+declare module "bun" {
+  export type ShellOutput = {
+    exitCode: number;
+    stderr: Buffer;
+    stdout: Buffer;
+  };
+
+  export type ShellPromise = Promise<ShellOutput> & {
+    cwd(directory: string): ShellPromise;
+    env(environment: Record<string, string | undefined>): ShellPromise;
+    json(): Promise<unknown>;
+    nothrow(): ShellPromise;
+    quiet(): ShellPromise;
+    text(): Promise<string>;
+  };
+
+  export type Shell = (
+    strings: TemplateStringsArray,
+    ...expressions: unknown[]
+  ) => ShellPromise;
+
+  export const $: Shell;
+
+  export type Subprocess = {
+    exitCode: number | null;
+    exited: Promise<number>;
+    kill(signal?: NodeJS.Signals): void;
+    pid: number;
+    stderr: ReadableStream<Uint8Array>;
+    stdin: { end(): void; write(chunk: string | Uint8Array): void };
+    stdout: ReadableStream<Uint8Array>;
+  };
+}
+
+declare module "bun:test" {
+  export const describe: (name: string, suite: () => void) => void;
+  export const expect: (value: unknown) => any;
+  export const test: (name: string, testCase: () => unknown | Promise<unknown>) => void;
+}
+
+declare namespace Bun {
+  type FileHandle = {
+    exists(): Promise<boolean>;
+    json(): Promise<any>;
+    readonly size: number;
+    text(): Promise<string>;
+  };
+
+  type SpawnOptions = {
+    cwd?: string;
+    env?: Record<string, string | undefined>;
+    stderr?: "ignore" | "inherit" | "pipe" | FileHandle;
+    stdin?: "ignore" | "inherit" | "pipe";
+    stdout?: "ignore" | "inherit" | "pipe" | FileHandle;
+  };
+
+  const argv: string[];
+  function file(path: string): FileHandle;
+  function sleep(milliseconds: number): Promise<void>;
+  function spawn(argv: string[], options?: SpawnOptions): import("bun").Subprocess;
+  function write(destination: string | FileHandle, data: string | Uint8Array): Promise<number>;
+}
+
+interface ImportMeta {
+  readonly dir: string;
+}

qa/native-vnc/cli/ci.ts

@@ -0,0 +1,62 @@
+#!/usr/bin/env bun
+
+import { $ } from "bun";
+import { mkdir, rm } from "node:fs/promises";
+import path from "node:path";
+import { repoRoot } from "../core/paths.ts";
+import { waitUntil } from "../core/processes.ts";
+
+const artifactDir = path.resolve(
+  repoRoot,
+  process.env.CAVE_VNC_ARTIFACT_DIR ?? "artifacts/openvnc-qa",
+);
+const launcherLog = path.join(artifactDir, "launcher.log");
+const launcherErrorLog = path.join(artifactDir, "launcher.stderr.log");
+const readyMarker = path.join(artifactDir, "native-window.ready");
+const env = {
+  ...process.env,
+  CAVE_VNC_ARTIFACT_DIR: artifactDir,
+  CAVE_VNC_FAKE_RUNTIMES: "1",
+  CAVE_VNC_READY_TIMEOUT: process.env.CAVE_VNC_READY_TIMEOUT ?? "900",
+  CAVE_VNC_RECORD_SCENARIOS: process.env.CAVE_VNC_RECORD_SCENARIOS ?? "1",
+};
+
+await rm(artifactDir, { recursive: true, force: true });
+await mkdir(artifactDir, { recursive: true });
+
+const launcher = Bun.spawn(["bun", "qa/native-vnc/cli/start.ts"], {
+  cwd: repoRoot,
+  env,
+  stdin: "ignore",
+  stdout: Bun.file(launcherLog),
+  stderr: Bun.file(launcherErrorLog),
+});
+
+try {
+  await waitUntil(
+    "native window",
+    async () => {
+      if (launcher.exitCode !== null) {
+        const [output, errors] = await Promise.all([
+          Bun.file(launcherLog).text(),
+          Bun.file(launcherErrorLog).text(),
+        ]);
+        throw new Error(`VNC launcher exited before readiness:\n${output}${errors}`);
+      }
+      return Bun.file(readyMarker).exists();
+    },
+    { attempts: Number(env.CAVE_VNC_READY_TIMEOUT), intervalMs: 1_000 },
+  );
+  await $`bun qa/native-vnc/cli/smoke.ts`.cwd(repoRoot).env(env);
+  await $`bun qa/native-vnc/cli/record-scenarios.ts`.cwd(repoRoot).env(env);
+} finally {
+  if (launcher.exitCode === null) launcher.kill("SIGTERM");
+  const stopped = await Promise.race([
+    launcher.exited.then(() => true),
+    Bun.sleep(10_000).then(() => false),
+  ]);
+  if (!stopped && launcher.exitCode === null) {
+    launcher.kill("SIGKILL");
+    await launcher.exited;
+  }
+}

qa/native-vnc/cli/record-scenarios.ts

@@ -0,0 +1,20 @@
+#!/usr/bin/env bun
+
+import { ScenarioContext } from "../scenarios/context.ts";
+import { runScenarioCases } from "../scenarios/cases.ts";
+
+const label = "[qa:native-vnc:scenarios]";
+const context = await ScenarioContext.create();
+
+try {
+  await runScenarioCases(context);
+} finally {
+  await context.close();
+}
+
+const failed = context.results.filter((result) => result.status !== "passed");
+console.log(`${label} ${context.results.length - failed.length}/${context.results.length} scenarios passed`);
+console.log(`${label} manifest: ${context.manifestPath}`);
+if (context.recordVideos) console.log(`${label} videos: ${context.videosDir}`);
+for (const failure of failed) console.error(`${label} FAIL ${failure.id}: ${failure.summary}`);
+if (failed.length > 0) process.exitCode = 1;

qa/native-vnc/cli/smoke.ts

@@ -0,0 +1,122 @@
+#!/usr/bin/env bun
+
+import { $ } from "bun";
+import assert from "node:assert/strict";
+import { mkdir, rm } from "node:fs/promises";
+import path from "node:path";
+import { repoRoot } from "../core/paths.ts";
+
+const label = "[qa:native-vnc:smoke]";
+
+function positiveInteger(name: string, fallback: number, minimum = 1): number {
+  const value = Number(process.env[name] ?? fallback);
+  if (!Number.isInteger(value) || value < minimum) throw new Error(`${name} must be at least ${minimum}`);
+  return value;
+}
+
+async function runVnc(args: string[], options: { quiet?: boolean } = {}) {
+  const version = process.env.CAVE_VNC_CLIENT_VERSION ?? "0.2.1";
+  const local = process.env.CAVE_VNC_FORCE_UVX !== "1"
+    && (await $`which vnc`.quiet().nothrow()).exitCode === 0;
+  const command = local ? ["vnc", ...args] : ["uvx", "--from", `vnc-computer-use==${version}`, "vnc", ...args];
+  const result = await $`${command}`.cwd(repoRoot).quiet().nothrow();
+  if (!options.quiet) process.stdout.write(result.stdout);
+  if (result.exitCode !== 0) {
+    throw new Error(`VNC command failed: ${result.stderr.toString().trim()}`);
+  }
+  return result;
+}
+
+if (process.platform !== "linux") throw new Error("the native VNC smoke test requires Linux");
+
+const vncPort = positiveInteger("CAVE_VNC_PORT", 5900);
+const viewerPort = positiveInteger("CAVE_VNC_VIEWER_PORT", 6080);
+const readyTimeout = positiveInteger("CAVE_VNC_READY_TIMEOUT", 300);
+const minimumBytes = positiveInteger("CAVE_VNC_MIN_SCREENSHOT_BYTES", 16_384);
+const minimumColors = positiveInteger("CAVE_VNC_MIN_SCREENSHOT_COLORS", 32, 2);
+const artifactDir = path.resolve(repoRoot, process.env.CAVE_VNC_ARTIFACT_DIR ?? "artifacts/openvnc-qa");
+const session = process.env.CAVE_VNC_SESSION ?? `covencave-qa-${process.pid}`;
+const files = {
+  ready: path.join(artifactDir, "native-window.ready"),
+  viewerUrl: path.join(artifactDir, "viewer.url"),
+  daemonReady: path.join(artifactDir, "daemon.ready"),
+  daemonHealth: path.join(artifactDir, "daemon-health.json"),
+  desktopReady: path.join(artifactDir, "desktop.ready"),
+  screenshot: path.join(artifactDir, "native-window.png"),
+  viewerProbe: path.join(artifactDir, "novnc-viewer.html"),
+  websocketProbe: path.join(artifactDir, "novnc-websocket.json"),
+};
+
+await mkdir(artifactDir, { recursive: true });
+await Promise.all([files.screenshot, files.viewerProbe, files.websocketProbe].map((file) => rm(file, { force: true })));
+
+let connected = false;
+try {
+  for (let second = 0; second < readyTimeout; second += 1) {
+    if (await Bun.file(files.ready).exists()) break;
+    await Bun.sleep(1_000);
+  }
+  assert.equal(await Bun.file(files.ready).exists(), true, "native window readiness marker was not created");
+  for (const evidence of [files.viewerUrl, files.daemonReady, files.daemonHealth, files.desktopReady]) {
+    assert.equal(await Bun.file(evidence).exists(), true, `missing readiness evidence: ${path.basename(evidence)}`);
+  }
+
+  const health = await Bun.file(files.daemonHealth).json();
+  assert.equal(health.ok, true);
+  assert.equal(health.apiVersion, "coven.daemon.v1");
+
+  const viewerUrl = (await Bun.file(files.viewerUrl).text()).trim();
+  const viewerResponse = await fetch(viewerUrl);
+  assert.equal(viewerResponse.ok, true, `noVNC viewer returned HTTP ${viewerResponse.status}`);
+  const viewerHtml = await viewerResponse.text();
+  assert.match(viewerHtml, /noVNC/);
+  await Bun.write(files.viewerProbe, viewerHtml);
+
+  const websocketUrl = `ws://127.0.0.1:${viewerPort}/websockify`;
+  const socket = new WebSocket(websocketUrl);
+  socket.binaryType = "arraybuffer";
+  const banner = await new Promise<string>((resolve, reject) => {
+    const timeout = setTimeout(() => reject(new Error("noVNC WebSocket probe timed out")), 5_000);
+    socket.addEventListener("message", (event) => {
+      clearTimeout(timeout);
+      const text = Buffer.from(event.data).toString("ascii").trim();
+      text.startsWith("RFB ") ? resolve(text) : reject(new Error(`unexpected VNC banner: ${text}`));
+    }, { once: true });
+    socket.addEventListener("error", () => {
+      clearTimeout(timeout);
+      reject(new Error("noVNC WebSocket could not reach the VNC server"));
+    }, { once: true });
+  });
+  socket.close();
+  await Bun.write(files.websocketProbe, `${JSON.stringify({ url: websocketUrl, banner }, null, 2)}\n`);
+
+  await runVnc(["--help"], { quiet: true });
+  for (let attempt = 0; attempt < 30; attempt += 1) {
+    try {
+      await runVnc(["-s", session, "connect", `127.0.0.1::${vncPort}`], { quiet: true });
+      connected = true;
+      break;
+    } catch {
+      await Bun.sleep(1_000);
+    }
+  }
+  assert.equal(connected, true, `could not connect to 127.0.0.1::${vncPort}`);
+  await runVnc(["-s", session, "get_screen_size"]);
+  await runVnc(["-s", session, "mouse_move", "40", "40"]);
+
+  let bytes = 0;
+  let colors = 0;
+  for (let attempt = 0; attempt < 90; attempt += 1) {
+    await runVnc(["-s", session, "get_screenshot", "-o", files.screenshot], { quiet: true });
+    bytes = Bun.file(files.screenshot).size;
+    const probe = await $`identify -format %k ${files.screenshot}`.quiet().nothrow();
+    colors = probe.exitCode === 0 ? Number(probe.stdout.toString()) : 0;
+    if (bytes >= minimumBytes && colors >= minimumColors) break;
+    await Bun.sleep(1_000);
+  }
+  assert.ok(bytes >= minimumBytes && colors >= minimumColors, `last frame: ${bytes} bytes, ${colors} colors`);
+  console.log(`${label} native VNC control verified (${bytes} bytes, ${colors} colors)`);
+  console.log(`${label} live noVNC viewer verified at ${viewerUrl}`);
+} finally {
+  if (connected) await runVnc(["-s", session, "disconnect"], { quiet: true }).catch(() => undefined);
+}