From 8220d9c26b271d389e491d6f93fd6053e028ee2a Mon Sep 17 00:00:00 2001 From: ox <286927284+oxfern@users.noreply.github.com> Date: Fri, 17 Jul 2026 04:30:14 +0300 Subject: [PATCH 1/2] feat(qa): add Bun native VNC harness --- qa/native-vnc/bun-env.d.ts | 66 +++++ qa/native-vnc/cli/ci.ts | 62 +++++ qa/native-vnc/cli/record-scenarios.ts | 20 ++ qa/native-vnc/cli/smoke.ts | 122 +++++++++ qa/native-vnc/cli/start.ts | 184 +++++++++++++ qa/native-vnc/cli/view.ts | 32 +++ qa/native-vnc/core/config.ts | 140 ++++++++++ qa/native-vnc/core/daemon.ts | 81 ++++++ qa/native-vnc/core/network.ts | 34 +++ qa/native-vnc/core/paths.ts | 12 + qa/native-vnc/core/processes.ts | 73 ++++++ qa/native-vnc/core/state.ts | 138 ++++++++++ qa/native-vnc/desktop/lxpanel.conf | 79 ++++++ qa/native-vnc/runtimes/fake-coven.ts | 140 ++++++++++ qa/native-vnc/runtimes/protocol.ts | 41 +++ qa/native-vnc/runtimes/vendor-runtime.ts | 105 ++++++++ qa/native-vnc/scenarios/cases.ts | 312 +++++++++++++++++++++++ qa/native-vnc/scenarios/context.ts | 277 ++++++++++++++++++++ qa/native-vnc/scenarios/types.ts | 50 ++++ qa/native-vnc/tests/protocol.test.ts | 19 ++ qa/native-vnc/tests/structure.test.ts | 58 +++++ 21 files changed, 2045 insertions(+) create mode 100644 qa/native-vnc/bun-env.d.ts create mode 100644 qa/native-vnc/cli/ci.ts create mode 100644 qa/native-vnc/cli/record-scenarios.ts create mode 100644 qa/native-vnc/cli/smoke.ts create mode 100644 qa/native-vnc/cli/start.ts create mode 100644 qa/native-vnc/cli/view.ts create mode 100644 qa/native-vnc/core/config.ts create mode 100644 qa/native-vnc/core/daemon.ts create mode 100644 qa/native-vnc/core/network.ts create mode 100644 qa/native-vnc/core/paths.ts create mode 100644 qa/native-vnc/core/processes.ts create mode 100644 qa/native-vnc/core/state.ts create mode 100644 qa/native-vnc/desktop/lxpanel.conf create mode 100644 qa/native-vnc/runtimes/fake-coven.ts create mode 100644 qa/native-vnc/runtimes/protocol.ts create mode 100644 qa/native-vnc/runtimes/vendor-runtime.ts create mode 100644 qa/native-vnc/scenarios/cases.ts create mode 100644 qa/native-vnc/scenarios/context.ts create mode 100644 qa/native-vnc/scenarios/types.ts create mode 100644 qa/native-vnc/tests/protocol.test.ts create mode 100644 qa/native-vnc/tests/structure.test.ts diff --git a/qa/native-vnc/bun-env.d.ts b/qa/native-vnc/bun-env.d.ts new file mode 100644 index 000000000..d24fab889 --- /dev/null +++ b/qa/native-vnc/bun-env.d.ts @@ -0,0 +1,66 @@ +declare module "bun" { + export type ShellOutput = { + exitCode: number; + stderr: Buffer; + stdout: Buffer; + }; + + export type ShellPromise = Promise & { + cwd(directory: string): ShellPromise; + env(environment: Record): ShellPromise; + json(): Promise; + nothrow(): ShellPromise; + quiet(): ShellPromise; + text(): Promise; + }; + + export type Shell = ( + strings: TemplateStringsArray, + ...expressions: unknown[] + ) => ShellPromise; + + export const $: Shell; + + export type Subprocess = { + exitCode: number | null; + exited: Promise; + kill(signal?: NodeJS.Signals): void; + pid: number; + stderr: ReadableStream; + stdin: { end(): void; write(chunk: string | Uint8Array): void }; + stdout: ReadableStream; + }; +} + +declare module "bun:test" { + export const describe: (name: string, suite: () => void) => void; + export const expect: (value: unknown) => any; + export const test: (name: string, testCase: () => unknown | Promise) => void; +} + +declare namespace Bun { + type FileHandle = { + exists(): Promise; + json(): Promise; + readonly size: number; + text(): Promise; + }; + + type SpawnOptions = { + cwd?: string; + env?: Record; + stderr?: "ignore" | "inherit" | "pipe" | FileHandle; + stdin?: "ignore" | "inherit" | "pipe"; + stdout?: "ignore" | "inherit" | "pipe" | FileHandle; + }; + + const argv: string[]; + function file(path: string): FileHandle; + function sleep(milliseconds: number): Promise; + function spawn(argv: string[], options?: SpawnOptions): import("bun").Subprocess; + function write(destination: string | FileHandle, data: string | Uint8Array): Promise; +} + +interface ImportMeta { + readonly dir: string; +} diff --git a/qa/native-vnc/cli/ci.ts b/qa/native-vnc/cli/ci.ts new file mode 100644 index 000000000..52194356d --- /dev/null +++ b/qa/native-vnc/cli/ci.ts @@ -0,0 +1,62 @@ +#!/usr/bin/env bun + +import { $ } from "bun"; +import { mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import { repoRoot } from "../core/paths.ts"; +import { waitUntil } from "../core/processes.ts"; + +const artifactDir = path.resolve( + repoRoot, + process.env.CAVE_VNC_ARTIFACT_DIR ?? "artifacts/openvnc-qa", +); +const launcherLog = path.join(artifactDir, "launcher.log"); +const launcherErrorLog = path.join(artifactDir, "launcher.stderr.log"); +const readyMarker = path.join(artifactDir, "native-window.ready"); +const env = { + ...process.env, + CAVE_VNC_ARTIFACT_DIR: artifactDir, + CAVE_VNC_FAKE_RUNTIMES: "1", + CAVE_VNC_READY_TIMEOUT: process.env.CAVE_VNC_READY_TIMEOUT ?? "900", + CAVE_VNC_RECORD_SCENARIOS: process.env.CAVE_VNC_RECORD_SCENARIOS ?? "1", +}; + +await rm(artifactDir, { recursive: true, force: true }); +await mkdir(artifactDir, { recursive: true }); + +const launcher = Bun.spawn(["bun", "qa/native-vnc/cli/start.ts"], { + cwd: repoRoot, + env, + stdin: "ignore", + stdout: Bun.file(launcherLog), + stderr: Bun.file(launcherErrorLog), +}); + +try { + await waitUntil( + "native window", + async () => { + if (launcher.exitCode !== null) { + const [output, errors] = await Promise.all([ + Bun.file(launcherLog).text(), + Bun.file(launcherErrorLog).text(), + ]); + throw new Error(`VNC launcher exited before readiness:\n${output}${errors}`); + } + return Bun.file(readyMarker).exists(); + }, + { attempts: Number(env.CAVE_VNC_READY_TIMEOUT), intervalMs: 1_000 }, + ); + await $`bun qa/native-vnc/cli/smoke.ts`.cwd(repoRoot).env(env); + await $`bun qa/native-vnc/cli/record-scenarios.ts`.cwd(repoRoot).env(env); +} finally { + if (launcher.exitCode === null) launcher.kill("SIGTERM"); + const stopped = await Promise.race([ + launcher.exited.then(() => true), + Bun.sleep(10_000).then(() => false), + ]); + if (!stopped && launcher.exitCode === null) { + launcher.kill("SIGKILL"); + await launcher.exited; + } +} diff --git a/qa/native-vnc/cli/record-scenarios.ts b/qa/native-vnc/cli/record-scenarios.ts new file mode 100644 index 000000000..0dd6ea1f9 --- /dev/null +++ b/qa/native-vnc/cli/record-scenarios.ts @@ -0,0 +1,20 @@ +#!/usr/bin/env bun + +import { ScenarioContext } from "../scenarios/context.ts"; +import { runScenarioCases } from "../scenarios/cases.ts"; + +const label = "[qa:native-vnc:scenarios]"; +const context = await ScenarioContext.create(); + +try { + await runScenarioCases(context); +} finally { + await context.close(); +} + +const failed = context.results.filter((result) => result.status !== "passed"); +console.log(`${label} ${context.results.length - failed.length}/${context.results.length} scenarios passed`); +console.log(`${label} manifest: ${context.manifestPath}`); +if (context.recordVideos) console.log(`${label} videos: ${context.videosDir}`); +for (const failure of failed) console.error(`${label} FAIL ${failure.id}: ${failure.summary}`); +if (failed.length > 0) process.exitCode = 1; diff --git a/qa/native-vnc/cli/smoke.ts b/qa/native-vnc/cli/smoke.ts new file mode 100644 index 000000000..624b65c9b --- /dev/null +++ b/qa/native-vnc/cli/smoke.ts @@ -0,0 +1,122 @@ +#!/usr/bin/env bun + +import { $ } from "bun"; +import assert from "node:assert/strict"; +import { mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import { repoRoot } from "../core/paths.ts"; + +const label = "[qa:native-vnc:smoke]"; + +function positiveInteger(name: string, fallback: number, minimum = 1): number { + const value = Number(process.env[name] ?? fallback); + if (!Number.isInteger(value) || value < minimum) throw new Error(`${name} must be at least ${minimum}`); + return value; +} + +async function runVnc(args: string[], options: { quiet?: boolean } = {}) { + const version = process.env.CAVE_VNC_CLIENT_VERSION ?? "0.2.1"; + const local = process.env.CAVE_VNC_FORCE_UVX !== "1" + && (await $`which vnc`.quiet().nothrow()).exitCode === 0; + const command = local ? ["vnc", ...args] : ["uvx", "--from", `vnc-computer-use==${version}`, "vnc", ...args]; + const result = await $`${command}`.cwd(repoRoot).quiet().nothrow(); + if (!options.quiet) process.stdout.write(result.stdout); + if (result.exitCode !== 0) { + throw new Error(`VNC command failed: ${result.stderr.toString().trim()}`); + } + return result; +} + +if (process.platform !== "linux") throw new Error("the native VNC smoke test requires Linux"); + +const vncPort = positiveInteger("CAVE_VNC_PORT", 5900); +const viewerPort = positiveInteger("CAVE_VNC_VIEWER_PORT", 6080); +const readyTimeout = positiveInteger("CAVE_VNC_READY_TIMEOUT", 300); +const minimumBytes = positiveInteger("CAVE_VNC_MIN_SCREENSHOT_BYTES", 16_384); +const minimumColors = positiveInteger("CAVE_VNC_MIN_SCREENSHOT_COLORS", 32, 2); +const artifactDir = path.resolve(repoRoot, process.env.CAVE_VNC_ARTIFACT_DIR ?? "artifacts/openvnc-qa"); +const session = process.env.CAVE_VNC_SESSION ?? `covencave-qa-${process.pid}`; +const files = { + ready: path.join(artifactDir, "native-window.ready"), + viewerUrl: path.join(artifactDir, "viewer.url"), + daemonReady: path.join(artifactDir, "daemon.ready"), + daemonHealth: path.join(artifactDir, "daemon-health.json"), + desktopReady: path.join(artifactDir, "desktop.ready"), + screenshot: path.join(artifactDir, "native-window.png"), + viewerProbe: path.join(artifactDir, "novnc-viewer.html"), + websocketProbe: path.join(artifactDir, "novnc-websocket.json"), +}; + +await mkdir(artifactDir, { recursive: true }); +await Promise.all([files.screenshot, files.viewerProbe, files.websocketProbe].map((file) => rm(file, { force: true }))); + +let connected = false; +try { + for (let second = 0; second < readyTimeout; second += 1) { + if (await Bun.file(files.ready).exists()) break; + await Bun.sleep(1_000); + } + assert.equal(await Bun.file(files.ready).exists(), true, "native window readiness marker was not created"); + for (const evidence of [files.viewerUrl, files.daemonReady, files.daemonHealth, files.desktopReady]) { + assert.equal(await Bun.file(evidence).exists(), true, `missing readiness evidence: ${path.basename(evidence)}`); + } + + const health = await Bun.file(files.daemonHealth).json(); + assert.equal(health.ok, true); + assert.equal(health.apiVersion, "coven.daemon.v1"); + + const viewerUrl = (await Bun.file(files.viewerUrl).text()).trim(); + const viewerResponse = await fetch(viewerUrl); + assert.equal(viewerResponse.ok, true, `noVNC viewer returned HTTP ${viewerResponse.status}`); + const viewerHtml = await viewerResponse.text(); + assert.match(viewerHtml, /noVNC/); + await Bun.write(files.viewerProbe, viewerHtml); + + const websocketUrl = `ws://127.0.0.1:${viewerPort}/websockify`; + const socket = new WebSocket(websocketUrl); + socket.binaryType = "arraybuffer"; + const banner = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error("noVNC WebSocket probe timed out")), 5_000); + socket.addEventListener("message", (event) => { + clearTimeout(timeout); + const text = Buffer.from(event.data).toString("ascii").trim(); + text.startsWith("RFB ") ? resolve(text) : reject(new Error(`unexpected VNC banner: ${text}`)); + }, { once: true }); + socket.addEventListener("error", () => { + clearTimeout(timeout); + reject(new Error("noVNC WebSocket could not reach the VNC server")); + }, { once: true }); + }); + socket.close(); + await Bun.write(files.websocketProbe, `${JSON.stringify({ url: websocketUrl, banner }, null, 2)}\n`); + + await runVnc(["--help"], { quiet: true }); + for (let attempt = 0; attempt < 30; attempt += 1) { + try { + await runVnc(["-s", session, "connect", `127.0.0.1::${vncPort}`], { quiet: true }); + connected = true; + break; + } catch { + await Bun.sleep(1_000); + } + } + assert.equal(connected, true, `could not connect to 127.0.0.1::${vncPort}`); + await runVnc(["-s", session, "get_screen_size"]); + await runVnc(["-s", session, "mouse_move", "40", "40"]); + + let bytes = 0; + let colors = 0; + for (let attempt = 0; attempt < 90; attempt += 1) { + await runVnc(["-s", session, "get_screenshot", "-o", files.screenshot], { quiet: true }); + bytes = Bun.file(files.screenshot).size; + const probe = await $`identify -format %k ${files.screenshot}`.quiet().nothrow(); + colors = probe.exitCode === 0 ? Number(probe.stdout.toString()) : 0; + if (bytes >= minimumBytes && colors >= minimumColors) break; + await Bun.sleep(1_000); + } + assert.ok(bytes >= minimumBytes && colors >= minimumColors, `last frame: ${bytes} bytes, ${colors} colors`); + console.log(`${label} native VNC control verified (${bytes} bytes, ${colors} colors)`); + console.log(`${label} live noVNC viewer verified at ${viewerUrl}`); +} finally { + if (connected) await runVnc(["-s", session, "disconnect"], { quiet: true }).catch(() => undefined); +} diff --git a/qa/native-vnc/cli/start.ts b/qa/native-vnc/cli/start.ts new file mode 100644 index 000000000..ae1b275c6 --- /dev/null +++ b/qa/native-vnc/cli/start.ts @@ -0,0 +1,184 @@ +#!/usr/bin/env bun + +import { $ } from "bun"; +import { randomBytes } from "node:crypto"; +import path from "node:path"; +import { loadConfig } from "../core/config.ts"; +import { startDaemon, stopDaemon } from "../core/daemon.ts"; +import { isPortListening } from "../core/network.ts"; +import { + startManaged, + stopManaged, + stopProcessGroup, + waitUntil, + type ManagedProcess, +} from "../core/processes.ts"; +import { appEnvironment, createState, removeState } from "../core/state.ts"; + +const label = "[qa:native-vnc]"; + +async function main(): Promise { + const config = await loadConfig(); + const state = await createState(config); + const env = appEnvironment(config, state); + const displayEnv = { ...env, DISPLAY: config.display, XAUTHORITY: state.xauthority }; + const services: ManagedProcess[] = []; + let daemonStarted = false; + let app: ManagedProcess | null = null; + + const addService = (service: ManagedProcess) => { + services.push(service); + return service; + }; + + try { + const cookie = randomBytes(16).toString("hex"); + await $`xauth -f ${state.xauthority} add ${config.display} . ${cookie}`.quiet(); + + const xvfb = addService(startManaged( + "Xvfb", + ["Xvfb", config.display, "-screen", "0", config.geometry, "-auth", state.xauthority, "-nolisten", "tcp", "-noreset"], + { env: displayEnv, logPath: path.join(config.artifactDir, "xvfb.log"), cwd: config.repoRoot }, + )); + await waitUntil( + "X display", + async () => { + if (xvfb.process.exitCode !== null) throw new Error("Xvfb exited during startup"); + return (await $`xdpyinfo -display ${config.display}`.env(displayEnv).quiet().nothrow()).exitCode === 0; + }, + { attempts: 50, intervalMs: 100 }, + ); + + addService(startManaged( + "Openbox", + ["openbox"], + { env: displayEnv, logPath: path.join(config.artifactDir, "openbox.log"), cwd: config.repoRoot }, + )); + + const wallpaper = path.join(config.artifactDir, "desktop-wallpaper.png"); + const wallpaperIcon = path.join(config.artifactDir, "desktop-icon.png"); + await $`convert ${config.wallpaperIcon} -resize 72x72 ${wallpaperIcon}`.quiet(); + await $`convert -size ${`${config.screenWidth}x${config.screenHeight}`} ${"gradient:#101b2b-#25122f"} ${wallpaperIcon} -gravity northwest -geometry +28+28 -composite ${wallpaper}`.quiet(); + + const panel = addService(startManaged( + "LXPanel", + ["lxpanel", "--profile", "covencave"], + { env: displayEnv, logPath: path.join(config.artifactDir, "lxpanel.log"), cwd: config.repoRoot }, + )); + await Bun.sleep(500); + if (panel.process.exitCode !== null) throw new Error("LXPanel exited during desktop startup"); + await $`feh --no-fehbg --bg-fill ${wallpaper}`.env(displayEnv).quiet(); + await Bun.write(state.artifacts.desktopReady, ""); + + const vnc = addService(startManaged( + "x11vnc", + [ + "x11vnc", "-display", config.display, "-auth", state.xauthority, + "-rfbport", String(config.vncPort), "-localhost", "-nopw", "-forever", + "-shared", "-noxdamage", "-o", path.join(config.artifactDir, "x11vnc.log"), + ], + { env: displayEnv, logPath: path.join(config.artifactDir, "x11vnc-process.log"), cwd: config.repoRoot }, + )); + await waitUntil( + "VNC endpoint", + async () => { + if (vnc.process.exitCode !== null) throw new Error("x11vnc exited during startup"); + return isPortListening(config.vncPort); + }, + { attempts: 100, intervalMs: 100 }, + ); + + const viewer = addService(startManaged( + "noVNC websockify", + [ + "websockify", "--web", config.noVncRoot, + `127.0.0.1:${config.viewerPort}`, `127.0.0.1:${config.vncPort}`, + ], + { env: displayEnv, logPath: path.join(config.artifactDir, "novnc.log"), cwd: config.repoRoot }, + )); + await waitUntil( + "noVNC viewer", + async () => { + if (viewer.process.exitCode !== null) throw new Error("websockify exited during startup"); + return isPortListening(config.viewerPort); + }, + { attempts: 100, intervalMs: 100 }, + ); + + const viewerUrl = `http://127.0.0.1:${config.viewerPort}/vnc.html?host=127.0.0.1&port=${config.viewerPort}&autoconnect=1&resize=scale`; + const appUrl = `http://127.0.0.1:${config.appPort}`; + await Promise.all([ + Bun.write(state.artifacts.viewerUrl, `${viewerUrl}\n`), + Bun.write(state.artifacts.appUrl, `${appUrl}\n`), + Bun.write(state.artifacts.runtime, `${JSON.stringify({ + stateDir: state.root, + home: state.home, + covenHome: env.COVEN_HOME, + caveHome: env.COVEN_CAVE_HOME, + xauthority: state.xauthority, + display: config.display, + geometry: config.geometry, + appUrl, + realCovenBin: config.covenBin, + }, null, 2)}\n`), + ]); + + await startDaemon(config, state, env); + daemonStarted = true; + console.log(`${label} isolated Coven daemon is healthy`); + console.log(`${label} endpoint: 127.0.0.1::${config.vncPort} (loopback only)`); + console.log(`${label} watch: ${viewerUrl} (loopback only)`); + console.log(`${label} app: ${appUrl}`); + console.log(`${label} open: bun qa/native-vnc/cli/view.ts`); + console.log(`${label} artifacts: ${config.artifactDir}`); + + app = startManaged( + "CovenCave", + ["setsid", "dbus-run-session", "--", "bun", "run", "dev:app", ...Bun.argv.slice(2)], + { + env: { ...env, PORT: String(config.appPort) }, + logPath: path.join(config.artifactDir, "app.log"), + cwd: config.repoRoot, + }, + ); + + let windowId = ""; + await waitUntil( + "native CovenCave window", + async () => { + if (app?.process.exitCode !== null) { + throw new Error(`Tauri exited before the native window was ready; inspect ${app?.logPath} and ${app?.stderrPath}`); + } + const result = await $`xdotool search --onlyvisible --name ${"^CovenCave$"}` + .env(displayEnv) + .quiet() + .nothrow(); + windowId = result.stdout.toString().trim().split(/\r?\n/)[0] ?? ""; + return result.exitCode === 0 && Boolean(windowId); + }, + { attempts: config.readyTimeoutSeconds, intervalMs: 1_000 }, + ); + await $`xdotool windowsize --sync ${windowId} ${config.windowWidth} ${config.windowHeight}`.env(displayEnv).quiet(); + await $`xdotool windowmove --sync ${windowId} ${config.windowX} ${config.windowY}`.env(displayEnv).quiet(); + await Bun.write(state.artifacts.ready, ""); + console.log(`${label} native CovenCave window is ready (${config.windowWidth}x${config.windowHeight}, centered)`); + + const signal = new Promise((resolve) => { + process.once("SIGINT", () => resolve(130)); + process.once("SIGTERM", () => resolve(143)); + }); + return await Promise.race([app.process.exited, signal]); + } finally { + await stopProcessGroup(app?.process.pid); + if (daemonStarted) await stopDaemon(config, env); + for (const service of services.reverse()) await stopManaged(service); + await removeState(state); + } +} + +try { + process.exitCode = await main(); +} catch (error) { + console.error(`${label} ERROR:`, error instanceof Error ? error.message : error); + process.exitCode = 1; +} diff --git a/qa/native-vnc/cli/view.ts b/qa/native-vnc/cli/view.ts new file mode 100644 index 000000000..0941237ea --- /dev/null +++ b/qa/native-vnc/cli/view.ts @@ -0,0 +1,32 @@ +#!/usr/bin/env bun + +import { $ } from "bun"; + +const label = "[qa:native-vnc:view]"; +const viewerPort = Number(process.env.CAVE_VNC_VIEWER_PORT ?? 6080); +if (!Number.isInteger(viewerPort) || viewerPort < 1 || viewerPort > 65_535) { + throw new Error("CAVE_VNC_VIEWER_PORT must be between 1 and 65535"); +} + +const viewerUrl = process.env.CAVE_VNC_VIEWER_URL + ?? `http://127.0.0.1:${viewerPort}/vnc.html?host=127.0.0.1&port=${viewerPort}&autoconnect=1&resize=scale`; +const response = await fetch(viewerUrl, { signal: AbortSignal.timeout(3_000) }).catch(() => null); +if (!response?.ok || !(await response.text()).includes("noVNC")) { + throw new Error(`the live viewer is not responding at ${viewerUrl}`); +} + +switch (process.platform) { + case "darwin": + await $`open ${viewerUrl}`.quiet(); + break; + case "linux": + await $`xdg-open ${viewerUrl}`.quiet(); + break; + case "win32": + await $`cmd.exe /c start "" ${viewerUrl}`.quiet(); + break; + default: + throw new Error(`unsupported operating system: ${process.platform}`); +} + +console.log(`${label} opened ${viewerUrl}`); diff --git a/qa/native-vnc/core/config.ts b/qa/native-vnc/core/config.ts new file mode 100644 index 000000000..2a28832be --- /dev/null +++ b/qa/native-vnc/core/config.ts @@ -0,0 +1,140 @@ +import { $ } from "bun"; +import { access } from "node:fs/promises"; +import path from "node:path"; +import { constants } from "node:fs"; +import { commandExists } from "./processes.ts"; +import { firstFreePort, isPortFree } from "./network.ts"; +import { fromHarness, repoRoot } from "./paths.ts"; + +export type HarnessConfig = { + display: string; + vncPort: number; + viewerPort: number; + appPort: number; + geometry: string; + screenWidth: number; + screenHeight: number; + windowWidth: number; + windowHeight: number; + windowX: number; + windowY: number; + readyTimeoutSeconds: number; + daemonTimeoutSeconds: number; + artifactDir: string; + noVncRoot: string; + covenBin: string; + fakeRuntimes: boolean; + wallpaperIcon: string; + panelProfile: string; + repoRoot: string; +}; + +function integer(name: string, fallback: number, minimum = 1, maximum = Number.MAX_SAFE_INTEGER): number { + const raw = process.env[name]; + const value = raw == null || raw === "" ? fallback : Number(raw); + if (!Number.isInteger(value) || value < minimum || value > maximum) { + throw new Error(`${name} must be an integer between ${minimum} and ${maximum}`); + } + return value; +} + +function dimensions(name: string, value: string, includeDepth: boolean): number[] { + const pattern = includeDepth ? /^(\d+)x(\d+)x(16|24|32)$/ : /^(\d+)x(\d+)$/; + const match = value.match(pattern); + if (!match) throw new Error(`${name} has an invalid format: ${value}`); + return match.slice(1).map(Number); +} + +async function executable(value: string | undefined, fallback: string): Promise { + const requested = value?.trim() || fallback; + if (requested.includes("/")) { + const resolved = path.resolve(repoRoot, requested); + await access(resolved, constants.X_OK); + return resolved; + } + const found = (await $`which ${requested}`.text()).trim(); + if (!found) throw new Error(`executable not found: ${requested}`); + return found; +} + +async function findNoVncRoot(): Promise { + const configured = process.env.CAVE_VNC_NOVNC_ROOT?.trim(); + const candidates = configured + ? [configured] + : ["/usr/share/novnc", "/usr/share/noVNC", "/usr/local/share/novnc"]; + for (const candidate of candidates) { + if (await Bun.file(path.join(candidate, "vnc.html")).exists()) return candidate; + } + throw new Error("noVNC vnc.html was not found; install noVNC or set CAVE_VNC_NOVNC_ROOT"); +} + +export async function loadConfig(): Promise { + if (process.platform !== "linux") throw new Error("the native VNC harness requires Linux"); + + const display = process.env.CAVE_VNC_DISPLAY ?? ":99"; + if (!/^:\d+$/.test(display)) throw new Error("CAVE_VNC_DISPLAY must look like :99"); + + const vncPort = integer("CAVE_VNC_PORT", 5900, 1, 65_535); + const viewerPort = integer("CAVE_VNC_VIEWER_PORT", 6080, 1, 65_535); + const requestedAppPort = process.env.CAVE_VNC_APP_PORT ?? process.env.PORT; + const appPort = requestedAppPort + ? integer("CAVE_VNC_APP_PORT", Number(requestedAppPort), 1, 65_535) + : await firstFreePort() ?? 0; + if (!appPort) throw new Error("no app port is free in 3000..3010"); + if (new Set([vncPort, viewerPort, appPort]).size !== 3) { + throw new Error("app, VNC, and noVNC ports must be different"); + } + for (const port of [vncPort, viewerPort, appPort]) { + if (!(await isPortFree(port))) throw new Error(`127.0.0.1:${port} is already in use`); + } + + const geometry = process.env.CAVE_VNC_GEOMETRY ?? "1440x900x24"; + const [screenWidth, screenHeight] = dimensions("CAVE_VNC_GEOMETRY", geometry, true); + const windowSize = process.env.CAVE_VNC_WINDOW_SIZE ?? "1180x760"; + const [windowWidth, windowHeight] = dimensions("CAVE_VNC_WINDOW_SIZE", windowSize, false); + if (windowWidth > screenWidth || windowHeight > screenHeight) { + throw new Error("CAVE_VNC_WINDOW_SIZE must fit inside CAVE_VNC_GEOMETRY"); + } + + const fakeMode = process.env.CAVE_VNC_FAKE_RUNTIMES ?? "0"; + if (!/^[01]$/.test(fakeMode)) throw new Error("CAVE_VNC_FAKE_RUNTIMES must be 0 or 1"); + + const required = [ + "Xvfb", "convert", "dbus-run-session", "feh", "lxpanel", "openbox", "setsid", + "websockify", "x11vnc", "xauth", "xdpyinfo", "xdotool", + ]; + const missing = []; + for (const command of required) { + if (!(await commandExists(command))) missing.push(command); + } + if (missing.length > 0) throw new Error(`missing required commands: ${missing.join(", ")}`); + + const artifactDir = path.resolve(repoRoot, process.env.CAVE_VNC_ARTIFACT_DIR ?? "artifacts/openvnc-qa"); + const wallpaperIcon = path.resolve(repoRoot, process.env.CAVE_VNC_WALLPAPER_ICON ?? "assets/brand/cave-icon.png"); + const panelProfile = fromHarness("desktop", "lxpanel.conf"); + if (!(await Bun.file(wallpaperIcon).exists())) throw new Error(`wallpaper icon not found: ${wallpaperIcon}`); + if (!(await Bun.file(panelProfile).exists())) throw new Error(`LXPanel profile not found: ${panelProfile}`); + + return { + display, + vncPort, + viewerPort, + appPort, + geometry, + screenWidth, + screenHeight, + windowWidth, + windowHeight, + windowX: Math.floor((screenWidth - windowWidth) / 2), + windowY: Math.floor((screenHeight - windowHeight) / 2), + readyTimeoutSeconds: integer("CAVE_VNC_READY_TIMEOUT", 300), + daemonTimeoutSeconds: integer("CAVE_VNC_DAEMON_TIMEOUT", 30), + artifactDir, + noVncRoot: await findNoVncRoot(), + covenBin: await executable(process.env.CAVE_VNC_COVEN_BIN ?? process.env.COVEN_BIN, "coven"), + fakeRuntimes: fakeMode === "1", + wallpaperIcon, + panelProfile, + repoRoot, + }; +} diff --git a/qa/native-vnc/core/daemon.ts b/qa/native-vnc/core/daemon.ts new file mode 100644 index 000000000..a110a9a37 --- /dev/null +++ b/qa/native-vnc/core/daemon.ts @@ -0,0 +1,81 @@ +import { $ } from "bun"; +import { request } from "node:http"; +import path from "node:path"; +import type { HarnessConfig } from "./config.ts"; +import type { HarnessState } from "./state.ts"; +import { waitUntil } from "./processes.ts"; + +export async function readDaemonHealth(state: HarnessState): Promise> { + const socketPath = path.join(state.home, ".coven", "coven.sock"); + return new Promise((resolve, reject) => { + const req = request( + { socketPath, path: "/api/v1/health", method: "GET", timeout: 1_000 }, + (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + response.on("end", () => { + try { + const body = JSON.parse(Buffer.concat(chunks).toString("utf8")); + const status = response.statusCode ?? 0; + if (status < 200 || status >= 300 || body.ok !== true) { + reject(new Error("daemon health endpoint is not ready")); + return; + } + resolve(body); + } catch (error) { + reject(error); + } + }); + }, + ); + req.on("timeout", () => req.destroy(new Error("daemon health timeout"))); + req.on("error", reject); + req.end(); + }); +} + +export async function startDaemon( + config: HarnessConfig, + state: HarnessState, + env: Record, +): Promise { + const log = Bun.file(path.join(config.artifactDir, "daemon-start.log")); + const start = await $`${config.covenBin} daemon start --color never` + .env(env) + .cwd(config.repoRoot) + .quiet() + .nothrow(); + await Bun.write(log, Buffer.concat([start.stdout, start.stderr])); + if (start.exitCode !== 0) throw new Error("the isolated Coven daemon failed to start"); + + let health: Record | null = null; + await waitUntil( + "isolated Coven daemon", + async () => { + try { + health = await readDaemonHealth(state); + return true; + } catch { + return false; + } + }, + { attempts: config.daemonTimeoutSeconds * 4, intervalMs: 250 }, + ); + await Bun.write(state.artifacts.daemonHealth, `${JSON.stringify(health, null, 2)}\n`); + await Bun.write(state.artifacts.daemonReady, ""); +} + +export async function stopDaemon( + config: HarnessConfig, + env: Record, +): Promise { + const result = await $`${config.covenBin} daemon stop --color never` + .env(env) + .cwd(config.repoRoot) + .quiet() + .nothrow(); + await Bun.write( + path.join(config.artifactDir, "daemon-stop.log"), + Buffer.concat([result.stdout, result.stderr]), + ); +} diff --git a/qa/native-vnc/core/network.ts b/qa/native-vnc/core/network.ts new file mode 100644 index 000000000..75a9b595c --- /dev/null +++ b/qa/native-vnc/core/network.ts @@ -0,0 +1,34 @@ +import net from "node:net"; + +export async function isPortFree(port: number): Promise { + return new Promise((resolve) => { + const server = net.createServer(); + server.once("error", () => resolve(false)); + server.listen({ host: "127.0.0.1", port }, () => { + server.close(() => resolve(true)); + }); + }); +} + +export async function isPortListening(port: number): Promise { + return new Promise((resolve) => { + const socket = net.connect({ host: "127.0.0.1", port }); + socket.setTimeout(300); + socket.once("connect", () => { + socket.destroy(); + resolve(true); + }); + socket.once("timeout", () => { + socket.destroy(); + resolve(false); + }); + socket.once("error", () => resolve(false)); + }); +} + +export async function firstFreePort(start = 3000, end = 3010): Promise { + for (let port = start; port <= end; port += 1) { + if (await isPortFree(port)) return port; + } + return null; +} diff --git a/qa/native-vnc/core/paths.ts b/qa/native-vnc/core/paths.ts new file mode 100644 index 000000000..9a9c79268 --- /dev/null +++ b/qa/native-vnc/core/paths.ts @@ -0,0 +1,12 @@ +import path from "node:path"; + +export const harnessRoot = path.resolve(import.meta.dir, ".."); +export const repoRoot = path.resolve(harnessRoot, "../.."); + +export function fromRepo(...parts: string[]): string { + return path.join(repoRoot, ...parts); +} + +export function fromHarness(...parts: string[]): string { + return path.join(harnessRoot, ...parts); +} diff --git a/qa/native-vnc/core/processes.ts b/qa/native-vnc/core/processes.ts new file mode 100644 index 000000000..416ca3715 --- /dev/null +++ b/qa/native-vnc/core/processes.ts @@ -0,0 +1,73 @@ +import { $ } from "bun"; +import type { Subprocess } from "bun"; + +export type ManagedProcess = { + logPath: string; + name: string; + process: Subprocess; + stderrPath: string; +}; + +export function startManaged( + name: string, + argv: string[], + options: { env?: Record; logPath: string; cwd?: string }, +): ManagedProcess { + const extension = options.logPath.endsWith(".log") ? ".log" : ""; + const stderrPath = extension + ? `${options.logPath.slice(0, -extension.length)}.stderr${extension}` + : `${options.logPath}.stderr`; + return { + logPath: options.logPath, + name, + stderrPath, + process: Bun.spawn(argv, { + cwd: options.cwd, + env: options.env, + stdin: "ignore", + stdout: Bun.file(options.logPath), + stderr: Bun.file(stderrPath), + }), + }; +} + +export async function stopManaged(service: ManagedProcess | null): Promise { + if (!service || service.process.exitCode !== null) return; + service.process.kill("SIGTERM"); + const completed = await Promise.race([ + service.process.exited.then(() => true), + Bun.sleep(2_000).then(() => false), + ]); + if (!completed && service.process.exitCode === null) { + service.process.kill("SIGKILL"); + await service.process.exited; + } +} + +export async function stopProcessGroup(pid: number | undefined): Promise { + if (!pid) return; + const group = `-${pid}`; + await $`kill -TERM -- ${group}`.quiet().nothrow(); + for (let attempt = 0; attempt < 20; attempt += 1) { + const probe = await $`kill -0 -- ${group}`.quiet().nothrow(); + if (probe.exitCode !== 0) return; + await Bun.sleep(100); + } + await $`kill -KILL -- ${group}`.quiet().nothrow(); +} + +export async function waitUntil( + description: string, + predicate: () => Promise, + options: { attempts: number; intervalMs: number }, +): Promise { + for (let attempt = 0; attempt < options.attempts; attempt += 1) { + if (await predicate()) return; + await Bun.sleep(options.intervalMs); + } + throw new Error(`${description} did not become ready`); +} + +export async function commandExists(command: string): Promise { + return (await $`which ${command}`.quiet().nothrow()).exitCode === 0; +} diff --git a/qa/native-vnc/core/state.ts b/qa/native-vnc/core/state.ts new file mode 100644 index 000000000..f3078a010 --- /dev/null +++ b/qa/native-vnc/core/state.ts @@ -0,0 +1,138 @@ +import { access, chmod, copyFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { HarnessConfig } from "./config.ts"; +import { fromHarness } from "./paths.ts"; + +export type ArtifactPaths = { + ready: string; + viewerUrl: string; + appUrl: string; + runtime: string; + daemonReady: string; + daemonHealth: string; + desktopReady: string; +}; + +export type HarnessState = { + root: string; + home: string; + temp: string; + xauthority: string; + fakeBin: string; + artifacts: ArtifactPaths; +}; + +const runtimeIds = ["codex", "claude", "copilot", "hermes", "hermes-coven", "opencode", "openclaw"]; + +export async function createState(config: HarnessConfig): Promise { + await mkdir(config.artifactDir, { recursive: true }); + const root = await mkdtemp(path.join(os.tmpdir(), "covencave-vnc-qa.")); + const home = path.join(root, "home"); + const temp = path.join(root, "tmp"); + const xauthority = path.join(root, "Xauthority"); + const fakeBin = path.join(root, "fake-bin"); + const panelDir = path.join(root, "xdg", "config", "lxpanel", "covencave", "panels"); + + await Promise.all([ + mkdir(path.join(home, ".coven", "cave"), { recursive: true }), + mkdir(path.join(home, ".openclaw", "agents", "qa-openclaw"), { recursive: true }), + mkdir(path.join(home, "Desktop"), { recursive: true }), + mkdir(path.join(home, "Templates"), { recursive: true }), + mkdir(temp, { recursive: true }), + mkdir(path.join(root, "xdg", "cache"), { recursive: true }), + mkdir(panelDir, { recursive: true }), + mkdir(path.join(root, "xdg", "data"), { recursive: true }), + mkdir(path.join(root, "xdg", "runtime"), { recursive: true }), + ]); + await copyFile(config.panelProfile, path.join(panelDir, "panel")); + await chmod(path.join(root, "xdg", "runtime"), 0o700); + await writeFile(xauthority, ""); + await chmod(xauthority, 0o600); + + if (config.fakeRuntimes) { + await mkdir(fakeBin, { recursive: true }); + const entry = fromHarness("runtimes", "vendor-runtime.ts"); + for (const runtimeId of runtimeIds) { + const shim = path.join(fakeBin, runtimeId); + await writeFile( + shim, + `#!/usr/bin/env bun\nimport { runVendorRuntime } from ${JSON.stringify(entry)};\nawait runVendorRuntime(${JSON.stringify(runtimeId)});\n`, + ); + await chmod(shim, 0o755); + } + const covenShim = path.join(fakeBin, "coven"); + await writeFile( + covenShim, + `#!/usr/bin/env bun\nimport ${JSON.stringify(fromHarness("runtimes", "fake-coven.ts"))};\n`, + ); + await chmod(covenShim, 0o755); + } + + const artifact = (name: string) => path.join(config.artifactDir, name); + const artifacts = { + ready: artifact("native-window.ready"), + viewerUrl: artifact("viewer.url"), + appUrl: artifact("app.url"), + runtime: artifact("runtime.json"), + daemonReady: artifact("daemon.ready"), + daemonHealth: artifact("daemon-health.json"), + desktopReady: artifact("desktop.ready"), + }; + await Promise.all(Object.values(artifacts).map((file) => rm(file, { force: true }))); + + return { root, home, temp, xauthority, fakeBin, artifacts }; +} + +export async function removeState(state: HarnessState): Promise { + for (let attempt = 0; attempt < 3; attempt += 1) { + await rm(state.root, { recursive: true, force: true }).catch(() => undefined); + if (!(await access(state.root).then(() => true).catch(() => false))) return; + await Bun.sleep(100); + } +} + +export function appEnvironment(config: HarnessConfig, state: HarnessState): Record { + const originalHome = process.env.HOME ?? config.repoRoot; + const env: Record = { + ...Object.fromEntries(Object.entries(process.env).filter((entry): entry is [string, string] => entry[1] != null)), + DISPLAY: config.display, + XAUTHORITY: state.xauthority, + HOME: state.home, + TMPDIR: state.temp, + CARGO_HOME: process.env.CARGO_HOME ?? path.join(originalHome, ".cargo"), + RUSTUP_HOME: process.env.RUSTUP_HOME ?? path.join(originalHome, ".rustup"), + XDG_CACHE_HOME: path.join(state.root, "xdg", "cache"), + XDG_CONFIG_HOME: path.join(state.root, "xdg", "config"), + XDG_DATA_HOME: path.join(state.root, "xdg", "data"), + XDG_RUNTIME_DIR: path.join(state.root, "xdg", "runtime"), + COVEN_HOME: path.join(state.home, ".coven"), + COVEN_CAVE_HOME: path.join(state.home, ".coven", "cave"), + COVEN_PREFERENCES_PATH: path.join(state.root, "preferences.json"), + COVEN_BACKDROP_PATH: path.join(state.root, "backdrop.jpg"), + COVEN_THEME_PATH: path.join(state.root, "theme.json"), + COVEN_CAVE_E2E: "1", + NEXT_TELEMETRY_DISABLED: "1", + HOSTNAME: "127.0.0.1", + GDK_BACKEND: "x11", + LIBGL_ALWAYS_SOFTWARE: "1", + NO_AT_BRIDGE: "1", + WEBKIT_DISABLE_COMPOSITING_MODE: "1", + WEBKIT_DISABLE_DMABUF_RENDERER: "1", + }; + + if (config.fakeRuntimes) { + env.PATH = `${state.fakeBin}:${env.PATH ?? ""}`; + env.COVEN_BIN = path.join(state.fakeBin, "coven"); + env.COVEN_QA_REAL_COVEN_BIN = config.covenBin; + env.COVEN_QA_TRACE_DIR = path.join(config.artifactDir, "runtime-traces"); + env.OPENCLAW_BIN = path.join(state.fakeBin, "openclaw"); + env.COVEN_CAVE_DEV_INIT_SCRIPT = process.env.COVEN_CAVE_DEV_INIT_SCRIPT + ?? 'window.setTimeout(()=>window.dispatchEvent(new Event("cave:onboarding-open")),1500)'; + } else { + env.COVEN_BIN = config.covenBin; + env.COVEN_CAVE_DEV_INIT_SCRIPT = process.env.COVEN_CAVE_DEV_INIT_SCRIPT + ?? 'try{window.localStorage.setItem("cave:onboarding:dismissed","1")}catch{}'; + } + return env; +} diff --git a/qa/native-vnc/desktop/lxpanel.conf b/qa/native-vnc/desktop/lxpanel.conf new file mode 100644 index 000000000..72af80a18 --- /dev/null +++ b/qa/native-vnc/desktop/lxpanel.conf @@ -0,0 +1,79 @@ +Global { + edge=bottom + allign=left + margin=0 + widthtype=percent + width=100 + height=32 + transparent=0 + tintcolor=#130d1f + alpha=255 + setdocktype=1 + setpartialstrut=1 + usefontcolor=1 + fontcolor=#f4efff + usefontsize=1 + fontsize=10 + background=0 +} + +Plugin { + type=menu + Config { + image=start-here + system { + } + separator { + } + item { + command=run + } + } +} + +Plugin { + type=space + Config { + Size=6 + } +} + +Plugin { + type=launchbar + Config { + Button { + id=lxde-x-terminal-emulator.desktop + } + } +} + +Plugin { + type=taskbar + expand=1 + Config { + tooltips=1 + IconsOnly=0 + AcceptSkipPager=1 + ShowIconified=1 + ShowMapped=1 + ShowAllDesks=0 + UseMouseWheel=1 + UseUrgencyHint=1 + FlatButton=1 + MaxTaskWidth=180 + spacing=2 + } +} + +Plugin { + type=tray +} + +Plugin { + type=dclock + Config { + ClockFmt=%R + TooltipFmt=%A %x + BoldFont=0 + } +} diff --git a/qa/native-vnc/runtimes/fake-coven.ts b/qa/native-vnc/runtimes/fake-coven.ts new file mode 100644 index 000000000..fcba423ce --- /dev/null +++ b/qa/native-vnc/runtimes/fake-coven.ts @@ -0,0 +1,140 @@ +#!/usr/bin/env bun + +import { $ } from "bun"; +import { randomUUID } from "node:crypto"; +import { homedir } from "node:os"; +import path from "node:path"; +import { emit, optionValue, rootHelpText, scenarioFrom, trace, writeJson } from "./protocol.ts"; + +const args = Bun.argv.slice(2); + +function promptValue(): string { + const separator = args.indexOf("--"); + return separator >= 0 ? args.slice(separator + 1).join(" ") : args.at(-1) ?? ""; +} + +function runHelp(): void { + process.stdout.write(`Usage: coven run [OPTIONS] -- + +Options: + --stream-json + --continue + --model + --permission + --add-dir + --familiar +`); +} + +function rootHelp(): void { + process.stdout.write(rootHelpText()); +} + +async function delegateToRealCoven(): Promise { + const real = process.env.COVEN_QA_REAL_COVEN_BIN?.trim(); + if (!real) throw new Error("COVEN_QA_REAL_COVEN_BIN is required for this command"); + const result = await $`${[real, ...args]}`.env(process.env).quiet().nothrow(); + process.stdout.write(result.stdout); + process.stderr.write(result.stderr); + process.exit(result.exitCode); +} + +if (args.length === 0 || args[0] === "--help" || args[0] === "-h") { + rootHelp(); + process.exit(0); +} +if (args[0] === "--version" || args[0] === "-V") { + process.stdout.write("coven 0.1.1-qa\n"); + process.exit(0); +} +if (args[0] === "adapter" && args[1] === "list" && args.includes("--json")) { + writeJson([{ + id: "moonbeam", + label: "Moonbeam external manifest", + executable: "moonbeam", + available: true, + install_hint: "QA-only untrusted adapter", + source: "external-manifest", + manifest_path: "/qa/adapters/moonbeam.json", + }]); + process.exit(0); +} +if (args[0] !== "run") await delegateToRealCoven(); +if (args.includes("--help")) { + runHelp(); + process.exit(0); +} + +const harness = args[1] ?? "unknown"; +const prompt = promptValue(); +const scenario = scenarioFrom(prompt); +const continuedSession = optionValue(args, "--continue"); +const model = optionValue(args, "--model") ?? `qa/${harness}-deterministic`; +const permission = optionValue(args, "--permission") ?? "default"; +const addDirs = args.flatMap((arg, index) => arg === "--add-dir" ? [args[index + 1]] : []).filter(Boolean); +const sessionId = continuedSession ?? `qa-${harness}-${randomUUID()}`; + +await trace("fake-coven.jsonl", { + command: "run", + harness, + scenario, + continuedSession, + model, + permission, + addDirs, + familiar: optionValue(args, "--familiar"), +}); + +if (scenario === "resume-recovery" && continuedSession) { + process.stderr.write(`session ${continuedSession} not found in local store\n`); + process.exit(1); +} +if (scenario === "failure-ux") { + process.stderr.write("401 unauthorized: QA provider credentials were intentionally not supplied\n"); + await emit({ type: "result", duration_ms: 120, is_error: true }, 10); + process.exit(1); +} + +await emit({ type: "system", subtype: "init", session_id: sessionId, model }); + +if (scenario === "trust-boundary") { + const boundaryPath = path.join(homedir(), "outside-granted-roots", "secret.txt"); + await emit({ + type: "assistant", + message: { + content: [ + { type: "tool_use", id: "qa-boundary-read", name: "Read", input: { file_path: boundaryPath } }, + { type: "text", text: `${harness} QA attempted an out-of-scope read so Cave can surface the boundary warning.` }, + ], + }, + }); + await emit({ + type: "user", + message: { + content: [{ + type: "tool_result", + tool_use_id: "qa-boundary-read", + content: "blocked by the QA boundary", + is_error: true, + }], + }, + }); +} else { + const detail = scenario === "permissions-scope" + ? `Permission ${permission}; ${addDirs.length} additional trusted director${addDirs.length === 1 ? "y" : "ies"}.` + : `Deterministic ${harness} response streamed without provider credentials.`; + if (harness === "hermes" || harness === "opencode") { + process.stdout.write(`${detail}\n`); + await Bun.sleep(90); + } else { + await emit({ type: "assistant", message: { content: [{ type: "text", text: detail }] } }); + } +} + +await emit({ + type: "result", + duration_ms: 260, + is_error: false, + usage: { input_tokens: 12, output_tokens: 18 }, + total_cost_usd: 0, +}, 10); diff --git a/qa/native-vnc/runtimes/protocol.ts b/qa/native-vnc/runtimes/protocol.ts new file mode 100644 index 000000000..67997caeb --- /dev/null +++ b/qa/native-vnc/runtimes/protocol.ts @@ -0,0 +1,41 @@ +import { appendFile, mkdir } from "node:fs/promises"; +import path from "node:path"; + +export function optionValue(args: string[], name: string): string | null { + const index = args.indexOf(name); + return index >= 0 ? args[index + 1] ?? null : null; +} + +export function rootHelpText(): string { + return `Coven QA process double + +Commands: + run Run a supported harness + adapter Inspect harness adapters + daemon Manage the daemon +`; +} + +export function scenarioFrom(prompt: string): string { + return prompt.match(/\[qa:([a-z0-9-]+)\]/i)?.[1]?.toLowerCase() ?? "chat-round-trip"; +} + +export async function trace(file: string, payload: Record): Promise { + const traceDir = process.env.COVEN_QA_TRACE_DIR?.trim(); + if (!traceDir) return; + await mkdir(traceDir, { recursive: true }); + await appendFile( + path.join(traceDir, file), + `${JSON.stringify({ at: new Date().toISOString(), ...payload })}\n`, + "utf8", + ); +} + +export function writeJson(value: unknown): void { + process.stdout.write(`${JSON.stringify(value)}\n`); +} + +export async function emit(value: unknown, delayMs = 90): Promise { + writeJson(value); + await Bun.sleep(delayMs); +} diff --git a/qa/native-vnc/runtimes/vendor-runtime.ts b/qa/native-vnc/runtimes/vendor-runtime.ts new file mode 100644 index 000000000..29f4f3394 --- /dev/null +++ b/qa/native-vnc/runtimes/vendor-runtime.ts @@ -0,0 +1,105 @@ +#!/usr/bin/env bun + +import { randomUUID } from "node:crypto"; +import { homedir } from "node:os"; +import path from "node:path"; +import { emit, optionValue, scenarioFrom, trace, writeJson } from "./protocol.ts"; + +export async function runVendorRuntime(runtime: string): Promise { + const args = Bun.argv.slice(2); + + const promptValue = () => { + const promptIndex = args.findIndex((arg) => arg === "-p" || arg === "--message"); + return promptIndex >= 0 ? args[promptIndex + 1] ?? "" : args.at(-1) ?? ""; + }; + + if (args.includes("--version") || args.includes("-V") || args.includes("version")) { + process.stdout.write(`${runtime} 1.0.0-qa\n`); + return; + } + + if (runtime === "openclaw") { + if (args[0] === "agents" && args[1] === "list") { + writeJson([{ id: "qa-openclaw", name: "QA OpenClaw", identityName: "QA OpenClaw", isDefault: true }]); + return; + } + if (args[0] === "agent") { + const prompt = promptValue(); + const scenario = scenarioFrom(prompt); + const sessionId = optionValue(args, "--session-id") ?? `qa-openclaw-${randomUUID()}`; + const warnings = { + localFileAttachments: /image attachments are not supported by this harness|cannot access local image files|not available to this runtime/i.test(prompt), + modelOverride: /model override|agent-owned/i.test(prompt), + }; + await trace("fake-runtime.jsonl", { + runtime, + command: "agent", + scenario, + sessionId, + modelArgument: optionValue(args, "--model"), + warnings, + }); + writeJson({ + status: "ok", + sessionId: `gateway-${sessionId}`, + result: { + payloads: [{ + text: scenario === "openclaw-warnings" + ? `OpenClaw bridge stayed local. Attachment warning: ${warnings.localFileAttachments ? "present" : "missing"}; model selection remains agent-owned.` + : "Deterministic OpenClaw bridge response without provider credentials.", + }], + }, + }); + return; + } + } + + if (runtime !== "copilot") { + const prompt = promptValue(); + await trace("fake-runtime.jsonl", { runtime, command: "one-shot", scenario: scenarioFrom(prompt) }); + process.stdout.write(`Deterministic ${runtime} one-shot response without provider credentials.\n`); + return; + } + + const prompt = promptValue(); + const scenario = scenarioFrom(prompt); + const resumedSession = optionValue(args, "--resume"); + const sessionId = resumedSession ?? optionValue(args, "--session-id") ?? `qa-copilot-${randomUUID()}`; + const model = optionValue(args, "--model") ?? "qa/copilot-deterministic"; + const addDirs = args.flatMap((arg, index) => arg === "--add-dir" ? [args[index + 1]] : []).filter(Boolean); + const readOnly = args.includes("--deny-tool"); + await trace("fake-runtime.jsonl", { runtime, command: "stream", scenario, sessionId, model, addDirs, readOnly }); + + if (scenario === "resume-recovery" && resumedSession) { + process.stderr.write(`No session, task, or name matched '${resumedSession}'\n`); + process.exitCode = 1; + return; + } + if (scenario === "failure-ux") { + process.stderr.write("401 unauthorized: QA Copilot credentials were intentionally not supplied\n"); + writeJson({ type: "result", sessionId, exitCode: 1, usage: { sessionDurationMs: 120 } }); + process.exitCode = 1; + return; + } + + const messageId = `qa-message-${randomUUID()}`; + const text = scenario === "permissions-scope" + ? `Copilot read-only=${readOnly}; ${addDirs.length} trusted director${addDirs.length === 1 ? "y" : "ies"}.` + : "Deterministic Copilot JSONL response without provider credentials."; + if (scenario === "trust-boundary") { + const boundaryPath = path.join(homedir(), "outside-granted-roots", "secret.txt"); + await emit({ + type: "tool.execution_start", + data: { toolCallId: "qa-copilot-boundary", toolName: "Read", arguments: { file_path: boundaryPath }, model }, + }); + await emit({ + type: "tool.execution_complete", + data: { toolCallId: "qa-copilot-boundary", success: false, result: { content: "blocked by the QA boundary" }, model }, + }); + } + const split = Math.max(1, Math.floor(text.length / 2)); + await emit({ type: "assistant.message_delta", data: { messageId, deltaContent: text.slice(0, split), model } }); + await emit({ type: "assistant.message_delta", data: { messageId, deltaContent: text.slice(split), model } }); + await emit({ type: "assistant.message", data: { messageId, content: text, toolRequests: [], model } }); + await emit({ type: "result", sessionId, exitCode: 0, usage: { sessionDurationMs: 280 } }, 10); +} diff --git a/qa/native-vnc/scenarios/cases.ts b/qa/native-vnc/scenarios/cases.ts new file mode 100644 index 000000000..7ffa2ba4e --- /dev/null +++ b/qa/native-vnc/scenarios/cases.ts @@ -0,0 +1,312 @@ +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { ScenarioContext } from "./context.ts"; + +export const scenarioIds = [ + "01-first-launch", + "02-familiar-creation", + "03-daemon-recovery", + "04-runtime-inventory", + "05-summon-choices", + "06-chat-round-trip", + "07-resume-recovery", + "08-permissions-scope", + "09-failure-ux", + "10-trust-boundary", + "11-openclaw-warnings", +] as const; + +export async function runScenarioCases(context: ScenarioContext): Promise { + await context.runScenario( + { id: scenarioIds[0], number: 1, title: "First launch", showRunning: false, preRollMs: 1_800 }, + async () => { + const { body } = await context.requestJson("/api/onboarding/status"); + assert.equal(body.ok, true); + assert.equal(body.steps.daemon.ok, true); + assert.equal(body.steps.adapters.ok, true); + return { + summary: "Fresh profile opened the setup surface with a healthy daemon and credential-free runtimes.", + assertions: ["fresh isolated HOME", "setup surface visible", "daemon healthy", "runtime source ready"], + }; + }, + ); + + await context.activateCave(); + await context.xdotool("key", "Escape"); + await Bun.sleep(1_000); + + await context.runScenario( + { id: scenarioIds[1], number: 2, title: "Create a familiar", showRunning: false }, + async () => { + await context.createFamiliar({ + id: "qa-codex", + displayName: "QA Codex", + role: "Test Familiar", + description: "Exercises the credential-free Codex stream contract.", + glyph: "ph:test-tube-fill", + harness: "codex", + model: "qa/codex-deterministic", + runtime: { kind: "local" }, + }); + await Bun.sleep(4_500); + const { body } = await context.requestJson("/api/familiars"); + assert.ok(body.familiars.some((familiar: { id: string }) => familiar.id === "qa-codex")); + return { + summary: "QA Codex was persisted through the real familiar-creation API and appeared in the roster.", + assertions: ["POST /api/familiars", "config binding persisted", "roster refreshed"], + }; + }, + ); + + await context.runScenario( + { id: scenarioIds[2], number: 3, title: "Daemon crash and recovery", showRunning: false, passHoldMs: 1_800 }, + async () => { + await context.runRealCoven(["daemon", "stop", "--color", "never"]); + await context.waitForDaemon(false); + await Bun.sleep(4_500); + await context.runRealCoven(["daemon", "start", "--color", "never"]); + await context.waitForDaemon(true); + await Bun.sleep(4_500); + return { + summary: "Cave surfaced the real daemon outage and recovered after the isolated daemon restarted.", + assertions: ["daemon stopped", "offline state observed", "daemon restarted", "health restored"], + }; + }, + ); + + await context.runScenario( + { id: scenarioIds[3], number: 4, title: "Runtime inventory" }, + async () => { + const { body } = await context.requestJson("/api/harnesses"); + const byId = new Map(body.harnesses.map((entry: { id: string }) => [entry.id, entry])); + for (const id of ["codex", "claude", "copilot", "hermes", "opencode", "openclaw"]) { + const entry = byId.get(id) as { installed?: boolean; chatSupported?: boolean } | undefined; + assert.equal(entry?.installed, true, `${id} must be detected`); + assert.equal(entry?.chatSupported, true, `${id} must be chat-trusted`); + } + const moonbeam = byId.get("moonbeam") as { installed?: boolean; chatSupported?: boolean } | undefined; + assert.equal(moonbeam?.installed, true); + assert.equal(moonbeam?.chatSupported, false); + return { + summary: "Six supported runtimes were detected; the external Moonbeam manifest stayed visible but untrusted.", + assertions: ["six supported runtimes installed", "external manifest discovered", "external manifest not chat-trusted"], + }; + }, + ); + + await context.runScenario( + { id: scenarioIds[4], number: 5, title: "Summon choices", showRunning: false }, + async () => { + const { SUMMONABLE_LOCAL_HARNESS_IDS } = await import("../../../src/lib/harness-adapters.ts"); + assert.deepEqual([...SUMMONABLE_LOCAL_HARNESS_IDS], ["codex", "claude", "copilot", "hermes"]); + const geometry = await context.caveWindowGeometry(); + await context.xdotool( + "mousemove", + String(geometry.X + geometry.WIDTH - 170), + String(geometry.Y + 118), + "click", + "1", + ); + await Bun.sleep(2_200); + return { + summary: "The summon flow offers Codex, Claude Code, Copilot, and Hermes; OpenCode and OpenClaw keep their distinct paths.", + assertions: ["four first-class summon runtimes", "OpenCode excluded from summon", "OpenClaw uses agent vessel"], + }; + }, + ); + + await context.activateCave(); + await context.xdotool("key", "Escape"); + await Bun.sleep(700); + + await context.runScenario( + { id: scenarioIds[5], number: 6, title: "Chat round trips", showRunning: false, passHoldMs: 1_800 }, + async () => { + for (const familiar of [ + { id: "qa-claude", displayName: "QA Claude", harness: "claude", model: "qa/claude-deterministic" }, + { id: "qa-copilot", displayName: "QA Copilot", harness: "copilot", model: "qa/copilot-deterministic" }, + { id: "qa-hermes", displayName: "QA Hermes", harness: "hermes", model: "qa/hermes-deterministic" }, + { id: "qa-opencode", displayName: "QA OpenCode", harness: "opencode", model: "qa/opencode-deterministic" }, + { id: "qa-openclaw", displayName: "QA OpenClaw", harness: "openclaw", model: "qa/openclaw-agent", openclawAgentId: "qa-openclaw" }, + ]) { + await context.createFamiliar({ + ...familiar, + role: "Test Familiar", + description: `Exercises the ${familiar.harness} integration contract.`, + glyph: "ph:test-tube-fill", + runtime: { kind: "local" }, + }); + } + + await context.activateCave(); + await context.xdotool("key", "ctrl+j"); + await Bun.sleep(900); + await context.xdotool("type", "--delay", "10", "[qa:chat-round-trip] Say hello from the VNC harness."); + await context.xdotool("key", "Return"); + await Bun.sleep(3_200); + + for (const id of ["qa-codex", "qa-claude", "qa-copilot", "qa-hermes", "qa-opencode", "qa-openclaw"]) { + const response = await context.sendChat({ familiarId: id, prompt: "[qa:chat-round-trip] identify your runtime class" }); + assert.equal(response.status, 200); + assert.match(context.assistantText(response.events), /Deterministic|OpenClaw bridge/i); + assert.equal(context.doneEvent(response.events)?.isError, false); + } + return { + summary: "Generic streams, Copilot JSONL, manifest one-shots, and the OpenClaw bridge all completed without keys.", + assertions: ["Codex", "Claude Code", "Copilot JSONL", "Hermes one-shot", "OpenCode one-shot", "OpenClaw bridge"], + }; + }, + ); + + await context.runScenario( + { id: scenarioIds[6], number: 7, title: "Resume and recovery" }, + async () => { + const first = await context.sendChat({ familiarId: "qa-codex", prompt: "[qa:resume-seed] remember this turn" }); + const sessionId = context.doneEvent(first.events)?.sessionId; + assert.ok(sessionId); + const resumed = await context.sendChat({ + familiarId: "qa-codex", + sessionId, + prompt: "[qa:resume-recovery] continue after a missing vendor session", + }); + assert.equal(context.doneEvent(resumed.events)?.isError, false); + assert.ok(resumed.events.some((event) => event.kind === "progress" && event.id === "resume-retry")); + assert.match(context.assistantText(resumed.events), /Deterministic codex response/i); + return { + summary: "A missing vendor session triggered Cave’s transparent replay into a fresh successful session.", + assertions: ["resume attempted", "missing session detected", "recent context replayed", "fresh response completed"], + }; + }, + ); + + await context.runScenario( + { id: scenarioIds[7], number: 8, title: "Permissions and scope" }, + async () => { + const projectRoot = path.join(context.runtime.covenHome, "workspaces", "projects", "qa-project"); + await mkdir(projectRoot, { recursive: true }); + const created = await context.requestJson( + "/api/projects", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ name: "QA Project", root: projectRoot }), + }, + [201], + ); + await context.requestJson("/api/project-grants", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + targetFamiliarId: "qa-codex", + projectId: created.body.project.id, + access: "write", + }), + }); + const response = await context.sendChat({ + familiarId: "qa-codex", + projectRoot, + permissionMode: "read", + prompt: "[qa:permissions-scope] report the effective process flags", + }); + const text = context.assistantText(response.events); + assert.match(text, /Permission read-only/); + assert.match(text, /1 additional trusted directory/); + assert.equal(context.doneEvent(response.events)?.responseMetadata?.runtime, `local:${projectRoot}`); + return { + summary: "Read-only permission and the familiar-workspace grant reached the runtime process boundary.", + assertions: ["--permission read-only", "repeatable --add-dir", "project cwd preserved", "metadata reports local scope"], + }; + }, + ); + + await context.runScenario( + { id: scenarioIds[8], number: 9, title: "Failure UX" }, + async () => { + const response = await context.sendChat({ familiarId: "qa-codex", prompt: "[qa:failure-ux] simulate missing credentials" }); + assert.equal(context.doneEvent(response.events)?.isError, true); + const text = context.assistantText(response.events); + assert.match(text, /harness errored/i); + assert.match(text, /401 unauthorized/i); + return { + summary: "A credential-style process failure became an actionable in-chat diagnostic instead of an empty bubble.", + assertions: ["non-zero runtime exit", "stderr retained", "error progress emitted", "diagnostic assistant bubble"], + }; + }, + ); + + await context.runScenario( + { id: scenarioIds[9], number: 10, title: "Trust boundary" }, + async () => { + const configPath = path.join(context.runtime.caveHome, "config.json"); + const config = JSON.parse(await readFile(configPath, "utf8")); + config.familiars["qa-moonbeam"] = { harness: "moonbeam", model: "qa/moonbeam" }; + await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`); + + const rejected = await context.sendChat( + { familiarId: "qa-moonbeam", prompt: "external manifest should not run" }, + [403], + ); + assert.equal(rejected.status, 403); + assert.match(String(rejected.body?.error), /not trusted/i); + + const observed = await context.sendChat({ familiarId: "qa-codex", prompt: "[qa:trust-boundary] exercise the path sentinel" }); + assert.ok(observed.events.some( + (event) => event.kind === "progress" && event.id === "boundary-sentinel" && event.status === "error", + )); + return { + summary: "Unaccepted manifests were blocked before spawn, while an out-of-scope tool path was surfaced by the boundary sentinel.", + assertions: ["external manifest visible", "native chat returned 403", "untrusted process never spawned", "path violation surfaced"], + }; + }, + ); + + await context.runScenario( + { id: scenarioIds[10], number: 11, title: "OpenClaw limitations" }, + async () => { + const local = await context.sendChat({ + familiarId: "qa-openclaw", + prompt: "[qa:openclaw-warnings] inspect Cave bridge limitations", + modelOverride: "qa/ignored-model", + modelOverrideScope: "next-message", + attachments: [{ + name: "one-pixel.png", + type: "image/png", + mimeType: "image/png", + size: 8, + dataUrl: "data:image/png;base64,iVBORw0KGgo=", + }], + }); + assert.match(context.assistantText(local.events), /Attachment warning: present/i); + const metadata = context.doneEvent(local.events)?.responseMetadata; + assert.equal(metadata?.modelApplicationState, "saved"); + assert.equal(metadata?.confirmedModel, undefined); + const trace = await context.readJsonLines(path.join(context.artifactDir, "runtime-traces", "fake-runtime.jsonl")); + const openClawTrace = trace.findLast( + (entry) => entry.runtime === "openclaw" && entry.scenario === "openclaw-warnings", + ); + assert.equal(openClawTrace?.modelArgument, null); + + await context.createFamiliar({ + id: "qa-openclaw-ssh", + displayName: "QA OpenClaw SSH", + role: "Test Familiar", + description: "Proves that OpenClaw does not silently cross an SSH boundary.", + glyph: "ph:test-tube-fill", + harness: "openclaw", + model: "qa/openclaw-agent", + openclawAgentId: "qa-openclaw", + runtime: { kind: "ssh", host: "qa.example", cwd: "/workspace", command: "openclaw" }, + }); + const remote = await context.sendChat( + { familiarId: "qa-openclaw-ssh", prompt: "do not bridge over SSH" }, + [501], + ); + assert.match(String(remote.body?.error), /OpenClaw SSH runtime is not supported/i); + return { + summary: "Cave warned about local image delivery, saved model intent without forwarding it, and refused an SSH OpenClaw bridge.", + assertions: ["image limitation in prompt", "model intent saved but not forwarded", "OpenClaw session stays Cave-owned", "SSH bridge returned 501"], + }; + }, + ); +} diff --git a/qa/native-vnc/scenarios/context.ts b/qa/native-vnc/scenarios/context.ts new file mode 100644 index 000000000..bfc4716f9 --- /dev/null +++ b/qa/native-vnc/scenarios/context.ts @@ -0,0 +1,277 @@ +import { $ } from "bun"; +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { repoRoot } from "../core/paths.ts"; +import type { + ChatResponse, + FamiliarInput, + RuntimeState, + ScenarioEvidence, + ScenarioResult, + ScenarioSpec, + StreamEvent, +} from "./types.ts"; + +type JsonResponse = { body: Record; status: number }; + +export class ScenarioContext { + readonly appUrl: string; + readonly artifactDir: string; + readonly displayEnv: Record; + readonly manifestPath: string; + readonly recordVideos: boolean; + readonly results: ScenarioResult[] = []; + readonly runtime: RuntimeState; + readonly videosDir: string; + private cardProcess: ReturnType | null = null; + + private constructor(runtime: RuntimeState, artifactDir: string) { + this.runtime = runtime; + this.artifactDir = artifactDir; + this.appUrl = process.env.CAVE_VNC_APP_URL ?? runtime.appUrl; + this.videosDir = path.join(artifactDir, "videos"); + this.manifestPath = path.join(artifactDir, "scenario-manifest.json"); + this.recordVideos = process.env.CAVE_VNC_RECORD_SCENARIOS !== "0"; + this.displayEnv = { + ...process.env, + DISPLAY: runtime.display, + XAUTHORITY: runtime.xauthority, + }; + } + + static async create(): Promise { + const artifactDir = path.resolve( + repoRoot, + process.env.CAVE_VNC_ARTIFACT_DIR ?? "artifacts/openvnc-qa", + ); + const runtime = JSON.parse( + await readFile(path.join(artifactDir, "runtime.json"), "utf8"), + ) as RuntimeState; + const context = new ScenarioContext(runtime, artifactDir); + await mkdir(context.videosDir, { recursive: true }); + return context; + } + + async requestJson(route: string, init: RequestInit = {}, expected = [200]): Promise { + const response = await fetch(`${this.appUrl}${route}`, init); + const body = await response.json().catch(() => ({})) as Record; + if (!expected.includes(response.status)) { + throw new Error(`${init.method ?? "GET"} ${route} returned HTTP ${response.status}: ${JSON.stringify(body)}`); + } + return { status: response.status, body }; + } + + async sendChat(body: Record, expected = [200]): Promise { + const response = await fetch(`${this.appUrl}/api/chat/send`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const raw = await response.text(); + if (!expected.includes(response.status)) { + throw new Error(`POST /api/chat/send returned HTTP ${response.status}: ${raw}`); + } + if (!response.headers.get("content-type")?.includes("text/event-stream")) { + const parsed = (() => { + try { + return JSON.parse(raw) as Record; + } catch { + return { raw }; + } + })(); + return { status: response.status, body: parsed, events: [] }; + } + const events = raw + .split(/\r?\n/) + .filter((line) => line.startsWith("data: ")) + .map((line) => JSON.parse(line.slice(6)) as StreamEvent); + return { status: response.status, events, raw }; + } + + assistantText(events: StreamEvent[]): string { + return events + .filter((event) => event.kind === "assistant_chunk") + .map((event) => event.text ?? "") + .join(""); + } + + doneEvent(events: StreamEvent[]): StreamEvent | undefined { + return events.findLast((event) => event.kind === "done"); + } + + async readJsonLines(filePath: string): Promise[]> { + return (await readFile(filePath, "utf8")) + .split(/\r?\n/) + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); + } + + async createFamiliar(input: FamiliarInput): Promise { + const response = await this.requestJson( + "/api/familiars", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ familiar: input }), + }, + [200, 201, 409], + ); + if (response.status === 200 || response.status === 201) assert.equal(response.body.ok, true); + return input.id; + } + + async xdotool(...args: string[]): Promise { + await $`${["xdotool", ...args]}`.env(this.displayEnv).quiet(); + await Bun.sleep(250); + } + + async caveWindowId(): Promise { + const output = await $`xdotool search --onlyvisible --name ${"^CovenCave$"}` + .env(this.displayEnv) + .text(); + return output.trim().split(/\r?\n/)[0] ?? ""; + } + + async caveWindowGeometry(): Promise> { + const output = await $`xdotool getwindowgeometry --shell ${await this.caveWindowId()}` + .env(this.displayEnv) + .text(); + return Object.fromEntries(output.trim().split(/\r?\n/).map((line) => { + const [key, value] = line.split("="); + return [key, Number(value)]; + })); + } + + async activateCave(): Promise { + await this.xdotool("windowactivate", "--sync", await this.caveWindowId()); + } + + async runRealCoven(args: string[]): Promise { + const env = { + ...process.env, + HOME: this.runtime.home, + COVEN_HOME: this.runtime.covenHome, + COVEN_CAVE_HOME: this.runtime.caveHome, + }; + const result = await $`${[this.runtime.realCovenBin, ...args]}`.env(env).quiet().nothrow(); + if (result.exitCode !== 0) { + throw new Error(`coven ${args.join(" ")} failed: ${(result.stderr.toString() || result.stdout.toString()).trim()}`); + } + } + + async waitForDaemon(expected: boolean, timeoutMs = 15_000): Promise> { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const { body } = await this.requestJson("/api/onboarding/status"); + if (body.steps?.daemon?.ok === expected) return body; + } catch { + // The app remains available while the isolated daemon cycles. + } + await Bun.sleep(500); + } + throw new Error(`daemon did not become ${expected ? "ready" : "offline"}`); + } + + async runScenario(spec: ScenarioSpec, test: () => Promise): Promise { + const videoPath = path.join(this.videosDir, `${spec.id}.webm`); + const startedAt = new Date().toISOString(); + const recorder = this.startRecorder(videoPath); + let result: ScenarioResult; + try { + await this.activateCave(); + await Bun.sleep(spec.preRollMs ?? 700); + if (spec.showRunning !== false) await this.showCard(`${spec.number}. ${spec.title}`, "RUNNING"); + const evidence = await test(); + await this.showCard(`${spec.number}. ${spec.title}`, `PASS\n${evidence.summary}`); + await Bun.sleep(spec.passHoldMs ?? 1_400); + result = { + ...spec, + ...evidence, + status: "passed", + video: this.recordVideos ? `videos/${spec.id}.webm` : null, + startedAt, + finishedAt: new Date().toISOString(), + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await this.showCard(`${spec.number}. ${spec.title}`, `FAIL\n${message}`); + await Bun.sleep(1_800); + result = { + ...spec, + assertions: [], + summary: message, + status: "failed", + video: this.recordVideos ? `videos/${spec.id}.webm` : null, + startedAt, + finishedAt: new Date().toISOString(), + }; + } finally { + await this.closeCard(); + await this.stopRecorder(recorder); + } + this.results.push(result); + await this.writeManifest(); + } + + async close(): Promise { + await this.closeCard(); + } + + private startRecorder(videoPath: string) { + if (!this.recordVideos) return null; + const displaySize = this.runtime.geometry.split("x").slice(0, 2).join("x"); + return Bun.spawn( + [ + "ffmpeg", "-y", "-f", "x11grab", "-draw_mouse", "1", "-framerate", "12", + "-video_size", displaySize, "-i", this.runtime.display, "-c:v", "libvpx-vp9", + "-crf", "42", "-b:v", "0", "-deadline", "realtime", "-cpu-used", "8", + "-pix_fmt", "yuv420p", videoPath, + ], + { + env: this.displayEnv, + stdin: "pipe", + stdout: "ignore", + stderr: Bun.file(`${videoPath}.ffmpeg.log`), + }, + ); + } + + private async stopRecorder(recorder: ReturnType): Promise { + if (!recorder) return; + recorder.stdin.write("q\n"); + recorder.stdin.end(); + const status = await recorder.exited; + if (status !== 0) throw new Error(`ffmpeg exited with status ${status}`); + } + + private async closeCard(): Promise { + if (!this.cardProcess) return; + this.cardProcess.kill("SIGTERM"); + await this.cardProcess.exited; + this.cardProcess = null; + } + + private async showCard(title: string, detail: string): Promise { + await this.closeCard(); + this.cardProcess = Bun.spawn( + [ + "xmessage", "-title", "Cave VNC QA", "-buttons", "Dismiss:0", + "-geometry", "620x150+790+690", `${title}\n\n${detail}`, + ], + { env: this.displayEnv, stdin: "ignore", stdout: "ignore", stderr: "ignore" }, + ); + await Bun.sleep(500); + } + + private async writeManifest(): Promise { + await writeFile(this.manifestPath, `${JSON.stringify({ + schemaVersion: 1, + appUrl: this.appUrl, + credentialMode: "deterministic-process-doubles", + generatedAt: new Date().toISOString(), + scenarios: this.results, + }, null, 2)}\n`); + } +} diff --git a/qa/native-vnc/scenarios/types.ts b/qa/native-vnc/scenarios/types.ts new file mode 100644 index 000000000..fd42f8a10 --- /dev/null +++ b/qa/native-vnc/scenarios/types.ts @@ -0,0 +1,50 @@ +export type RuntimeState = { + appUrl: string; + caveHome: string; + covenHome: string; + display: string; + geometry: string; + home: string; + realCovenBin: string; + xauthority: string; +}; + +export type ScenarioSpec = { + id: string; + number: number; + title: string; + passHoldMs?: number; + preRollMs?: number; + showRunning?: boolean; +}; + +export type ScenarioEvidence = { + assertions: string[]; + summary: string; +}; + +export type ScenarioResult = ScenarioSpec & ScenarioEvidence & { + finishedAt: string; + startedAt: string; + status: "passed" | "failed"; + video: string | null; +}; + +export type StreamEvent = { + id?: string; + isError?: boolean; + kind?: string; + responseMetadata?: Record; + sessionId?: string; + status?: string; + text?: string; +}; + +export type ChatResponse = { + body?: Record; + events: StreamEvent[]; + raw?: string; + status: number; +}; + +export type FamiliarInput = Record & { id: string }; diff --git a/qa/native-vnc/tests/protocol.test.ts b/qa/native-vnc/tests/protocol.test.ts new file mode 100644 index 000000000..c62721a8f --- /dev/null +++ b/qa/native-vnc/tests/protocol.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from "bun:test"; +import { covenHelpSupportsAdapterList } from "../../../src/lib/harness-adapters.ts"; +import { optionValue, rootHelpText, scenarioFrom } from "../runtimes/protocol.ts"; + +describe("runtime-double protocol", () => { + test("extracts scenario markers from prompts", () => { + expect(scenarioFrom("please run [qa:resume-recovery] now")).toBe("resume-recovery"); + expect(scenarioFrom("ordinary prompt")).toBe("chat-round-trip"); + }); + + test("reads process option values", () => { + expect(optionValue(["run", "codex", "--model", "qa/model"], "--model")).toBe("qa/model"); + expect(optionValue(["run", "codex"], "--model")).toBeNull(); + }); + + test("advertises the adapter inventory command", () => { + expect(covenHelpSupportsAdapterList(rootHelpText())).toBe(true); + }); +}); diff --git a/qa/native-vnc/tests/structure.test.ts b/qa/native-vnc/tests/structure.test.ts new file mode 100644 index 000000000..2adbc41af --- /dev/null +++ b/qa/native-vnc/tests/structure.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, test } from "bun:test"; +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { harnessRoot } from "../core/paths.ts"; +import { scenarioIds } from "../scenarios/cases.ts"; + +async function filesUnder(directory: string): Promise { + const entries = await readdir(directory, { withFileTypes: true }); + const nested = await Promise.all(entries.map((entry) => { + const absolute = path.join(directory, entry.name); + return entry.isDirectory() ? filesUnder(absolute) : [absolute]; + })); + return nested.flat(); +} + +describe("native VNC harness structure", () => { + test("contains no shell scripts or checked-in media", async () => { + const files = await filesUnder(harnessRoot); + const forbidden = files.filter((file) => /\.(?:sh|webm|mp4|mov|gif)$/i.test(file)); + expect(forbidden).toEqual([]); + }); + + test("keeps the workflow template inactive", async () => { + const workflow = await readFile( + path.join(harnessRoot, "workflow-template", "native-vnc.yml"), + "utf8", + ); + expect(workflow).toContain("if: ${{ false }}"); + expect(workflow).toContain("bun qa/native-vnc/cli/ci.ts"); + }); + + test("uses Bun Shell instead of child_process", async () => { + const files = (await filesUnder(harnessRoot)).filter((file) => file.endsWith(".ts")); + const sources = await Promise.all(files.map((file) => readFile(file, "utf8"))); + const forbiddenImport = ["node", "child_process"].join(":"); + expect(sources.some((source) => source.includes(forbiddenImport))).toBe(false); + for (const entrypoint of ["start.ts", "view.ts", "smoke.ts", "ci.ts"]) { + const source = await readFile(path.join(harnessRoot, "cli", entrypoint), "utf8"); + expect(source).toContain('import { $ } from "bun"'); + } + }); + + test("retains the complete scenario contract", () => { + expect(scenarioIds).toEqual([ + "01-first-launch", + "02-familiar-creation", + "03-daemon-recovery", + "04-runtime-inventory", + "05-summon-choices", + "06-chat-round-trip", + "07-resume-recovery", + "08-permissions-scope", + "09-failure-ux", + "10-trust-boundary", + "11-openclaw-warnings", + ]); + }); +}); From dc179b3c7dbcc2640fb97f3b494fffc5776dc5a4 Mon Sep 17 00:00:00 2001 From: ox <286927284+oxfern@users.noreply.github.com> Date: Fri, 17 Jul 2026 04:30:26 +0300 Subject: [PATCH 2/2] docs(qa): document VNC harness adoption --- qa/README.md | 83 +++++++++++++++++++ qa/native-vnc/docs/adding-scenarios.md | 37 +++++++++ qa/native-vnc/docs/architecture.md | 43 ++++++++++ qa/native-vnc/docs/future-steps.md | 50 +++++++++++ .../workflow-template/native-vnc.yml | 64 ++++++++++++++ 5 files changed, 277 insertions(+) create mode 100644 qa/README.md create mode 100644 qa/native-vnc/docs/adding-scenarios.md create mode 100644 qa/native-vnc/docs/architecture.md create mode 100644 qa/native-vnc/docs/future-steps.md create mode 100644 qa/native-vnc/workflow-template/native-vnc.yml diff --git a/qa/README.md b/qa/README.md new file mode 100644 index 000000000..16f158d92 --- /dev/null +++ b/qa/README.md @@ -0,0 +1,83 @@ +# Coven Cave QA + +This directory contains opt-in quality harnesses that are intentionally kept +separate from the application and its existing CI. The native VNC harness runs +the real Linux Tauri window inside an isolated desktop, exposes that desktop to +VNC automation, and serves the same screen through noVNC for a human observer. + +## Native VNC harness + +The harness creates a temporary home directory and XDG profile, starts a real +Coven daemon, opens Cave as a centered `1180x760` window on a `1440x900` +Openbox desktop, and binds VNC and noVNC to loopback only. The desktop includes +a wallpaper, Applications menu, taskbar, and clock, so app activity stays +visible in context instead of filling the screen. + +Install the Linux dependencies: + +```text +build-essential dbus-x11 feh ffmpeg imagemagick +libayatana-appindicator3-dev libgtk-3-dev libjavascriptcoregtk-4.1-dev +librsvg2-dev libsoup-3.0-dev libssl-dev libwebkit2gtk-4.1-dev libxdo-dev +lxpanel novnc openbox pkg-config websockify x11-utils x11vnc xauth xdotool xvfb +``` + +Install a compatible Coven CLI, then start the harness in the foreground: + +```bash +bun qa/native-vnc/cli/start.ts +``` + +The launcher prints the loopback VNC endpoint, noVNC URL, app URL, and artifact +directory. Open the viewer in the operating system's default browser: + +```bash +bun qa/native-vnc/cli/view.ts +``` + +From another terminal, verify connectivity, input, daemon health, and a real +screenshot: + +```bash +bun qa/native-vnc/cli/smoke.ts +``` + +The principal overrides are `CAVE_VNC_DISPLAY`, `CAVE_VNC_PORT`, +`CAVE_VNC_VIEWER_PORT`, `CAVE_VNC_APP_PORT`, `CAVE_VNC_GEOMETRY`, +`CAVE_VNC_WINDOW_SIZE`, `CAVE_VNC_COVEN_BIN`, `CAVE_VNC_NOVNC_ROOT`, and +`CAVE_VNC_ARTIFACT_DIR`. + +## Credential-free runtime matrix + +Set `CAVE_VNC_FAKE_RUNTIMES=1` before starting the harness to replace only the +credential-bearing vendor processes with deterministic local doubles. The +Tauri shell, Next server, Coven daemon, application routes, persistence, VNC +transport, and desktop remain real. + +```bash +CAVE_VNC_FAKE_RUNTIMES=1 bun qa/native-vnc/cli/start.ts +bun qa/native-vnc/cli/record-scenarios.ts +``` + +The runtime contract is deliberately asymmetric: + +| Runtime | Inventory | Summon flow | Chat contract | +| --- | --- | --- | --- | +| Codex | Supported | Supported | Generic stream | +| Claude Code | Supported | Supported | Generic stream | +| Copilot | Supported | Supported | Copilot JSONL | +| Hermes | Supported | Supported | One-shot manifest | +| OpenCode | Supported | Not yet offered | One-shot manifest | +| OpenClaw | Supported | Dedicated agent path | Local OpenClaw bridge | +| External manifests | Visible | Not trusted | Rejected before spawn | + +The 11 scenarios cover first launch, familiar creation, daemon recovery, +runtime inventory, summon choices, chat round trips, resume fallback, +permissions and directory scope, process failure UX, trust boundaries, and +OpenClaw limitations. Results are written to `artifacts/openvnc-qa/`, including +`scenario-manifest.json` and optional recordings. Generated media is evidence, +not source: keep it out of Git and link externally hosted previews from the PR. + +For the design and process model, see [architecture](native-vnc/docs/architecture.md). +For extending coverage, see [adding scenarios](native-vnc/docs/adding-scenarios.md). +For the hardening roadmap, see [future steps](native-vnc/docs/future-steps.md). diff --git a/qa/native-vnc/docs/adding-scenarios.md b/qa/native-vnc/docs/adding-scenarios.md new file mode 100644 index 000000000..3b4516f5a --- /dev/null +++ b/qa/native-vnc/docs/adding-scenarios.md @@ -0,0 +1,37 @@ +# Adding a native VNC scenario + +Add scenarios when a behavior crosses a native boundary, depends on a real +desktop or daemon, or benefits from visible evidence. Prefer faster unit or API +tests for behavior that does not need the Tauri window or VNC transport. + +1. Add a stable, numbered identifier to `scenarioIds` in + `scenarios/cases.ts`. +2. Add one `context.runScenario(...)` block. Keep setup inside the scenario or + make its dependency on an earlier case explicit. +3. Drive product state through the real API where possible. Use + `context.xdotool(...)` only when the visible native interaction is part of + the contract. +4. Return a short summary and concrete assertion labels. They become the + durable manifest and the pass card in recorded evidence. +5. If a vendor process response is needed, extend the appropriate protocol in + `runtimes/`. Do not add credentials or imitate application internals. +6. Add a fast test for any new parser, invariant, or structural rule. +7. Run the checks below, then run the full native matrix on Linux. + +```bash +bun test qa/native-vnc/tests +bun build qa/native-vnc/cli/start.ts qa/native-vnc/cli/view.ts \ + qa/native-vnc/cli/smoke.ts qa/native-vnc/cli/record-scenarios.ts \ + qa/native-vnc/cli/ci.ts qa/native-vnc/runtimes/fake-coven.ts \ + --target=bun --outdir=/tmp/coven-cave-native-vnc-build +``` + +For a local matrix, start Cave with `CAVE_VNC_FAKE_RUNTIMES=1`, run the smoke +entrypoint, then run `record-scenarios.ts`. Inspect the manifest, the final +screenshot, service logs, and each recording. A passing manifest alone is not +enough when the scenario's purpose is visual UX. + +Keep generated screenshots, recordings, traces, and logs under +`artifacts/openvnc-qa/`. Before review, verify that no `.webm`, `.mp4`, `.mov`, +`.gif`, or generated artifact directory is staged. Publish review evidence as +external PR-linked assets instead. diff --git a/qa/native-vnc/docs/architecture.md b/qa/native-vnc/docs/architecture.md new file mode 100644 index 000000000..9f985a738 --- /dev/null +++ b/qa/native-vnc/docs/architecture.md @@ -0,0 +1,43 @@ +# Native VNC harness architecture + +The harness separates lifecycle, runtime behavior, scenario assertions, and +operator commands so each layer can change without turning the suite into one +large orchestration script. + +```text +qa/native-vnc/ +├── cli/ operator entrypoints +├── core/ configuration, isolation, daemon, ports, processes +├── desktop/ the lightweight LXPanel profile +├── runtimes/ deterministic Coven and vendor process doubles +├── scenarios/ scenario context, assertions, and manifest contract +├── tests/ fast structural and protocol tests +└── workflow-template/ inactive CI proposal +``` + +Commands use Bun Shell for readable, escaped, cross-platform process +invocation. `Bun.spawn` is reserved for services that must stay alive while the +harness supervises them: Xvfb, the window manager, VNC servers, Cave, ffmpeg, +and the temporary evidence card. This gives long-lived processes explicit PIDs, +logs, signals, and shutdown deadlines. + +The launcher uses three containment boundaries: + +- Xvfb disables its TCP listener, while x11vnc and noVNC bind only to + `127.0.0.1`. +- Cave and the daemon receive a temporary `HOME`, `COVEN_HOME`, + `COVEN_CAVE_HOME`, and XDG tree that is removed at shutdown. +- Credential-free mode prepends generated runtime shims from the temporary + state directory. No generated executables or user credentials enter Git. + +The runtime doubles implement process protocols, not product behavior. They +exercise generic JSON streams, Copilot JSONL, one-shot manifests, OpenClaw's +local bridge, resume errors, provider-style failures, permission flags, and +path-boundary events. Authenticated provider canaries belong in a separate, +secret-backed check because their availability and billing should not control +the deterministic PR gate. + +Every scenario produces structured assertions in `scenario-manifest.json`. +Video is optional and lives under the ignored runtime artifact directory. A PR +may link a release-hosted recording and animated preview; the repository should +never contain those binaries. diff --git a/qa/native-vnc/docs/future-steps.md b/qa/native-vnc/docs/future-steps.md new file mode 100644 index 000000000..826d7a5f7 --- /dev/null +++ b/qa/native-vnc/docs/future-steps.md @@ -0,0 +1,50 @@ +# Native VNC hardening roadmap + +The current harness is useful as an opt-in Linux native smoke and deterministic +runtime matrix. The following steps would make it a dependable gate without +mixing experimental infrastructure into the existing CI. + +## Before enabling CI + +- Run the inactive workflow on a fork by temporarily copying it into + `.github/workflows/` and removing the job-level `if: false` guard. +- Pin every action, Bun version, Coven CLI version, uv version, and Linux image; + update them intentionally with a recorded successful matrix. +- Measure cold and warm durations, then set the timeout and artifact retention + from observed data. +- Add a failure-path run that proves cleanup removes the daemon, X server, + websockify process, temporary home, and occupied ports. +- Decide whether recordings are needed on every run or only on failures and + manual evidence runs. + +## Reliability improvements + +- Add schema validation for `runtime.json` and `scenario-manifest.json`. +- Replace fixed UI delays with observable state or accessibility conditions + where Cave exposes them. +- Record process PIDs and exit reasons in the manifest for faster diagnosis. +- Add a disk-space preflight before Rust compilation and video capture. +- Exercise non-default display numbers and occupied-port selection in a + container-level test. +- Test SIGINT, SIGTERM, app crash, daemon crash, and ffmpeg failure cleanup. +- Make the evidence card position derive from the selected desktop geometry. + +## Coverage growth + +- Add keyboard-only navigation and focus-order scenarios. +- Add window resize, high-DPI, and smaller-desktop cases. +- Add clipboard, drag/drop, attachment, and native dialog coverage where those + paths are supported on Linux. +- Add one authenticated canary per supported vendor as a manual, secret-backed + workflow. Keep it outside the deterministic PR matrix. +- Add adapter conformance tests before promoting any external manifest into the + trusted runtime registry or summon flow. + +## Enablement decision + +The template at `workflow-template/native-vnc.yml` is inert both because it is +outside `.github/workflows/` and because the job has `if: false`. The repository +owner can enable it by copying the file into `.github/workflows/`, reviewing the +versions and permissions, removing the guard, and choosing `workflow_dispatch`, +scheduled, or pull-request triggers. Existing CI should remain untouched until +that decision is made. diff --git a/qa/native-vnc/workflow-template/native-vnc.yml b/qa/native-vnc/workflow-template/native-vnc.yml new file mode 100644 index 000000000..7a4491c70 --- /dev/null +++ b/qa/native-vnc/workflow-template/native-vnc.yml @@ -0,0 +1,64 @@ +name: Native VNC QA (inactive template) + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + native-vnc: + if: ${{ false }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: 24 + cache: pnpm + + - uses: oven-sh/setup-bun@735343b667d3e6f658f44d0eca948eb6282f2b76 + with: + bun-version: 1.3.14 + + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + with: + version: "0.11.23" + + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 + with: + workspaces: src-tauri -> target + + - name: Install native desktop and virtual-display dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential dbus-x11 feh ffmpeg imagemagick \ + libayatana-appindicator3-dev libgtk-3-dev libjavascriptcoregtk-4.1-dev \ + librsvg2-dev libsoup-3.0-dev libssl-dev libwebkit2gtk-4.1-dev \ + libxdo-dev lxpanel novnc openbox pkg-config websockify x11-utils \ + x11vnc xauth xdotool xvfb + + - run: pnpm install --frozen-lockfile + + - name: Install the Coven CLI used by the isolated daemon + run: npm install --global @opencoven/cli@0.1.1 + + - name: Run native VNC smoke and scenario matrix + env: + CAVE_VNC_FORCE_UVX: "1" + run: bun qa/native-vnc/cli/ci.ts + + - name: Retain native VNC evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: native-vnc-qa + path: artifacts/openvnc-qa + if-no-files-found: error + retention-days: 14