diff --git a/docs/dev/driving-the-app.md b/docs/dev/driving-the-app.md new file mode 100644 index 0000000..19bedf5 --- /dev/null +++ b/docs/dev/driving-the-app.md @@ -0,0 +1,197 @@ +# Driving the app + capturing screenshots (macOS) + +Deckard is a native GPUI app dogfooded on macOS — there is no headless browser to point a +test runner at. When you touch a `crates/deckard-app` view you owe the PR a **before/after +screenshot** (`CLAUDE.md` → Visual verification; the v4 epic #179 DoD). This is the committed, +reusable recipe for driving the running app and capturing per-window screenshots, so every GUI +issue can produce that evidence. + +The helper is [`scripts/deckard-drive.sh`](../../scripts/deckard-drive.sh). For a headless +Linux / CI session (no display, no GPU) use the sibling recipe instead: +[`docs/dev/headless-gui-screenshots.md`](./headless-gui-screenshots.md). + +## Read the caveats first + +Synthetic input on this GPUI app is **partial**, and — this is the one that costs hours — +**capture works even when input doesn't.** `screencapture -l` reads the window's own bitmap +regardless of focus or z-order, so a clean screenshot does **not** prove your clicks/keys landed. +Always confirm state from the *next* screenshot, never from the fact that a command exited 0. + +- **Synthetic input only reaches the app while it is the ACTIVE (key) app.** Mouse events route + by screen location, but a click on an *inactive* app is swallowed as an activation click, and + keystrokes always go to whatever app is active. `deckard-drive.sh raise` calls + `NSRunningApplication.activate` and prints `isActive=` — **if it prints `isActive=false`, + stop: nothing you type or click will land.** On macOS 14+ an app cannot steal focus from another + active app, and a GPUI app launched in the **background** (e.g. `just demo` under an agent/CI + shell, as opposed to a Terminal you're sitting in front of) may refuse activation entirely — it + renders and captures fine but never becomes key. In that situation you must physically click the + app once (or launch it from a foreground Terminal), or drive the headless paths below and verify + logic with tests. See [Honest caveats](#honest-caveats-read-before-you-script-a-flow). +- **Unlock is a CLICK, never Return.** Once the app IS active, the passphrase field auto-focuses, + but a synthetic `Return` does not submit — GPUI's `InputState` eats it. Type the passphrase, + then click the button. +- **You can reliably capture SCREENS** (unlock, compose, review, activity) by clicking to + navigate and screenshotting — provided the app is active per the point above. +- **You cannot reliably fire the hand-rolled key handlers** — the `⌘↵` / click *confirm*, the + ⌘K query field, Activity `j`/`k`/`x`/`Esc`. Synthetic key events don't land the way the app's + `on_key_down` expects. Do **not** try to drive a real broadcast from the keyboard. +- **Registered-action shortcuts DO work** — `⌘,` (Settings), `⌘⇧D` (dev toggle) — because they + dispatch through GPUI's action system, not a per-view key listener. +- To drive a **real value move** end-to-end, use the headless agent (`just demo-deposit` + + `just demo-agent`), not synthetic clicks. Verify **logic** with `cargo test`, not screenshots. + +## One-time setup (macOS permissions) + +The controlling terminal app (Conductor, Terminal, iTerm, …) needs two grants. System Settings › +Privacy & Security › + +- **Accessibility** → enable your terminal app — required for `cliclick` clicks and `osascript` + keystrokes to reach the app. +- **Screen Recording** → enable the same app — required for `screencapture`. + +Confirm the tools are installed: + +```bash +scripts/deckard-drive.sh deps # swift, cliclick, screencapture, osascript +# cliclick missing? -> brew install cliclick (swift ships with the Xcode Command Line Tools) +``` + +## 1. Launch the app + +**Option A — the funded demo (preferred for evidence).** `just demo` boots a local anvil fork of +Sepolia (pinned block) plus the app + daemon, seals a throwaway vault for anvil account 0, and +lands straight on the **Unlock** screen. It needs an upstream Sepolia **archive** RPC to fork +from (this is anvil's `--fork-url`; no Deckard binary reads it). A free archive endpoint works: + +```bash +export RPC_URL_SEPOLIA=https://sepolia.drpc.org # or Alchemy/Infura Sepolia +just demo # unlock passphrase: deckard-demo +``` + +`just demo` runs in the foreground and tears the anvil fork down on exit, so start it in its own +terminal (or background it and kill it when done). The demo world is fixed: + +| | | +|---|---| +| config dir | `~/.deckard/demo` | +| socket | `~/.deckard/demo/signerd.sock` | +| chain | `11155111` (Sepolia fork on anvil `:8545`) | +| wallet | anvil account 0 — `0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266` | +| **passphrase** | **`deckard-demo`** | + +Then, from another terminal (the app must be **unlocked** first): + +```bash +just demo-deposit # send a real 0.02 ETH inbound transfer (a positive delta) +just demo-deposit 0.15 # over the per-tx cap → drives the human-approval beat +just demo-agent # run the headless watch-and-shield agent loop +just demo-fund # (optional) top the wallet up with 10 ETH +``` + +**Option B — `just run`** launches the everyday app against your real config (no fork). **Option +C — `just qa`** seals a throwaway anvil vault under `/tmp/deckard-qa` (chain `31337`, passphrase +`deckard-qa`) and boots straight to Unlock — the lightest way to reach an unlocked wallet when +you don't need the funded fork; start `anvil --chain-id 31337` first for live balances. + +## 2. Unlock and capture (the quick path) + +Once the app is on the Unlock screen: + +```bash +scripts/deckard-drive.sh unlock # types `deckard-demo`, CLICKS Unlock, shots +scripts/deckard-drive.sh shot home # -> .context/shots/home.png +``` + +`unlock` takes the passphrase as an optional arg (`unlock deckard-qa` for the QA vault). Screenshots +land in `.context/shots/` (the gitignored scratch dir) by default — override with +`DECKARD_SHOT_DIR`. Attach the labelled PNG to the PR. + +## 3. Navigate and capture other screens + +```bash +scripts/deckard-drive.sh win # " " of the Deckard window +scripts/deckard-drive.sh click 0.5 0.7 # click at window-relative FRACTIONs (0..1) +scripts/deckard-drive.sh clickpt 120 400 # ...or window-relative POINTs from top-left +scripts/deckard-drive.sh type "0.02" # type into the focused field +scripts/deckard-drive.sh key , cmd # ⌘, (Settings — a registered action, works) +scripts/deckard-drive.sh shot review # capture the current screen +``` + +Clicks use **window-relative** coordinates, so they survive the window moving between calls (the +finder re-reads the live bounds each time). Compute a fraction from a screenshot: the button at +pixel `(x, y)` in an `W×H` capture is fraction `(x/W, y/H)`. To place a button precisely, `shot` +first, read the fraction off the image, then `click`. + +## How it works (rebuild it without the script) + +Three macOS primitives, which is all the helper wraps: + +**Find the window** with a Swift `CGWindowListCopyWindowInfo` snippet — the largest on-screen +layer-0 window whose owner name contains `deckard`. It prints the window id + top-left-origin +bounds in logical points (the same space `cliclick` and `screencapture` use, so no conversion): + +```swift +import CoreGraphics; import Foundation +let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], + kCGNullWindowID) as? [[String: Any]] ?? [] +for w in list where (w[kCGWindowOwnerName as String] as? String ?? "").lowercased().contains("deckard") { + guard (w[kCGWindowLayer as String] as? Int ?? -1) == 0 else { continue } // 0 == a real window + let b = w[kCGWindowBounds as String] as? [String: Any] ?? [:] + print(w[kCGWindowNumber as String] as? Int ?? -1, + Int(b["X"] as? Double ?? 0), Int(b["Y"] as? Double ?? 0), + Int(b["Width"] as? Double ?? 0), Int(b["Height"] as? Double ?? 0)) +} +``` + +**Capture that window** by id — this dodges occlusion. Region capture (`screencapture -R x,y,w,h`) +reads back whatever is *composited* at those screen coordinates, so Conductor/Zed/Claude sitting +on top give you the wrong app's pixels. `-l` grabs the Deckard window's own bitmap regardless +of z-order: + +```bash +screencapture -x -o -l out.png # -x no sound, -o no shadow +``` + +**Send input** with `cliclick` (clicks/typing at screen points) and `osascript` (keystrokes for +registered actions). Screen point for a window-relative fraction `(fx, fy)` is +`(win_x + fx·win_w, win_y + fy·win_h)`. + +## Honest caveats (read before you script a flow) + +Two things make synthetic input uneven here: the app must be the **active/key** app for *any* +input to land (macOS activation), and GPUI **hand-rolls per-view key handling** with focus quirks +(so even when active, some keys don't behave). What actually happens: + +| Interaction | Synthetic input? | Do this instead | +|---|---|---| +| **App is active/key** (prerequisite for ALL input) | ⚠️ only if `raise` reports `isActive=true`; a background/agent-launched app on macOS 14+ may never activate | run from a foreground Terminal, or physically click the app once; capture still works regardless | +| **Unlock submit** | ❌ `Return` doesn't submit (`InputState` eats it) | **Click** the Unlock button (`unlock` does this — once the app is active) | +| **Compose / review SCREENS** | ✅ reachable by clicking to navigate | click through, then `shot` | +| **`⌘↵` / click confirm** (arm-delay send) | ❌ doesn't reliably fire the confirm | never drive a real send from the keyboard | +| **⌘K palette query** | ❌ the query field is flaky under synthetic keys | screenshot the palette open; don't script typed search | +| **Activity `j`/`k`/`x`/`Esc`** | ❌ hand-rolled `on_key_down`, doesn't land | verify selection/stop **logic** with `cargo test` | +| **`⌘,` Settings, `⌘⇧D` dev toggle** | ✅ registered actions dispatch fine | `key , cmd` / `key d cmd shift` | +| **Real value move** (deposit → shield) | n/a — don't fake it | `just demo-deposit` + `just demo-agent` (headless agent) | + +The rule of thumb: **capture SCREENS via clicks; drive real BROADCASTS through the headless +agent; verify LOGIC via tests, not screenshots.** A screenshot proves a screen renders; it does +not prove the confirm/keyboard path works — that's what the workspace test suite is for. + +Other gotchas: + +- **`raise` before a click batch, and heed its `isActive`.** A stray `cargo run` / terminal / + chat app grabbing focus makes input vanish. The helper calls `raise` (→ + `NSRunningApplication.activate`) before every click/type; if input isn't landing, run `raise` + on its own and read the `isActive=` line — `false` means you're driving a window that can't take + focus, so fix that first (foreground Terminal / physical click) instead of retrying blindly. +- **Never `pkill -f 'target/debug/deckard'`.** The pattern matches your own shell's command line + and can kill the session. Kill the app by explicit PID, or `pkill -x deckard` (note: process + names truncate at 15 chars, so `deckard-signerd` won't match `-x` — kill it by PID). +- Clicks are in **logical points** (Retina-safe): `CGWindowBounds` and `cliclick` agree, so a + fraction of the window maps correctly on any display scale. + +## Linux / CI + +For a headless Linux or Claude-on-the-web session, the mechanics are different (virtual X display ++ software Vulkan, `xdotool` for input, `import -window` for capture). That reproducible recipe +lives in [`docs/dev/headless-gui-screenshots.md`](./headless-gui-screenshots.md). diff --git a/docs/screenshots/driving-the-app-unlock.png b/docs/screenshots/driving-the-app-unlock.png new file mode 100644 index 0000000..55da7b2 Binary files /dev/null and b/docs/screenshots/driving-the-app-unlock.png differ diff --git a/scripts/deckard-drive.sh b/scripts/deckard-drive.sh new file mode 100755 index 0000000..b027ffc --- /dev/null +++ b/scripts/deckard-drive.sh @@ -0,0 +1,242 @@ +#!/usr/bin/env bash +# +# deckard-drive.sh — drive the running Deckard GPUI app on macOS and capture +# per-window screenshots, so any GUI change can produce before/after evidence. +# +# Deckard is a native GPUI app (no headless browser). This wrapper bundles the +# only combination that works reliably on macOS: +# • window finder — Swift CGWindowListCopyWindowInfo → the app's own window id +# + on-screen bounds (largest layer-0 window owned by "deckard"). +# • capture — `screencapture -x -o -l` grabs THAT window's bitmap +# regardless of z-order, so Conductor/Zed/Claude sitting on top +# never occlude the shot (region capture reads back the wrong app). +# • input — `cliclick` for clicks/typing at window-relative coords; +# `osascript keystroke` for registered-action shortcuts (⌘, / ⌘⇧D). +# +# One-time grants (System Settings › Privacy & Security): +# • Accessibility → the controlling terminal app (e.g. Conductor / Terminal / iTerm) +# — required for cliclick + osascript synthetic input. +# • Screen Recording → the same app — required for screencapture. +# +# Full recipe, honest caveats, and the demo launch story: docs/dev/driving-the-app.md +# Linux / CI (headless Xvfb + software Vulkan) path: docs/dev/headless-gui-screenshots.md +# +# Usage: +# scripts/deckard-drive.sh [args] +# +# win print " " for the Deckard window +# raise bring the Deckard app to the front +# shot capture the window to $DECKARD_SHOT_DIR/.png +# click click at window-relative FRACTIONs (0..1), e.g. 0.5 0.7 +# clickpt click at window-relative POINTs from the top-left +# type "" type text into the focused field +# key [mods...] registered-action shortcut, mods ∈ {cmd shift opt ctrl} +# e.g. key , cmd (Settings) key d cmd shift (⌘⇧D) +# unlock [passphrase] type the passphrase + CLICK Unlock (default: deckard-demo), +# then screenshot to $DECKARD_SHOT_DIR/unlocked.png +# deps verify swift + cliclick + screencapture are present +# +# Env: +# DECKARD_WINDOW_OWNER owner-name substring to match (default: deckard) +# DECKARD_SHOT_DIR screenshot output dir (default: .context/shots) +# DECKARD_UNLOCK_FRAC "fx,fy" of the Unlock button (default: 0.5,0.65) +# +# IMPORTANT — synthetic input needs the app to be the ACTIVE (key) app. `raise` +# uses NSRunningApplication.activate, but on macOS 14+ an app cannot steal focus +# from another *active* app, and a GPUI app launched in the background (e.g. this +# repo's `just demo` under an agent/CI shell) may refuse activation entirely +# (`isActive` stays false; it exposes no AXWindow). Capture (`screencapture -l`) +# works regardless of focus; clicks/keys do NOT. See docs/dev/driving-the-app.md +# → "Honest caveats" for when this bites and what to do (physically click; or +# verify logic via `cargo test`, not screenshots). +# +set -euo pipefail + +OWNER="${DECKARD_WINDOW_OWNER:-deckard}" +SHOT_DIR="${DECKARD_SHOT_DIR:-.context/shots}" +UNLOCK_FRAC="${DECKARD_UNLOCK_FRAC:-0.5,0.65}" + +die() { printf 'deckard-drive: %s\n' "$*" >&2; exit 1; } + +# ── window finder ───────────────────────────────────────────────────────────── +# Print " " for the largest on-screen layer-0 window whose +# owner name contains $OWNER. Coordinates are top-left-origin logical points — +# the same space cliclick and screencapture use, so no conversion is needed. +find_window() { + swift - "$OWNER" <<'SWIFT' +import CoreGraphics +import Foundation + +let needle = (CommandLine.arguments.dropFirst().first ?? "deckard").lowercased() +guard let list = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID + ) as? [[String: Any]] else { exit(1) } + +var best: (id: Int, x: Int, y: Int, w: Int, h: Int, area: Double)? +for win in list { + let owner = (win[kCGWindowOwnerName as String] as? String ?? "").lowercased() + guard owner.contains(needle) else { continue } + // Layer 0 == a normal app window; title-bar/shadow helpers live on other layers. + guard (win[kCGWindowLayer as String] as? Int ?? -1) == 0 else { continue } + let b = win[kCGWindowBounds as String] as? [String: Any] ?? [:] + let x = Int(b["X"] as? Double ?? 0), y = Int(b["Y"] as? Double ?? 0) + let w = Int(b["Width"] as? Double ?? 0), h = Int(b["Height"] as? Double ?? 0) + let area = Double(w * h) + let id = win[kCGWindowNumber as String] as? Int ?? -1 + if best == nil || area > best!.area { best = (id, x, y, w, h, area) } +} +guard let b = best else { + FileHandle.standardError.write( + "no on-screen layer-0 window owned by '\(needle)' — is the app running?\n".data(using: .utf8)!) + exit(2) +} +print("\(b.id) \(b.x) \(b.y) \(b.w) \(b.h)") +SWIFT +} + +# Raise the app so synthetic input reaches it. Uses the modern activation API +# (NSRunningApplication.activate) — the right primitive when the app CAN be +# activated. Best-effort: on macOS 14+ this cannot steal focus from another +# active app, and a background-launched GPUI app may not activate at all (see the +# header note + docs/dev/driving-the-app.md). Prints "isActive=" so a caller +# can tell whether input will actually land. +raise_app() { + local pid + pid="$(pgrep -x "$OWNER" | head -1)" + [ -n "$pid" ] || { echo "isActive=false (no '$OWNER' process)"; return 0; } + swift - "$pid" <<'SWIFT' 2>/dev/null || true +import AppKit +let pid = Int32(CommandLine.arguments.dropFirst().first ?? "0") ?? 0 +guard let app = NSRunningApplication(processIdentifier: pid) else { print("isActive=false"); exit(0) } +app.activate() +usleep(400_000) +print("isActive=\(app.isActive)") +SWIFT +} + +# ── commands ────────────────────────────────────────────────────────────────── +cmd_win() { find_window; } + +cmd_raise() { raise_app; echo "raised $OWNER"; } + +cmd_shot() { + local name="${1:-shot}" + mkdir -p "$SHOT_DIR" + read -r id _ _ _ _ < <(find_window) + local out="$SHOT_DIR/${name}.png" + screencapture -x -o -l"$id" "$out" + echo "$out" +} + +# Click at window-relative fractions (0..1 of width/height). +cmd_click() { + [ $# -eq 2 ] || die "click needs (fractions 0..1)" + raise_app + read -r _ x y w h < <(find_window) + local sx sy + sx=$(awk -v x="$x" -v w="$w" -v f="$1" 'BEGIN{printf "%d", x + f*w}') + sy=$(awk -v y="$y" -v h="$h" -v f="$2" 'BEGIN{printf "%d", y + f*h}') + cliclick "c:${sx},${sy}" + echo "clicked frac($1,$2) -> screen($sx,$sy)" +} + +# Click at window-relative points (from the top-left corner of the window). +cmd_clickpt() { + [ $# -eq 2 ] || die "clickpt needs (points from window top-left)" + raise_app + read -r _ x y _ _ < <(find_window) + cliclick "c:$((x + $1)),$((y + $2))" + echo "clicked pt($1,$2) -> screen($((x + $1)),$((y + $2)))" +} + +cmd_type() { + [ $# -ge 1 ] || die "type needs \"\"" + raise_app + osascript -e "tell application \"System Events\" to keystroke \"$1\"" + echo "typed: $1" +} + +# Registered-action shortcut, e.g. `key , cmd` or `key d cmd shift`. +cmd_key() { + [ $# -ge 1 ] || die "key needs [mods...]" + local char="$1"; shift + raise_app + if [ $# -eq 0 ]; then + osascript -e "tell application \"System Events\" to keystroke \"$char\"" + else + local clause="" m + for m in "$@"; do + case "$m" in + cmd) clause+="command down, " ;; + shift) clause+="shift down, " ;; + opt) clause+="option down, " ;; + ctrl) clause+="control down, " ;; + *) die "unknown modifier '$m' (use cmd|shift|opt|ctrl)" ;; + esac + done + clause="${clause%, }" + osascript -e "tell application \"System Events\" to keystroke \"$char\" using {$clause}" + fi + echo "key: $char ${*:-}" +} + +# Unlock the demo/QA vault: the passphrase field auto-focuses, so type it and then +# CLICK the Unlock button — a synthetic Return does NOT submit (GPUI InputState eats +# it). Argon2 + the daemon round-trip take ~1s (PRODUCTION KDF) before the app lands. +cmd_unlock() { + local pass="${1:-deckard-demo}" + mkdir -p "$SHOT_DIR" + local active; active="$(raise_app)"; echo "$active" + case "$active" in + *isActive=false*) + echo "deckard-drive: WARNING — Deckard is not the active app, so the passphrase and" >&2 + echo " Unlock click will NOT land (you'll get an empty field). Give the window focus" >&2 + echo " first — run it from a foreground Terminal, or physically click the app once —" >&2 + echo " then re-run. See docs/dev/driving-the-app.md → Honest caveats." >&2 ;; + esac + osascript -e "tell application \"System Events\" to keystroke \"$pass\"" >/dev/null + local fx="${UNLOCK_FRAC%,*}" fy="${UNLOCK_FRAC#*,}" + read -r _ x y w h < <(find_window) + local sx sy + sx=$(awk -v x="$x" -v w="$w" -v f="$fx" 'BEGIN{printf "%d", x + f*w}') + sy=$(awk -v y="$y" -v h="$h" -v f="$fy" 'BEGIN{printf "%d", y + f*h}') + cliclick "c:${sx},${sy}" + # Give the KDF + daemon unlock a moment before capturing the unlocked screen. + osascript -e 'delay 2' >/dev/null 2>&1 || true + local out="$SHOT_DIR/unlocked.png" + read -r id _ _ _ _ < <(find_window) + screencapture -x -o -l"$id" "$out" + echo "typed passphrase, clicked Unlock at frac($fx,$fy) -> screen($sx,$sy); shot: $out" +} + +cmd_deps() { + local ok=1 + for bin in swift cliclick screencapture osascript; do + if command -v "$bin" >/dev/null 2>&1; then + echo "ok $bin -> $(command -v "$bin")" + else + echo "MISS $bin"; ok=0 + fi + done + [ $ok -eq 1 ] || die "install missing tools (cliclick: brew install cliclick; swift ships with Xcode CLT)" +} + +main() { + [ $# -ge 1 ] || { grep -E '^#( |$)' "$0" | sed -E 's/^# ?//'; exit 0; } + local cmd="$1"; shift + case "$cmd" in + win|find) cmd_win "$@" ;; + raise) cmd_raise "$@" ;; + shot) cmd_shot "$@" ;; + click) cmd_click "$@" ;; + clickpt) cmd_clickpt "$@" ;; + type) cmd_type "$@" ;; + key) cmd_key "$@" ;; + unlock) cmd_unlock "$@" ;; + deps) cmd_deps "$@" ;; + -h|--help|help) grep -E '^#( |$)' "$0" | sed -E 's/^# ?//' ;; + *) die "unknown command '$cmd' (try: help)" ;; + esac +} + +main "$@"