From c2b6b4e6099d8bd8a4c5f279f2e1f166c645d105 Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:47:58 -0600 Subject: [PATCH 1/9] desktop: wait_for and stable element selectors (17 tools) wait_for (observing tier) waits, bounded, until a control matching a selector appears or goes, a window appears or closes, or the screen fingerprint changes. timeout_ms defaults to 5000 and is held to 60000; poll_ms 50..2000; no poll starts after the deadline; the stop is asked every 100 ms. A timeout is a JSON reply (result=timeout, last_seen), not a tool error. Selectors (key=value;... : app, window, role, name, aid, path, nth) are printed by ui_tree beside every element and accepted as `selector` by click_element, set_value, element_action and wait_for in place of `id`. Resolved against the live tree at call time; none or several matches is an error naming which, never a guess. eN ids are unchanged. Element gains automation_id (UIA AutomationId, AX AXIdentifier, AT-SPI AccessibleId). Tier tables, incode's for_desktop, toolbox OBSERVING and the parity test cover wait_for; docs updated. --- Cargo.toml | 2 +- README.md | 38 +- crates/desktop/Cargo.toml | 2 +- crates/desktop/src/fingerprint.rs | 166 +++ crates/desktop/src/lib.rs | 5 +- crates/desktop/src/linux.rs | 20 + crates/desktop/src/macos.rs | 4 + crates/desktop/src/mcp.rs | 6 + crates/desktop/src/mcp/definitions.rs | 154 ++- crates/desktop/src/mcp/gate.rs | 44 +- crates/desktop/src/mcp/selector_tests.rs | 484 ++++++++ crates/desktop/src/mcp/tools.rs | 222 +++- crates/desktop/src/mcp/wait_tool.rs | 279 +++++ crates/desktop/src/permission.rs | 15 +- crates/desktop/src/selector.rs | 963 +++++++++++++++ crates/desktop/src/uia.rs | 41 +- crates/desktop/src/uia/resolve.rs | 367 ++++++ crates/desktop/src/uia_windows.rs | 4 + crates/desktop/src/wait.rs | 1402 ++++++++++++++++++++++ docs/ARCHITECTURE.md | 6 +- docs/COMPUTER_USE.md | 95 +- docs/incord-memory-integration.md | 2 +- src/desktop.rs | 18 +- src/permissions/mod.rs | 7 +- src/toolbox/desktop.rs | 3 + 25 files changed, 4245 insertions(+), 104 deletions(-) create mode 100644 crates/desktop/src/fingerprint.rs create mode 100644 crates/desktop/src/mcp/selector_tests.rs create mode 100644 crates/desktop/src/mcp/wait_tool.rs create mode 100644 crates/desktop/src/selector.rs create mode 100644 crates/desktop/src/uia/resolve.rs create mode 100644 crates/desktop/src/wait.rs diff --git a/Cargo.toml b/Cargo.toml index bd854e3..416241b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,7 +90,7 @@ agent-client-protocol-schema = { version = "=1.9.1", default-features = false } # no new package; it adds the seccomp filter, the host-allowlist proxy and the # `.git` protection the built-in engine lacks. incord-sandbox = { path = "crates/sandbox" } -# Computer use (`crates/desktop`, binary `incu`): the 16-tool MCP server that +# Computer use (`crates/desktop`, binary `incu`): the 17-tool MCP server that # `incode --desktop-mcp` wraps, and the journal/stop-file/capture code the run # server and the toolbox call. A path dependency, so no new package: it carries # the screen-capture, accessibility and input dependencies that used to be diff --git a/README.md b/README.md index bc4dc64..27327cd 100644 --- a/README.md +++ b/README.md @@ -850,7 +850,7 @@ anything runs. | `Screenshot` | observe | A picture of a public page or a workspace HTML/SVG file. Offered only when the headless browser is found. | | `RenderPdf` | change | A public page or a workspace HTML/SVG file printed to PDF. Offered only when a printer is found (below). | | `ConvertDocument` | change | An office document to PDF through LibreOffice. Offered only when LibreOffice is found. | -| `screen_info` · `list_windows` · `screenshot` · `ui_tree` · `cursor_position` | observe | Look at the desktop. Offered only when `INCODE_DESKTOP=1` — see [Computer use](#computer-use). | +| `screen_info` · `list_windows` · `screenshot` · `ui_tree` · `cursor_position` · `wait_for` | observe | Look at the desktop, or wait (bounded) for a control, a window or a change of the screen. Offered only when `INCODE_DESKTOP=1` — see [Computer use](#computer-use). | | `click_element` · `set_value` · `element_action` · `focus_window` · `type_text` · `paste_text` · `key` · `scroll` · `drag` · `move_mouse` · `click` | run | Act on the desktop. Same condition; refused in read-only. | | `Skill` | observe | Read one skill's text from the shared store. | | `Agent` | run | Hand a self-contained task to a sub-agent (a second headless run of the session's CLI, or the API route), with the session's mode, guards and cancel; at most 2 levels deep. Takes `agent: ` for a [user-defined sub-agent](#user-defined-sub-agents). | @@ -1075,12 +1075,13 @@ plain screenshots and pixel clicks still work. |---|---| | `screen_info` | Displays, their sizes and scale, and on macOS any missing grant. | | `list_windows` | The open windows: `id`, title, app, pid, focused, rectangle, and `tree` (false when the window exposes no accessibility tree). A blocked app's window is listed without its title. | -| `ui_tree` | One window's controls as `[e] Role "name" (x,y,w×h)`, with `password` on secret fields. `window` picks the window (default: the focused one); `depth` limits the walk. Controls at no place on screen are counted on one line, not listed. A window with no tree is refused with a pointer to `screenshot` and `click`. | +| `ui_tree` | One window's controls as `[e] Role "name" (x,y,w×h) sel: `, with `password` on secret fields; `sel:` names the control without its id (see [Selectors](#selectors-and-wait_for)). `window` picks the window (default: the focused one); `depth` limits the walk. Controls at no place on screen are counted on one line, not listed. A window with no tree is refused with a pointer to `screenshot` and `click`. | | `screenshot` | The primary display, or one `window`. The reply carries the region and origin, so a point in the picture maps back to the screen. | | `cursor_position` | Where the pointer is. | -| `click_element` | Click a control by id, at its centre as it is *now*. | -| `set_value` | Set a text field's value through the accessibility interface: no typing, no focus, no clipboard. Read back and marked `(verified)`, or `(unverified: why)`. A password field is never read back. | -| `element_action` | Run one of a control's own actions without the mouse: `press`, `toggle`, `expand`, `collapse`, `select` and `focus` everywhere, plus what the platform adds (`scroll_into_view` on Windows; `show_menu`, `increment`, `decrement`, `confirm`, `cancel` on macOS). An action the control lacks is refused, and the refusal lists the ones it offers. | +| `wait_for` | Wait, bounded, until a control matching a `selector` appears or goes, a window appears or closes, or the screen changes. Returns what it found, or a `timeout` reply with what it last saw. | +| `click_element` | Click a control by id or `selector`, at its centre as it is *now*. | +| `set_value` | Set a text field's value (by id or `selector`) through the accessibility interface: no typing, no focus, no clipboard. Read back and marked `(verified)`, or `(unverified: why)`. A password field is never read back. | +| `element_action` | Run one of a control's own actions (by id or `selector`) without the mouse: `press`, `toggle`, `expand`, `collapse`, `select` and `focus` everywhere, plus what the platform adds (`scroll_into_view` on Windows; `show_menu`, `increment`, `decrement`, `confirm`, `cancel` on macOS). An action the control lacks is refused, and the refusal lists the ones it offers. | | `focus_window` | Bring a window to the front so typing reaches it. | | `type_text` · `key` · `paste_text` | Keystrokes, a chord (`["cmdorctrl","s"]`: Cmd on macOS, Ctrl elsewhere), or a paste. `paste_text` puts the old clipboard back after `paste_restore_delay_ms`, and refuses to paste at all when the clipboard cannot be read. | | `scroll` | Wheel steps (`dy` down, `dx` right) over a control by `id`, or under the pointer. | @@ -1091,6 +1092,33 @@ Every acting tool also takes `window`; typing into a window that is not the focused one is refused rather than sent elsewhere. It also takes `screenshot_after`, which returns a picture once the screen has settled. +### Selectors and `wait_for` + +An `eN` id is a position in one `ui_tree` result. A **selector** names a +control by what it is, and is resolved against the live tree when the call is +made, so it still means something in the next call, and after the program has +been restarted. `ui_tree` prints one beside every element (`sel: …`) and the +window part once at the top; `click_element`, `set_value`, `element_action` +and `wait_for` take it as `selector` in place of `id`. It must match exactly one +control: none, or several, is an error that says which, never a guess. + +``` +role=button;name=Save one control of the focused window +role=edit;aid=searchBox by the program's own automation id +role=button;name=Save;nth=1 the second of two look-alikes +app=notepad;window~=Save As;role=button;name=Save in another window +``` + +Keys: `app`, `window` / `window~` (exact / contains), `role`, `name` / `name~`, +`aid` (UIA `AutomationId`, AX `AXIdentifier`, AT-SPI `AccessibleId`), `path` +(child indices, `0.2.1`), `nth`. `;` and `\` in a value are written `\;` and +`\\`. `wait_for` takes `condition` (`element_appears`, `element_gone`, +`window_appears`, `window_gone`, `screen_changes`), a `selector` (window +conditions take only `app`/`window`), `timeout_ms` (default 5000, held to +60000) and `poll_ms` (250, 50–2000), and answers JSON whose `result` is `found` +or `timeout`. The full grammar and what each platform supplies is in +`docs/COMPUTER_USE.md`. + ### What it will not do - **Act on a control it did not show you.** An id is a position in one diff --git a/crates/desktop/Cargo.toml b/crates/desktop/Cargo.toml index 7bbb019..e914d57 100644 --- a/crates/desktop/Cargo.toml +++ b/crates/desktop/Cargo.toml @@ -5,7 +5,7 @@ version = "0.1.0" edition = "2021" rust-version = "1.88" license = "Apache-2.0" -description = "Computer use for an agent: screenshots, accessibility trees, mouse and keyboard, as a 16-tool MCP server with a kill switch and a watch journal" +description = "Computer use for an agent: screenshots, accessibility trees, mouse and keyboard, as a 17-tool MCP server with a kill switch and a watch journal" repository = "https://github.com/incordai/Incode" keywords = ["computer-use", "mcp", "accessibility", "agents", "automation"] categories = ["command-line-utilities", "accessibility"] diff --git a/crates/desktop/src/fingerprint.rs b/crates/desktop/src/fingerprint.rs new file mode 100644 index 0000000..c5476b5 --- /dev/null +++ b/crates/desktop/src/fingerprint.rs @@ -0,0 +1,166 @@ +//! A small fingerprint of a picture of the screen, for asking "did it change?" +//! without keeping or sending the picture (`wait_for` `screen_changes`). +//! +//! # Why two parts +//! +//! A difference hash (does each cell get brighter or darker than its +//! neighbour) sees edges, which is what a dialog opening or a page loading +//! changes. It is blind to a change of brightness alone: a white screen and a +//! black one have no edges, so they hash the same. So the fingerprint is 128 +//! bits: that gradient hash (64) beside the average brightness of a 4x4 grid, +//! four bits a cell (64). The screens are compared by how many bits differ +//! ([`distance`]), so a blinking cursor or a clock tick that moves a bit or two +//! is below a tolerance, and a window opening is far above it. +//! +//! Pure over pixels, so tested on every platform; the capture that feeds it is +//! `mcp::wait_tool`. + +use image::imageops::{grayscale, resize, FilterType}; +use image::RgbaImage; + +/// 128 bits: gradient hash in the high half, brightness grid in the low half. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Fingerprint(pub u128); + +/// The fingerprint of `picture`. +#[must_use] +pub fn of(picture: &RgbaImage) -> Fingerprint { + let grey = grayscale(picture); + + // Gradient: 9x8 cells, one bit for each of the 8x8 left-brighter-than-right. + let cells = resize(&grey, 9, 8, FilterType::Triangle); + let mut gradient = 0u64; + for y in 0..8 { + for x in 0..8 { + gradient <<= 1; + if cells.get_pixel(x, y).0[0] > cells.get_pixel(x + 1, y).0[0] { + gradient |= 1; + } + } + } + + // Brightness: 4x4 cells, the top four bits of each average. + let grid = resize(&grey, 4, 4, FilterType::Triangle); + let mut levels = 0u64; + for y in 0..4 { + for x in 0..4 { + levels = (levels << 4) | u64::from(grid.get_pixel(x, y).0[0] >> 4); + } + } + + Fingerprint((u128::from(gradient) << 64) | u128::from(levels)) +} + +/// How many of the 128 bits differ. +#[must_use] +pub fn distance(a: Fingerprint, b: Fingerprint) -> u32 { + (a.0 ^ b.0).count_ones() +} + +impl Fingerprint { + /// 32 hex digits, for a reply the next call can pass back as a baseline. + #[must_use] + pub fn hex(self) -> String { + format!("{:032x}", self.0) + } + + /// [`Self::hex`]'s inverse; `None` for anything that is not 32 hex digits. + #[must_use] + pub fn from_hex(text: &str) -> Option { + let text = text.trim(); + (text.len() == 32 && text.chars().all(|c| c.is_ascii_hexdigit())) + .then(|| u128::from_str_radix(text, 16).ok().map(Fingerprint)) + .flatten() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use image::Rgba; + + fn flat(width: u32, height: u32, grey: u8) -> RgbaImage { + RgbaImage::from_pixel(width, height, Rgba([grey, grey, grey, 255])) + } + + /// A page: light, with a dark bar across the top and a dark box inside. + fn page() -> RgbaImage { + let mut img = flat(320, 200, 235); + for y in 0..20 { + for x in 0..320 { + img.put_pixel(x, y, Rgba([40, 40, 40, 255])); + } + } + for y in 80..140 { + for x in 100..220 { + img.put_pixel(x, y, Rgba([90, 120, 200, 255])); + } + } + img + } + + #[test] + fn the_same_picture_is_the_same_fingerprint() { + assert_eq!(distance(of(&page()), of(&page())), 0); + assert_eq!(of(&page()), of(&page())); + } + + #[test] + fn white_and_black_differ_though_neither_has_an_edge() { + let d = distance(of(&flat(160, 90, 255)), of(&flat(160, 90, 0))); + assert!( + d >= 16, + "a whole-screen brightness change is far apart: {d}" + ); + } + + #[test] + fn a_dialog_opening_is_far_above_a_tolerance_of_a_few_bits() { + let before = page(); + let mut after = page(); + // A dark dialog over the middle third. + for y in 50..150 { + for x in 60..260 { + after.put_pixel(x, y, Rgba([20, 20, 30, 255])); + } + } + let d = distance(of(&before), of(&after)); + assert!(d > 8, "{d}"); + } + + #[test] + fn a_blinking_cursor_is_below_the_default_tolerance() { + let before = page(); + let mut after = page(); + for y in 100..112 { + for x in 150..152 { + after.put_pixel(x, y, Rgba([0, 0, 0, 255])); + } + } + let d = distance(of(&before), of(&after)); + assert!(d <= 4, "a 2x12 pixel mark moved {d} bits"); + } + + #[test] + fn the_size_of_the_picture_does_not_matter() { + let small = of(&image::imageops::resize( + &page(), + 160, + 100, + FilterType::Triangle, + )); + let large = of(&page()); + assert!(distance(small, large) <= 4); + } + + #[test] + fn hex_round_trips_and_refuses_anything_else() { + let f = of(&page()); + assert_eq!(f.hex().len(), 32); + assert_eq!(Fingerprint::from_hex(&f.hex()), Some(f)); + assert_eq!(Fingerprint::from_hex(&format!(" {} ", f.hex())), Some(f)); + assert_eq!(Fingerprint::from_hex("abc"), None); + assert_eq!(Fingerprint::from_hex(&"z".repeat(32)), None); + assert_eq!(Fingerprint(0).hex(), "0".repeat(32)); + } +} diff --git a/crates/desktop/src/lib.rs b/crates/desktop/src/lib.rs index 50eaff1..493b12f 100644 --- a/crates/desktop/src/lib.rs +++ b/crates/desktop/src/lib.rs @@ -8,7 +8,7 @@ //! //! - **This is offered to a DELEGATED agent over MCP.** It is not dispatched by //! a loop, because there is no such loop: the agent runs in its own process -//! and reaches the screen only through the 16 tools of [`mcp`]. +//! and reaches the screen only through the 17 tools of [`mcp`]. //! //! - **This crate does not depend on incode.** It began as a module of incode //! (`src/desktop/`) and moved out whole. What it used to take from the rest @@ -45,14 +45,17 @@ pub mod cli; pub mod config; pub mod coords; pub mod dpi; +pub mod fingerprint; pub mod hooks; pub mod journal; pub mod limits; pub mod mcp; pub mod permission; pub mod safety; +pub mod selector; pub mod thread; pub mod uia; +pub mod wait; #[cfg(target_os = "linux")] pub mod linux; diff --git a/crates/desktop/src/linux.rs b/crates/desktop/src/linux.rs index d04aece..440c0b3 100644 --- a/crates/desktop/src/linux.rs +++ b/crates/desktop/src/linux.rs @@ -486,9 +486,28 @@ fn describe(bus: &Connection, node: &Node, id: u32, depth: u32) -> Element { rect, enabled: is_enabled(&states(bus, node)), secret: role_name == "password text", + automation_id: accessible_id(bus, node), } } +/// The object's `AccessibleId` property (AT-SPI 2.46 and later; GTK 4 and Qt +/// fill it from the widget's id), empty when the toolkit has none or the +/// registry is older. UNVERIFIED against a live bus: this reads the property +/// the same way [`name`] reads `Name`, and an error is the empty answer. +fn accessible_id(bus: &Connection, node: &Node) -> String { + bus.call_method( + Some(node.0.as_str()), + node.1.as_str(), + Some("org.freedesktop.DBus.Properties"), + "Get", + &(ACCESSIBLE, "AccessibleId"), + ) + .and_then(|reply| reply.body().deserialize::()) + .ok() + .and_then(|value| String::try_from(value).ok()) + .unwrap_or_default() +} + fn extents(bus: &Connection, node: &Node) -> Rect { bus.call_method( Some(node.0.as_str()), @@ -1511,6 +1530,7 @@ mod tests { }, enabled: true, secret: false, + automation_id: String::new(), } } diff --git a/crates/desktop/src/macos.rs b/crates/desktop/src/macos.rs index ecae417..16596f4 100644 --- a/crates/desktop/src/macos.rs +++ b/crates/desktop/src/macos.rs @@ -375,6 +375,10 @@ fn describe(element: &Owned, id: u32, depth: u32) -> Element { // that says it is disabled is marked so. enabled: bool_attribute(element, "AXEnabled").unwrap_or(true), secret, + // AXIdentifier: the program's own name for the control (a Cocoa + // accessibilityIdentifier). Few controls set one; it names the + // control, it is not its content. + automation_id: text_attribute(element, "AXIdentifier").unwrap_or_default(), } } diff --git a/crates/desktop/src/mcp.rs b/crates/desktop/src/mcp.rs index 4ee3bdc..aa07719 100644 --- a/crates/desktop/src/mcp.rs +++ b/crates/desktop/src/mcp.rs @@ -52,6 +52,8 @@ //! - [`tools`]: `run_tool` and one step per tool. //! - [`guard`]: the blocked-application, focus, point and region refusals a //! tool asks before it acts. +//! - `wait_tool`: `wait_for`, the one tool that polls: its arguments and the +//! live desktop, over the timing rules in [`crate::wait`]. //! - `tests`: the tests, at their old path `mcp::tests`. //! //! # Wired to @@ -75,10 +77,14 @@ mod gate; mod guard; mod tools; mod transport; +mod wait_tool; #[cfg(test)] mod tests; +#[cfg(test)] +mod selector_tests; + pub use definitions::tool_definitions; pub use transport::run_stdio; diff --git a/crates/desktop/src/mcp/definitions.rs b/crates/desktop/src/mcp/definitions.rs index 15c6c41..3170e3f 100644 --- a/crates/desktop/src/mcp/definitions.rs +++ b/crates/desktop/src/mcp/definitions.rs @@ -30,6 +30,7 @@ pub fn tool_definitions() -> Vec { const WINDOWED: &[&str] = &[ "screenshot", "ui_tree", + "wait_for", "click_element", "set_value", "element_action", @@ -51,6 +52,12 @@ pub fn tool_definitions() -> Vec { focused window the call is REFUSED rather than typing somewhere else." } "screenshot" => "Capture only this window, from list_windows.", + "wait_for" => { + "The window to watch, from list_windows: for element_* the one the \ + selector is looked for in, for screen_changes the one whose area is \ + compared. Default: the window the selector names, else the focused one \ + when the wait begins, else the whole screen." + } _ => "The window, from list_windows. Default: the focused window.", }; properties["window"] = json!({"type": "string", "description": what}); @@ -96,35 +103,38 @@ fn base_definitions() -> Vec { json!({ "name": "set_value", "description": "Set a control's value (a text field, a slider) directly through \ - the accessibility interface, by the id ui_tree gave it — no \ - typing, no focus needed, no clipboard. PREFER THIS over \ - type_text for a field. The value is read back where allowed and \ - the result says whether it matched (a password field is never \ - read back).", + the accessibility interface, by the id ui_tree gave it or by a \ + `selector` — no typing, no focus needed, no clipboard. PREFER \ + THIS over type_text for a field. The value is read back where \ + allowed and the result says whether it matched (a password \ + field is never read back).", "inputSchema": { "type": "object", "properties": { "id": {"type": "integer", "description": "The eN number from ui_tree."}, + "selector": selector_property(), "value": {"type": "string"} }, - "required": ["id", "value"], + "required": ["value"], }, }), json!({ "name": "element_action", - "description": "Run a control's own action by the id ui_tree gave it, without \ - the mouse: the cursor does not move and no other window gets \ - the click. Actions: press, toggle, expand, collapse, select, \ - scroll_into_view, focus, default (macOS also show_menu, \ - confirm, increment, decrement, cancel, raise). An action the \ - control does not have is refused with the ones it does.", + "description": "Run a control's own action by the id ui_tree gave it, or by a \ + `selector`, without the mouse: the cursor does not move and no \ + other window gets the click. Actions: press, toggle, expand, \ + collapse, select, scroll_into_view, focus, default (macOS also \ + show_menu, confirm, increment, decrement, cancel, raise). An \ + action the control does not have is refused with the ones it \ + does.", "inputSchema": { "type": "object", "properties": { "id": {"type": "integer", "description": "The eN number from ui_tree."}, + "selector": selector_property(), "action": {"type": "string"} }, - "required": ["id", "action"], + "required": ["action"], }, }), json!({ @@ -169,8 +179,13 @@ fn base_definitions() -> Vec { names, roles and exact rectangles; password fields are marked and never \ read. PREFER THIS OVER screenshot: clicking by element id hits the true centre \ of a named control, where a coordinate read off a downscaled image is a \ - guess that can miss silently. Falls back to screenshot for canvas, images \ - and custom-drawn UI, which expose no tree. {UNTRUSTED}" + guess that can miss silently. Each element's `sel:` is a selector that names \ + it without its eN id, which holds only for this one result: pass it as \ + `selector` to click_element, set_value, element_action and wait_for, \ + resolved against the live tree when called (the line at the top gives the \ + app/window part for a window that is not focused). Falls back to \ + screenshot for canvas, images and custom-drawn UI, which expose no tree. \ + {UNTRUSTED}" ), "inputSchema": { "type": "object", @@ -185,10 +200,10 @@ fn base_definitions() -> Vec { }), json!({ "name": "click_element", - "description": "Click a control by the id ui_tree gave it. The element is \ - looked up again first, so a control that moved or vanished is \ - reported instead of clicked at its old position. Deterministic \ - where click(x,y) is a guess.", + "description": "Click a control by the id ui_tree gave it, or by a `selector`. \ + The element is looked up again first, so a control that moved \ + or vanished is reported instead of clicked at its old position. \ + Deterministic where click(x,y) is a guess.", "inputSchema": { "type": "object", "properties": { @@ -196,9 +211,9 @@ fn base_definitions() -> Vec { "type": "integer", "description": "The eN number from ui_tree, without the 'e'." }, + "selector": selector_property(), "button": {"type": "string", "enum": ["left", "right", "middle"]} }, - "required": ["id"], }, }), json!({ @@ -309,5 +324,104 @@ fn base_definitions() -> Vec { "required": ["x", "y"], }, }), + wait_for_definition(), ] } + +/// The `selector` argument of the tools that act on one control, which they +/// take in place of `id`. +fn selector_property() -> Value { + json!({ + "type": "string", + "description": "Instead of `id`: name the control the way ui_tree's `sel:` does, e.g. \ + role=button;name=Save (also aid=, nth=, path=; app= and window= pick \ + a window other than the focused one). It is resolved against the \ + live tree now and must match exactly one control: none, or several, \ + is an error that says which, never a guess. Give id or selector, \ + not both." + }) +} + +/// `wait_for`, apart from the list above because its schema is the longest. +fn wait_for_definition() -> Value { + json!({ + "name": "wait_for", + "description": format!( + "Wait, bounded, until something is true, instead of sleeping and hoping: a \ + control matching a `selector` appears (element_appears) or goes \ + (element_gone), a window appears (window_appears) or closes (window_gone), or \ + the screen changes (screen_changes). Returns at once when it already holds. \ + The reply is JSON whose `result` is `found` (with what it found: for an \ + element its window, role, name and a `selector` for it) or `timeout` (with \ + `last_seen`, what the last look saw) — a timeout is a reply, not an error, so \ + read `result`. `timeout_ms` defaults to {default} and is held to {max}; the \ + reply lists any bound it changed. A selector is as ui_tree prints it: \ + role=button;name=Save, adding aid=, nth= or path=<0.2.1> to \ + pick one of several; add app=;window= (or window~= for \ + contains) to name a window other than the focused one, or pass `window`. For \ + an element condition with no window given, the window focused when the wait \ + begins is the one watched. Window conditions take only app and window. A \ + control that matches several is reported as `matches` > 1, not picked. \ + screen_changes compares a 128-bit fingerprint of the screen (or of `window`) \ + with `baseline` — a `hash` an earlier wait_for returned — or else with how it \ + is when the wait begins; pass the returned `hash` back as `baseline` to go on \ + waiting from there. {untrusted}", + default = crate::wait::DEFAULT_TIMEOUT_MS, + max = crate::wait::MAX_TIMEOUT_MS, + untrusted = UNTRUSTED, + ), + "inputSchema": { + "type": "object", + "properties": { + "condition": { + "type": "string", + "enum": [ + "element_appears", "element_gone", "window_appears", "window_gone", + "screen_changes" + ], + }, + "selector": { + "type": "string", + "description": "key=value pairs joined by ';' — control: role, name \ + (name~ for contains), aid, path, nth; window: app, \ + window (window~ for contains). E.g. \ + role=button;name=Save or app=notepad;window~=Save As. \ + Escape ';' and '\\' in a value as '\\;' and '\\\\'. \ + For element_* conditions it must name a control; for \ + window_* only app and/or window.", + }, + "timeout_ms": { + "type": "integer", + "description": format!( + "Longest to wait. Default {}, at most {}.", + crate::wait::DEFAULT_TIMEOUT_MS, + crate::wait::MAX_TIMEOUT_MS + ), + }, + "poll_ms": { + "type": "integer", + "description": format!( + "How often to look. Default {}, from {} to {}.", + crate::wait::DEFAULT_POLL_MS, + crate::wait::MIN_POLL_MS, + crate::wait::MAX_POLL_MS + ), + }, + "baseline": { + "type": "string", + "description": "screen_changes: the `hash` of an earlier reply to compare \ + with. Default: the screen as the wait begins.", + }, + "tolerance": { + "type": "integer", + "description": format!( + "screen_changes: how many of the 128 bits may differ before the \ + screen counts as changed. Default {}; 0 is any difference.", + crate::wait::DEFAULT_TOLERANCE + ), + }, + }, + "required": ["condition"], + }, + }) +} diff --git a/crates/desktop/src/mcp/gate.rs b/crates/desktop/src/mcp/gate.rs index 530eeb3..f25ac50 100644 --- a/crates/desktop/src/mcp/gate.rs +++ b/crates/desktop/src/mcp/gate.rs @@ -31,6 +31,33 @@ use serde_json::Value; use super::tools::window_arg; use super::Server; +use crate::selector::Selector; + +/// The selector of a call as the journal may carry it: see +/// [`Selector::render_redacted`]. Empty when the call has none. +fn selector_ref(args: &Value) -> String { + match args.get("selector").and_then(Value::as_str) { + None => String::new(), + Some(raw) => match Selector::parse(raw) { + Ok(selector) => format!("selector {}", selector.render_redacted()), + // Not journalled as given: a malformed one may be anything. + Err(_) => "selector (unreadable)".to_string(), + }, + } +} + +/// Which control an element tool was aimed at: its selector when it had one, +/// else the `ui_tree` number, which is all that call carries. +fn element_ref(args: &Value) -> String { + if args.get("selector").is_some() { + selector_ref(args) + } else { + format!( + "element e{}", + args.get("id").and_then(Value::as_i64).unwrap_or_default() + ) + } +} impl Server { /// The operator's stop file, turned into a stop of this server. @@ -219,14 +246,23 @@ impl Server { // schema is `{id, button}`. The element's role and name live in the // `ui_tree` the agent read to get the id, which is itself a // journalled call, so an operator can follow `e16` back to it. - "click_element" => format!("element e{}", n("id")), + "click_element" => element_ref(args), // LENGTH ONLY, like typing: a value set may be a password. "set_value" => format!( - "element e{}, {} characters", - n("id"), + "{}, {} characters", + element_ref(args), s("value").chars().count() ), - "element_action" => format!("element e{} {}", n("id"), s("action")), + "element_action" => format!("{} {}", element_ref(args), s("action")), + // The condition and the selector as `Selector::render_redacted` + // has it (no text field's name), and how long it may wait. + "wait_for" => format!( + "{} {} timeout {}", + s("condition"), + selector_ref(args), + args.get("timeout_ms") + .map_or("default".to_string(), |ms| format!("{ms} ms")) + ), "focus_window" => format!("window {}", window_arg(args).unwrap_or_default()), "click" | "move_mouse" => format!("({}, {})", n("x"), n("y")), "drag" => format!( diff --git a/crates/desktop/src/mcp/selector_tests.rs b/crates/desktop/src/mcp/selector_tests.rs new file mode 100644 index 0000000..8c38063 --- /dev/null +++ b/crates/desktop/src/mcp/selector_tests.rs @@ -0,0 +1,484 @@ +//! Tests of `wait_for` and of acting by selector, as a client meets them: the +//! tool definitions, the argument rules, the journal line, and (on Windows, +//! where the existing live tests run) the real server answering over its own +//! protocol path. None of them moves the mouse or types. +//! +//! The timing rules themselves are in `crate::wait`'s tests, against a scripted +//! desktop with a virtual clock; the selector grammar and matching are in +//! `crate::selector`'s. + +use super::tools::{element_target, Target}; +use super::*; + +#[cfg(windows)] +use crate::uia; + +fn definition(name: &str) -> Value { + tool_definitions() + .into_iter() + .find(|t| t["name"] == name) + .unwrap_or_else(|| panic!("no tool named {name}")) +} + +fn required(tool: &Value) -> Vec<String> { + tool["inputSchema"]["required"] + .as_array() + .map(|r| { + r.iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default() +} + +#[test] +fn wait_for_is_offered_with_its_conditions_and_its_caps() { + let tool = definition("wait_for"); + assert_eq!(required(&tool), ["condition"]); + let conditions: Vec<&str> = tool["inputSchema"]["properties"]["condition"]["enum"] + .as_array() + .expect("an enum") + .iter() + .filter_map(Value::as_str) + .collect(); + assert_eq!( + conditions, + [ + "element_appears", + "element_gone", + "window_appears", + "window_gone", + "screen_changes" + ] + ); + for property in [ + "selector", + "timeout_ms", + "poll_ms", + "baseline", + "tolerance", + "window", + ] { + assert!( + tool["inputSchema"]["properties"][property].is_object(), + "wait_for has no `{property}`" + ); + } + let said = tool["description"].as_str().unwrap(); + assert!(said.contains("60000"), "the cap is told: {said}"); + assert!(said.contains("UNTRUSTED"), "{said}"); + // It acts on nothing, so it does not offer a screenshot afterwards. + assert!(tool["inputSchema"]["properties"]["screenshot_after"].is_null()); +} + +#[test] +fn the_element_tools_take_an_id_or_a_selector() { + for (name, needs) in [ + ("click_element", vec![]), + ("set_value", vec!["value"]), + ("element_action", vec!["action"]), + ] { + let tool = definition(name); + assert!( + tool["inputSchema"]["properties"]["selector"].is_object(), + "{name} takes no selector" + ); + assert!( + tool["inputSchema"]["properties"]["id"].is_object(), + "{name} lost its id" + ); + // Neither alone is required: one of them is, and the step says so. + assert_eq!(required(&tool), needs, "{name}"); + } +} + +#[test] +fn an_id_alone_is_the_old_behaviour_and_a_selector_alone_is_the_new() { + assert!(matches!( + element_target(&json!({"id": 5})).unwrap(), + Target::Id(5) + )); + match element_target(&json!({"selector": "role=button;name=Save"})).unwrap() { + Target::Selector(s) => assert_eq!(s.render(), "role=button;name=Save"), + other => panic!("{other:?}"), + } +} + +#[test] +fn an_id_and_a_selector_together_are_refused_and_so_is_neither() { + let both = element_target(&json!({"id": 5, "selector": "role=button"})).unwrap_err(); + assert!(both.contains("not both"), "{both}"); + let neither = element_target(&json!({})).unwrap_err(); + assert!(neither.contains("selector"), "{neither}"); + assert!(neither.contains("id"), "{neither}"); +} + +#[test] +fn a_selector_that_does_not_name_a_control_is_refused_before_anything_is_read() { + for bad in ["nth=2", "app=notepad", "colour=red", ""] { + let why = element_target(&json!({"selector": bad})).unwrap_err(); + assert!( + why.contains("selector"), + "`{bad}` was not refused by name: {why}" + ); + } + let why = element_target(&json!({"selector": 7})).unwrap_err(); + assert!(why.contains("text"), "{why}"); +} + +#[test] +fn the_journal_names_the_selector_but_never_a_text_fields_name() { + let line = |name: &str, args: Value| Server::describe(name, &args); + // The id form is as it was. + assert_eq!(line("click_element", json!({"id": 5})), "element e5"); + assert_eq!( + line("set_value", json!({"id": 5, "value": "abc"})), + "element e5, 3 characters" + ); + // A button's name is journalled; a text field's is not (on macOS it is + // what the user typed). + assert_eq!( + line( + "click_element", + json!({"selector": "role=button;name=Save"}) + ), + "selector role=button;name=Save" + ); + let edit = line( + "set_value", + json!({"selector": "role=edit;name=hunter2", "value": "x"}), + ); + assert!(!edit.contains("hunter2"), "{edit}"); + assert!(edit.contains("1 characters"), "{edit}"); + let unroled = line( + "element_action", + json!({"selector": "name=hunter2", "action": "press"}), + ); + assert!(!unroled.contains("hunter2"), "{unroled}"); + // A selector that does not parse is not echoed. + let bad = line("click_element", json!({"selector": "name=hunter2;;;bad"})); + assert!(!bad.contains("hunter2"), "{bad}"); +} + +#[test] +fn wait_for_is_journalled_by_condition_selector_and_timeout() { + let said = Server::describe( + "wait_for", + &json!({"condition": "element_appears", "selector": "role=button;name=Save", "timeout_ms": 3000}), + ); + assert_eq!( + said, + "element_appears selector role=button;name=Save timeout 3000 ms" + ); + let said = Server::describe("wait_for", &json!({"condition": "screen_changes"})); + assert_eq!(said, "screen_changes timeout default"); +} + +#[test] +fn a_tree_prints_each_selector_beside_its_element_and_the_window_part_once() { + use crate::selector::Selector; + use crate::uia::{Element, Role, Tree}; + use crate::Rect; + let el = |id, role, name: &str, aid: &str, disabled: bool| Element { + id, + depth: 1, + role, + name: name.to_string(), + rect: Rect { + x: 0, + y: 0, + width: 10, + height: 10, + }, + enabled: !disabled, + secret: false, + automation_id: aid.to_string(), + }; + let tree = Tree { + root_name: "Untitled - Notepad".to_string(), + window: "42".to_string(), + app: "notepad".to_string(), + no_positions: None, + elements: vec![ + el(1, Role::Button, "Save", "", false), + el(2, Role::Button, "Save", "", true), + el(3, Role::Edit, "", "editor", false), + ], + truncated: false, + offscreen: 0, + }; + let shown = tree.render(); + assert!( + shown.contains("Selector window part: app=notepad;window=Untitled - Notepad"), + "{shown}" + ); + assert_eq!(shown.matches("Selector window part").count(), 1); + let line = |id: u32| { + shown + .lines() + .find(|l| l.contains(&format!("[e{id}]"))) + .unwrap() + .to_string() + }; + // The ids are as they were, and each line says how to name its control. + assert!( + line(1).contains("sel: role=button;name=Save;nth=0"), + "{}", + line(1) + ); + assert!( + line(2).contains("sel: role=button;name=Save;nth=1"), + "{}", + line(2) + ); + assert!(line(3).contains("sel: role=edit;aid=editor"), "{}", line(3)); + // The flags still end the line. + assert!(line(2).ends_with(" disabled"), "{}", line(2)); + // And what a line says is what the matcher resolves. + for (id, said) in [ + (1, "role=button;name=Save;nth=0"), + (3, "role=edit;aid=editor"), + ] { + let got = Selector::parse(said) + .unwrap() + .resolve_in(&tree.elements) + .unwrap(); + assert_eq!(got.id, id); + } +} + +/// What needs the real input thread and the real desktop. +#[cfg(windows)] +mod live { + use super::*; + use std::time::Instant; + + fn server() -> Server { + Server::new(InputThread::start().expect("input thread")) + } + + fn call(server: &mut Server, name: &str, args: Value) -> Value { + let line = json!({ + "jsonrpc": "2.0", "id": 1, "method": "tools/call", + "params": {"name": name, "arguments": args} + }) + .to_string(); + let reply = server.handle_line(&line).expect("a reply was expected"); + serde_json::from_str(&reply).expect("the reply must be valid JSON") + } + + /// The JSON the tool answered, when it did not refuse. + fn answer(reply: &Value) -> Option<Value> { + if reply["result"]["isError"] == true { + return None; + } + let text = reply["result"]["content"][0]["text"].as_str()?; + // The JSON first, then the untrusted-data note. + let end = text.rfind('}')?; + serde_json::from_str(&text[..=end]).ok() + } + + fn refusal(reply: &Value) -> String { + reply["result"]["content"][0]["text"] + .as_str() + .unwrap_or_default() + .to_string() + } + + const NO_SUCH_WINDOW: &str = "window=zz no such window 7f3a9c1e"; + + #[test] + fn a_window_that_never_comes_is_a_timeout_reply_not_an_error() { + let mut s = server(); + let started = Instant::now(); + let reply = call( + &mut s, + "wait_for", + json!({"condition": "window_appears", "selector": NO_SUCH_WINDOW, + "timeout_ms": 1500, "poll_ms": 100}), + ); + let waited = started.elapsed(); + let got = answer(&reply).unwrap_or_else(|| panic!("a timeout is a reply: {reply}")); + assert_eq!(got["result"], "timeout", "{got}"); + assert_eq!(got["condition"], "window_appears"); + // Listing the windows takes a few hundred ms on a busy desktop, so + // a 1.5 s wait is a handful of looks, not a dozen. + assert!(got["polls"].as_u64().unwrap() >= 2, "{got}"); + assert!(got["last_seen"].as_str().is_some(), "{got}"); + assert!(waited.as_millis() >= 1500, "returned early: {waited:?}"); + assert!( + waited.as_secs() < 10, + "a 1.5 s wait took {waited:?}: nothing bounded it" + ); + } + + #[test] + fn a_window_already_gone_is_found_at_once_and_a_huge_timeout_is_held_to_the_cap() { + let mut s = server(); + let started = Instant::now(); + let reply = call( + &mut s, + "wait_for", + json!({"condition": "window_gone", "selector": NO_SUCH_WINDOW, + "timeout_ms": 99_999_999u64}), + ); + let got = answer(&reply).unwrap_or_else(|| panic!("{reply}")); + assert_eq!(got["result"], "found", "{got}"); + assert_eq!(got["polls"], 1); + assert_eq!(got["timeout_ms"], 60_000, "held to the cap"); + assert!( + got["notes"][0].as_str().unwrap().contains("over the cap"), + "the reply says it was held: {got}" + ); + assert!(started.elapsed().as_secs() < 10); + } + + #[test] + fn the_window_in_front_is_found_by_its_program() { + let Ok(Some(front)) = uia::foreground() else { + eprintln!("no window in front in this session"); + return; + }; + if crate::limits::get().blocked(&front.app).is_some() { + return; + } + let mut s = server(); + let selector = format!("app={}", front.app); + let reply = call( + &mut s, + "wait_for", + json!({"condition": "window_appears", "selector": selector, "timeout_ms": 1000}), + ); + let got = answer(&reply).unwrap_or_else(|| panic!("{reply}")); + assert_eq!(got["result"], "found", "{got}"); + let ids: Vec<&str> = got["windows"] + .as_array() + .unwrap() + .iter() + .filter_map(|w| w["id"].as_str()) + .collect(); + assert!( + ids.contains(&front.id.as_str()), + "{ids:?} lacks {}", + front.id + ); + } + + /// Every selector `ui_tree` prints for a REAL window picks its own element + /// back out of the same tree, whatever odd names the application has. + #[test] + fn every_selector_a_real_tree_prints_resolves_to_its_own_element() { + let tree = match uia::window_tree(None, 12) { + Ok(tree) => tree, + Err(e) => { + eprintln!("no focused window in this session: {e}"); + return; + } + }; + if crate::limits::get().blocked(&tree.app).is_some() { + return; + } + let selectors = crate::selector::selectors_for(&tree.elements, uia::NAMES_ARE_VALUES); + for (e, s) in tree.elements.iter().zip(&selectors) { + let again = crate::selector::Selector::parse(&s.render()).unwrap_or_else(|why| { + panic!("`{}` for {e:?} does not read back: {why}", s.render()) + }); + assert_eq!(&again, s, "{e:?}"); + let got = again + .resolve_in(&tree.elements) + .unwrap_or_else(|why| panic!("`{}` for {e:?}: {why}", s.render())); + assert_eq!(got.id, e.id, "`{}`", s.render()); + } + let shown = tree.render(); + assert!(shown.contains("Selector window part:"), "{shown}"); + assert_eq!( + shown.matches(" sel: ").count(), + tree.elements.len(), + "every element line carries its selector" + ); + } + + #[test] + fn a_selector_that_matches_nothing_is_a_clear_error_and_clicks_nothing() { + let mut s = server(); + let reply = call( + &mut s, + "click_element", + json!({"selector": "role=button;name=zz no such button 7f3a9c1e"}), + ); + if reply["result"]["isError"] != true { + panic!("a selector that matches nothing was acted on: {reply}"); + } + let why = refusal(&reply); + // A session with no focused window refuses earlier, in its own words; + // otherwise the miss is named. + assert!( + why.contains("matches no element") || why.contains("window"), + "{why}" + ); + assert!(!why.contains("clicked"), "{why}"); + } + + #[test] + fn a_read_only_session_may_wait_and_a_wait_is_stopped_by_the_stop() { + let mut s = Server::new(InputThread::start().expect("input thread")) + .with_tier(crate::permission::Tier::Observe); + let safety = s.safety(); + let stopper = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(250)); + safety.stop(); + }); + let started = Instant::now(); + let reply = call( + &mut s, + "wait_for", + json!({"condition": "window_appears", "selector": NO_SUCH_WINDOW, + "timeout_ms": 30_000}), + ); + stopper.join().unwrap(); + // It ran at the read-only tier (a permission refusal would have + // named the mode), and the stop ended it long before its 30 s. + let why = refusal(&reply); + assert_eq!(reply["result"]["isError"], true, "{reply}"); + assert!(!why.contains("read-only"), "refused by the tier: {why}"); + assert!(why.contains("stopped"), "{why}"); + assert!( + started.elapsed().as_secs() < 5, + "the stop took {:?} to end the wait", + started.elapsed() + ); + } + + #[test] + fn the_screen_is_compared_with_a_baseline_the_caller_gives() { + let mut s = server(); + // All ones: the real screen is not. + let far = "f".repeat(32); + let reply = call( + &mut s, + "wait_for", + json!({"condition": "screen_changes", "baseline": far, + "tolerance": 0, "timeout_ms": 1000}), + ); + let Some(got) = answer(&reply) else { + eprintln!("no capture in this session: {}", refusal(&reply)); + return; + }; + assert_eq!(got["result"], "found", "{got}"); + assert_eq!(got["polls"], 1); + assert_eq!(got["hash"].as_str().unwrap().len(), 32); + assert!(got["distance"].as_u64().unwrap() > 0); + + // And a tolerance of every bit can never be exceeded: a timeout that + // hands back the last hash to go on from. + let reply = call( + &mut s, + "wait_for", + json!({"condition": "screen_changes", "tolerance": 128, + "timeout_ms": 300, "poll_ms": 100}), + ); + let got = answer(&reply).unwrap_or_else(|| panic!("{reply}")); + assert_eq!(got["result"], "timeout", "{got}"); + assert_eq!(got["hash"].as_str().unwrap().len(), 32, "{got}"); + } +} diff --git a/crates/desktop/src/mcp/tools.rs b/crates/desktop/src/mcp/tools.rs index aa8de6f..173b65b 100644 --- a/crates/desktop/src/mcp/tools.rs +++ b/crates/desktop/src/mcp/tools.rs @@ -30,6 +30,7 @@ use serde_json::{json, Value}; use crate::capture::{self, CaptureOptions, ShotFormat}; use crate::coords::PhysicalPoint; +use crate::selector::Selector; use crate::thread::{InputThread, MouseButton}; use crate::uia::{self, WindowInfo}; @@ -76,6 +77,15 @@ impl Server { "key" => key(input, args), "scroll" => scroll(input, args), "drag" => drag(input, args), + "wait_for" => { + // Asked between polls and between sleeps, so the operator's + // stop ends a wait within a fraction of a second. + let stopped = || { + self.catch_stop_file(); + self.safety.is_stopped() + }; + super::wait_tool::wait_for(args, &stopped) + } other => Err(format!("no tool named '{other}'")), } } @@ -157,30 +167,48 @@ fn focus_window(args: &Value) -> Result<Vec<Value>, String> { /// Called by [`Server::run_tool`]. fn set_value(args: &Value) -> Result<Vec<Value>, String> { let window = window_arg(args); - let id = element_id(args)?; + let target = element_target(args)?; let value = args .get("value") .and_then(Value::as_str) .ok_or("value is required")?; - let info = uia::window_info(window.as_deref()).map_err(|e| e.to_string())?; - refuse_blocked(&info)?; - let done = uia::set_value(Some(&info.id), id, value).map_err(|e| e.to_string())?; - Ok(vec![text(done.render())]) + match target { + Target::Id(id) => { + let info = uia::window_info(window.as_deref()).map_err(|e| e.to_string())?; + refuse_blocked(&info)?; + let done = uia::set_value(Some(&info.id), id, value).map_err(|e| e.to_string())?; + Ok(vec![text(done.render())]) + } + Target::Selector(selector) => { + let found = resolve_selector(window.as_deref(), &selector)?; + let done = uia::set_value_on(&found, value).map_err(|e| e.to_string())?; + Ok(vec![text(by_selector(done.render(), &found))]) + } + } } /// `element_action`: run an element's own action, without the mouse. /// Called by [`Server::run_tool`]. fn element_action(args: &Value) -> Result<Vec<Value>, String> { let window = window_arg(args); - let id = element_id(args)?; + let target = element_target(args)?; let action = args .get("action") .and_then(Value::as_str) .ok_or("action is required, e.g. \"press\"")?; - let info = uia::window_info(window.as_deref()).map_err(|e| e.to_string())?; - refuse_blocked(&info)?; - let done = uia::act(Some(&info.id), id, action).map_err(|e| e.to_string())?; - Ok(vec![text(done.render())]) + match target { + Target::Id(id) => { + let info = uia::window_info(window.as_deref()).map_err(|e| e.to_string())?; + refuse_blocked(&info)?; + let done = uia::act(Some(&info.id), id, action).map_err(|e| e.to_string())?; + Ok(vec![text(done.render())]) + } + Target::Selector(selector) => { + let found = resolve_selector(window.as_deref(), &selector)?; + let done = uia::act_on(&found, action).map_err(|e| e.to_string())?; + Ok(vec![text(by_selector(done.render(), &found))]) + } + } } /// `move_mouse`: move the pointer, the focused window checked first. @@ -231,19 +259,45 @@ fn ui_tree(args: &Value) -> Result<Vec<Value>, String> { /// now. Called by [`Server::run_tool`]. fn click_element(input: &InputThread, args: &Value) -> Result<Vec<Value>, String> { let window = window_arg(args); - let id = element_id(args)?; + let target = element_target(args)?; let button = mouse_button(args); - let info = uia::window_info(window.as_deref()).map_err(|e| e.to_string())?; - refuse_blocked(&info)?; // RE-READ, never trust the id's remembered rectangle. Between // the agent reading a tree and acting on it a list can scroll // or a dialog can move, and clicking the old position would hit // whatever took its place — silently, and on the wrong control. - let element = live_element(&info, id)?; + // + // A selector is resolved against the live tree here, the same way: its + // one match (or an error saying none or several) is the control as it is + // now. + let (element, by, label, cut) = match target { + Target::Id(id) => { + let info = uia::window_info(window.as_deref()).map_err(|e| e.to_string())?; + refuse_blocked(&info)?; + let element = live_element(&info, id)?; + let label = element.label(); + (element, format!("e{id}"), label, String::new()) + } + Target::Selector(selector) => { + let found = resolve_selector(window.as_deref(), &selector)?; + if let Some(why) = &found.no_positions { + return Err(format!( + "{why} so the control cannot be found on screen; use element_action or \ + set_value on it" + )); + } + let by = format!("selector `{}`", selector.render()); + let label = format!( + "{by} (\"{}\", {:?})", + found.element.name, found.element.role + ); + let cut = with_cut_note(String::new(), &found); + (found.element, by, label, cut) + } + }; if !element.enabled { return Err(format!( - "e{id} \"{}\" is disabled; clicking it would do nothing", + "{by} \"{}\" is disabled; clicking it would do nothing", element.name )); } @@ -255,11 +309,9 @@ fn click_element(input: &InputThread, args: &Value) -> Result<Vec<Value>, String input.click(at, button, 1).map_err(|e| e.to_string())?; Ok(vec![text(format!( - "clicked {} at {},{} (unverified: a click has no state to read back; look \ - at the result)", - element.label(), - at.x, - at.y + "clicked {label} at {},{} (unverified: a click has no state to read back; look \ + at the result){cut}", + at.x, at.y ))]) } @@ -489,40 +541,106 @@ fn live_element(window: &WindowInfo, id: u32) -> Result<uia::Element, String> { }) } -/// A screenshot of one window, or of the primary display — refused when a -/// blocked application's window is in what would be captured. -fn shoot(window: Option<&str>, options: CaptureOptions) -> Result<Vec<Value>, String> { - let (shot, of) = match window { - Some(id) => { - let info = uia::window_info(Some(id)).map_err(|e| e.to_string())?; - refuse_blocked(&info)?; - if let Some(why) = uia::window_position_unknown(&info) { - return Err(format!( - "{why} so that window's area cannot be cut out of the screen; take a \ - screenshot without `window`" - )); - } - refuse_region(info.rect)?; - let shot = capture::capture_region(info.rect, options).map_err(|e| e.to_string())?; - (shot, Some(info)) +/// How a call names a control: the number of the last `ui_tree`, or a selector +/// (`crate::selector`) resolved against the live tree. +#[derive(Debug)] +pub(super) enum Target { + Id(u32), + Selector(Selector), +} + +/// The `id` or the `selector` of an element tool, one of them and not both. +pub(super) fn element_target(args: &Value) -> Result<Target, String> { + match (args.get("id"), args.get("selector")) { + (Some(_), Some(_)) => Err("give `id` or `selector`, not both".to_string()), + (None, Some(raw)) => { + let raw = raw + .as_str() + .ok_or("selector must be text like role=button;name=Save")?; + let selector = Selector::parse(raw).map_err(|e| format!("selector: {e}"))?; + selector.require_element()?; + Ok(Target::Selector(selector)) } - None => { - // Every window on screen is in a full capture, so every one is - // checked — and a desktop that cannot list them is refused. - if let Some(blocked) = blocked_windows()?.first() { - return Err(format!( - "refused: a window of \"{}\" is on screen, and it is blocked; a full \ - screenshot would show it. Pass `window` to capture one window, or close \ - or minimise it.", - blocked.app - )); - } - ( - capture::capture_primary(options).map_err(|e| e.to_string())?, - None, - ) + _ => element_id(args).map(Target::Id).map_err(|_| { + "id (the eN number from ui_tree) or selector (as ui_tree prints it, \ + e.g. role=button;name=Save) is required" + .to_string() + }), + } +} + +/// The one control `selector` names, in the window it names (or `window`, or +/// the focused one), refused before anything is read when that window is a +/// blocked application's. +fn resolve_selector(window: Option<&str>, selector: &Selector) -> Result<uia::Resolved, String> { + let info = uia::window_for(window, selector).map_err(|e| e.to_string())?; + refuse_blocked(&info)?; + uia::resolve_in(info, selector).map_err(|e| e.to_string()) +} + +/// `done`, with what only a selector adds: its `eN` numbers this call's own +/// walk and not the agent's last `ui_tree`, and a cut tree cannot rule out a +/// second match. +fn by_selector(done: String, found: &uia::Resolved) -> String { + format!( + "{done} (found by selector; any eN here numbers this call's own walk, not your last \ + ui_tree){}", + with_cut_note(String::new(), found) + ) +} + +/// `base`, and the warning that the tree was cut when it was. +fn with_cut_note(base: String, found: &uia::Resolved) -> String { + if found.truncated { + format!( + "{base} (the tree was cut at its element limit, so a second control that fits the \ + selector may lie beyond it)" + ) + } else { + base + } +} + +/// What must hold before the screen, or one window of it, is captured: +/// refused when that window is a blocked application's, when its position is +/// not known, or when a blocked window is in what the picture would show. The +/// window it is, when one was named. Called by [`shoot`] and `wait_for`'s +/// screen condition, so both are held to the same rule. +pub(super) fn guard_capture(window: Option<&str>) -> Result<Option<WindowInfo>, String> { + let Some(id) = window else { + // Every window on screen is in a full capture, so every one is + // checked — and a desktop that cannot list them is refused. + if let Some(blocked) = blocked_windows()?.first() { + return Err(format!( + "refused: a window of \"{}\" is on screen, and it is blocked; a full \ + screenshot would show it. Pass `window` to capture one window, or close \ + or minimise it.", + blocked.app + )); } + return Ok(None); }; + let info = uia::window_info(Some(id)).map_err(|e| e.to_string())?; + refuse_blocked(&info)?; + if let Some(why) = uia::window_position_unknown(&info) { + return Err(format!( + "{why} so that window's area cannot be cut out of the screen; take a \ + screenshot without `window`" + )); + } + refuse_region(info.rect)?; + Ok(Some(info)) +} + +/// A screenshot of one window, or of the primary display — refused when a +/// blocked application's window is in what would be captured. +fn shoot(window: Option<&str>, options: CaptureOptions) -> Result<Vec<Value>, String> { + let of = guard_capture(window)?; + let shot = match &of { + Some(info) => capture::capture_region(info.rect, options), + None => capture::capture_primary(options), + } + .map_err(|e| e.to_string())?; let (width, height) = shot.model_size(); let t = shot.transform; Ok(vec![ @@ -560,7 +678,7 @@ fn after_shot(args: &Value) -> Vec<Value> { } /// A text content block. -fn text(body: String) -> Value { +pub(super) fn text(body: String) -> Value { json!({"type": "text", "text": body}) } diff --git a/crates/desktop/src/mcp/wait_tool.rs b/crates/desktop/src/mcp/wait_tool.rs new file mode 100644 index 0000000..e0bfa17 --- /dev/null +++ b/crates/desktop/src/mcp/wait_tool.rs @@ -0,0 +1,279 @@ +//! The `wait_for` tool: its arguments read, the live desktop handed to +//! [`crate::wait`], and what that found rendered as the reply. +//! +//! The waiting itself, and every rule about time, is in `crate::wait` and is +//! tested there against a scripted desktop. This file is the part that touches +//! the real one, and it holds each look to the same guards the one-shot tools +//! keep: a blocked application's window is never listed, walked or captured, +//! and a full capture is refused while one is on screen. +//! +//! # Wired to +//! +//! - **Called by** `Server::run_tool` in [`super::tools`]. +//! - **Calls** [`crate::wait::run`] with [`Live`], which reads +//! [`crate::uia`] and [`crate::capture`] through [`super::guard`]. + +use std::time::{Duration, Instant}; + +use serde_json::Value; + +use crate::capture::{self, CaptureOptions, ShotFormat}; +use crate::fingerprint::{self, Fingerprint}; +use crate::selector::Selector; +use crate::uia::{self, Tree, WindowInfo}; +use crate::wait::{self, Budget, Condition, Env, Outcome, DEFAULT_TOLERANCE}; + +use super::guard::{refuse_app, refuse_blocked}; +use super::tools::{guard_capture, text, window_arg}; + +/// Longest edge of the picture a fingerprint is made from: the fingerprint +/// is 128 bits, so more pixels only cost. +const FINGERPRINT_EDGE: u32 = 256; + +/// The desktop as this process sees it, with a clock that starts now. +struct Live { + started: Instant, +} + +fn text_of(e: impl ToString) -> String { + e.to_string() +} + +impl Env for Live { + fn now_ms(&mut self) -> u64 { + u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX) + } + + fn sleep_ms(&mut self, ms: u64) { + std::thread::sleep(Duration::from_millis(ms)); + } + + fn windows(&mut self) -> Result<Vec<WindowInfo>, String> { + // Not the windows of a blocked application: they are not listed, and a + // selector must not be able to find one by its title. + Ok(listed()? + .into_iter() + .filter(|w| crate::limits::get().blocked(&w.app).is_none()) + .collect()) + } + + fn focused_window(&mut self) -> Result<WindowInfo, String> { + let window = uia::window_info(None).map_err(text_of)?; + refuse_blocked(&window)?; + Ok(window) + } + + fn tree(&mut self, window: &str) -> Result<Tree, String> { + let info = uia::window_info(Some(window)).map_err(text_of)?; + refuse_blocked(&info)?; + let tree = uia::tree_now(window).map_err(text_of)?; + // Walked in this process, and discarded unread when blocked. + refuse_app(&tree.app)?; + Ok(tree) + } + + fn screen(&mut self, window: Option<&str>) -> Result<Fingerprint, String> { + let of = guard_capture(window)?; + let options = CaptureOptions { + max_edge: FINGERPRINT_EDGE, + format: ShotFormat::Png, + ..CaptureOptions::default() + }; + let shot = match &of { + Some(info) => capture::capture_region(info.rect, options), + None => capture::capture_primary(options), + } + .map_err(text_of)?; + let picture = image::load_from_memory(&shot.bytes) + .map_err(|e| format!("could not read the capture back to fingerprint it: {e}"))? + .to_rgba8(); + Ok(fingerprint::of(&picture)) + } +} + +/// The windows, asked again a few times when the desktop will not say. +/// +/// A wait is exactly what meets a window closing in the middle of the +/// enumeration, and UI Automation answers that with an "unspecified error" for +/// that one call (measured: it failed the live wait tests on a busy Windows +/// desktop). One failed listing must not end a wait that is for the very thing +/// happening; a listing that keeps failing does. +fn listed() -> Result<Vec<WindowInfo>, String> { + const TRIES: u32 = 4; + let mut last = String::new(); + for attempt in 0..TRIES { + match uia::list_windows() { + Ok(windows) => return Ok(windows), + Err(e) => last = e.to_string(), + } + if attempt + 1 < TRIES { + std::thread::sleep(Duration::from_millis(50)); + } + } + Err(format!("{last} (after {TRIES} tries)")) +} + +/// A non-negative whole number argument, when it was given. +fn whole(args: &Value, key: &str) -> Result<Option<u64>, String> { + match args.get(key) { + None | Some(Value::Null) => Ok(None), + Some(v) => v + .as_u64() + .map(Some) + .ok_or_else(|| format!("{key} must be a whole number of milliseconds, 0 or more")), + } +} + +fn selector_arg(args: &Value, condition: &str) -> Result<Selector, String> { + let raw = args + .get("selector") + .and_then(Value::as_str) + .ok_or_else(|| format!("{condition} needs a `selector`, e.g. role=button;name=Save"))?; + let selector = Selector::parse(raw).map_err(|e| format!("selector: {e}"))?; + // A blocked application is refused by name before anything looks for it. + if let Some(app) = &selector.app { + refuse_app(app)?; + } + Ok(selector) +} + +/// What the call waits for. +fn condition_of(args: &Value) -> Result<Condition, String> { + let which = args.get("condition").and_then(Value::as_str).ok_or( + "condition is required: element_appears, element_gone, window_appears, \ + window_gone or screen_changes", + )?; + match which { + "element_appears" | "element_gone" => Ok(Condition::Element { + selector: selector_arg(args, which)?, + window: window_arg(args), + appears: which == "element_appears", + }), + "window_appears" | "window_gone" => Ok(Condition::Window { + selector: selector_arg(args, which)?, + appears: which == "window_appears", + }), + "screen_changes" => { + let baseline = match args.get("baseline").and_then(Value::as_str) { + None => None, + Some(hex) => Some(Fingerprint::from_hex(hex).ok_or( + "baseline must be a `hash` an earlier wait_for returned (32 hex digits)", + )?), + }; + let tolerance = match args.get("tolerance") { + None | Some(Value::Null) => DEFAULT_TOLERANCE, + Some(v) => v + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .filter(|n| *n <= 128) + .ok_or("tolerance must be a whole number of bits, 0 to 128")?, + }; + Ok(Condition::Screen { + baseline, + tolerance, + window: window_arg(args), + }) + } + other => Err(format!( + "unknown condition '{other}': use element_appears, element_gone, window_appears, \ + window_gone or screen_changes" + )), + } +} + +/// `wait_for`: wait, bounded, until the condition holds. `stopped` is the +/// operator's stop, asked between polls. Called by `Server::run_tool`. +pub(super) fn wait_for(args: &Value, stopped: &dyn Fn() -> bool) -> Result<Vec<Value>, String> { + let condition = condition_of(args)?; + let budget = Budget::new(whole(args, "timeout_ms")?, whole(args, "poll_ms")?); + let mut env = Live { + started: Instant::now(), + }; + let outcome = wait::run(&mut env, &condition, &budget, stopped)?; + if matches!(outcome, Outcome::Stopped { .. }) { + return Err(crate::safety::Refusal::Stopped.message()); + } + let reply = outcome.to_json(&condition, &budget); + Ok(vec![text(format!( + "{}\n{}", + serde_json::to_string_pretty(&reply).unwrap_or_default(), + super::UNTRUSTED + ))]) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn the_conditions_are_read_from_the_arguments() { + let c = condition_of(&json!({"condition": "element_gone", "selector": "role=button;name=Save", "window": "7"})) + .unwrap(); + match c { + Condition::Element { + selector, + window, + appears, + } => { + assert_eq!(selector.render(), "role=button;name=Save"); + assert_eq!(window.as_deref(), Some("7")); + assert!(!appears); + } + other => panic!("{other:?}"), + } + let c = + condition_of(&json!({"condition": "window_appears", "selector": "window~=Save As"})) + .unwrap(); + assert!(matches!(c, Condition::Window { appears: true, .. })); + let c = condition_of(&json!({"condition": "screen_changes"})).unwrap(); + assert!(matches!( + c, + Condition::Screen { + baseline: None, + tolerance: DEFAULT_TOLERANCE, + window: None + } + )); + } + + #[test] + fn a_bad_condition_is_refused_by_name() { + let bad = |v: Value| condition_of(&v).unwrap_err(); + assert!(bad(json!({})).contains("condition is required")); + assert!(bad(json!({"condition": "soon"})).contains("unknown condition 'soon'")); + assert!(bad(json!({"condition": "element_appears"})).contains("needs a `selector`")); + assert!( + bad(json!({"condition": "element_appears", "selector": "colour=red"})) + .contains("unknown key") + ); + assert!( + bad(json!({"condition": "screen_changes", "baseline": "xyz"})).contains("baseline") + ); + assert!( + bad(json!({"condition": "screen_changes", "tolerance": 129})).contains("tolerance") + ); + assert!(bad(json!({"condition": "screen_changes", "tolerance": -1})).contains("tolerance")); + } + + #[test] + fn a_blocked_application_is_refused_before_anything_looks_for_it() { + let why = + condition_of(&json!({"condition": "window_appears", "selector": "app=KeePassXC"})) + .unwrap_err(); + assert!(why.contains("blocked"), "{why}"); + } + + #[test] + fn a_timeout_must_be_a_whole_non_negative_number() { + assert_eq!(whole(&json!({}), "timeout_ms"), Ok(None)); + assert_eq!(whole(&json!({"timeout_ms": 0}), "timeout_ms"), Ok(Some(0))); + assert_eq!( + whole(&json!({"timeout_ms": 1500}), "timeout_ms"), + Ok(Some(1500)) + ); + assert!(whole(&json!({"timeout_ms": -5}), "timeout_ms").is_err()); + assert!(whole(&json!({"timeout_ms": 1.5}), "timeout_ms").is_err()); + assert!(whole(&json!({"timeout_ms": "soon"}), "timeout_ms").is_err()); + } +} diff --git a/crates/desktop/src/permission.rs b/crates/desktop/src/permission.rs index 482b2d1..b58e24e 100644 --- a/crates/desktop/src/permission.rs +++ b/crates/desktop/src/permission.rs @@ -12,14 +12,14 @@ //! //! Until the desktop became a crate of its own this was incode's //! `PermissionPolicy::for_desktop(PermissionMode)`. The crate cannot depend on -//! incode, so the tier is restated here as the two rungs the 16 tools actually +//! incode, so the tier is restated here as the two rungs the 17 tools actually //! use, and [`Tier::from_mode_word`] reads incode's own mode words so the argv //! incode has always passed (`-p read-only`) keeps meaning the same thing: //! //! | mode word | tier | reaches | //! |---|---|---| -//! | `read-only` | [`Tier::Observe`] | the 5 observing tools | -//! | `workspace-write` (the default), `danger-full-access`, `allow` | [`Tier::Act`] | all 16 | +//! | `read-only` | [`Tier::Observe`] | the 6 observing tools | +//! | `workspace-write` (the default), `danger-full-access`, `allow` | [`Tier::Act`] | all 17 | //! | `prompt` | [`Tier::Deny`] | nothing | //! //! `prompt` denies everything, and that is correct rather than unfortunate: @@ -44,6 +44,9 @@ pub const OBSERVING: &[&str] = &[ "screenshot", "ui_tree", "list_windows", + // Waits and looks: it polls the tree, the window list and the screen and + // changes nothing, so it is held to the same tier as the tools it is made of. + "wait_for", ]; /// The tools that change the state of a machine the operator is signed into, and @@ -157,14 +160,14 @@ mod tests { /// The tier table must be the server's tool list, or a tool is either /// ungoverned or unreachable. Checked against what `tools/list` offers, so a - /// seventeenth tool cannot be added without deciding its tier. + /// eighteenth tool cannot be added without deciding its tier. #[test] - fn every_listed_tool_has_a_tier_and_there_are_sixteen() { + fn every_listed_tool_has_a_tier_and_there_are_seventeen() { let listed: Vec<String> = crate::mcp::tool_definitions() .iter() .map(|t| t["name"].as_str().unwrap_or_default().to_string()) .collect(); - assert_eq!(listed.len(), 16, "{listed:?}"); + assert_eq!(listed.len(), 17, "{listed:?}"); for name in &listed { assert!( OBSERVING.contains(&name.as_str()) ^ ACTING.contains(&name.as_str()), diff --git a/crates/desktop/src/selector.rs b/crates/desktop/src/selector.rs new file mode 100644 index 0000000..fd75c22 --- /dev/null +++ b/crates/desktop/src/selector.rs @@ -0,0 +1,963 @@ +//! Stable element selectors: naming a control by what it is, not by where the +//! last `ui_tree` happened to number it. +//! +//! # Why this exists next to the `eN` ids +//! +//! An id is a position in ONE walk (`uia.rs`, `Shown`): a row inserted above +//! it, or a walk to another depth, renumbers everything after. That is right +//! for an agent acting a moment after reading the tree, and useless for a +//! flow written down and run later (`docs/COMPUTER_USE.md`, replay). A +//! selector says which window, and which control in it, in terms that survive a +//! restart of the program: the role, the name, the automation id where the +//! platform has one, and failing those a position among look-alikes. +//! +//! It is resolved against the LIVE tree at call time (`uia::resolve`), and the +//! rule is the one the ids already keep: zero matches or several is a clear +//! error that says which, never a guess at the nearest or the first. +//! +//! # The text form +//! +//! `key=value` pairs joined by `;`, no spaces around the `;` (a value keeps its +//! own spaces): +//! +//! | key | meaning | +//! |---|---| +//! | `app` | the owning program, exact, case-insensitive, a trailing `.exe` ignored | +//! | `window` / `window~` | the window title: exact / contains (case-insensitive) | +//! | `role` | `button`, `edit`, `listitem`, ... (the `role` names of `ui_tree`, lower case) | +//! | `name` / `name~` | the control's name: exact / contains (case-insensitive) | +//! | `aid` | the automation id: UIA `AutomationId`, AX `AXIdentifier`, AT-SPI `AccessibleId` | +//! | `path` | child indices from the window down, dot separated (`0.2.1`), among the controls `ui_tree` lists | +//! | `nth` | which of several matches, from 0, in `ui_tree` order | +//! +//! A value escapes `\` as `\\`, `;` as `\;` and a line break as `\n` / `\r`. +//! `app` and `window` choose the WINDOW; the rest choose the CONTROL inside it. +//! `ui_tree` prints the control part beside every element, and the window part +//! once at the top. +//! +//! This module is pure, so it is tested on every platform. + +use crate::uia::{Element, Role, WindowInfo}; + +/// Every role, with the lower-case word a selector spells it as. +const ROLES: &[(&str, Role)] = &[ + ("button", Role::Button), + ("edit", Role::Edit), + ("text", Role::Text), + ("list", Role::List), + ("listitem", Role::ListItem), + ("checkbox", Role::CheckBox), + ("radiobutton", Role::RadioButton), + ("combobox", Role::ComboBox), + ("menu", Role::Menu), + ("menuitem", Role::MenuItem), + ("tab", Role::Tab), + ("tabitem", Role::TabItem), + ("link", Role::Link), + ("image", Role::Image), + ("window", Role::Window), + ("pane", Role::Pane), + ("tree", Role::Tree), + ("treeitem", Role::TreeItem), + ("table", Role::Table), + ("slider", Role::Slider), + ("group", Role::Group), + ("other", Role::Other), +]; + +impl Role { + /// The word a selector uses for this role. + #[must_use] + pub fn word(self) -> &'static str { + ROLES + .iter() + .find(|(_, role)| *role == self) + .map_or("other", |(word, _)| word) + } + + /// The role a selector's `role=` word names. + #[must_use] + pub fn from_word(word: &str) -> Option<Role> { + let word = word.trim().to_ascii_lowercase(); + ROLES + .iter() + .find(|(w, _)| *w == word) + .map(|(_, role)| *role) + } +} + +/// How a selector compares a piece of text. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Text { + /// The whole text, character for character. + Exact(String), + /// A part of it, ignoring case. + Contains(String), +} + +impl Text { + fn matches(&self, actual: &str) -> bool { + match self { + Text::Exact(want) => actual == want, + Text::Contains(part) => actual.to_lowercase().contains(&part.to_lowercase()), + } + } +} + +/// A window and/or a control, as text. See the module header for the form. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Selector { + pub app: Option<String>, + pub window: Option<Text>, + pub role: Option<Role>, + pub name: Option<Text>, + pub automation_id: Option<String>, + pub path: Option<Vec<u32>>, + pub nth: Option<u32>, +} + +/// Names longer than this are written into a derived selector as their first +/// this-many characters, matched as a part: a paragraph of static text is not +/// a name anyone should have to copy out whole. +const LONGEST_NAME: usize = 60; + +/// Most look-alikes an ambiguity error lists. +const EXAMPLES: usize = 4; + +/// Why a selector did not pick exactly one thing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SelectError { + /// Nothing matched. + Missing, + /// `nth` asked for a match there are not that many of. + NthOutOfRange { nth: u32, count: usize }, + /// Several matched and the selector does not say which. + Ambiguous { count: usize, examples: Vec<String> }, +} + +impl std::fmt::Display for SelectError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SelectError::Missing => write!(f, "matches no element"), + SelectError::NthOutOfRange { nth, count } => { + write!(f, "has nth={nth} but only {count} element(s) match") + } + SelectError::Ambiguous { count, examples } => { + write!(f, "matches {count} elements, so it does not say which")?; + if !examples.is_empty() { + write!(f, " (for example")?; + for e in examples { + write!(f, " `{e}`")?; + } + write!(f, ")")?; + } + write!(f, "; add name, aid, path or nth to pick one") + } + } + } +} + +fn escape(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for c in value.chars() { + match c { + '\\' => out.push_str("\\\\"), + ';' => out.push_str("\\;"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + c => out.push(c), + } + } + out +} + +fn unescape(raw: &str) -> Result<String, String> { + let mut out = String::with_capacity(raw.len()); + let mut chars = raw.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('\\') => out.push('\\'), + Some(';') => out.push(';'), + Some('n') => out.push('\n'), + Some('r') => out.push('\r'), + Some(other) => { + return Err(format!( + "unknown escape `\\{other}` (only `\\\\`, `\\;`, `\\n` and `\\r` exist)" + )) + } + None => return Err("a value ends in a lone `\\`".to_string()), + } + } + Ok(out) +} + +/// `raw` cut at each `;` that is not escaped. +fn split_pairs(raw: &str) -> Vec<String> { + let mut pieces = vec![String::new()]; + let mut escaped = false; + for c in raw.chars() { + let piece = pieces.last_mut().expect("one piece always exists"); + if escaped { + piece.push(c); + escaped = false; + } else if c == '\\' { + piece.push(c); + escaped = true; + } else if c == ';' { + pieces.push(String::new()); + } else { + piece.push(c); + } + } + pieces +} + +/// Store `value` in `slot`, refusing a key that appears twice. `key` is the +/// name reported, so `name` and `name~` count as one key. +fn once<T>(slot: &mut Option<T>, key: &str, value: T) -> Result<(), String> { + if slot.is_some() { + return Err(format!("`{key}` appears twice")); + } + *slot = Some(value); + Ok(()) +} + +impl Selector { + /// Read the text form. Strict: an unknown key, a repeated one, a bad number + /// or role is an error that names it, never skipped. + pub fn parse(raw: &str) -> Result<Selector, String> { + let mut s = Selector::default(); + for piece in split_pairs(raw) { + if piece.trim().is_empty() { + continue; + } + let Some((key, value)) = piece.split_once('=') else { + return Err(format!( + "`{piece}` is not key=value (keys: app, window, window~, role, name, \ + name~, aid, path, nth)" + )); + }; + let key = key.trim(); + let value = unescape(value).map_err(|e| format!("in `{key}`: {e}"))?; + match key { + "app" => once(&mut s.app, key, value)?, + "window" => once(&mut s.window, key, Text::Exact(value))?, + "window~" => once(&mut s.window, "window", Text::Contains(value))?, + "role" => { + let role = Role::from_word(&value).ok_or_else(|| { + format!( + "unknown role `{value}` (roles: {})", + ROLES.iter().map(|(w, _)| *w).collect::<Vec<_>>().join(", ") + ) + })?; + once(&mut s.role, key, role)?; + } + "name" => once(&mut s.name, key, Text::Exact(value))?, + "name~" => once(&mut s.name, "name", Text::Contains(value))?, + "aid" => once(&mut s.automation_id, key, value)?, + "path" => { + let path = value + .split('.') + .map(|n| n.trim().parse::<u32>()) + .collect::<Result<Vec<_>, _>>() + .map_err(|_| format!("path `{value}` must be child indices like 0.2.1"))?; + once(&mut s.path, key, path)?; + } + "nth" => { + let nth = value + .trim() + .parse::<u32>() + .map_err(|_| format!("nth `{value}` must be a whole number from 0"))?; + once(&mut s.nth, key, nth)?; + } + other => { + return Err(format!( + "unknown key `{other}` (keys: app, window, window~, role, name, \ + name~, aid, path, nth)" + )) + } + } + } + if s == Selector::default() { + return Err("the selector is empty".to_string()); + } + Ok(s) + } + + /// The text form, [`Self::parse`]'s inverse. + #[must_use] + pub fn render(&self) -> String { + let mut parts = Vec::new(); + self.push_window_parts(&mut parts); + self.push_element_parts(&mut parts, true); + parts.join(";") + } + + /// Only the window part (`app`, `window`), as the header of a tree shows it. + #[must_use] + pub fn render_window_part(&self) -> String { + let mut parts = Vec::new(); + self.push_window_parts(&mut parts); + parts.join(";") + } + + /// Only the control part, as `ui_tree` shows it beside each element. + #[must_use] + pub fn render_element_part(&self) -> String { + let mut parts = Vec::new(); + self.push_element_parts(&mut parts, true); + parts.join(";") + } + + fn push_window_parts(&self, parts: &mut Vec<String>) { + if let Some(app) = &self.app { + parts.push(format!("app={}", escape(app))); + } + match &self.window { + Some(Text::Exact(t)) => parts.push(format!("window={}", escape(t))), + Some(Text::Contains(t)) => parts.push(format!("window~={}", escape(t))), + None => {} + } + } + + fn push_element_parts(&self, parts: &mut Vec<String>, with_name: bool) { + if let Some(role) = self.role { + parts.push(format!("role={}", role.word())); + } + if with_name { + match &self.name { + Some(Text::Exact(t)) => parts.push(format!("name={}", escape(t))), + Some(Text::Contains(t)) => parts.push(format!("name~={}", escape(t))), + None => {} + } + } + if let Some(aid) = &self.automation_id { + parts.push(format!("aid={}", escape(aid))); + } + if let Some(path) = &self.path { + let path: Vec<String> = path.iter().map(u32::to_string).collect(); + parts.push(format!("path={}", path.join("."))); + } + if let Some(nth) = self.nth { + parts.push(format!("nth={nth}")); + } + } + + /// The selector as a journal may carry it. A control's name is left out + /// when it might be what the user typed: an edit field's (on macOS the + /// name of a text field IS its value), or any control whose role is not + /// given. Everything else is the same as [`Self::render`]. + #[must_use] + pub fn render_redacted(&self) -> String { + let keep_name = self.role.is_some_and(|r| r != Role::Edit); + let mut parts = Vec::new(); + self.push_window_parts(&mut parts); + self.push_element_parts(&mut parts, keep_name); + if !keep_name && self.name.is_some() { + parts.push("name=<not journalled>".to_string()); + } + parts.join(";") + } + + /// Whether it names a window (`app` or `window`). + #[must_use] + pub fn has_window_part(&self) -> bool { + self.app.is_some() || self.window.is_some() + } + + /// Whether it names a control (`role`, `name`, `aid` or `path`). + #[must_use] + pub fn has_element_part(&self) -> bool { + self.role.is_some() + || self.name.is_some() + || self.automation_id.is_some() + || self.path.is_some() + } + + /// Fit for acting on a control: it names one. `nth` alone is "the third of + /// anything", which is a guess. + pub fn require_element(&self) -> Result<(), String> { + if self.has_element_part() { + Ok(()) + } else { + Err(format!( + "selector `{}` names no control: give at least one of role, name, aid, path", + self.render() + )) + } + } + + /// Fit for naming a window: only `app`/`window`. + pub fn require_window_only(&self) -> Result<(), String> { + if !self.has_window_part() { + return Err(format!( + "selector `{}` names no window: give app and/or window (or window~)", + self.render() + )); + } + if self.has_element_part() || self.nth.is_some() { + return Err(format!( + "selector `{}` names a control too, but this asks about a window: use only \ + app and window", + self.render() + )); + } + Ok(()) + } + + /// Whether `window` is what the window part names. With no window part, + /// any window is. + #[must_use] + pub fn matches_window(&self, window: &WindowInfo) -> bool { + let app_ok = self + .app + .as_deref() + .is_none_or(|want| same_app(want, &window.app)); + let title_ok = self + .window + .as_ref() + .is_none_or(|text| text.matches(&window.title)); + app_ok && title_ok + } + + /// The windows of `windows` this names. + #[must_use] + pub fn windows_named<'a>(&self, windows: &'a [WindowInfo]) -> Vec<&'a WindowInfo> { + windows.iter().filter(|w| self.matches_window(w)).collect() + } + + /// Whether `element` fits the control part, `nth` aside. `path` is the + /// element's own path ([`paths`]) when the selector gives one. + fn matches_element(&self, element: &Element, path: Option<&[u32]>) -> bool { + self.role.is_none_or(|r| r == element.role) + && self + .name + .as_ref() + .is_none_or(|text| text.matches(&element.name)) + && self + .automation_id + .as_deref() + .is_none_or(|aid| aid == element.automation_id) + && match (&self.path, path) { + (Some(want), Some(have)) => want.as_slice() == have, + (Some(_), None) => false, + (None, _) => true, + } + } + + /// The places in `elements` that fit the control part, `nth` not yet applied. + #[must_use] + pub fn candidates(&self, elements: &[Element]) -> Vec<usize> { + let own_paths = self.path.as_ref().map(|_| paths(elements)); + elements + .iter() + .enumerate() + .filter(|(i, e)| self.matches_element(e, own_paths.as_ref().map(|p| p[*i].as_slice()))) + .map(|(i, _)| i) + .collect() + } + + /// The one element this picks out of `elements` (a fresh walk): the single + /// match, or the `nth` of several. Zero, or several without `nth`, is an + /// error and never a choice. + pub fn resolve_in<'a>(&self, elements: &'a [Element]) -> Result<&'a Element, SelectError> { + let found = self.candidates(elements); + let at = match (found.len(), self.nth) { + (0, _) => return Err(SelectError::Missing), + (1, None) => found[0], + (count, None) => { + let examples = found + .iter() + .take(EXAMPLES) + .map(|&i| selector_for(elements, i, false).render_element_part()) + .collect(); + return Err(SelectError::Ambiguous { count, examples }); + } + (count, Some(nth)) => *found + .get(nth as usize) + .ok_or(SelectError::NthOutOfRange { nth, count })?, + }; + Ok(&elements[at]) + } +} + +/// Whether two program names are the same one: case-insensitive, and a +/// trailing `.exe` (Windows) is not part of the name. +fn same_app(want: &str, actual: &str) -> bool { + fn norm(s: &str) -> String { + let s = s.trim().to_lowercase(); + s.strip_suffix(".exe").map_or(s.clone(), str::to_string) + } + norm(want) == norm(actual) +} + +/// Every element's path: its index among its siblings, then its parent's, up +/// to the window. Siblings are the elements `ui_tree` LISTS (a wrapper it +/// drops is not a level), so the path reads the same on every platform and +/// does not depend on what the toolkit calls a wrapper. +#[must_use] +pub fn paths(elements: &[Element]) -> Vec<Vec<u32>> { + // (depth, path, children so far) of the elements still open. + let mut open: Vec<(u32, Vec<u32>, u32)> = Vec::new(); + let mut roots = 0u32; + let mut out = Vec::with_capacity(elements.len()); + for e in elements { + while open.last().is_some_and(|(depth, _, _)| *depth >= e.depth) { + open.pop(); + } + let (index, mut path) = match open.last_mut() { + Some((_, parent, children)) => { + let index = *children; + *children += 1; + (index, parent.clone()) + } + None => { + let index = roots; + roots += 1; + (index, Vec::new()) + } + }; + path.push(index); + open.push((e.depth, path.clone(), 0)); + out.push(path); + } + out +} + +/// A name as a selector writes it: whole when short, else its beginning as a +/// "contains". +fn name_text(name: &str) -> Text { + if name.chars().count() > LONGEST_NAME { + let head: String = name.chars().take(LONGEST_NAME).collect(); + Text::Contains(head) + } else { + Text::Exact(name.to_string()) + } +} + +/// The shortest control selector that picks `elements[index]` and nothing +/// else, in the order of what changes least: the automation id, then the name, +/// then — when several look alike — their order (`nth`) among those. +/// +/// `names_are_values`: the platform names a text field by its content (macOS), +/// which changes as it is typed in, so the field's name is not used. +#[must_use] +pub fn selector_for(elements: &[Element], index: usize, names_are_values: bool) -> Selector { + let e = &elements[index]; + let name = if (names_are_values && e.role == Role::Edit) || e.name.trim().is_empty() { + None + } else { + Some(name_text(&e.name)) + }; + let aid = Some(e.automation_id.clone()).filter(|a| !a.is_empty()); + let base = Selector { + role: Some(e.role), + ..Selector::default() + }; + let by_aid = Selector { + automation_id: aid.clone(), + ..base.clone() + }; + let by_name = Selector { + name: name.clone(), + ..base.clone() + }; + for attempt in [by_aid, by_name] { + if attempt.has_more_than_role() && attempt.candidates(elements).len() == 1 { + return attempt; + } + } + let mut picked = Selector { + name, + automation_id: aid, + ..base + }; + picked.nth = picked + .candidates(elements) + .iter() + .position(|&i| i == index) + .map(|n| n as u32); + picked +} + +impl Selector { + fn has_more_than_role(&self) -> bool { + self.name.is_some() || self.automation_id.is_some() + } +} + +/// [`selector_for`] every element: what `ui_tree` prints beside each. +#[must_use] +pub fn selectors_for(elements: &[Element], names_are_values: bool) -> Vec<Selector> { + (0..elements.len()) + .map(|i| selector_for(elements, i, names_are_values)) + .collect() +} + +/// The selector that records WHERE the element is as well as what it is: the +/// short one plus its `path`, which is a hard condition, so a control that has +/// moved is NOT found. For a flow written down to replay later, which wants to +/// notice that. The path already says which of several look-alikes, so the +/// `nth` of the short form is dropped (it would count the matches before the +/// path narrowed them). +#[must_use] +pub fn selector_with_path(elements: &[Element], index: usize, names_are_values: bool) -> Selector { + let mut s = selector_for(elements, index, names_are_values); + s.path = Some(paths(elements)[index].clone()); + s.nth = None; + s +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Rect; + + fn el(id: u32, depth: u32, role: Role, name: &str) -> Element { + Element { + id, + depth, + role, + name: name.to_string(), + rect: Rect { + x: 0, + y: 0, + width: 10, + height: 10, + }, + enabled: true, + secret: false, + automation_id: String::new(), + } + } + + fn aid(mut e: Element, id: &str) -> Element { + e.automation_id = id.to_string(); + e + } + + /// A small dialog: two Save buttons (one in each pane), an OK button, two + /// unnamed edits, a long paragraph. + fn dialog() -> Vec<Element> { + vec![ + el(1, 1, Role::Pane, "Left"), + aid(el(2, 2, Role::Button, "Save"), "saveLeft"), + el(3, 2, Role::Edit, ""), + el(4, 1, Role::Pane, "Right"), + el(5, 2, Role::Button, "Save"), + el(6, 2, Role::Edit, ""), + el(7, 1, Role::Button, "OK"), + el(8, 1, Role::Text, &"x".repeat(200)), + ] + } + + fn window(app: &str, title: &str) -> WindowInfo { + WindowInfo { + id: "1".to_string(), + title: title.to_string(), + app: app.to_string(), + pid: 1, + focused: false, + rect: Rect { + x: 0, + y: 0, + width: 10, + height: 10, + }, + has_tree: true, + } + } + + #[test] + fn a_selector_parses_every_key_and_renders_back_the_same() { + let text = "app=notepad;window=Untitled - Notepad;role=button;name=Save;aid=btn1;\ + path=0.2.1;nth=1"; + let s = Selector::parse(text).unwrap(); + assert_eq!(s.app.as_deref(), Some("notepad")); + assert_eq!(s.window, Some(Text::Exact("Untitled - Notepad".into()))); + assert_eq!(s.role, Some(Role::Button)); + assert_eq!(s.name, Some(Text::Exact("Save".into()))); + assert_eq!(s.automation_id.as_deref(), Some("btn1")); + assert_eq!(s.path, Some(vec![0, 2, 1])); + assert_eq!(s.nth, Some(1)); + assert_eq!(s.render(), text); + } + + #[test] + fn contains_keys_round_trip_and_keep_their_meaning() { + let s = Selector::parse("window~=save as;name~=ok").unwrap(); + assert_eq!(s.window, Some(Text::Contains("save as".into()))); + assert_eq!(s.render(), "window~=save as;name~=ok"); + } + + #[test] + fn values_escape_the_separator_and_line_breaks() { + let s = Selector { + name: Some(Text::Exact("a;b\\c\nd=e".into())), + role: Some(Role::Text), + ..Selector::default() + }; + let text = s.render(); + assert_eq!(text, "role=text;name=a\\;b\\\\c\\nd=e"); + assert_eq!(Selector::parse(&text).unwrap(), s); + } + + #[test] + fn a_value_may_contain_equals_signs_and_keep_its_spaces() { + let s = Selector::parse("name= a = b ").unwrap(); + assert_eq!(s.name, Some(Text::Exact(" a = b ".into()))); + } + + #[test] + fn a_bad_selector_is_refused_by_name_never_skipped() { + let bad = |t: &str| Selector::parse(t).unwrap_err(); + assert!(bad("").contains("empty")); + assert!(bad(";;").contains("empty")); + assert!(bad("colour=red").contains("unknown key `colour`")); + assert!(bad("role=gizmo").contains("unknown role `gizmo`")); + assert!(bad("role=button;role=edit").contains("twice")); + assert!(bad("name=a;name~=b").contains("twice")); + assert!(bad("nth=-1").contains("nth")); + assert!(bad("path=1..2").contains("path")); + assert!(bad("justaword").contains("not key=value")); + assert!(bad("name=a\\qb").contains("unknown escape")); + assert!(bad("name=a\\").contains("lone")); + } + + #[test] + fn every_role_has_one_word_that_reads_back() { + for (word, role) in ROLES { + assert_eq!(role.word(), *word); + assert_eq!(Role::from_word(word), Some(*role)); + } + assert_eq!(Role::from_word(" Button "), Some(Role::Button)); + assert_eq!(ROLES.len(), 22, "a new Role needs a selector word"); + } + + #[test] + fn nth_alone_or_a_window_alone_does_not_name_a_control() { + assert!(Selector::parse("nth=2").unwrap().require_element().is_err()); + assert!(Selector::parse("app=x;window=y") + .unwrap() + .require_element() + .is_err()); + assert!(Selector::parse("role=button") + .unwrap() + .require_element() + .is_ok()); + } + + #[test] + fn a_window_question_takes_only_window_keys() { + assert!(Selector::parse("app=notepad") + .unwrap() + .require_window_only() + .is_ok()); + assert!(Selector::parse("role=button") + .unwrap() + .require_window_only() + .is_err()); + assert!(Selector::parse("app=a;role=button") + .unwrap() + .require_window_only() + .is_err()); + } + + #[test] + fn windows_match_by_program_and_title() { + let w = window("Notepad.exe", "Untitled - Notepad"); + let m = |t: &str| Selector::parse(t).unwrap().matches_window(&w); + assert!(m("app=notepad")); + assert!(m("app=NOTEPAD.EXE")); + assert!(!m("app=note"), "app is exact, not a prefix"); + assert!(m("window=Untitled - Notepad")); + assert!(!m("window=untitled - notepad"), "exact is case-sensitive"); + assert!(m("window~=UNTITLED")); + assert!(m("app=notepad;window~=untitled")); + assert!(!m("app=notepad;window~=report")); + assert_eq!( + Selector::parse("app=notepad") + .unwrap() + .windows_named(&[w.clone(), window("calc", "Calculator")]) + .len(), + 1 + ); + } + + #[test] + fn one_match_resolves_and_none_or_many_is_an_error() { + let tree = dialog(); + let get = |t: &str| Selector::parse(t).unwrap().resolve_in(&tree); + assert_eq!(get("role=button;name=OK").unwrap().id, 7); + assert_eq!(get("role=button;name=Nope"), Err(SelectError::Missing)); + match get("role=button;name=Save") { + Err(SelectError::Ambiguous { count, examples }) => { + assert_eq!(count, 2); + assert_eq!(examples.len(), 2); + } + other => panic!("two Save buttons must be ambiguous, got {other:?}"), + } + } + + #[test] + fn ambiguity_is_never_settled_by_taking_the_first() { + let tree = dialog(); + let err = Selector::parse("role=edit") + .unwrap() + .resolve_in(&tree) + .unwrap_err(); + let said = err.to_string(); + assert!(said.contains("matches 2 elements"), "{said}"); + assert!(said.contains("nth"), "{said}"); + } + + #[test] + fn nth_picks_among_several_and_out_of_range_is_refused() { + let tree = dialog(); + let nth = |n: u32| { + Selector::parse(&format!("role=button;name=Save;nth={n}")) + .unwrap() + .resolve_in(&tree) + }; + assert_eq!(nth(0).unwrap().id, 2); + assert_eq!(nth(1).unwrap().id, 5); + assert_eq!(nth(2), Err(SelectError::NthOutOfRange { nth: 2, count: 2 })); + } + + #[test] + fn the_automation_id_tells_look_alikes_apart_and_must_match_exactly() { + let tree = dialog(); + let by = |t: &str| Selector::parse(t).unwrap().resolve_in(&tree); + assert_eq!(by("role=button;aid=saveLeft").unwrap().id, 2); + assert_eq!(by("aid=saveLeft").unwrap().id, 2); + assert_eq!(by("aid=saveleft"), Err(SelectError::Missing)); + } + + #[test] + fn contains_ignores_case_and_exact_does_not() { + let tree = dialog(); + let by = |t: &str| Selector::parse(t).unwrap().resolve_in(&tree); + assert_eq!(by("name~=ok").unwrap().id, 7); + assert_eq!(by("name=ok"), Err(SelectError::Missing)); + } + + #[test] + fn paths_follow_the_listed_hierarchy_and_pick_one_element() { + let tree = dialog(); + let p = paths(&tree); + assert_eq!(p[0], vec![0]); // Left pane + assert_eq!(p[1], vec![0, 0]); // its Save + assert_eq!(p[2], vec![0, 1]); // its edit + assert_eq!(p[3], vec![1]); // Right pane + assert_eq!(p[4], vec![1, 0]); + assert_eq!(p[6], vec![2]); // OK + let by = Selector::parse("role=button;path=1.0").unwrap(); + assert_eq!(by.resolve_in(&tree).unwrap().id, 5); + } + + #[test] + fn a_path_skips_wrappers_the_tree_does_not_list() { + // A depth-3 control under a dropped depth-2 wrapper is the first child + // of its nearest listed ancestor. + let tree = vec![ + el(1, 1, Role::Pane, "P"), + el(2, 3, Role::Button, "A"), + el(3, 3, Role::Button, "B"), + el(4, 1, Role::Pane, "Q"), + ]; + let p = paths(&tree); + assert_eq!(p[1], vec![0, 0]); + assert_eq!(p[2], vec![0, 1]); + assert_eq!(p[3], vec![1]); + } + + #[test] + fn a_derived_selector_resolves_back_to_its_own_element() { + let mut tree = dialog(); + tree.push(el(9, 1, Role::Button, "OK")); // a second OK + tree.push(aid(el(10, 1, Role::Edit, "Name"), "nameBox")); + for names_are_values in [false, true] { + for i in 0..tree.len() { + let s = selector_for(&tree, i, names_are_values); + let got = s.resolve_in(&tree).unwrap_or_else(|e| { + panic!("`{}` for {:?} did not resolve: {e}", s.render(), tree[i]) + }); + assert_eq!(got.id, tree[i].id, "`{}`", s.render()); + // And through the text form an agent would copy. + let again = Selector::parse(&s.render()).unwrap(); + assert_eq!(again, s); + assert_eq!(again.resolve_in(&tree).unwrap().id, tree[i].id); + let full = selector_with_path(&tree, i, names_are_values); + assert_eq!(full.resolve_in(&tree).unwrap().id, tree[i].id); + } + } + } + + #[test] + fn a_derived_selector_is_the_shortest_that_is_unique() { + let tree = dialog(); + // OK is the only button of that name: role and name are enough. + assert_eq!( + selector_for(&tree, 6, false).render_element_part(), + "role=button;name=OK" + ); + // The Save with an automation id is told by it, not by its name. + assert_eq!( + selector_for(&tree, 1, false).render_element_part(), + "role=button;aid=saveLeft" + ); + // The other Save has only a name shared with the first: its place. + assert_eq!( + selector_for(&tree, 4, false).render_element_part(), + "role=button;name=Save;nth=1" + ); + // Unnamed look-alikes are told by their order. + assert_eq!( + selector_for(&tree, 5, false).render_element_part(), + "role=edit;nth=1" + ); + } + + #[test] + fn a_text_field_named_by_its_value_is_not_selected_by_it() { + let mut tree = dialog(); + tree[2].name = "what the user typed".to_string(); + let on_macos = selector_for(&tree, 2, true).render_element_part(); + assert!(!on_macos.contains("typed"), "{on_macos}"); + let elsewhere = selector_for(&tree, 2, false).render_element_part(); + assert!(elsewhere.contains("what the user typed"), "{elsewhere}"); + } + + #[test] + fn a_long_name_is_selected_by_its_beginning() { + let tree = dialog(); + let s = selector_for(&tree, 7, false); + match &s.name { + Some(Text::Contains(head)) => assert_eq!(head.chars().count(), LONGEST_NAME), + other => panic!("expected a prefix match, got {other:?}"), + } + assert_eq!(s.resolve_in(&tree).unwrap().id, 8); + } + + #[test] + fn the_journal_form_drops_a_text_fields_name_and_keeps_a_buttons() { + let button = Selector::parse("role=button;name=Save").unwrap(); + assert_eq!(button.render_redacted(), "role=button;name=Save"); + let edit = Selector::parse("role=edit;name=hunter2;nth=0").unwrap(); + let said = edit.render_redacted(); + assert!(!said.contains("hunter2"), "{said}"); + let no_role = Selector::parse("name=hunter2").unwrap(); + assert!(!no_role.render_redacted().contains("hunter2")); + } + + #[test] + fn the_parts_render_separately() { + let s = Selector::parse("app=a;window=W;role=edit;nth=1").unwrap(); + assert_eq!(s.render_window_part(), "app=a;window=W"); + assert_eq!(s.render_element_part(), "role=edit;nth=1"); + } +} diff --git a/crates/desktop/src/uia.rs b/crates/desktop/src/uia.rs index 88f7846..17c8b86 100644 --- a/crates/desktop/src/uia.rs +++ b/crates/desktop/src/uia.rs @@ -25,8 +25,14 @@ use serde::{Deserialize, Serialize}; use super::coords::PhysicalPoint; +use super::selector::{selectors_for, Selector, Text}; use super::{DesktopError, Rect}; +/// Whether the platform names a text field by its content, so a selector must +/// not use a field's name to find it: macOS (`describe` falls back to +/// `AXValue`), where the name changes as the field is typed into. +pub(crate) const NAMES_ARE_VALUES: bool = cfg!(target_os = "macos"); + /// How deep to walk by default. /// /// Deep enough for real dialogs and settings panes, shallow enough that a @@ -119,6 +125,13 @@ pub struct Element { /// and the agent knows what it would be typing into. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub secret: bool, + /// The program's own name for the control, where the toolkit has one: UIA + /// `AutomationId`, AX `AXIdentifier`, AT-SPI `AccessibleId`. Empty when it + /// has none (most controls of most programs). Unlike `name` it does not + /// change with the language or the content, so a selector prefers it + /// (`selector.rs`). + #[serde(default, skip_serializing_if = "String::is_empty")] + pub automation_id: String, } impl Element { @@ -222,15 +235,30 @@ impl Tree { out.push_str(&format!(" [window {}, app {}]", self.window, self.app)); } out.push('\n'); + if !self.window.is_empty() { + // The window part of a selector, once: each element's own part + // follows it as `sel:`. Together they name a control without + // an eN id, which holds only for this one tree. + let window_part = Selector { + app: Some(self.app.clone()).filter(|a| !a.is_empty()), + window: Some(Text::Exact(self.root_name.clone())), + ..Selector::default() + }; + out.push_str(&format!( + "Selector window part: {} (control selectors: see `sel:`; `;` joins them)\n", + window_part.render_window_part() + )); + } if let Some(why) = &self.no_positions { out.push_str(&format!( "NOTE: {why} Positions below are not real: use set_value and element_action \ by id; click_element, scroll and drag by id are refused.\n" )); } - for e in &self.elements { + let selectors = selectors_for(&self.elements, NAMES_ARE_VALUES); + for (e, selector) in self.elements.iter().zip(&selectors) { out.push_str(&format!( - "{:indent$}[e{}] {:<10} \"{}\" ({},{},{}x{}){}{}{}\n", + "{:indent$}[e{}] {:<10} \"{}\" ({},{},{}x{}) sel: {}{}{}{}\n", "", e.id, format!("{:?}", e.role), @@ -239,6 +267,9 @@ impl Tree { e.rect.y, e.rect.width, e.rect.height, + // Before the flags, so a line still ends with `disabled`, + // `password` or `off-screen` when it has one. + selector.render_element_part(), if e.enabled { "" } else { " disabled" }, if e.secret { " password" } else { "" }, if e.beyond_screens(screens) { @@ -580,6 +611,10 @@ pub fn focus_window(window: &str) -> Result<Done, DesktopError> { platform::focus_window(window) } +// Acting on a control by selector instead of by id: `resolve.rs`. +mod resolve; +pub use resolve::{act_on, resolve, resolve_in, set_value_on, tree_now, window_for, Resolved}; + #[cfg(windows)] #[path = "uia_windows.rs"] mod platform; @@ -772,6 +807,7 @@ mod tests { rect, enabled: true, secret: false, + automation_id: String::new(), } } @@ -799,6 +835,7 @@ mod tests { }, enabled: true, secret: false, + automation_id: String::new(), } } diff --git a/crates/desktop/src/uia/resolve.rs b/crates/desktop/src/uia/resolve.rs new file mode 100644 index 0000000..e750251 --- /dev/null +++ b/crates/desktop/src/uia/resolve.rs @@ -0,0 +1,367 @@ +//! Finding a control by selector, in the live tree, at the moment of the call. +//! +//! The ids `ui_tree` prints are positions in one walk. A selector +//! (`crate::selector`) is resolved the other way round: walk the window now, +//! and keep what it says. Exactly one control must fit; none, or several, is an +//! error that names which. What is returned carries the same `Want` the id path +//! builds, so the platform layers act on it through the code they already have +//! (`set_value`, `act`) and keep the check that the control in that place is +//! still the one that was resolved. +//! +//! Wired to: `mcp::tools` (`click_element`, `set_value`, `element_action`) and +//! `mcp::wait_tool` (`wait_for`), through [`resolve`], [`window_for`] and +//! [`tree_now`]. + +use super::{bounded, platform, same_control, Done, Element, Tree, Want, WindowInfo}; +use crate::selector::Selector; +use crate::DesktopError; + +/// Windows listed in an error about a window that was not found. +const LISTED: usize = 8; + +/// A control found by selector, and what acting on it takes. +#[derive(Debug, Clone)] +pub struct Resolved { + /// The window it is in. + pub window: WindowInfo, + /// The control as the live tree has it now (its `id` is this walk's own). + pub element: Element, + /// The walk stopped at a bound, so a second match beyond it cannot be + /// ruled out. Reported with the result, since an error cannot be. + pub truncated: bool, + /// Why the rectangle is not a position, when it is not (GTK 4). + pub no_positions: Option<String>, + depth: u32, + want: Want, +} + +fn os(message: String) -> DesktopError { + DesktopError::Os(message) +} + +/// `id "title" (app)` for a list in a message. +fn describe(w: &WindowInfo) -> String { + format!("{} \"{}\" ({})", w.id, w.title, w.app) +} + +/// The one window a selector's `app`/`window` part names among `windows`. +/// Windows of blocked applications are never candidates: their titles are not +/// shown, and a title must not be learned by matching against it. +pub(crate) fn pick_named_window( + selector: &Selector, + windows: Vec<WindowInfo>, +) -> Result<WindowInfo, DesktopError> { + if let Some(app) = &selector.app { + if crate::limits::get().blocked(app).is_some() { + return Err(os(format!( + "refused: \"{app}\" is a blocked application (a password manager, or \ + desktop_limits.blocked_apps); incord never reads or acts on it" + ))); + } + } + let open: Vec<WindowInfo> = windows + .into_iter() + .filter(|w| crate::limits::get().blocked(&w.app).is_none()) + .collect(); + let mut named: Vec<WindowInfo> = selector.windows_named(&open).into_iter().cloned().collect(); + match named.len() { + 1 => Ok(named.remove(0)), + 0 => Err(os(format!( + "no window matches `{}`; open: {}", + selector.render_window_part(), + open.iter() + .take(LISTED) + .map(describe) + .collect::<Vec<_>>() + .join(", ") + ))), + n => Err(os(format!( + "`{}` matches {n} windows, so it does not say which: {}. Pass `window` (an id \ + from list_windows) or make the app/window more specific", + selector.render_window_part(), + named + .iter() + .take(LISTED) + .map(describe) + .collect::<Vec<_>>() + .join(", ") + ))), + } +} + +/// The window a call acts in: the `window` id when one is given (and the +/// selector's own `app`/`window` must then agree with it), else the one the +/// selector names, else the focused window. +pub fn window_for(window: Option<&str>, selector: &Selector) -> Result<WindowInfo, DesktopError> { + match window { + Some(id) => { + let info = platform::window_info(Some(id))?; + if selector.has_window_part() && !selector.matches_window(&info) { + return Err(os(format!( + "window {id} is \"{}\" ({}), which the selector's window part `{}` does \ + not name; drop `window` or fix the selector", + info.title, + info.app, + selector.render_window_part() + ))); + } + Ok(info) + } + None if selector.has_window_part() => { + pick_named_window(selector, platform::list_windows()?) + } + None => platform::window_info(None), + } +} + +/// A fresh walk of `window` as deep as the walk may go, for a caller that only +/// looks (`wait_for`). Not recorded as shown: an id is not being handed out. +pub fn tree_now(window: &str) -> Result<Tree, DesktopError> { + platform::tree(Some(window), bounded(u32::MAX)) +} + +/// The control `selector` names in `tree`, which was walked at `depth`. +/// Separate from the walk so it is tested without a desktop. +pub(crate) fn resolved_from( + window: WindowInfo, + tree: Tree, + selector: &Selector, + depth: u32, +) -> Result<Resolved, DesktopError> { + let element = selector + .resolve_in(&tree.elements) + .map_err(|e| { + let cut = if tree.truncated { + format!( + " (the tree was cut at {} elements, so it may lie beyond the cut)", + super::max_elements() + ) + } else { + String::new() + }; + os(format!( + "selector `{}` in window \"{}\" {e}{cut}", + selector.render(), + window.title + )) + })? + .clone(); + Ok(Resolved { + want: Want { + id: element.id, + expect: element.clone(), + }, + window, + element, + truncated: tree.truncated, + no_positions: tree.no_positions, + depth, + }) +} + +/// Walk the window the selector names and return the one control it picks. +/// +/// Ambiguous or missing is an error, never a guess: see [`Selector::resolve_in`]. +pub fn resolve(window: Option<&str>, selector: &Selector) -> Result<Resolved, DesktopError> { + selector.require_element().map_err(os)?; + resolve_in(window_for(window, selector)?, selector) +} + +/// [`resolve`] in a window already chosen, so a caller can refuse the window +/// (a blocked application's) before anything is read from it. +pub fn resolve_in(info: WindowInfo, selector: &Selector) -> Result<Resolved, DesktopError> { + selector.require_element().map_err(os)?; + let depth = bounded(u32::MAX); + let tree = platform::tree(Some(&info.id), depth)?; + resolved_from(info, tree, selector, depth) +} + +/// [`super::set_value`] on a control found by selector. +pub fn set_value_on(found: &Resolved, value: &str) -> Result<Done, DesktopError> { + platform::set_value(Some(&found.window.id), &found.want, found.depth, value) +} + +/// [`super::act`] on a control found by selector. +pub fn act_on(found: &Resolved, action: &str) -> Result<Done, DesktopError> { + platform::act(Some(&found.window.id), &found.want, found.depth, action) +} + +impl Resolved { + /// Whether the live control is still the one that was resolved: the same + /// check the id path makes before it acts. + #[must_use] + pub fn is_same_as(&self, now: &Element) -> bool { + same_control(now, &self.want.expect) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::uia::Role; + use crate::Rect; + + fn rect() -> Rect { + Rect { + x: 0, + y: 0, + width: 10, + height: 10, + } + } + + fn window(id: &str, app: &str, title: &str) -> WindowInfo { + WindowInfo { + id: id.to_string(), + title: title.to_string(), + app: app.to_string(), + pid: 1, + focused: false, + rect: rect(), + has_tree: true, + } + } + + fn button(id: u32, name: &str) -> Element { + Element { + id, + depth: 1, + role: Role::Button, + name: name.to_string(), + rect: rect(), + enabled: true, + secret: false, + automation_id: String::new(), + } + } + + fn tree_of(elements: Vec<Element>, truncated: bool) -> Tree { + Tree { + root_name: "Doc".to_string(), + window: "7".to_string(), + app: "app".to_string(), + no_positions: None, + elements, + truncated, + offscreen: 0, + } + } + + fn sel(text: &str) -> Selector { + Selector::parse(text).unwrap() + } + + #[test] + fn a_unique_match_resolves_with_its_own_id_and_the_walks_depth() { + let tree = tree_of(vec![button(4, "Save"), button(9, "Open")], false); + let found = resolved_from( + window("7", "app", "Doc"), + tree, + &sel("role=button;name=Open"), + 5, + ) + .unwrap(); + assert_eq!(found.element.id, 9); + assert_eq!(found.want.id, 9); + assert_eq!(found.depth, 5); + assert!(!found.truncated); + } + + #[test] + fn a_missing_control_is_an_error_naming_the_selector_and_the_window() { + let tree = tree_of(vec![button(4, "Save")], false); + let err = resolved_from( + window("7", "app", "Doc"), + tree, + &sel("role=button;name=Nope"), + 5, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("role=button;name=Nope"), "{err}"); + assert!(err.contains("\"Doc\""), "{err}"); + assert!(err.contains("matches no element"), "{err}"); + } + + #[test] + fn two_matches_are_an_error_and_never_the_first() { + let tree = tree_of(vec![button(4, "Save"), button(9, "Save")], false); + let err = resolved_from(window("7", "app", "Doc"), tree, &sel("name=Save"), 5) + .unwrap_err() + .to_string(); + assert!(err.contains("matches 2 elements"), "{err}"); + assert!(err.contains("nth"), "{err}"); + } + + #[test] + fn a_cut_tree_says_a_miss_may_lie_beyond_the_cut() { + let tree = tree_of(vec![button(4, "Save")], true); + let err = resolved_from( + window("7", "app", "Doc"), + tree.clone(), + &sel("name=Nope"), + 5, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("cut at"), "{err}"); + // A unique match in a cut tree is still returned, and flagged. + let found = resolved_from(window("7", "app", "Doc"), tree, &sel("name=Save"), 5).unwrap(); + assert!(found.truncated); + } + + #[test] + fn the_window_part_picks_one_window_or_says_why_not() { + let open = || { + vec![ + window("1", "notepad", "a.txt - Notepad"), + window("2", "notepad", "b.txt - Notepad"), + window("3", "calc", "Calculator"), + ] + }; + let picked = pick_named_window(&sel("app=calc"), open()).unwrap(); + assert_eq!(picked.id, "3"); + let picked = pick_named_window(&sel("app=notepad;window~=b.txt"), open()).unwrap(); + assert_eq!(picked.id, "2"); + + let both = pick_named_window(&sel("app=notepad"), open()) + .unwrap_err() + .to_string(); + assert!(both.contains("matches 2 windows"), "{both}"); + assert!(both.contains("a.txt") && both.contains("b.txt"), "{both}"); + + let none = pick_named_window(&sel("window=Nope"), open()) + .unwrap_err() + .to_string(); + assert!(none.contains("no window matches"), "{none}"); + assert!( + none.contains("Calculator"), + "the open windows are listed: {none}" + ); + } + + #[test] + fn a_blocked_application_is_neither_named_nor_matched_by_its_title() { + // `keepass` is in the built-in blocklist (limits.rs). + let open = vec![ + window("1", "keepass", "Passwords - Vault"), + window("2", "notepad", "notes"), + ]; + let err = pick_named_window(&sel("app=keepass"), open.clone()) + .unwrap_err() + .to_string(); + assert!(err.contains("blocked"), "{err}"); + // Probing for its title by a title-only selector finds nothing. + let probe = pick_named_window(&sel("window~=Vault"), open) + .unwrap_err() + .to_string(); + assert!(probe.contains("no window matches"), "{probe}"); + assert!(!probe.contains("Passwords"), "{probe}"); + } + + #[test] + fn a_control_selector_needs_a_control_part() { + let err = resolve(None, &sel("app=notepad;nth=0")).unwrap_err(); + assert!(err.to_string().contains("names no control"), "{err}"); + } +} diff --git a/crates/desktop/src/uia_windows.rs b/crates/desktop/src/uia_windows.rs index 0cf7ff3..39088f8 100644 --- a/crates/desktop/src/uia_windows.rs +++ b/crates/desktop/src/uia_windows.rs @@ -91,6 +91,10 @@ fn describe(element: &UIElement, id: u32, depth: u32) -> Element { rect: rect_of(element), enabled: element.is_enabled().unwrap_or(false), secret: element.is_password().unwrap_or(false), + // UIA's AutomationId: the program's own name for the control, empty + // for most. Read for a password field too: it names the control, it + // is not the content. + automation_id: element.get_automation_id().unwrap_or_default(), } } diff --git a/crates/desktop/src/wait.rs b/crates/desktop/src/wait.rs new file mode 100644 index 0000000..be6f1f8 --- /dev/null +++ b/crates/desktop/src/wait.rs @@ -0,0 +1,1402 @@ +//! `wait_for`: wait, bounded, until the screen is in a state the caller names. +//! +//! # Why this is the missing primitive +//! +//! `screenshot_after` sleeps a fixed `settle` time and hopes. That is a guess +//! at how long an application takes, and it is the guess a flow replayed without +//! a model cannot make (`docs/COMPUTER_USE.md`, replay): too short and the next +//! step runs against a dialog that is not there yet, too long and every step +//! pays for the slowest one. This waits for the thing itself: a control that +//! matches a selector appears or goes, a window with a title appears or goes, +//! or the picture of the screen changes. It reports what it found, or that it +//! timed out and what it saw last. +//! +//! # The bounds +//! +//! The caller gives `timeout_ms`; it is held to [`MAX_TIMEOUT_MS`] and the reply +//! says so. The polling interval is held to [`MIN_POLL_MS`]..=[`MAX_POLL_MS`]. +//! No poll STARTS after the deadline, and the last one starts exactly at it, so +//! a condition that comes true at the deadline is not missed; a poll already +//! under way is not interrupted, so the call can end later than the timeout by +//! the length of one poll (a walk of a big window is the slow one). The stop +//! switch is asked between every poll and every [`SLICE_MS`] of sleeping. +//! +//! # Why a trait +//! +//! [`Env`] is everything the wait touches: the clock, the windows, a tree, a +//! picture. The live one is `mcp::wait_tool`; the tests here run the same code +//! against a scripted desktop with a virtual clock, so the timing rules are +//! tested to the millisecond and without sleeping. + +use serde_json::{json, Value}; + +use crate::fingerprint::{self, Fingerprint}; +use crate::selector::{selector_for, Selector}; +use crate::uia::{Element, Tree, WindowInfo, NAMES_ARE_VALUES}; + +/// The timeout when the call gives none. +pub const DEFAULT_TIMEOUT_MS: u64 = 5_000; +/// The longest one call may wait. A longer request is held to this and says so. +pub const MAX_TIMEOUT_MS: u64 = 60_000; +/// How often the condition is checked when the call does not say. +pub const DEFAULT_POLL_MS: u64 = 250; +/// The fastest and slowest checking interval a call may ask for. +pub const MIN_POLL_MS: u64 = 50; +pub const MAX_POLL_MS: u64 = 2_000; +/// Bits of the 128 that may differ before the screen counts as changed. +pub const DEFAULT_TOLERANCE: u32 = 4; +/// Sleeps are cut into slices this long so a stop is noticed within one. +const SLICE_MS: u64 = 100; +/// Look-alikes listed beside a match. +const OTHERS: usize = 4; + +/// How long to wait and how often to look, after the bounds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Budget { + pub timeout_ms: u64, + pub poll_ms: u64, + /// What was changed from what was asked, for the reply. + pub notes: Vec<String>, +} + +impl Budget { + /// `timeout_ms` and `poll_ms` as the call gave them (or did not), held to + /// the bounds. + #[must_use] + pub fn new(timeout_ms: Option<u64>, poll_ms: Option<u64>) -> Budget { + let mut notes = Vec::new(); + let asked = timeout_ms.unwrap_or(DEFAULT_TIMEOUT_MS); + let timeout_ms = if asked > MAX_TIMEOUT_MS { + notes.push(format!( + "timeout_ms {asked} is over the cap; waited at most {MAX_TIMEOUT_MS} ms" + )); + MAX_TIMEOUT_MS + } else { + asked + }; + let wanted = poll_ms.unwrap_or(DEFAULT_POLL_MS); + let poll = wanted.clamp(MIN_POLL_MS, MAX_POLL_MS); + if poll != wanted { + notes.push(format!( + "poll_ms {wanted} is outside {MIN_POLL_MS}..={MAX_POLL_MS}; polled every {poll} ms" + )); + } + Budget { + timeout_ms, + poll_ms: poll, + notes, + } + } +} + +/// What a wait touches. See the module header. +pub trait Env { + /// Milliseconds since some fixed moment of this wait's own. + fn now_ms(&mut self) -> u64; + fn sleep_ms(&mut self, ms: u64); + /// The windows there are to look at: those of blocked applications are not + /// among them. + fn windows(&mut self) -> Result<Vec<WindowInfo>, String>; + /// The window keystrokes go to now. + fn focused_window(&mut self) -> Result<WindowInfo, String>; + /// A fresh walk of one window. + fn tree(&mut self, window: &str) -> Result<Tree, String>; + /// The fingerprint of the screen, or of one window's area. + fn screen(&mut self, window: Option<&str>) -> Result<Fingerprint, String>; +} + +/// What to wait for. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Condition { + /// A control that matches `selector` is there (`appears`) or is not + /// (`gone`). `window` is a window id; without it the selector's own + /// `app`/`window` part names the window, and without that the window that + /// has focus when the wait begins, which is then the one watched. + Element { + selector: Selector, + window: Option<String>, + appears: bool, + }, + /// A window matching the selector's `app`/`window` is there, or is not. + Window { selector: Selector, appears: bool }, + /// The screen (or `window`'s area) differs from `baseline` by more than + /// `tolerance` bits; with no baseline, from how it is when the wait begins. + Screen { + baseline: Option<Fingerprint>, + tolerance: u32, + window: Option<String>, + }, +} + +impl Condition { + /// The word the call used. + #[must_use] + pub fn name(&self) -> &'static str { + match self { + Condition::Element { appears: true, .. } => "element_appears", + Condition::Element { appears: false, .. } => "element_gone", + Condition::Window { appears: true, .. } => "window_appears", + Condition::Window { appears: false, .. } => "window_gone", + Condition::Screen { .. } => "screen_changes", + } + } +} + +/// What a satisfied wait found. +#[derive(Debug, Clone)] +pub enum Found { + Element { + window: WindowInfo, + element: Box<Element>, + /// The control part of a selector that picks it out of its window. + selector: String, + matches: usize, + /// Others that fit, as selectors, when there are several. + others: Vec<String>, + /// The walk was cut, so more may lie beyond. + truncated: bool, + }, + ElementGone { + why: String, + }, + Windows(Vec<WindowInfo>), + WindowGone, + Screen { + hash: Fingerprint, + baseline: Fingerprint, + distance: u32, + tolerance: u32, + }, +} + +/// How a wait ended. +#[derive(Debug, Clone)] +pub enum Outcome { + Met { + found: Found, + waited_ms: u64, + polls: u32, + }, + TimedOut { + waited_ms: u64, + polls: u32, + /// What the last look saw, in words. + last: String, + /// The last picture's fingerprint, for `screen_changes`: pass it back + /// as `baseline` to go on waiting from where this left off. + hash: Option<Fingerprint>, + }, + Stopped { + waited_ms: u64, + }, +} + +/// One look at the condition. +enum Poll<T> { + Ready(T), + /// Not yet, and what was seen. + Pending(String), +} + +enum Waited<T> { + Met(T, u64, u32), + TimedOut(u64, u32, String), + Stopped(u64), +} + +/// Check, and while the answer is "not yet" and time is left, sleep and check +/// again: the loop every condition shares. The first check is at once. +fn wait_until<E: Env, T>( + env: &mut E, + budget: &Budget, + stopped: &dyn Fn() -> bool, + mut check: impl FnMut(&mut E) -> Result<Poll<T>, String>, +) -> Result<Waited<T>, String> { + let start = env.now_ms(); + let mut polls = 0u32; + loop { + if stopped() { + return Ok(Waited::Stopped(env.now_ms() - start)); + } + polls += 1; + let last = match check(env)? { + Poll::Ready(found) => return Ok(Waited::Met(found, env.now_ms() - start, polls)), + Poll::Pending(seen) => seen, + }; + let elapsed = env.now_ms() - start; + if elapsed >= budget.timeout_ms { + return Ok(Waited::TimedOut(elapsed, polls, last)); + } + // The next look is at the interval, or at the deadline when that is + // sooner: the last look is AT the deadline, not after it. + let mut nap = budget.poll_ms.min(budget.timeout_ms - elapsed); + while nap > 0 { + if stopped() { + return Ok(Waited::Stopped(env.now_ms() - start)); + } + let slice = nap.min(SLICE_MS); + env.sleep_ms(slice); + nap -= slice; + } + } +} + +/// Which window an element wait watches. +enum Target { + /// This one, by id, whatever happens to focus. + Pinned(String), + /// Whichever the selector's `app`/`window` names, looked for again each time. + Named, +} + +/// What one look at an element condition saw. +enum Look { + NoWindow, + Seen(Box<Seen>), +} + +/// A window that was there, its tree, and where in it the selector matched. +struct Seen { + window: WindowInfo, + tree: Tree, + hits: Vec<usize>, +} + +fn windows_named(env: &mut impl Env, selector: &Selector) -> Result<Vec<WindowInfo>, String> { + let all = env.windows()?; + Ok(selector.windows_named(&all).into_iter().cloned().collect()) +} + +/// The window the wait watches, decided once at the start. +fn target_of( + env: &mut impl Env, + selector: &Selector, + window: Option<&str>, +) -> Result<Target, String> { + if let Some(id) = window { + let all = env.windows()?; + let Some(w) = all.iter().find(|w| w.id == id) else { + return Err(format!( + "no window has the id {id}; take one from list_windows" + )); + }; + if selector.has_window_part() && !selector.matches_window(w) { + return Err(format!( + "window {id} is \"{}\" ({}), which the selector's window part `{}` does not name", + w.title, + w.app, + selector.render_window_part() + )); + } + return Ok(Target::Pinned(id.to_string())); + } + if selector.has_window_part() { + return Ok(Target::Named); + } + // No window said: the one in front NOW. Watched by id from here on, so a + // change of focus mid-wait is not taken for the control going away. + Ok(Target::Pinned(env.focused_window()?.id)) +} + +fn look(env: &mut impl Env, target: &Target, selector: &Selector) -> Result<Look, String> { + let window = match target { + Target::Pinned(id) => env.windows()?.into_iter().find(|w| &w.id == id), + Target::Named => { + let mut named = windows_named(env, selector)?; + match named.len() { + 0 => None, + 1 => Some(named.remove(0)), + n => { + return Err(format!( + "`{}` matches {n} windows, so it does not say which; pass `window` \ + (an id from list_windows) or make the app/window more specific", + selector.render_window_part() + )) + } + } + } + }; + let Some(window) = window else { + return Ok(Look::NoWindow); + }; + let tree = match env.tree(&window.id) { + Ok(tree) => tree, + Err(why) => { + // A window that closes while it is being walked fails the walk: + // that is the window going away, which is an answer, not a + // failure. Only a window that is still there makes it an error. + let still_there = env.windows()?.iter().any(|w| w.id == window.id); + if still_there { + return Err(why); + } + return Ok(Look::NoWindow); + } + }; + let mut hits = selector.candidates(&tree.elements); + if let Some(nth) = selector.nth { + hits = hits.get(nth as usize).copied().into_iter().collect(); + } + Ok(Look::Seen(Box::new(Seen { window, tree, hits }))) +} + +fn cut_note(tree: &Tree) -> String { + if tree.truncated { + " (the tree was cut, so more may lie beyond)".to_string() + } else { + String::new() + } +} + +fn found_element(window: WindowInfo, tree: Tree, hits: &[usize]) -> Found { + let first = hits[0]; + let others = hits + .iter() + .skip(1) + .take(OTHERS) + .map(|&i| selector_for(&tree.elements, i, NAMES_ARE_VALUES).render_element_part()) + .collect(); + Found::Element { + selector: selector_for(&tree.elements, first, NAMES_ARE_VALUES).render_element_part(), + element: Box::new(tree.elements[first].clone()), + matches: hits.len(), + others, + truncated: tree.truncated, + window, + } +} + +fn element_check( + env: &mut impl Env, + target: &Target, + selector: &Selector, + appears: bool, +) -> Result<Poll<Found>, String> { + Ok(match (look(env, target, selector)?, appears) { + (Look::NoWindow, true) => Poll::Pending("the window is not there".to_string()), + (Look::NoWindow, false) => Poll::Ready(Found::ElementGone { + why: "its window is not there".to_string(), + }), + (Look::Seen(seen), true) => { + let Seen { window, tree, hits } = *seen; + if hits.is_empty() { + Poll::Pending(format!( + "window \"{}\" has {} elements, none matches{}", + window.title, + tree.elements.len(), + cut_note(&tree) + )) + } else { + Poll::Ready(found_element(window, tree, &hits)) + } + } + (Look::Seen(seen), false) => { + let Seen { window, tree, hits } = *seen; + if !hits.is_empty() { + Poll::Pending(format!( + "{} element(s) in window \"{}\" still match", + hits.len(), + window.title + )) + } else if tree.truncated { + // Absent from the part that was walked is not absent. + Poll::Pending(format!( + "no match in window \"{}\", but the tree was cut, so its absence cannot be \ + told", + window.title + )) + } else { + Poll::Ready(Found::ElementGone { + why: "no element matches".to_string(), + }) + } + } + }) +} + +fn window_check( + env: &mut impl Env, + selector: &Selector, + appears: bool, +) -> Result<Poll<Found>, String> { + let named = windows_named(env, selector)?; + Ok(match (named.is_empty(), appears) { + (false, true) => Poll::Ready(Found::Windows(named)), + (true, true) => Poll::Pending("no window matches yet".to_string()), + (true, false) => Poll::Ready(Found::WindowGone), + (false, false) => Poll::Pending(format!("{} window(s) still match", named.len())), + }) +} + +fn into_outcome(waited: Waited<Found>, hash: Option<Fingerprint>) -> Outcome { + match waited { + Waited::Met(found, waited_ms, polls) => Outcome::Met { + found, + waited_ms, + polls, + }, + Waited::TimedOut(waited_ms, polls, last) => Outcome::TimedOut { + waited_ms, + polls, + last, + hash, + }, + Waited::Stopped(waited_ms) => Outcome::Stopped { waited_ms }, + } +} + +/// Wait for `condition`. `Err` is a refusal or a failure of the desktop (a +/// selector that names two windows, a window that cannot be walked); a wait +/// that ran out of time is [`Outcome::TimedOut`], not an error. +pub fn run<E: Env>( + env: &mut E, + condition: &Condition, + budget: &Budget, + stopped: &dyn Fn() -> bool, +) -> Result<Outcome, String> { + match condition { + Condition::Element { + selector, + window, + appears, + } => { + selector.require_element()?; + let target = target_of(env, selector, window.as_deref())?; + let waited = wait_until(env, budget, stopped, |env| { + element_check(env, &target, selector, *appears) + })?; + Ok(into_outcome(waited, None)) + } + Condition::Window { selector, appears } => { + selector.require_window_only()?; + let waited = wait_until(env, budget, stopped, |env| { + window_check(env, selector, *appears) + })?; + Ok(into_outcome(waited, None)) + } + Condition::Screen { + baseline, + tolerance, + window, + } => { + let mut from = *baseline; + let mut last = None; + let waited = wait_until(env, budget, stopped, |env| { + let now = env.screen(window.as_deref())?; + last = Some(now); + let Some(base) = from else { + // The baseline is how it is as the wait begins. + from = Some(now); + return Ok(Poll::Pending("took the baseline".to_string())); + }; + let distance = fingerprint::distance(base, now); + Ok(if distance > *tolerance { + Poll::Ready(Found::Screen { + hash: now, + baseline: base, + distance, + tolerance: *tolerance, + }) + } else { + Poll::Pending(format!( + "the screen is {distance} bit(s) from the baseline; it counts as \ + changed above {tolerance}" + )) + }) + })?; + Ok(into_outcome(waited, last)) + } + } +} + +fn window_json(w: &WindowInfo) -> Value { + json!({"id": w.id, "title": w.title, "app": w.app, "focused": w.focused}) +} + +impl Found { + fn fields(&self) -> Vec<(&'static str, Value)> { + match self { + Found::Element { + window, + element, + selector, + matches, + others, + truncated, + } => { + let mut e = json!({ + "role": element.role.word(), + "name": element.name, + "rect": element.rect, + "enabled": element.enabled, + "selector": selector, + }); + if !element.automation_id.is_empty() { + e["automation_id"] = json!(element.automation_id); + } + let mut w = window_json(window); + let window_part = Selector { + app: Some(window.app.clone()).filter(|a| !a.is_empty()), + window: Some(crate::selector::Text::Exact(window.title.clone())), + ..Selector::default() + }; + w["selector"] = json!(window_part.render_window_part()); + let mut out = vec![ + ("window", w), + ("element", e), + ("matches", json!(matches)), + ("unique", json!(*matches == 1)), + ]; + if !others.is_empty() { + out.push(("others", json!(others))); + } + if *truncated { + out.push(("truncated", json!(true))); + } + out + } + Found::ElementGone { why } => vec![("why", json!(why))], + Found::Windows(ws) => vec![( + "windows", + json!(ws.iter().map(window_json).collect::<Vec<_>>()), + )], + Found::WindowGone => vec![("why", json!("no window matches"))], + Found::Screen { + hash, + baseline, + distance, + tolerance, + } => vec![ + ("hash", json!(hash.hex())), + ("baseline", json!(baseline.hex())), + ("distance", json!(distance)), + ("tolerance", json!(tolerance)), + ], + } + } +} + +impl Outcome { + /// The reply: a JSON object whose `result` is `found` or `timeout`. + /// + /// A timeout is a REPLY, not a tool error: the desktop worked, and what the + /// caller wanted did not happen in time. It is no refusal either, which is + /// what the journal would call an error. + #[must_use] + pub fn to_json(&self, condition: &Condition, budget: &Budget) -> Value { + let mut out = serde_json::Map::new(); + let mut put = |k: &str, v: Value| { + out.insert(k.to_string(), v); + }; + match self { + Outcome::Met { + found, + waited_ms, + polls, + } => { + put("result", json!("found")); + put("condition", json!(condition.name())); + put("waited_ms", json!(waited_ms)); + put("polls", json!(polls)); + for (k, v) in found.fields() { + put(k, v); + } + } + Outcome::TimedOut { + waited_ms, + polls, + last, + hash, + } => { + put("result", json!("timeout")); + put("condition", json!(condition.name())); + put("waited_ms", json!(waited_ms)); + put("polls", json!(polls)); + put("last_seen", json!(last)); + if let Some(h) = hash { + put("hash", json!(h.hex())); + } + } + Outcome::Stopped { waited_ms } => { + put("result", json!("stopped")); + put("condition", json!(condition.name())); + put("waited_ms", json!(waited_ms)); + } + } + put("timeout_ms", json!(budget.timeout_ms)); + put("poll_ms", json!(budget.poll_ms)); + if !budget.notes.is_empty() { + put("notes", json!(budget.notes)); + } + Value::Object(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::uia::Role; + use crate::Rect; + use std::cell::Cell; + + fn rect() -> Rect { + Rect { + x: 0, + y: 0, + width: 10, + height: 10, + } + } + + fn window(id: &str, app: &str, title: &str, focused: bool) -> WindowInfo { + WindowInfo { + id: id.to_string(), + title: title.to_string(), + app: app.to_string(), + pid: 1, + focused, + rect: rect(), + has_tree: true, + } + } + + fn button(id: u32, name: &str) -> Element { + Element { + id, + depth: 1, + role: Role::Button, + name: name.to_string(), + rect: rect(), + enabled: true, + secret: false, + automation_id: String::new(), + } + } + + type At<T> = Box<dyn Fn(u64) -> T>; + + /// A desktop that is a function of time, on a clock that only moves when + /// something sleeps or something slow is asked. + struct Fake { + now: u64, + /// What each tree or picture costs, in virtual milliseconds. + cost: u64, + windows: At<Vec<WindowInfo>>, + /// window id -> its tree at a time. + elements: At<Vec<Element>>, + truncated: bool, + /// From this time on a walk fails, as one does of a closing window. + tree_fails_from: Option<u64>, + screen: At<u128>, + sleeps: Vec<u64>, + trees: u32, + } + + impl Fake { + fn new() -> Fake { + Fake { + now: 0, + cost: 0, + windows: Box::new(|_| vec![window("1", "app", "Doc", true)]), + elements: Box::new(|_| Vec::new()), + truncated: false, + tree_fails_from: None, + screen: Box::new(|_| 0), + sleeps: Vec::new(), + trees: 0, + } + } + } + + impl Env for Fake { + fn now_ms(&mut self) -> u64 { + self.now + } + fn sleep_ms(&mut self, ms: u64) { + self.sleeps.push(ms); + self.now += ms; + } + fn windows(&mut self) -> Result<Vec<WindowInfo>, String> { + Ok((self.windows)(self.now)) + } + fn focused_window(&mut self) -> Result<WindowInfo, String> { + (self.windows)(self.now) + .into_iter() + .find(|w| w.focused) + .ok_or_else(|| "nothing has focus".to_string()) + } + fn tree(&mut self, window: &str) -> Result<Tree, String> { + self.trees += 1; + if self.tree_fails_from.is_some_and(|t| self.now >= t) { + return Err("no desktop element: Unspecified error".to_string()); + } + let at = self.now; + self.now += self.cost; + Ok(Tree { + root_name: "t".to_string(), + window: window.to_string(), + app: "app".to_string(), + no_positions: None, + elements: (self.elements)(at), + truncated: self.truncated, + offscreen: 0, + }) + } + fn screen(&mut self, _: Option<&str>) -> Result<Fingerprint, String> { + let at = self.now; + self.now += self.cost; + Ok(Fingerprint((self.screen)(at))) + } + } + + fn never() -> bool { + false + } + + fn sel(text: &str) -> Selector { + Selector::parse(text).unwrap() + } + + fn appears(text: &str) -> Condition { + Condition::Element { + selector: sel(text), + window: None, + appears: true, + } + } + + fn gone(text: &str) -> Condition { + Condition::Element { + selector: sel(text), + window: None, + appears: false, + } + } + + fn budget(timeout: u64, poll: u64) -> Budget { + Budget::new(Some(timeout), Some(poll)) + } + + // ---- the bounds ---- + + #[test] + fn the_defaults_are_five_seconds_every_quarter_second() { + let b = Budget::new(None, None); + assert_eq!((b.timeout_ms, b.poll_ms), (5_000, 250)); + assert!(b.notes.is_empty()); + } + + #[test] + fn a_timeout_over_the_cap_is_held_to_it_and_says_so() { + let b = Budget::new(Some(10 * MAX_TIMEOUT_MS), None); + assert_eq!(b.timeout_ms, MAX_TIMEOUT_MS); + assert_eq!(b.notes.len(), 1); + assert!(b.notes[0].contains("over the cap"), "{:?}", b.notes); + assert_eq!(Budget::new(Some(MAX_TIMEOUT_MS), None).notes.len(), 0); + assert_eq!(Budget::new(Some(u64::MAX), None).timeout_ms, MAX_TIMEOUT_MS); + } + + #[test] + fn the_polling_interval_is_held_between_its_bounds() { + assert_eq!(Budget::new(None, Some(1)).poll_ms, MIN_POLL_MS); + assert_eq!(Budget::new(None, Some(0)).poll_ms, MIN_POLL_MS); + assert_eq!(Budget::new(None, Some(1_000_000)).poll_ms, MAX_POLL_MS); + assert_eq!(Budget::new(None, Some(500)).poll_ms, 500); + assert_eq!(Budget::new(None, Some(1)).notes.len(), 1); + } + + // ---- the loop ---- + + /// Checks at 0, 250, 500, 750 ...; true from `t` on. + fn true_from(t: u64) -> impl FnMut(&mut Fake) -> Result<Poll<u64>, String> { + move |env: &mut Fake| { + let now = env.now_ms(); + Ok(if now >= t { + Poll::Ready(now) + } else { + Poll::Pending(format!("not yet at {now}")) + }) + } + } + + #[test] + fn a_condition_already_true_returns_at_once_without_sleeping() { + let mut env = Fake::new(); + let got = wait_until(&mut env, &budget(5_000, 250), &never, true_from(0)).unwrap(); + assert!(matches!(got, Waited::Met(0, 0, 1))); + assert!(env.sleeps.is_empty()); + } + + #[test] + fn a_condition_that_comes_true_is_caught_at_the_next_poll() { + let mut env = Fake::new(); + let got = wait_until(&mut env, &budget(5_000, 250), &never, true_from(700)).unwrap(); + // Looked at 0, 250, 500, 750. + assert!(matches!(got, Waited::Met(750, 750, 4)), "{:?}", { + match got { + Waited::Met(a, b, c) => (a, b, c), + _ => (0, 0, 0), + } + }); + } + + #[test] + fn a_condition_that_never_holds_times_out_at_exactly_the_timeout() { + let mut env = Fake::new(); + let got = wait_until(&mut env, &budget(1_000, 300), &never, true_from(u64::MAX)).unwrap(); + match got { + Waited::TimedOut(waited, polls, last) => { + assert_eq!(waited, 1_000); + // 0, 300, 600, 900, then the last look AT 1000. + assert_eq!(polls, 5); + assert_eq!(last, "not yet at 1000"); + } + _ => panic!("must time out"), + } + } + + #[test] + fn a_condition_true_exactly_at_the_deadline_is_not_missed() { + let mut env = Fake::new(); + let got = wait_until(&mut env, &budget(1_000, 300), &never, true_from(1_000)).unwrap(); + assert!(matches!(got, Waited::Met(1_000, 1_000, 5))); + } + + #[test] + fn a_zero_timeout_looks_once() { + let mut env = Fake::new(); + let got = wait_until(&mut env, &budget(0, 250), &never, true_from(1)).unwrap(); + assert!(matches!(got, Waited::TimedOut(0, 1, _))); + assert!(env.sleeps.is_empty()); + let mut env = Fake::new(); + let got = wait_until(&mut env, &budget(0, 250), &never, true_from(0)).unwrap(); + assert!(matches!(got, Waited::Met(0, 0, 1))); + } + + #[test] + fn no_look_starts_after_the_deadline_even_when_looking_is_slow() { + // Each look takes 400 ms. Looks start at 0 and 650; the second ends at + // 1050, past the 1000 deadline, so there is no third. + let mut env = Fake::new(); + let got = wait_until(&mut env, &budget(1_000, 250), &never, |env: &mut Fake| { + env.now += 400; + Ok(Poll::<()>::Pending("slow".to_string())) + }) + .unwrap(); + match got { + Waited::TimedOut(waited, polls, _) => { + assert_eq!(polls, 2); + assert_eq!(waited, 1_050, "ends late by at most the look in flight"); + } + _ => panic!("must time out"), + } + } + + #[test] + fn sleeping_is_cut_into_slices_so_a_stop_is_noticed_within_one() { + let mut env = Fake::new(); + let _ = wait_until(&mut env, &budget(2_000, 1_000), &never, true_from(u64::MAX)); + assert!(!env.sleeps.is_empty()); + assert!( + env.sleeps.iter().all(|&s| s <= SLICE_MS), + "{:?}", + env.sleeps + ); + assert_eq!(env.sleeps.iter().sum::<u64>(), 2_000); + } + + #[test] + fn a_stop_ends_the_wait_between_two_slices() { + let mut env = Fake::new(); + let asked = Cell::new(0u32); + let stop_after_a_few = || { + asked.set(asked.get() + 1); + asked.get() > 4 + }; + let got = wait_until( + &mut env, + &budget(60_000, 2_000), + &stop_after_a_few, + true_from(u64::MAX), + ) + .unwrap(); + assert!(matches!(got, Waited::Stopped(_))); + assert!(env.now < 1_000, "stopped promptly, not after the interval"); + } + + #[test] + fn a_stop_before_the_first_look_looks_at_nothing() { + let mut env = Fake::new(); + let mut looked = false; + let got = wait_until(&mut env, &budget(1_000, 250), &|| true, |_: &mut Fake| { + looked = true; + Ok(Poll::<()>::Pending(String::new())) + }) + .unwrap(); + assert!(matches!(got, Waited::Stopped(0))); + assert!(!looked); + } + + #[test] + fn a_failing_look_ends_the_wait_with_its_error() { + let mut env = Fake::new(); + let got = wait_until(&mut env, &budget(1_000, 250), &never, |_: &mut Fake| { + Err::<Poll<()>, _>("the window cannot be walked".to_string()) + }); + assert_eq!(got.err().as_deref(), Some("the window cannot be walked")); + } + + // ---- elements ---- + + #[test] + fn an_element_that_shows_up_later_is_found_with_a_selector_for_it() { + let mut env = Fake::new(); + env.elements = Box::new(|t| { + let mut e = vec![button(1, "Open")]; + if t >= 600 { + e.push(button(2, "Save")); + } + e + }); + let out = run( + &mut env, + &appears("role=button;name=Save"), + &budget(5_000, 250), + &never, + ) + .unwrap(); + match out { + Outcome::Met { + found: + Found::Element { + selector, + matches, + window, + .. + }, + waited_ms, + polls, + } => { + assert_eq!(selector, "role=button;name=Save"); + assert_eq!(matches, 1); + assert_eq!(window.id, "1"); + assert_eq!(waited_ms, 750); + assert_eq!(polls, 4); + } + other => panic!("expected found, got {other:?}"), + } + } + + #[test] + fn an_element_that_never_comes_times_out_and_says_what_it_saw() { + let mut env = Fake::new(); + env.elements = Box::new(|_| vec![button(1, "Open")]); + let c = appears("name=Save"); + let out = run(&mut env, &c, &budget(1_000, 250), &never).unwrap(); + match &out { + Outcome::TimedOut { + waited_ms, last, .. + } => { + assert_eq!(*waited_ms, 1_000); + assert!(last.contains("none matches"), "{last}"); + } + other => panic!("expected timeout, got {other:?}"), + } + let json = out.to_json(&c, &budget(1_000, 250)); + assert_eq!(json["result"], "timeout"); + assert_eq!(json["timeout_ms"], 1_000); + } + + #[test] + fn several_matches_are_reported_as_several_and_not_resolved() { + let mut env = Fake::new(); + env.elements = Box::new(|_| vec![button(1, "Save"), button(2, "Save")]); + let out = run(&mut env, &appears("name=Save"), &budget(1_000, 250), &never).unwrap(); + let json = out.to_json(&appears("name=Save"), &budget(1_000, 250)); + assert_eq!(json["result"], "found"); + assert_eq!(json["matches"], 2); + assert_eq!(json["unique"], false); + assert_eq!(json["others"].as_array().unwrap().len(), 1); + } + + #[test] + fn nth_waits_for_that_many_matches() { + let mut env = Fake::new(); + env.elements = Box::new(|t| { + let mut e = vec![button(1, "Row")]; + if t >= 500 { + e.push(button(2, "Row")); + } + e + }); + let out = run( + &mut env, + &appears("name=Row;nth=1"), + &budget(5_000, 250), + &never, + ) + .unwrap(); + match out { + Outcome::Met { waited_ms, .. } => assert_eq!(waited_ms, 500), + other => panic!("{other:?}"), + } + } + + #[test] + fn an_element_that_goes_away_is_waited_for() { + let mut env = Fake::new(); + env.elements = Box::new(|t| { + if t < 400 { + vec![button(1, "Loading")] + } else { + vec![button(2, "Done")] + } + }); + let out = run(&mut env, &gone("name=Loading"), &budget(5_000, 250), &never).unwrap(); + match out { + Outcome::Met { + found: Found::ElementGone { .. }, + waited_ms, + .. + } => assert_eq!(waited_ms, 500), + other => panic!("{other:?}"), + } + } + + #[test] + fn an_element_that_stays_times_out_of_a_gone_wait() { + let mut env = Fake::new(); + env.elements = Box::new(|_| vec![button(1, "Loading")]); + let out = run(&mut env, &gone("name=Loading"), &budget(500, 250), &never).unwrap(); + match out { + Outcome::TimedOut { last, .. } => assert!(last.contains("still match"), "{last}"), + other => panic!("{other:?}"), + } + } + + #[test] + fn a_window_that_closes_takes_its_elements_with_it() { + let mut env = Fake::new(); + env.windows = Box::new(|t| { + if t < 300 { + vec![window("1", "app", "Doc", true)] + } else { + vec![window("2", "app", "Other", true)] + } + }); + env.elements = Box::new(|_| vec![button(1, "OK")]); + // Pinned to window 1 at the start (the focused one). + let out = run(&mut env, &gone("name=OK"), &budget(5_000, 250), &never).unwrap(); + match out { + Outcome::Met { + found: Found::ElementGone { why }, + .. + } => assert!(why.contains("window"), "{why}"), + other => panic!("{other:?}"), + } + } + + #[test] + fn a_walk_that_fails_because_the_window_closed_is_the_window_going_away() { + let mut env = Fake::new(); + env.windows = Box::new(|t| { + if t < 300 { + vec![window("1", "app", "Doc", true)] + } else { + vec![] + } + }); + env.elements = Box::new(|_| vec![button(1, "OK")]); + env.tree_fails_from = Some(300); + let out = run(&mut env, &gone("name=OK"), &budget(5_000, 250), &never).unwrap(); + assert!( + matches!( + out, + Outcome::Met { + found: Found::ElementGone { .. }, + .. + } + ), + "{out:?}" + ); + } + + #[test] + fn a_walk_that_fails_of_a_window_that_is_still_there_is_an_error() { + let mut env = Fake::new(); + env.elements = Box::new(|_| vec![button(1, "OK")]); + env.tree_fails_from = Some(0); + let err = run(&mut env, &gone("name=OK"), &budget(5_000, 250), &never).unwrap_err(); + assert!(err.contains("Unspecified error"), "{err}"); + } + + #[test] + fn a_cut_tree_cannot_prove_an_element_gone() { + let mut env = Fake::new(); + env.truncated = true; + env.elements = Box::new(|_| vec![button(1, "Other")]); + let out = run(&mut env, &gone("name=Loading"), &budget(500, 250), &never).unwrap(); + match out { + Outcome::TimedOut { last, .. } => assert!(last.contains("cut"), "{last}"), + other => panic!("a cut tree must not satisfy `gone`: {other:?}"), + } + } + + #[test] + fn the_watched_window_is_the_one_in_front_at_the_start_not_whichever_is_later() { + let mut env = Fake::new(); + // Focus moves to window 2 at once; the control exists "everywhere". + env.windows = Box::new(|t| { + vec![ + window("1", "app", "A", t < 100), + window("2", "app", "B", t >= 100), + ] + }); + env.elements = Box::new(|_| vec![button(1, "OK")]); + let out = run(&mut env, &appears("name=OK"), &budget(1_000, 250), &never).unwrap(); + match out { + Outcome::Met { + found: Found::Element { window, .. }, + .. + } => assert_eq!(window.id, "1", "pinned at the start"), + other => panic!("{other:?}"), + } + } + + #[test] + fn a_window_id_must_exist_and_agree_with_the_selectors_window_part() { + let mut env = Fake::new(); + let c = Condition::Element { + selector: sel("name=OK"), + window: Some("99".to_string()), + appears: true, + }; + let err = run(&mut env, &c, &budget(100, 50), &never).unwrap_err(); + assert!(err.contains("no window has the id 99"), "{err}"); + + let c = Condition::Element { + selector: sel("app=other;name=OK"), + window: Some("1".to_string()), + appears: true, + }; + let err = run(&mut env, &c, &budget(100, 50), &never).unwrap_err(); + assert!(err.contains("does not name"), "{err}"); + } + + #[test] + fn a_selector_naming_two_windows_is_refused_not_guessed() { + let mut env = Fake::new(); + env.windows = Box::new(|_| { + vec![ + window("1", "notepad", "a", true), + window("2", "notepad", "b", false), + ] + }); + let c = appears("app=notepad;name=OK"); + let err = run(&mut env, &c, &budget(100, 50), &never).unwrap_err(); + assert!(err.contains("matches 2 windows"), "{err}"); + } + + #[test] + fn a_selector_with_no_control_part_is_not_an_element_wait() { + let mut env = Fake::new(); + let err = run(&mut env, &appears("app=notepad"), &budget(100, 50), &never).unwrap_err(); + assert!(err.contains("names no control"), "{err}"); + assert_eq!(env.trees, 0); + } + + // ---- windows ---- + + #[test] + fn a_window_with_the_title_is_waited_for_and_listed_when_it_comes() { + let mut env = Fake::new(); + env.windows = Box::new(|t| { + let mut w = vec![window("1", "app", "Doc", true)]; + if t >= 300 { + w.push(window("2", "app", "Save As", false)); + } + w + }); + let c = Condition::Window { + selector: sel("window~=save as"), + appears: true, + }; + let out = run(&mut env, &c, &budget(5_000, 250), &never).unwrap(); + match out { + Outcome::Met { + found: Found::Windows(ws), + waited_ms, + .. + } => { + assert_eq!(ws.len(), 1); + assert_eq!(ws[0].id, "2"); + assert_eq!(waited_ms, 500); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn a_window_that_closes_is_waited_for_and_one_already_gone_returns_at_once() { + let mut env = Fake::new(); + env.windows = Box::new(|t| { + if t < 400 { + vec![window("2", "app", "Save As", false)] + } else { + vec![] + } + }); + let c = Condition::Window { + selector: sel("window=Save As"), + appears: false, + }; + let out = run(&mut env, &c, &budget(5_000, 250), &never).unwrap(); + assert!(matches!( + out, + Outcome::Met { + found: Found::WindowGone, + waited_ms: 500, + .. + } + )); + let out = run(&mut env, &c, &budget(5_000, 250), &never).unwrap(); + assert!(matches!( + out, + Outcome::Met { + found: Found::WindowGone, + polls: 1, + .. + } + )); + } + + #[test] + fn a_window_wait_takes_only_window_keys() { + let mut env = Fake::new(); + let c = Condition::Window { + selector: sel("app=x;role=button"), + appears: true, + }; + assert!(run(&mut env, &c, &budget(100, 50), &never) + .unwrap_err() + .contains("asks about a window")); + } + + // ---- the screen ---- + + fn screen(baseline: Option<u128>, tolerance: u32) -> Condition { + Condition::Screen { + baseline: baseline.map(Fingerprint), + tolerance, + window: None, + } + } + + #[test] + fn the_screen_is_compared_with_how_it_was_when_the_wait_began() { + let mut env = Fake::new(); + // Changes by 8 bits at 800. + env.screen = Box::new(|t| if t >= 800 { 0xff } else { 0 }); + let c = screen(None, 4); + let out = run(&mut env, &c, &budget(5_000, 250), &never).unwrap(); + match out { + Outcome::Met { + found: + Found::Screen { + distance, + baseline, + hash, + .. + }, + waited_ms, + .. + } => { + assert_eq!(distance, 8); + assert_eq!(baseline, Fingerprint(0)); + assert_eq!(hash, Fingerprint(0xff)); + assert_eq!(waited_ms, 1_000); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn a_change_within_the_tolerance_is_not_a_change() { + let mut env = Fake::new(); + env.screen = Box::new(|t| if t >= 300 { 0b1111 } else { 0 }); + let c = screen(None, 4); + let out = run(&mut env, &c, &budget(1_000, 250), &never).unwrap(); + match out { + Outcome::TimedOut { hash, last, .. } => { + assert_eq!( + hash, + Some(Fingerprint(0b1111)), + "the last picture is returned" + ); + assert!(last.contains("4 bit(s)"), "{last}"); + } + other => panic!("4 bits is within a tolerance of 4: {other:?}"), + } + } + + #[test] + fn a_baseline_from_an_earlier_call_is_compared_at_the_first_look() { + let mut env = Fake::new(); + env.screen = Box::new(|_| 0xffff); + let out = run(&mut env, &screen(Some(0), 4), &budget(5_000, 250), &never).unwrap(); + assert!(matches!( + out, + Outcome::Met { + waited_ms: 0, + polls: 1, + .. + } + )); + } + + #[test] + fn a_picture_that_takes_time_to_take_counts_against_the_timeout() { + let mut env = Fake::new(); + env.cost = 300; + env.screen = Box::new(|_| 0); + let out = run(&mut env, &screen(None, 4), &budget(1_000, 250), &never).unwrap(); + match out { + Outcome::TimedOut { waited_ms, .. } => { + // The last look starts at the deadline and takes its 300. + assert!((1_000..=1_300).contains(&waited_ms), "{waited_ms}") + } + other => panic!("{other:?}"), + } + } + + // ---- the reply ---- + + #[test] + fn a_found_element_reports_its_window_its_selector_and_the_bounds_used() { + let mut env = Fake::new(); + env.elements = Box::new(|_| vec![button(7, "Save")]); + let c = appears("name=Save"); + let b = Budget::new(Some(10 * MAX_TIMEOUT_MS), Some(1)); + let out = run(&mut env, &c, &b, &never).unwrap(); + let json = out.to_json(&c, &b); + assert_eq!(json["result"], "found"); + assert_eq!(json["condition"], "element_appears"); + assert_eq!(json["element"]["role"], "button"); + assert_eq!(json["element"]["name"], "Save"); + assert_eq!(json["element"]["selector"], "role=button;name=Save"); + assert_eq!(json["window"]["id"], "1"); + assert_eq!(json["window"]["selector"], "app=app;window=Doc"); + assert_eq!(json["unique"], true); + assert_eq!(json["timeout_ms"], MAX_TIMEOUT_MS); + assert_eq!(json["poll_ms"], MIN_POLL_MS); + assert_eq!(json["notes"].as_array().unwrap().len(), 2); + } + + #[test] + fn a_stopped_wait_says_so() { + let mut env = Fake::new(); + let c = appears("name=Save"); + let out = run(&mut env, &c, &budget(1_000, 250), &|| true).unwrap(); + assert!(matches!(out, Outcome::Stopped { waited_ms: 0 })); + assert_eq!(out.to_json(&c, &budget(1_000, 250))["result"], "stopped"); + } +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9186f4f..e6fe916 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1059,7 +1059,7 @@ was not probed. `team_read` are toolbox tools (`toolbox::team`, over `team_mcp::Server`), and the separate server is no longer handed to a bridged model. It is kept for codex, which still reaches the channel through `mcp_overrides_for`. -- The desktop: **fixed (phase 6b)** — its sixteen tools are toolbox tools, reached +- The desktop: **fixed (phase 6b)** — its seventeen tools are toolbox tools, reached through the bridge; `incode --desktop-mcp` is now only their executor, started by the toolbox. No desktop server is handed to the CLI. See §5b, "The desktop". - The engine's session-capture hooks: **fixed (phase 4)** — incode fires them itself on @@ -1117,7 +1117,7 @@ for a patch). The ACP kind is `acp_agent::present::tool_kind`. | `WebFetch` | observe | fetch | 20 s, 512 KiB read, 20 000 chars shown; rendered: 20 s + 10 s kill grace, 2 MiB read | private addresses refused by name and by resolution; carries no credential; an HTML answer becomes markdown through Read's HTML conversion (readability when the page reads as an article, the whole page less nav/scripts otherwise), anything else is returned as sent. Where the headless browser is found: `render_js` runs the page's script first, and a page that comes back as an app shell (script, under 200 characters of text) is rendered on its own; `render_js` is not advertised where there is no browser | | `Screenshot` | observe | other | `PAGE_DEADLINE` 60 s, load 30 s, settle 3 s; viewport 200–3840 x 200–2160 CSS px; the image bounds below | OFFERED ONLY when the headless browser is found. A public `url` (WebFetch's address rules) or an HTML/SVG `path` in the workspace; `width`/`height`, `scale` 1 or 2, `full_page` (clipped at `MAX_IMAGE_EDGE` px tall, said so). The answer is an image block. See "The headless browser" below | | `RenderPdf` | change | edit | `PDF_DEADLINE` 120 s, whole-tree kill | OFFERED ONLY when a printer is found (`pdf_renderer`, below). A public `url` or a workspace HTML/SVG `path`, to `output` (must end `.pdf`); the guards and the checkpoint see `output`; the engine's `code-edit` is told it. Printed in a private temp profile and copied into place only when it exists | -| `screen_info`, `screenshot`, `ui_tree`, `cursor_position` | observe | other | a call 120 s; the image bounds | OFFERED ONLY when the desktop is switched on (`INCODE_DESKTOP`). Executed by the desktop child — see "The desktop" below. `screenshot` answers with an image block | +| `screen_info`, `screenshot`, `ui_tree`, `cursor_position`, `wait_for` | observe | other | a call 120 s; the image bounds | OFFERED ONLY when the desktop is switched on (`INCODE_DESKTOP`). Executed by the desktop child — see "The desktop" below. `screenshot` answers with an image block | | `click_element`, `type_text`, `paste_text`, `key`, `scroll`, `drag`, `move_mouse`, `click` | run | other | a call 120 s | the same; refused read-only, asked under prompt, refused when the engine cannot decide | | `WebSearch` | observe | fetch | 15 s, 512 KiB, 8 results | provider from `admin.json` `web_search` — contract in `web::search_with` | | `Skill` | observe | other | — | | @@ -1290,7 +1290,7 @@ code names it. ### The desktop (phase 6b) -The sixteen desktop tools are toolbox tools when the desktop is switched on +The seventeen desktop tools are toolbox tools when the desktop is switched on (`INCODE_DESKTOP`; `cli::desktop_setup_for` → `Toolbox::enable_desktop`, on the bridged claude route and the API route). They are offered, schema-checked, held to the mode and the engine, and drawn as incode's own calls like every other tool, and no desktop server diff --git a/docs/COMPUTER_USE.md b/docs/COMPUTER_USE.md index d315c08..1b67d83 100644 --- a/docs/COMPUTER_USE.md +++ b/docs/COMPUTER_USE.md @@ -2,7 +2,7 @@ > **Status note, 2026-09-26.** The first line below is out of date. The Windows desktop > (now the `incord-desktop` crate, `crates/desktop`, binary `incu`; it was `src/desktop/`) -> is built, and its sixteen tools (five that observe, eleven that act) are Toolbox tools +> is built, and its seventeen tools (six that observe, eleven that act) are Toolbox tools > behind incode's gate, executed by `incode --desktop-mcp`, which is a thin wrapper over > `incu serve` (`ARCHITECTURE.md` §5b, "The desktop"). > macOS (Phase C) is built (2026-10-01, `src/desktop/macos.rs`): monitors and capture @@ -30,6 +30,10 @@ > An element id is acted on only while the control the last `ui_tree` showed there is > still there; clicks and captures never land inside a blocked window's rectangle. > +> **Third round, 2026-11-13: `wait_for` and stable selectors** (phase 2 of the +> computer-use plan; the first two things a flow run later without a model needs). Both are +> described under "Waiting and selectors" below. +> > **The invariant in 4c, as built:** one coordinate space at the tool surface — physical > pixels on Windows and X11, **points** on macOS (the space CGEvent and AXUIElement both > use), top-left origin. Each image carries its transform back to that space; on a @@ -206,7 +210,7 @@ Tools the MCP server exposes. Names are what the delegated agent sees. | `paste_text` | `text` | Sets the clipboard, sends Ctrl+V, restores the prior clipboard. **Strongly preferred over `type_text` for anything long or non-ASCII.** | | `key` | `keys: ["ctrl","c"]` | Chord. Presses in order, releases in reverse. | | `key_down` / `key_up` | `key` | For held modifiers spanning several actions. | -| `wait_for_change` | `timeout_ms`, `region?` | Polls a hash of the region. Replaces blind sleeps. | +| `wait_for_change` | `timeout_ms`, `region?` | Polls a hash of the region. Replaces blind sleeps. **Built as `wait_for` `screen_changes`, with more conditions: see "Waiting and selectors".** | ### Why `key` takes an array @@ -249,6 +253,87 @@ exhausts its context in ten actions. See section 8. --- +### Waiting and selectors (built 2026-11-13, phase 2 of the computer-use plan) + +Why these two: a flow that is replayed later with no model (`.claude/temp/computer-use-plan.md` +section 4) cannot use `screenshot_after`'s fixed sleep, and cannot use an `eN` id, which is +a position in one `ui_tree` result. It needs to wait for a state, and to name a control by +what it is. Replay itself is later; these are its two prerequisites, and each is useful to a +live agent now. + +**Selectors.** `ui_tree` prints, beside every element, `sel: <control part>`, and once at the +top `Selector window part: app=…;window=…`. `click_element`, `set_value`, `element_action` +and `wait_for` take `selector` in place of `id` (both is refused). The selector is resolved +against a fresh walk of the window **when the call is made**, and must match exactly one +control. None, or several, is an error that says which (and lists a few look-alikes with the +selector that would pick each). Nothing is ever taken to be the first or the nearest. `eN` +ids work exactly as before. + +| key | meaning | +|---|---| +| `app` | the owning program, exact, case-insensitive, a trailing `.exe` ignored | +| `window` / `window~` | window title: exact / contains (ignoring case) | +| `role` | `button`, `edit`, `text`, `list`, `listitem`, `checkbox`, `radiobutton`, `combobox`, `menu`, `menuitem`, `tab`, `tabitem`, `link`, `image`, `window`, `pane`, `tree`, `treeitem`, `table`, `slider`, `group`, `other` | +| `name` / `name~` | the control's name: exact / contains (ignoring case) | +| `aid` | the program's own id for the control, exact | +| `path` | child indices from the window down (`0.2.1`), counted among the controls `ui_tree` lists, so a wrapper it drops is not a level. A hard condition: a control that moved is not found | +| `nth` | which of several matches, from 0, in `ui_tree` order. Out of range is an error | + +`key=value` pairs joined by `;`, with no spaces around the `;`; `\\`, `\;`, `\n`, `\r` escape a +value. Unknown or repeated keys and bad numbers are refused by name. `app`/`window` pick the +window (the one named, else `window=<id>`, else the focused one); the other keys pick the +control. A window selector never matches a blocked application's window, and one naming a +blocked app is refused, so a title cannot be probed for. What `ui_tree` prints is the +**shortest** selector that is unique in that tree, in the order of what changes least: +`role`+`aid`, else `role`+`name`, else those plus `nth`. A name longer than 60 characters is +written as its first 60, as a "contains". On macOS an edit field's name IS its content +(`AXValue`), so its name is never used. `selector_with_path` (for a recorder) adds the +`path` as well. + +| platform | role / name | `aid` source | notes | +|---|---|---|---| +| Windows (UIA) | yes | `AutomationId` | measured: selector round trip over a real window's whole tree | +| macOS (AX) | yes | `AXIdentifier` | compiled and clippy-clean for `aarch64-apple-darwin`; **not run on a Mac** | +| Linux (AT-SPI) | yes | `AccessibleId` property (AT-SPI 2.46+; empty on older registries and toolkits that set none) | compiled for `x86_64-unknown-linux-musl`; **not run on a bus**. X11-only windows (no accessibility bridge: xterm) have no controls to select: `wait_for` can still wait on their *window* and on the *screen* | + +**`wait_for`** (observing tier: it looks and changes nothing, so a read-only session may use +it). `condition` is one of: + +| condition | holds when | +|---|---| +| `element_appears` | at least one control matches the selector (`matches` > 1 is reported, not resolved) | +| `element_gone` | none does, or its window is gone. Not satisfied by a **cut** tree: absence from a partial walk is not absence | +| `window_appears` / `window_gone` | a window matches `app`/`window` (only those keys) | +| `screen_changes` | a 128-bit fingerprint of the screen (or of `window`'s area) differs by more than `tolerance` bits (default 4) from `baseline`, else from how it was when the wait began. A difference hash (edges) beside a 4x4 brightness grid, so a dark-mode flip is seen as well as a dialog opening, and a blinking cursor is not | + +For an element condition with no window given, the window in front **when the wait begins** +is the one watched, by id, so a change of focus is not mistaken for the control going away. + +*Bounds.* `timeout_ms` default 5000, held to **60000** (a longer request is held to it and the +reply's `notes` says so); `poll_ms` default 250, held to 50 to 2000. The first look is at once; +no look starts after the deadline, and the last starts exactly at it, so a condition that +comes true at the deadline is caught; a look under way is not interrupted, so the call can end +later than `timeout_ms` by one look (listing the windows takes a few hundred ms on a busy +Windows desktop, measured). The operator's stop (stop file or `Safety`) is asked between +looks and every 100 ms of sleeping, and ends the wait with the stop's refusal. + +*Reply.* JSON whose `result` is `found` or `timeout` (a timeout is a reply, not a tool error, +so the journal does not call it a refusal): `condition`, `waited_ms`, `polls`, the bounds +used, and for `found` an `element` (`role`, `name`, `rect`, `enabled`, a `selector` for it), +its `window` (with the window selector part), `matches`, `unique` and `others`; for +`screen_changes`, `hash`, `baseline`, `distance` (pass `hash` back as `baseline` to go on from +there; a timeout returns the last `hash` too); for a `timeout`, `last_seen` in words. + +*Guards.* Each look goes through the same guards as the one-shot tools: a blocked +application's window is never listed or walked, and a full capture is refused while one is on +screen (`guard_capture`, shared with `screenshot`). The journal records a wait as +`condition selector timeout`, and a selector with the name of a text field left out. + +Code: `crates/desktop/src/selector.rs` (grammar, matching, derivation; pure), `wait.rs` (the +timing and the conditions, over an `Env` trait; tested with a scripted desktop and a virtual +clock), `fingerprint.rs`, `uia/resolve.rs` (resolving against the live tree), +`mcp/wait_tool.rs` (the live `Env`). + ## 4. Coordinates — the section that decides whether this works This is where implementations fail, and they fail *silently*: the click succeeds, it @@ -487,9 +572,9 @@ argv (`incu` also takes `--tier observe|act`). | Mode | The desktop it gets | |---|---| -| `read-only` | `screen_info`, `cursor_position`, `screenshot`, `ui_tree`, `list_windows`. Nothing that moves. | -| `workspace-write` (default) | all sixteen — unchanged from before | -| `danger-full-access`, `allow` | all sixteen | +| `read-only` | `screen_info`, `cursor_position`, `screenshot`, `ui_tree`, `list_windows`, `wait_for`. Nothing that moves. | +| `workspace-write` (default) | all seventeen | +| `danger-full-access`, `allow` | all seventeen | | `prompt` | nothing. stdin and stdout are the protocol, so there is no channel to ask a human on, and `authorize` with no prompter denies. | The acting tools require `WorkspaceWrite` rather than `DangerFullAccess`, even diff --git a/docs/incord-memory-integration.md b/docs/incord-memory-integration.md index ffd811d..869c8b9 100644 --- a/docs/incord-memory-integration.md +++ b/docs/incord-memory-integration.md @@ -90,7 +90,7 @@ may hear a session twice — once from incode, once from the CLI under the CLI's (Closed for codex in phase 8: it runs with `--disable hooks`. Still true for agy.) **Phase 6b (2026-09-25).** Three more kinds of call now pass the engine's pre-tool hook, -because they are Toolbox tools: the desktop's sixteen tools (previously run in +because they are Toolbox tools: the desktop's seventeen tools (previously run in `incode --desktop-mcp` with only the desktop's own hooks, which still run there as a second line), `Screenshot` (observe, fail open) and `RenderPdf` (change, fail closed; its OUTPUT path is in `tool_input.paths` and is reported to `code-edit` once written). A diff --git a/src/desktop.rs b/src/desktop.rs index 1c9bd2a..c7abe15 100644 --- a/src/desktop.rs +++ b/src/desktop.rs @@ -329,11 +329,27 @@ mod tests { /// **The crate's tier and incode's policy must say the same thing.** The /// server now runs the tier; `PermissionPolicy::for_desktop` stays as incode's /// statement of the same table, and this is what keeps the two from - /// drifting: every mode, every one of the 16 tools, the same verdict and the + /// drifting: every mode, every one of the 17 tools, the same verdict and the /// same words in a refusal. #[test] fn the_crates_tier_agrees_with_the_desktop_permission_policy() { use crate::permissions::{PermissionOutcome, PermissionPolicy}; + // The tables walked below are the tools the server offers: one added + // to the server and to neither table would be compared with nothing. + let offered: Vec<String> = incord_desktop::mcp::tool_definitions() + .iter() + .filter_map(|t| t["name"].as_str().map(str::to_string)) + .collect(); + assert_eq!(offered.len(), 17, "{offered:?}"); + for name in &offered { + assert!( + permission::OBSERVING.contains(&name.as_str()) + || permission::ACTING.contains(&name.as_str()), + "{name} is offered by the desktop server but has no tier" + ); + } + // `wait_for` only looks: a read-only session may use it. + assert!(permission::OBSERVING.contains(&"wait_for")); let modes = [ PermissionMode::ReadOnly, PermissionMode::WorkspaceWrite, diff --git a/src/permissions/mod.rs b/src/permissions/mod.rs index 8160ebb..5366269 100644 --- a/src/permissions/mod.rs +++ b/src/permissions/mod.rs @@ -196,7 +196,7 @@ impl PermissionPolicy { self } - /// The requirements for the sixteen tools of the desktop MCP server: five + /// The requirements for the seventeen tools of the desktop MCP server: six /// that observe and eleven that move something. /// /// The server is the `incord-desktop` crate now, and it runs its own @@ -220,7 +220,7 @@ impl PermissionPolicy { /// /// # The split, and why it is only two rungs /// - /// The five observing tools need `ReadOnly`; the eleven that move something + /// The six observing tools need `ReadOnly`; the eleven that move something /// need `WorkspaceWrite`. Not `DangerFullAccess`, even though a click /// reaches far outside any workspace — because `workspace-write` is the /// default mode, and putting the desktop above it would mean the feature @@ -245,6 +245,9 @@ impl PermissionPolicy { .with_tool_requirement("screenshot", PermissionMode::ReadOnly) .with_tool_requirement("ui_tree", PermissionMode::ReadOnly) .with_tool_requirement("list_windows", PermissionMode::ReadOnly) + // Waits for a control, a window or a change of the screen: it + // looks and changes nothing. + .with_tool_requirement("wait_for", PermissionMode::ReadOnly) // Acting. Each one changes the state of a machine the operator is // signed into. .with_tool_requirement("move_mouse", PermissionMode::WorkspaceWrite) diff --git a/src/toolbox/desktop.rs b/src/toolbox/desktop.rs index 3cf5b57..a15c9b5 100644 --- a/src/toolbox/desktop.rs +++ b/src/toolbox/desktop.rs @@ -40,6 +40,9 @@ pub(super) const OBSERVING: &[&str] = &[ "ui_tree", "cursor_position", "list_windows", + // Polls up to its own cap of 60 s (`incord_desktop::wait`), well inside + // `CALL_BOUND`. + "wait_for", ]; /// The desktop tools that act on the machine. pub(super) const ACTING: &[&str] = &[ From ebccc5a35892999cf46e8360329e0ca7696cf2a5 Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:28:36 -0600 Subject: [PATCH 2/9] desktop: fix four review findings in selectors and wait_for 1. A blocked application's window named by id with a selector was described in the "does not name" error (its title), and a match and a mismatch gave different refusals. window_for now takes the blocked guard and runs it before the selector's window part is compared, with the id path's own words. Tests on the selector path. 2. Errors are journalled word for word, so none quotes what it was given. Parse errors name the item by position; a selector in an error is shown redacted (resolve, require_*, the "is disabled" refusal); ambiguity examples name a text field by its place on every platform, not by its name. Tests. 3. nth/path were counted in the walk ui_tree was asked for but resolved in a deeper one, so nth=1 could be another control. Selectors gain `depth`: ui_tree stamps it when asked for a non-default depth and the resolver (and wait_for) walks as deep as the selector says, else to the default depth that ui_tree uses without being asked. Tests over several depths with a button nested below the default. 4. A derived name cut at a capital sigma lowercased to a different letter (final sigma) than inside the whole name and missed its own element. Contains now folds case one character at a time. Test. --- README.md | 4 +- crates/desktop/src/mcp/selector_tests.rs | 107 ++++++- crates/desktop/src/mcp/tools.rs | 54 +++- crates/desktop/src/mcp/wait_tool.rs | 4 +- crates/desktop/src/selector.rs | 222 ++++++++++++--- crates/desktop/src/uia.rs | 20 +- crates/desktop/src/uia/resolve.rs | 339 ++++++++++++++++++++--- crates/desktop/src/wait.rs | 28 +- docs/COMPUTER_USE.md | 8 +- 9 files changed, 688 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 27327cd..2f7f781 100644 --- a/README.md +++ b/README.md @@ -1111,7 +1111,9 @@ app=notepad;window~=Save As;role=button;name=Save in another window Keys: `app`, `window` / `window~` (exact / contains), `role`, `name` / `name~`, `aid` (UIA `AutomationId`, AX `AXIdentifier`, AT-SPI `AccessibleId`), `path` -(child indices, `0.2.1`), `nth`. `;` and `\` in a value are written `\;` and +(child indices, `0.2.1`), `nth`, `depth` (how deep to walk; `ui_tree` writes it +when asked for a depth other than the default, so `nth` counts the same +controls when resolved). `;` and `\` in a value are written `\;` and `\\`. `wait_for` takes `condition` (`element_appears`, `element_gone`, `window_appears`, `window_gone`, `screen_changes`), a `selector` (window conditions take only `app`/`window`), `timeout_ms` (default 5000, held to diff --git a/crates/desktop/src/mcp/selector_tests.rs b/crates/desktop/src/mcp/selector_tests.rs index 8c38063..658f209 100644 --- a/crates/desktop/src/mcp/selector_tests.rs +++ b/crates/desktop/src/mcp/selector_tests.rs @@ -160,6 +160,111 @@ fn the_journal_names_the_selector_but_never_a_text_fields_name() { assert!(!bad.contains("hunter2"), "{bad}"); } +fn window(app: &str, title: &str) -> crate::uia::WindowInfo { + crate::uia::WindowInfo { + id: "9".to_string(), + title: title.to_string(), + app: app.to_string(), + pid: 1, + focused: false, + rect: crate::Rect { + x: 0, + y: 0, + width: 10, + height: 10, + }, + has_tree: true, + } +} + +/// **A blocked application's window, named by id with a selector, is refused +/// by the same words the id path gives — before the selector is read.** The +/// guard here is the real `refuse_blocked` that `click_element`, `set_value` +/// and `element_action` hand to `window_for`. +#[test] +fn a_blocked_window_named_by_id_is_refused_as_the_id_path_refuses_it() { + use super::guard::refuse_blocked; + let keepass = || window("KeePassXC", "Bank login - entry"); + let the_id_path = refuse_blocked(&keepass()).unwrap_err(); + assert!(!the_id_path.contains("Bank login"), "{the_id_path}"); + for selector in [ + "window=x;role=button", + "window~=bank;role=button", + "app=notepad;role=button", + "role=button;name=Save", + ] { + let selector = crate::selector::Selector::parse(selector).unwrap(); + let said = crate::uia::confirm_window(keepass(), &selector, &refuse_blocked) + .unwrap_err() + .to_string(); + assert_eq!(said, the_id_path, "`{}`", selector.render()); + } + // And a window that is not blocked still reaches the selector's check. + let selector = crate::selector::Selector::parse("window=x;role=button").unwrap(); + let said = crate::uia::confirm_window(window("notepad", "notes"), &selector, &refuse_blocked) + .unwrap_err() + .to_string(); + assert!(said.contains("does not name"), "{said}"); +} + +#[test] +fn what_a_selector_error_says_is_never_what_was_typed() { + // The tool's own arguments: a stray word with no `=` is the case a + // secret typed into the wrong field would come in as. + for raw in [ + "role=edit;hunter2", + "hunter2=1", + "name=hunter2;nth=zz", + "role=hunter2", + ] { + let why = element_target(&json!({"selector": raw})).unwrap_err(); + assert!(!why.contains("hunter2"), "`{raw}` was quoted: {why}"); + } + // A selector that parses but names no control: shown redacted. + let why = element_target(&json!({"selector": "name=hunter2;nth=1;app=notes"})) + .map(|_| String::new()) + .unwrap_or_else(|e| e); + assert!(!why.contains("hunter2"), "{why}"); +} + +/// The "is disabled" refusal is journalled word for word. +#[test] +fn a_disabled_control_is_refused_without_naming_a_selected_text_field() { + use super::tools::disabled_error; + use crate::uia::{Element, Role}; + let element = |role, name: &str| Element { + id: 7, + depth: 1, + role, + name: name.to_string(), + rect: crate::Rect { + x: 0, + y: 0, + width: 10, + height: 10, + }, + enabled: false, + secret: false, + automation_id: String::new(), + }; + // By id, the message is what it always was. + assert_eq!( + disabled_error(&Target::Id(7), &element(Role::Button, "Save")), + "e7 \"Save\" is disabled; clicking it would do nothing" + ); + // By selector: redacted, and the control's own name is not repeated. + let typed = crate::selector::Selector::parse("role=edit;name=hunter2").unwrap(); + let why = disabled_error(&Target::Selector(typed), &element(Role::Edit, "hunter2")); + assert!(!why.contains("hunter2"), "{why}"); + assert!(why.contains("is disabled"), "{why}"); + let untyped = crate::selector::Selector::parse("name=hunter2").unwrap(); + let why = disabled_error(&Target::Selector(untyped), &element(Role::Edit, "hunter2")); + assert!(!why.contains("hunter2"), "{why}"); + let button = crate::selector::Selector::parse("role=button;name=Save").unwrap(); + let why = disabled_error(&Target::Selector(button), &element(Role::Button, "Save")); + assert!(why.contains("role=button;name=Save"), "{why}"); +} + #[test] fn wait_for_is_journalled_by_condition_selector_and_timeout() { let said = Server::describe( @@ -378,7 +483,7 @@ mod live { if crate::limits::get().blocked(&tree.app).is_some() { return; } - let selectors = crate::selector::selectors_for(&tree.elements, uia::NAMES_ARE_VALUES); + let selectors = crate::selector::selectors_for(&tree.elements, uia::NAMES_ARE_VALUES, None); for (e, s) in tree.elements.iter().zip(&selectors) { let again = crate::selector::Selector::parse(&s.render()).unwrap_or_else(|why| { panic!("`{}` for {e:?} does not read back: {why}", s.render()) diff --git a/crates/desktop/src/mcp/tools.rs b/crates/desktop/src/mcp/tools.rs index 173b65b..c57357f 100644 --- a/crates/desktop/src/mcp/tools.rs +++ b/crates/desktop/src/mcp/tools.rs @@ -249,9 +249,12 @@ fn ui_tree(args: &Value) -> Result<Vec<Value>, String> { // With no displays readable the tree is still shown, unmarked: // reading it is harmless, and every action re-checks its point. let screens = screen_rects().unwrap_or_default(); + // The depth the walk really went to (held to its bound), because the + // selectors printed are counted in that walk and say so when it is not + // the default one. Ok(vec![text(format!( "{}\n{UNTRUSTED}", - tree.render_on(&screens) + tree.render_at(&screens, uia::bounded(depth)) ))]) } @@ -270,36 +273,34 @@ fn click_element(input: &InputThread, args: &Value) -> Result<Vec<Value>, String // A selector is resolved against the live tree here, the same way: its // one match (or an error saying none or several) is the control as it is // now. - let (element, by, label, cut) = match target { + let (element, label, cut) = match &target { Target::Id(id) => { let info = uia::window_info(window.as_deref()).map_err(|e| e.to_string())?; refuse_blocked(&info)?; - let element = live_element(&info, id)?; + let element = live_element(&info, *id)?; let label = element.label(); - (element, format!("e{id}"), label, String::new()) + (element, label, String::new()) } Target::Selector(selector) => { - let found = resolve_selector(window.as_deref(), &selector)?; + let found = resolve_selector(window.as_deref(), selector)?; if let Some(why) = &found.no_positions { return Err(format!( "{why} so the control cannot be found on screen; use element_action or \ set_value on it" )); } - let by = format!("selector `{}`", selector.render()); let label = format!( - "{by} (\"{}\", {:?})", - found.element.name, found.element.role + "selector `{}` (\"{}\", {:?})", + selector.render_redacted(), + found.element.name, + found.element.role ); let cut = with_cut_note(String::new(), &found); - (found.element, by, label, cut) + (found.element, label, cut) } }; if !element.enabled { - return Err(format!( - "{by} \"{}\" is disabled; clicking it would do nothing", - element.name - )); + return Err(disabled_error(&target, &element)); } let at = element.click_point(); @@ -572,12 +573,35 @@ pub(super) fn element_target(args: &Value) -> Result<Target, String> { /// The one control `selector` names, in the window it names (or `window`, or /// the focused one), refused before anything is read when that window is a /// blocked application's. +/// +/// The blocked-application refusal is the window guard handed to +/// `uia::window_for`, so it comes BEFORE the selector's window part is +/// compared with a window named by id: a mismatch error describes the window, +/// and for a blocked one that is its title, and a match and a mismatch would +/// be told apart. fn resolve_selector(window: Option<&str>, selector: &Selector) -> Result<uia::Resolved, String> { - let info = uia::window_for(window, selector).map_err(|e| e.to_string())?; - refuse_blocked(&info)?; + let info = uia::window_for(window, selector, &refuse_blocked).map_err(|e| e.to_string())?; uia::resolve_in(info, selector).map_err(|e| e.to_string()) } +/// Why clicking a disabled control is refused. The id path names the control +/// as it always has. A selector is named redacted and the control is not named +/// at all: this error is journalled word for word, and a text field's name is +/// what was typed in it. +pub(super) fn disabled_error(target: &Target, element: &uia::Element) -> String { + match target { + Target::Id(id) => format!( + "e{id} \"{}\" is disabled; clicking it would do nothing", + element.name + ), + Target::Selector(selector) => format!( + "selector `{}` ({:?}) is disabled; clicking it would do nothing", + selector.render_redacted(), + element.role + ), + } +} + /// `done`, with what only a selector adds: its `eN` numbers this call's own /// walk and not the agent's last `ui_tree`, and a cut tree cannot rule out a /// second match. diff --git a/crates/desktop/src/mcp/wait_tool.rs b/crates/desktop/src/mcp/wait_tool.rs index e0bfa17..dddbe56 100644 --- a/crates/desktop/src/mcp/wait_tool.rs +++ b/crates/desktop/src/mcp/wait_tool.rs @@ -63,10 +63,10 @@ impl Env for Live { Ok(window) } - fn tree(&mut self, window: &str) -> Result<Tree, String> { + fn tree(&mut self, window: &str, depth: Option<u32>) -> Result<Tree, String> { let info = uia::window_info(Some(window)).map_err(text_of)?; refuse_blocked(&info)?; - let tree = uia::tree_now(window).map_err(text_of)?; + let tree = uia::tree_now(window, depth).map_err(text_of)?; // Walked in this process, and discarded unread when blocked. refuse_app(&tree.app)?; Ok(tree) diff --git a/crates/desktop/src/selector.rs b/crates/desktop/src/selector.rs index fd75c22..c1a4bf3 100644 --- a/crates/desktop/src/selector.rs +++ b/crates/desktop/src/selector.rs @@ -29,6 +29,7 @@ //! | `aid` | the automation id: UIA `AutomationId`, AX `AXIdentifier`, AT-SPI `AccessibleId` | //! | `path` | child indices from the window down, dot separated (`0.2.1`), among the controls `ui_tree` lists | //! | `nth` | which of several matches, from 0, in `ui_tree` order | +//! | `depth` | how deep the window is walked (default 12). `ui_tree` writes it into each selector when it was asked for another depth, so `nth` and `path` count the same controls when resolved | //! //! A value escapes `\` as `\\`, `;` as `\;` and a line break as `\n` / `\r`. //! `app` and `window` choose the WINDOW; the rest choose the CONTROL inside it. @@ -95,11 +96,24 @@ pub enum Text { Contains(String), } +/// `text` with case ignored, one character at a time. +/// +/// Not `str::to_lowercase`: that reads the text around a character, and a +/// capital sigma becomes `ς` at the end of a word and `σ` elsewhere. A name +/// cut off at a sigma (a derived selector keeps only the first +/// [`LONGEST_NAME`] characters) would then lowercase differently from the same +/// letters inside the whole name, and the selector would miss its own element. +/// Folding each character on its own gives the same letters wherever the text +/// is cut. +fn fold(text: &str) -> String { + text.chars().flat_map(char::to_lowercase).collect() +} + impl Text { fn matches(&self, actual: &str) -> bool { match self { Text::Exact(want) => actual == want, - Text::Contains(part) => actual.to_lowercase().contains(&part.to_lowercase()), + Text::Contains(part) => fold(actual).contains(&fold(part)), } } } @@ -114,6 +128,13 @@ pub struct Selector { pub automation_id: Option<String>, pub path: Option<Vec<u32>>, pub nth: Option<u32>, + /// How deep the window's tree is walked to find the control. `nth` and + /// `path` count the controls of THAT walk, so a selector derived from a + /// walk of one depth must be resolved by a walk of the same depth, or + /// `nth=1` is a different control. Absent: the default depth + /// (`desktop_limits.tree_depth`, 12). `ui_tree` writes it into each + /// selector when it was asked for another depth. + pub depth: Option<u32>, } /// Names longer than this are written into a derived selector as their first @@ -184,12 +205,12 @@ fn unescape(raw: &str) -> Result<String, String> { Some(';') => out.push(';'), Some('n') => out.push('\n'), Some('r') => out.push('\r'), - Some(other) => { - return Err(format!( - "unknown escape `\\{other}` (only `\\\\`, `\\;`, `\\n` and `\\r` exist)" - )) + Some(_) => { + return Err( + "has an unknown escape (only `\\\\`, `\\;`, `\\n` and `\\r` exist)".to_string(), + ) } - None => return Err("a value ends in a lone `\\`".to_string()), + None => return Err("ends in a lone `\\`".to_string()), } } Ok(out) @@ -228,21 +249,26 @@ fn once<T>(slot: &mut Option<T>, key: &str, value: T) -> Result<(), String> { impl Selector { /// Read the text form. Strict: an unknown key, a repeated one, a bad number - /// or role is an error that names it, never skipped. + /// or role is an error, never skipped. + /// + /// **An error never quotes what it was given.** It names the item by its + /// position (`item 2`) and says what is wrong with it, because the text of + /// a selector may be a typed value (`role=edit;hunter2`), and an error is + /// journalled word for word (`mcp/gate.rs`, `record_call`). Only the + /// fixed names of this grammar appear in one. pub fn parse(raw: &str) -> Result<Selector, String> { + const KEYS: &str = "app, window, window~, role, name, name~, aid, path, nth, depth"; let mut s = Selector::default(); - for piece in split_pairs(raw) { - if piece.trim().is_empty() { - continue; - } + let items = split_pairs(raw) + .into_iter() + .filter(|piece| !piece.trim().is_empty()); + for (at, piece) in items.enumerate() { + let item = at + 1; let Some((key, value)) = piece.split_once('=') else { - return Err(format!( - "`{piece}` is not key=value (keys: app, window, window~, role, name, \ - name~, aid, path, nth)" - )); + return Err(format!("item {item} is not key=value (keys: {KEYS})")); }; let key = key.trim(); - let value = unescape(value).map_err(|e| format!("in `{key}`: {e}"))?; + let value = unescape(value).map_err(|e| format!("item {item}: the value {e}"))?; match key { "app" => once(&mut s.app, key, value)?, "window" => once(&mut s.window, key, Text::Exact(value))?, @@ -250,7 +276,7 @@ impl Selector { "role" => { let role = Role::from_word(&value).ok_or_else(|| { format!( - "unknown role `{value}` (roles: {})", + "item {item}: the role is not one of {}", ROLES.iter().map(|(w, _)| *w).collect::<Vec<_>>().join(", ") ) })?; @@ -264,22 +290,26 @@ impl Selector { .split('.') .map(|n| n.trim().parse::<u32>()) .collect::<Result<Vec<_>, _>>() - .map_err(|_| format!("path `{value}` must be child indices like 0.2.1"))?; + .map_err(|_| { + format!("item {item}: path must be child indices like 0.2.1") + })?; once(&mut s.path, key, path)?; } "nth" => { let nth = value .trim() .parse::<u32>() - .map_err(|_| format!("nth `{value}` must be a whole number from 0"))?; + .map_err(|_| format!("item {item}: nth must be a whole number from 0"))?; once(&mut s.nth, key, nth)?; } - other => { - return Err(format!( - "unknown key `{other}` (keys: app, window, window~, role, name, \ - name~, aid, path, nth)" - )) + "depth" => { + let depth = value + .trim() + .parse::<u32>() + .map_err(|_| format!("item {item}: depth must be a whole number"))?; + once(&mut s.depth, key, depth)?; } + _ => return Err(format!("item {item} has an unknown key (keys: {KEYS})")), } } if s == Selector::default() { @@ -345,6 +375,9 @@ impl Selector { if let Some(nth) = self.nth { parts.push(format!("nth={nth}")); } + if let Some(depth) = self.depth { + parts.push(format!("depth={depth}")); + } } /// The selector as a journal may carry it. A control's name is left out @@ -380,13 +413,16 @@ impl Selector { /// Fit for acting on a control: it names one. `nth` alone is "the third of /// anything", which is a guess. + /// + /// These errors, like every one that carries a selector, show it as + /// [`Self::render_redacted`]: an error is journalled word for word. pub fn require_element(&self) -> Result<(), String> { if self.has_element_part() { Ok(()) } else { Err(format!( "selector `{}` names no control: give at least one of role, name, aid, path", - self.render() + self.render_redacted() )) } } @@ -396,14 +432,14 @@ impl Selector { if !self.has_window_part() { return Err(format!( "selector `{}` names no window: give app and/or window (or window~)", - self.render() + self.render_redacted() )); } - if self.has_element_part() || self.nth.is_some() { + if self.has_element_part() || self.nth.is_some() || self.depth.is_some() { return Err(format!( "selector `{}` names a control too, but this asks about a window: use only \ app and window", - self.render() + self.render_redacted() )); } Ok(()) @@ -470,10 +506,14 @@ impl Selector { (0, _) => return Err(SelectError::Missing), (1, None) => found[0], (count, None) => { + // The examples are in an error, which is journalled, so a text + // field is written by its place and never by its name: on + // macOS the name IS what was typed there (`names_are_values`, + // here on every platform because an example costs nothing). let examples = found .iter() .take(EXAMPLES) - .map(|&i| selector_for(elements, i, false).render_element_part()) + .map(|&i| selector_for(elements, i, true).render_element_part()) .collect(); return Err(SelectError::Ambiguous { count, examples }); } @@ -591,10 +631,22 @@ impl Selector { } /// [`selector_for`] every element: what `ui_tree` prints beside each. +/// +/// `depth` is how deep the walk that produced `elements` went, when that is +/// not the default one: it is written into every selector, because `nth` and +/// `path` count the controls of that walk and the resolver must walk as deep +/// to count the same ones (see [`Selector::depth`]). #[must_use] -pub fn selectors_for(elements: &[Element], names_are_values: bool) -> Vec<Selector> { +pub fn selectors_for( + elements: &[Element], + names_are_values: bool, + depth: Option<u32>, +) -> Vec<Selector> { (0..elements.len()) - .map(|i| selector_for(elements, i, names_are_values)) + .map(|i| Selector { + depth, + ..selector_for(elements, i, names_are_values) + }) .collect() } @@ -717,17 +769,119 @@ mod tests { let bad = |t: &str| Selector::parse(t).unwrap_err(); assert!(bad("").contains("empty")); assert!(bad(";;").contains("empty")); - assert!(bad("colour=red").contains("unknown key `colour`")); - assert!(bad("role=gizmo").contains("unknown role `gizmo`")); + assert!(bad("colour=red").contains("item 1 has an unknown key")); + assert!(bad("role=gizmo").contains("item 1: the role is not one of")); assert!(bad("role=button;role=edit").contains("twice")); assert!(bad("name=a;name~=b").contains("twice")); assert!(bad("nth=-1").contains("nth")); assert!(bad("path=1..2").contains("path")); - assert!(bad("justaword").contains("not key=value")); + assert!(bad("depth=deep").contains("depth")); + assert!(bad("justaword").contains("item 1 is not key=value")); assert!(bad("name=a\\qb").contains("unknown escape")); assert!(bad("name=a\\").contains("lone")); } + /// **An error never quotes what it was given.** A selector may carry a + /// typed value, and an error is journalled word for word. + #[test] + fn a_parse_error_names_the_position_and_never_quotes_the_text() { + for raw in [ + "role=edit;hunter2", + "hunter2=1", + "role=hunter2", + "nth=hunter2", + "path=hunter2", + "depth=hunter2", + "name=a\\hunter2", + "name=hunter2\\", + "role=edit;name=hunter2;name=hunter2", + ] { + let why = Selector::parse(raw).unwrap_err(); + assert!(!why.contains("hunter2"), "`{raw}` was quoted: {why}"); + } + let why = Selector::parse("role=edit;hunter2").unwrap_err(); + assert!(why.contains("item 2"), "{why}"); + } + + #[test] + fn the_requirement_errors_show_the_selector_redacted() { + let s = Selector::parse("role=edit;name=hunter2;nth=0;depth=3").unwrap(); + let none = Selector::parse("app=x;nth=1").unwrap(); + assert!(none.require_element().unwrap_err().contains("app=x")); + let why = s.require_window_only().unwrap_err(); + assert!(!why.contains("hunter2"), "{why}"); + let typed = Selector::parse("name=hunter2;nth=0").unwrap(); + let why = typed.require_window_only().unwrap_err(); + assert!(!why.contains("hunter2"), "{why}"); + } + + #[test] + fn an_ambiguity_error_lists_a_text_field_by_its_place_and_never_its_name() { + let tree = vec![ + el(1, 1, Role::Edit, "hunter2"), + el(2, 1, Role::Edit, "hunter3"), + el(3, 1, Role::Button, "OK"), + el(4, 1, Role::Button, "OK"), + ]; + let why = Selector::parse("role=edit") + .unwrap() + .resolve_in(&tree) + .unwrap_err() + .to_string(); + assert!(why.contains("matches 2 elements"), "{why}"); + assert!(!why.contains("hunter"), "{why}"); + assert!(why.contains("role=edit;nth=1"), "{why}"); + // A button's name is not content, and is still shown. + let why = Selector::parse("role=button") + .unwrap() + .resolve_in(&tree) + .unwrap_err() + .to_string(); + assert!(why.contains("name=OK"), "{why}"); + } + + #[test] + fn depth_is_part_of_the_text_form_and_not_a_control_part() { + let s = Selector::parse("role=button;name=OK;nth=1;depth=20").unwrap(); + assert_eq!(s.depth, Some(20)); + assert_eq!(s.render(), "role=button;name=OK;nth=1;depth=20"); + // It narrows nothing by itself. + assert!(Selector::parse("depth=20") + .unwrap() + .require_element() + .is_err()); + assert!(Selector::parse("role=button;depth=20") + .unwrap() + .require_window_only() + .is_err()); + let stamped = selectors_for(&dialog(), false, Some(20)); + assert!(stamped.iter().all(|s| s.depth == Some(20))); + let plain = selectors_for(&dialog(), false, None); + assert!(plain.iter().all(|s| s.depth.is_none())); + } + + /// A name cut off at a capital sigma lowercases to a different letter than + /// the same sigma inside the whole name: `ς` at the end of a word, `σ` + /// elsewhere. The derived selector must still find its own element. + #[test] + fn a_name_cut_at_a_sigma_still_finds_its_own_element() { + let name = format!("{}Σ{}", "a".repeat(LONGEST_NAME - 1), "bcdef"); + assert_eq!(name.chars().nth(LONGEST_NAME - 1), Some('Σ')); + let tree = vec![ + el(1, 1, Role::Text, &name), + el(2, 1, Role::Text, "something else"), + ]; + let s = selector_for(&tree, 0, false); + assert!(matches!(s.name, Some(Text::Contains(_))), "{s:?}"); + assert_eq!(s.resolve_in(&tree).unwrap().id, 1, "{}", s.render()); + // Through the text form an agent would copy. + let again = Selector::parse(&s.render()).unwrap(); + assert_eq!(again.resolve_in(&tree).unwrap().id, 1); + // And case is still ignored for the rest. + let upper = Selector::parse("name~=SOMETHING").unwrap(); + assert_eq!(upper.resolve_in(&tree).unwrap().id, 2); + } + #[test] fn every_role_has_one_word_that_reads_back() { for (word, role) in ROLES { diff --git a/crates/desktop/src/uia.rs b/crates/desktop/src/uia.rs index 17c8b86..462de04 100644 --- a/crates/desktop/src/uia.rs +++ b/crates/desktop/src/uia.rs @@ -229,6 +229,15 @@ impl Tree { /// /// Called by `mcp.rs`'s `ui_tree` with the displays `screen_info` reports. pub fn render_on(&self, screens: &[super::Rect]) -> String { + self.render_at(screens, super::limits::get().tree_depth) + } + + /// [`Self::render_on`] for a tree walked to `depth`. A selector printed + /// beside an element counts its `nth` and `path` in THIS walk, so when the + /// walk was not the default depth the selector says so (`depth=N`), and the + /// resolver then walks as deep (`resolve::resolve_depth`): otherwise it + /// would count different controls and `nth=1` could be another one. + pub fn render_at(&self, screens: &[super::Rect], depth: u32) -> String { let mut out = String::new(); out.push_str(&format!("Window \"{}\"", self.root_name)); if !self.window.is_empty() { @@ -255,7 +264,8 @@ impl Tree { by id; click_element, scroll and drag by id are refused.\n" )); } - let selectors = selectors_for(&self.elements, NAMES_ARE_VALUES); + let stamp = (depth != super::limits::get().tree_depth).then_some(depth); + let selectors = selectors_for(&self.elements, NAMES_ARE_VALUES, stamp); for (e, selector) in self.elements.iter().zip(&selectors) { out.push_str(&format!( "{:indent$}[e{}] {:<10} \"{}\" ({},{},{}x{}) sel: {}{}{}{}\n", @@ -445,7 +455,7 @@ pub(crate) fn gone(id: u32, window: &WindowInfo) -> DesktopError { } /// The depth a caller asked for, held to four times the configured default. -fn bounded(depth: u32) -> u32 { +pub(crate) fn bounded(depth: u32) -> u32 { depth.min(super::limits::get().tree_depth.saturating_mul(4)) } @@ -613,7 +623,11 @@ pub fn focus_window(window: &str) -> Result<Done, DesktopError> { // Acting on a control by selector instead of by id: `resolve.rs`. mod resolve; -pub use resolve::{act_on, resolve, resolve_in, set_value_on, tree_now, window_for, Resolved}; +pub use resolve::{act_on, resolve_in, set_value_on, tree_now, window_for, Resolved, WindowGuard}; +// The window check `window_for` makes, on a window already in hand: for the +// tests of `mcp::tools`, which hand it the real blocked-application guard. +#[cfg(test)] +pub(crate) use resolve::confirm_window; #[cfg(windows)] #[path = "uia_windows.rs"] diff --git a/crates/desktop/src/uia/resolve.rs b/crates/desktop/src/uia/resolve.rs index e750251..2d5ad0a 100644 --- a/crates/desktop/src/uia/resolve.rs +++ b/crates/desktop/src/uia/resolve.rs @@ -8,8 +8,25 @@ //! (`set_value`, `act`) and keep the check that the control in that place is //! still the one that was resolved. //! +//! # One walk for printing and for resolving +//! +//! `nth` and `path` count the controls of a walk, so the walk that prints a +//! selector (`ui_tree`, to the depth it was asked for) and the walk that +//! resolves it must be the same. The resolver walks to [`resolve_depth`]: the +//! selector's own `depth` when it has one (`ui_tree` writes it when it was +//! asked for another depth than the default), else the default depth, which is +//! the depth `ui_tree` walks without being asked. Any other pairing would let +//! `nth=1` mean one control to the tree and another to the resolver. +//! +//! # Errors are journalled +//! +//! Every error here goes to the operator's journal word for word, so a selector +//! in one is shown redacted ([`Selector::render_redacted`]): a text field's +//! name is what was typed in it. A blocked application's window is refused +//! before its title can appear in any message. +//! //! Wired to: `mcp::tools` (`click_element`, `set_value`, `element_action`) and -//! `mcp::wait_tool` (`wait_for`), through [`resolve`], [`window_for`] and +//! `mcp::wait_tool` (`wait_for`), through [`window_for`], [`resolve_in`] and //! [`tree_now`]. use super::{bounded, platform, same_control, Done, Element, Tree, Want, WindowInfo}; @@ -19,6 +36,10 @@ use crate::DesktopError; /// Windows listed in an error about a window that was not found. const LISTED: usize = 8; +/// What refuses a window before anything is said about it: the blocked +/// application check, in the words the id path gives (`mcp::guard`). +pub type WindowGuard<'a> = &'a dyn Fn(&WindowInfo) -> Result<(), String>; + /// A control found by selector, and what acting on it takes. #[derive(Debug, Clone)] pub struct Resolved { @@ -39,6 +60,13 @@ fn os(message: String) -> DesktopError { DesktopError::Os(message) } +/// How deep a selector's window is walked: its `depth`, else the default, +/// held to the bound every walk is. +#[must_use] +pub(crate) fn resolve_depth(depth: Option<u32>) -> u32 { + bounded(depth.unwrap_or(crate::limits::get().tree_depth)) +} + /// `id "title" (app)` for a list in a message. fn describe(w: &WindowInfo) -> String { format!("{} \"{}\" ({})", w.id, w.title, w.app) @@ -89,35 +117,63 @@ pub(crate) fn pick_named_window( } } +/// `info`, once `guard` has let it through and it is what the selector's +/// `app`/`window` part names. +/// +/// **The guard comes first.** A window that was named by id and does not match +/// is described in the error (`window 7 is "<title>" (<app>) ... does not +/// name`), which for a blocked application's window would put its title in the +/// message, and make "does it match `window~=a`?" an oracle on that title: a +/// different refusal for a match and for a mismatch. So a blocked window gets +/// the blocked refusal, the same in both cases, before the selector is looked +/// at. +pub(crate) fn confirm_window( + info: WindowInfo, + selector: &Selector, + guard: WindowGuard<'_>, +) -> Result<WindowInfo, DesktopError> { + guard(&info).map_err(os)?; + if selector.has_window_part() && !selector.matches_window(&info) { + return Err(os(format!( + "window {} is \"{}\" ({}), which the selector's window part `{}` does not name; \ + drop `window` or fix the selector", + info.id, + info.title, + info.app, + selector.render_window_part() + ))); + } + Ok(info) +} + /// The window a call acts in: the `window` id when one is given (and the /// selector's own `app`/`window` must then agree with it), else the one the -/// selector names, else the focused window. -pub fn window_for(window: Option<&str>, selector: &Selector) -> Result<WindowInfo, DesktopError> { +/// selector names, else the focused window. Every one of them passes `guard` +/// first. +pub fn window_for( + window: Option<&str>, + selector: &Selector, + guard: WindowGuard<'_>, +) -> Result<WindowInfo, DesktopError> { match window { - Some(id) => { - let info = platform::window_info(Some(id))?; - if selector.has_window_part() && !selector.matches_window(&info) { - return Err(os(format!( - "window {id} is \"{}\" ({}), which the selector's window part `{}` does \ - not name; drop `window` or fix the selector", - info.title, - info.app, - selector.render_window_part() - ))); - } - Ok(info) - } + Some(id) => confirm_window(platform::window_info(Some(id))?, selector, guard), None if selector.has_window_part() => { - pick_named_window(selector, platform::list_windows()?) + let info = pick_named_window(selector, platform::list_windows()?)?; + confirm_window(info, selector, guard) + } + None => { + let info = platform::window_info(None)?; + guard(&info).map_err(os)?; + Ok(info) } - None => platform::window_info(None), } } -/// A fresh walk of `window` as deep as the walk may go, for a caller that only -/// looks (`wait_for`). Not recorded as shown: an id is not being handed out. -pub fn tree_now(window: &str) -> Result<Tree, DesktopError> { - platform::tree(Some(window), bounded(u32::MAX)) +/// A fresh walk of `window` to the depth a selector resolves at (`depth`, its +/// own, or the default), for a caller that only looks (`wait_for`). Not +/// recorded as shown: an id is not being handed out. +pub fn tree_now(window: &str, depth: Option<u32>) -> Result<Tree, DesktopError> { + platform::tree(Some(window), resolve_depth(depth)) } /// The control `selector` names in `tree`, which was walked at `depth`. @@ -141,7 +197,7 @@ pub(crate) fn resolved_from( }; os(format!( "selector `{}` in window \"{}\" {e}{cut}", - selector.render(), + selector.render_redacted(), window.title )) })? @@ -159,19 +215,14 @@ pub(crate) fn resolved_from( }) } -/// Walk the window the selector names and return the one control it picks. +/// Walk `info`'s window to the selector's depth and return the one control it +/// picks. The window is already chosen ([`window_for`]), so a caller has +/// refused it, if it is a blocked application's, before anything is read. /// /// Ambiguous or missing is an error, never a guess: see [`Selector::resolve_in`]. -pub fn resolve(window: Option<&str>, selector: &Selector) -> Result<Resolved, DesktopError> { - selector.require_element().map_err(os)?; - resolve_in(window_for(window, selector)?, selector) -} - -/// [`resolve`] in a window already chosen, so a caller can refuse the window -/// (a blocked application's) before anything is read from it. pub fn resolve_in(info: WindowInfo, selector: &Selector) -> Result<Resolved, DesktopError> { selector.require_element().map_err(os)?; - let depth = bounded(u32::MAX); + let depth = resolve_depth(selector.depth); let tree = platform::tree(Some(&info.id), depth)?; resolved_from(info, tree, selector, depth) } @@ -198,7 +249,8 @@ impl Resolved { #[cfg(test)] mod tests { use super::*; - use crate::uia::Role; + use crate::selector::selectors_for; + use crate::uia::{walk, Role}; use crate::Rect; fn rect() -> Rect { @@ -293,6 +345,50 @@ mod tests { assert!(err.contains("nth"), "{err}"); } + /// An error is journalled word for word, and the selector in it may carry + /// what was typed into a field. + #[test] + fn an_error_shows_a_text_fields_name_redacted_whatever_the_selector_says() { + let edit = |id: u32, name: &str| Element { + role: Role::Edit, + ..button(id, name) + }; + let tree = tree_of(vec![edit(1, "x")], false); + // A miss. + let err = resolved_from( + window("7", "app", "Doc"), + tree.clone(), + &sel("role=edit;name=hunter2"), + 5, + ) + .unwrap_err() + .to_string(); + assert!(!err.contains("hunter2"), "{err}"); + assert!(err.contains("matches no element"), "{err}"); + // A miss with no role given, so the name might be anything. + let err = resolved_from(window("7", "app", "Doc"), tree, &sel("name~=hunter2"), 5) + .unwrap_err() + .to_string(); + assert!(!err.contains("hunter2"), "{err}"); + // An ambiguity that lists look-alikes: by their place, not their name. + let tree = tree_of(vec![edit(1, "hunter2"), edit(2, "hunter3")], false); + let err = resolved_from(window("7", "app", "Doc"), tree, &sel("role=edit"), 5) + .unwrap_err() + .to_string(); + assert!(!err.contains("hunter"), "{err}"); + // A button's name is not content, and the selector still says it. + let tree = tree_of(vec![button(1, "Save")], false); + let err = resolved_from( + window("7", "app", "Doc"), + tree, + &sel("role=button;name=Nope"), + 5, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("name=Nope"), "{err}"); + } + #[test] fn a_cut_tree_says_a_miss_may_lie_beyond_the_cut() { let tree = tree_of(vec![button(4, "Save")], true); @@ -359,9 +455,184 @@ mod tests { assert!(!probe.contains("Passwords"), "{probe}"); } + /// The guard of the id path, as `mcp::tools` hands it: refuses a blocked + /// application's window, in one fixed message. + fn blocked_guard(w: &WindowInfo) -> Result<(), String> { + if crate::limits::get().blocked(&w.app).is_some() { + Err("refused: that window belongs to a blocked application".to_string()) + } else { + Ok(()) + } + } + + /// **A blocked window named by id is refused before the selector is read.** + /// Otherwise the mismatch error says its title, and a match and a mismatch + /// are told apart: an oracle on the title of a password manager's entry. + #[test] + fn a_blocked_window_is_refused_before_its_title_can_be_told_or_probed() { + let keepass = || window("9", "KeePassXC", "Bank login - entry"); + let mut said = Vec::new(); + for selector in [ + "window=x;role=button", // a mismatch + "window~=bank;role=button", // a match + "window~=zzz;role=button", // another mismatch + "app=notepad;role=button", // the wrong program + "role=button", // no window part at all + ] { + let err = confirm_window(keepass(), &sel(selector), &blocked_guard) + .unwrap_err() + .to_string(); + assert!( + !err.contains("Bank login"), + "`{selector}` told the title: {err}" + ); + assert!(!err.contains("entry"), "{err}"); + said.push(err); + } + assert!( + said.windows(2).all(|pair| pair[0] == pair[1]), + "the refusal differs with the selector, which is an oracle: {said:?}" + ); + } + + #[test] + fn an_allowed_window_that_does_not_match_is_still_told_apart_from_one_that_does() { + let notes = || window("2", "notepad", "notes"); + assert!(confirm_window(notes(), &sel("window=notes;role=button"), &blocked_guard).is_ok()); + let err = confirm_window(notes(), &sel("window=other;role=button"), &blocked_guard) + .unwrap_err() + .to_string(); + assert!(err.contains("does not name"), "{err}"); + } + #[test] fn a_control_selector_needs_a_control_part() { - let err = resolve(None, &sel("app=notepad;nth=0")).unwrap_err(); + let err = resolve_in(window("1", "notepad", "n"), &sel("app=notepad;nth=0")).unwrap_err(); assert!(err.to_string().contains("names no control"), "{err}"); } + + // ---- one walk for printing and resolving ---- + + /// A window whose OK button sits below the default depth, with two more + /// OK buttons near the top that come after it in the tree's order. + #[derive(Clone)] + struct Node { + role: Role, + name: String, + kids: Vec<Node>, + } + + fn node(role: Role, name: &str, kids: Vec<Node>) -> Node { + Node { + role, + name: name.to_string(), + kids, + } + } + + fn deep_dialog(wrappers: usize) -> Node { + let mut chain = node(Role::Button, "OK", vec![]); + for i in (0..wrappers).rev() { + chain = node(Role::Group, &format!("wrap{i}"), vec![chain]); + } + node( + Role::Window, + "Dialog", + vec![ + chain, + node(Role::Button, "OK", vec![]), + node(Role::Button, "OK", vec![]), + ], + ) + } + + fn walked(root: &Node, depth: u32) -> Vec<Element> { + walk( + root, + depth, + &|n: &Node, id, d| Element { + id, + depth: d, + role: n.role, + name: n.name.clone(), + rect: rect(), + enabled: true, + secret: false, + automation_id: String::new(), + }, + &|n: &Node| n.kids.clone(), + ) + .elements + } + + /// **`nth` counts the walk that printed it.** `ui_tree` prints the + /// selector from the walk it was asked for; the resolver walks again. With + /// the OK button nested below the default depth, a walk to the default sees + /// two OKs (`nth=0` and `nth=1`) and a deeper one three, so a selector + /// printed by one and resolved by the other clicks the wrong button. + #[test] + fn a_printed_selector_resolves_to_the_control_it_was_printed_for_at_any_depth() { + let default = crate::limits::get().tree_depth; + let root = deep_dialog(default as usize + 2); + for asked in [1, 3, default, default + 4, default + 20] { + let shown = walked(&root, bounded(asked)); + let stamp = (bounded(asked) != default).then_some(bounded(asked)); + let selectors = selectors_for(&shown, false, stamp); + for (e, s) in shown.iter().zip(&selectors) { + // The selector as an agent copies it, from text. + let s = Selector::parse(&s.render()).unwrap(); + let again = walked(&root, resolve_depth(s.depth)); + let got = s.resolve_in(&again).unwrap_or_else(|why| { + panic!("depth {asked}: `{}` for {e:?}: {why}", s.render()) + }); + assert_eq!( + (got.id, got.depth, got.name.as_str()), + (e.id, e.depth, e.name.as_str()), + "depth {asked}: `{}` resolved to another control", + s.render() + ); + } + } + } + + #[test] + fn a_selector_printed_at_another_depth_says_so_and_the_default_does_not() { + let default = crate::limits::get().tree_depth; + let root = deep_dialog(default as usize + 2); + let shown = walked(&root, default); + let plain = selectors_for(&shown, false, None); + assert!(plain.iter().all(|s| !s.render().contains("depth="))); + // The two OKs in sight are told apart by nth, from the walk they are in. + let oks: Vec<String> = shown + .iter() + .zip(&plain) + .filter(|(e, _)| e.name == "OK") + .map(|(_, s)| s.render()) + .collect(); + assert_eq!( + oks, + ["role=button;name=OK;nth=0", "role=button;name=OK;nth=1"] + ); + + let deeper = walked(&root, default + 4); + let stamped = selectors_for(&deeper, false, Some(default + 4)); + assert!(stamped + .iter() + .all(|s| s.render().ends_with(&format!("depth={}", default + 4)))); + // The deeper walk has three OKs, so a selector from it says nth up to 2. + assert!(stamped.iter().any(|s| s.render().contains("nth=2"))); + } + + #[test] + fn the_resolver_walks_as_deep_as_the_selector_says_else_the_default() { + let default = crate::limits::get().tree_depth; + assert_eq!(resolve_depth(None), default); + assert_eq!(resolve_depth(Some(3)), 3); + assert_eq!(resolve_depth(Some(default + 4)), default + 4); + assert_eq!( + resolve_depth(Some(u32::MAX)), + default * 4, + "held to the bound every walk is" + ); + } } diff --git a/crates/desktop/src/wait.rs b/crates/desktop/src/wait.rs index be6f1f8..d9eeab7 100644 --- a/crates/desktop/src/wait.rs +++ b/crates/desktop/src/wait.rs @@ -100,7 +100,11 @@ pub trait Env { /// The window keystrokes go to now. fn focused_window(&mut self) -> Result<WindowInfo, String>; /// A fresh walk of one window. - fn tree(&mut self, window: &str) -> Result<Tree, String>; + /// + /// `depth` is how deep, when the selector gave one; else the default one. + /// The selector's `nth` and `path` count the controls of that walk + /// (`uia::resolve`). + fn tree(&mut self, window: &str, depth: Option<u32>) -> Result<Tree, String>; /// The fingerprint of the screen, or of one window's area. fn screen(&mut self, window: Option<&str>) -> Result<Fingerprint, String>; } @@ -319,7 +323,7 @@ fn look(env: &mut impl Env, target: &Target, selector: &Selector) -> Result<Look let Some(window) = window else { return Ok(Look::NoWindow); }; - let tree = match env.tree(&window.id) { + let tree = match env.tree(&window.id, selector.depth) { Ok(tree) => tree, Err(why) => { // A window that closes while it is being walked fails the walk: @@ -347,16 +351,26 @@ fn cut_note(tree: &Tree) -> String { } } -fn found_element(window: WindowInfo, tree: Tree, hits: &[usize]) -> Found { +/// `depth` is the walk's, as the wait's selector gave it: the selectors written +/// for what was found are counted in that walk, so they carry it (see +/// [`Selector::depth`]). +fn found_element(window: WindowInfo, tree: Tree, hits: &[usize], depth: Option<u32>) -> Found { let first = hits[0]; + let written = |i: usize| { + Selector { + depth, + ..selector_for(&tree.elements, i, NAMES_ARE_VALUES) + } + .render_element_part() + }; let others = hits .iter() .skip(1) .take(OTHERS) - .map(|&i| selector_for(&tree.elements, i, NAMES_ARE_VALUES).render_element_part()) + .map(|&i| written(i)) .collect(); Found::Element { - selector: selector_for(&tree.elements, first, NAMES_ARE_VALUES).render_element_part(), + selector: written(first), element: Box::new(tree.elements[first].clone()), matches: hits.len(), others, @@ -386,7 +400,7 @@ fn element_check( cut_note(&tree) )) } else { - Poll::Ready(found_element(window, tree, &hits)) + Poll::Ready(found_element(window, tree, &hits, selector.depth)) } } (Look::Seen(seen), false) => { @@ -724,7 +738,7 @@ mod tests { .find(|w| w.focused) .ok_or_else(|| "nothing has focus".to_string()) } - fn tree(&mut self, window: &str) -> Result<Tree, String> { + fn tree(&mut self, window: &str, _: Option<u32>) -> Result<Tree, String> { self.trees += 1; if self.tree_fails_from.is_some_and(|t| self.now >= t) { return Err("no desktop element: Unspecified error".to_string()); diff --git a/docs/COMPUTER_USE.md b/docs/COMPUTER_USE.md index 1b67d83..37b99b6 100644 --- a/docs/COMPUTER_USE.md +++ b/docs/COMPUTER_USE.md @@ -278,12 +278,18 @@ ids work exactly as before. | `aid` | the program's own id for the control, exact | | `path` | child indices from the window down (`0.2.1`), counted among the controls `ui_tree` lists, so a wrapper it drops is not a level. A hard condition: a control that moved is not found | | `nth` | which of several matches, from 0, in `ui_tree` order. Out of range is an error | +| `depth` | how deep the window is walked to find the control (default 12, held to 4x that). `nth` and `path` count the controls of a walk, so the walk that prints a selector and the one that resolves it must be the same: `ui_tree` writes `depth=N` into every selector when it was asked for a depth other than the default, and the resolver walks as deep as the selector says, else to the default. Not a control part on its own | `key=value` pairs joined by `;`, with no spaces around the `;`; `\\`, `\;`, `\n`, `\r` escape a value. Unknown or repeated keys and bad numbers are refused by name. `app`/`window` pick the window (the one named, else `window=<id>`, else the focused one); the other keys pick the control. A window selector never matches a blocked application's window, and one naming a -blocked app is refused, so a title cannot be probed for. What `ui_tree` prints is the +blocked app is refused, so a title cannot be probed for; a blocked window named by `window=<id>` +is refused with the id path's own words BEFORE the selector's window part is compared with it, +so neither its title nor whether a `window~=` matched can be learned from the error. Errors are +journalled word for word, so none quotes what it was given: a parse error names the item by its +position (`item 2 is not key=value`), and a selector shown in any other error is redacted (a +text field's name is left out, and a list of look-alikes names a text field by its place). What `ui_tree` prints is the **shortest** selector that is unique in that tree, in the order of what changes least: `role`+`aid`, else `role`+`name`, else those plus `nth`. A name longer than 60 characters is written as its first 60, as a "contains". On macOS an edit field's name IS its content From b56356a1d6595c18f5127fadb1e2df8e245933e0 Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:00:02 -0600 Subject: [PATCH 3/9] desktop: three review follow-ups (example depth, final sigma, field names in messages) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. An ambiguity error's example selectors carry the failing selector's depth, so their nth is counted in the walk the resolver will make. 2. fold() also maps a typed final sigma to sigma, so name~=λόγος matches ΛΌΓΟΣ as str::to_lowercase did. 3. Element::label (used in platform errors such as "has no value to set") and the id path's "is disabled" refusal no longer name an edit or text field: on macOS its name is its content and errors are journalled word for word. --- crates/desktop/src/mcp/selector_tests.rs | 40 +++++++++++++ crates/desktop/src/mcp/tools.rs | 33 +++++++---- crates/desktop/src/selector.rs | 75 +++++++++++++++++++++++- crates/desktop/src/uia.rs | 21 ++++++- 4 files changed, 152 insertions(+), 17 deletions(-) diff --git a/crates/desktop/src/mcp/selector_tests.rs b/crates/desktop/src/mcp/selector_tests.rs index 658f209..7461937 100644 --- a/crates/desktop/src/mcp/selector_tests.rs +++ b/crates/desktop/src/mcp/selector_tests.rs @@ -227,6 +227,46 @@ fn what_a_selector_error_says_is_never_what_was_typed() { assert!(!why.contains("hunter2"), "{why}"); } +/// **No message names an edit or text field, on either path.** Platform +/// errors (`set_value`, `element_action`: "`{what}` has no value to set") say +/// `Element::label`, the disabled refusal says the id, and on macOS the name of +/// such a field is what is in it. Every message goes to the journal as it was +/// returned. +#[test] +fn no_message_names_an_edit_or_a_text_field_whatever_the_platform() { + use super::tools::disabled_error; + use crate::uia::{Element, Role}; + let element = |role, name: &str| Element { + id: 7, + depth: 1, + role, + name: name.to_string(), + rect: crate::Rect { + x: 0, + y: 0, + width: 10, + height: 10, + }, + enabled: false, + secret: false, + automation_id: String::new(), + }; + for role in [Role::Edit, Role::Text] { + let typed = element(role, "hunter2"); + assert_eq!(typed.label(), format!("e7 ({role:?})")); + assert!(!typed.label().contains("hunter2")); + let why = disabled_error(&Target::Id(7), &typed); + assert!(!why.contains("hunter2"), "{why}"); + assert!(why.contains("e7 is disabled"), "{why}"); + } + // A control whose name is a label, not content, is named as before. + for role in [Role::Button, Role::CheckBox, Role::MenuItem, Role::Link] { + let named = element(role, "Save"); + assert_eq!(named.label(), format!("e7 \"Save\" ({role:?})")); + assert!(disabled_error(&Target::Id(7), &named).contains("\"Save\"")); + } +} + /// The "is disabled" refusal is journalled word for word. #[test] fn a_disabled_control_is_refused_without_naming_a_selected_text_field() { diff --git a/crates/desktop/src/mcp/tools.rs b/crates/desktop/src/mcp/tools.rs index c57357f..ee9636a 100644 --- a/crates/desktop/src/mcp/tools.rs +++ b/crates/desktop/src/mcp/tools.rs @@ -289,12 +289,18 @@ fn click_element(input: &InputThread, args: &Value) -> Result<Vec<Value>, String set_value on it" )); } - let label = format!( - "selector `{}` (\"{}\", {:?})", - selector.render_redacted(), - found.element.name, - found.element.role - ); + let label = match found.element.name_for_messages() { + Some(name) => format!( + "selector `{}` (\"{name}\", {:?})", + selector.render_redacted(), + found.element.role + ), + None => format!( + "selector `{}` ({:?})", + selector.render_redacted(), + found.element.role + ), + }; let cut = with_cut_note(String::new(), &found); (found.element, label, cut) } @@ -585,15 +591,16 @@ fn resolve_selector(window: Option<&str>, selector: &Selector) -> Result<uia::Re } /// Why clicking a disabled control is refused. The id path names the control -/// as it always has. A selector is named redacted and the control is not named -/// at all: this error is journalled word for word, and a text field's name is -/// what was typed in it. +/// as it always has, except that an edit or text field is not named +/// ([`uia::Element::name_for_messages`]). A selector is named redacted and the +/// control is not named at all: this error is journalled word for word, and a +/// text field's name is what was typed in it. pub(super) fn disabled_error(target: &Target, element: &uia::Element) -> String { match target { - Target::Id(id) => format!( - "e{id} \"{}\" is disabled; clicking it would do nothing", - element.name - ), + Target::Id(id) => match element.name_for_messages() { + Some(name) => format!("e{id} \"{name}\" is disabled; clicking it would do nothing"), + None => format!("e{id} is disabled; clicking it would do nothing"), + }, Target::Selector(selector) => format!( "selector `{}` ({:?}) is disabled; clicking it would do nothing", selector.render_redacted(), diff --git a/crates/desktop/src/selector.rs b/crates/desktop/src/selector.rs index c1a4bf3..e95105f 100644 --- a/crates/desktop/src/selector.rs +++ b/crates/desktop/src/selector.rs @@ -105,8 +105,15 @@ pub enum Text { /// letters inside the whole name, and the selector would miss its own element. /// Folding each character on its own gives the same letters wherever the text /// is cut. +/// +/// A final sigma `ς` is folded to `σ` as well. `char::to_lowercase` leaves a +/// typed `ς` alone, so `λόγος` would not match `ΛΌΓΟΣ` (whose capital Σ folds +/// to `σ`), which `str::to_lowercase` did match. fn fold(text: &str) -> String { - text.chars().flat_map(char::to_lowercase).collect() + text.chars() + .flat_map(char::to_lowercase) + .map(|c| if c == 'ς' { 'σ' } else { c }) + .collect() } impl Text { @@ -513,7 +520,17 @@ impl Selector { let examples = found .iter() .take(EXAMPLES) - .map(|&i| selector_for(elements, i, true).render_element_part()) + .map(|&i| { + // With the depth of THIS selector: an example's `nth` + // counts the controls of the walk that found the + // matches, and the resolver must walk as deep to count + // the same ones (see [`Selector::depth`]). + Selector { + depth: self.depth, + ..selector_for(elements, i, true) + } + .render_element_part() + }) .collect(); return Err(SelectError::Ambiguous { count, examples }); } @@ -860,6 +877,60 @@ mod tests { assert!(plain.iter().all(|s| s.depth.is_none())); } + /// An ambiguity error's examples count `nth` in the walk that found the + /// matches, so they carry that walk's depth. + #[test] + fn an_ambiguity_examples_carry_the_depth_of_the_selector_that_failed() { + let tree = vec![el(1, 1, Role::Button, "OK"), el(2, 1, Role::Button, "OK")]; + let at_three = Selector::parse("role=button;name=OK;depth=3").unwrap(); + match at_three.resolve_in(&tree) { + Err(SelectError::Ambiguous { examples, .. }) => { + assert_eq!(examples.len(), 2); + for example in &examples { + assert!(example.ends_with(";depth=3"), "{example}"); + } + assert_eq!(examples[1], "role=button;name=OK;nth=1;depth=3"); + // What the error suggests is a selector that resolves, in a + // walk of that depth, to the control it named. + let again = Selector::parse(&examples[1]).unwrap(); + assert_eq!(again.resolve_in(&tree).unwrap().id, 2); + } + other => panic!("{other:?}"), + } + // No depth said, none suggested. + let plain = Selector::parse("role=button;name=OK").unwrap(); + match plain.resolve_in(&tree) { + Err(SelectError::Ambiguous { examples, .. }) => { + assert!( + examples.iter().all(|e| !e.contains("depth")), + "{examples:?}" + ) + } + other => panic!("{other:?}"), + } + } + + /// A typed final sigma is the same letter as a capital one in a name: + /// `str::to_lowercase` made them match, so the per-character fold must. + #[test] + fn a_typed_final_sigma_matches_a_capital_sigma_in_the_name() { + let tree = vec![el(1, 1, Role::Text, "ΛΌΓΟΣ"), el(2, 1, Role::Text, "other")]; + for typed in ["name~=λόγος", "name~=λόγοσ", "name~=ΛΌΓΟΣ", "name~=γος"] { + let s = Selector::parse(typed).unwrap(); + assert_eq!(s.resolve_in(&tree).unwrap().id, 1, "{typed}"); + } + // And a lowercase name, with the final form, is found by a capital one. + let tree = vec![el(1, 1, Role::Text, "λόγος")]; + let s = Selector::parse("name~=ΛΌΓΟΣ").unwrap(); + assert_eq!(s.resolve_in(&tree).unwrap().id, 1); + // The derived selector for a name ending in ς still finds it. + let tree = vec![el(1, 1, Role::Text, &format!("{}ς", "a".repeat(80)))]; + assert_eq!( + selector_for(&tree, 0, false).resolve_in(&tree).unwrap().id, + 1 + ); + } + /// A name cut off at a capital sigma lowercases to a different letter than /// the same sigma inside the whole name: `ς` at the end of a word, `σ` /// elsewhere. The derived selector must still find its own element. diff --git a/crates/desktop/src/uia.rs b/crates/desktop/src/uia.rs index 462de04..8a69d87 100644 --- a/crates/desktop/src/uia.rs +++ b/crates/desktop/src/uia.rs @@ -435,9 +435,26 @@ impl Done { } impl Element { - /// `e7 "Save" (Button)`, for a message. + /// The name as a message may say it: `None` for an edit or a text field. + /// + /// Messages go to the operator's journal word for word (an error is + /// recorded as it was returned), and on macOS the name of such a field IS + /// what is in it (`AXValue`, the fallback of `describe`), which may have + /// been typed. So no message names one, on any platform. + pub(crate) fn name_for_messages(&self) -> Option<&str> { + match self.role { + Role::Edit | Role::Text => None, + _ => Some(&self.name), + } + } + + /// `e7 "Save" (Button)`, for a message. An edit or text field is `e7 (Edit)`, + /// without its name: see [`Self::name_for_messages`]. pub fn label(&self) -> String { - format!("e{} \"{}\" ({:?})", self.id, self.name, self.role) + match self.name_for_messages() { + Some(name) => format!("e{} \"{name}\" ({:?})", self.id, self.role), + None => format!("e{} ({:?})", self.id, self.role), + } } } From eaf09427e1216f7e2d40274e9a1414fad28e41ea Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:11:24 -0600 Subject: [PATCH 4/9] desktop: wait_for cannot hang on a window that closes Live test: after a click closed Notepad, wait_for element_gone on that window with timeout_ms 4000 did not return for more than five minutes. - A pinned window that no longer exists (IsWindow on Windows, asked of the window manager and nothing else) is element_gone: found at once, with no accessibility call. A closed window named by id is gone for element_gone and an error for element_appears. - Every look (list windows, walk, capture, focused window) runs on a thread of its own and is waited for until the wait's deadline plus 1 s. A look the desktop did not answer is a look that saw nothing (wait::UNANSWERED): the wait ends as `timeout` with last_seen saying so, never as `gone`, never as an error, and no look starts after the deadline. At most 4 abandoned looks are tolerated before none is started. - Tests: a fake desktop where the window is destroyed mid-wait or never answers, and a test that a look sleeping 5 s is given up on within its limit plus the grace. --- crates/desktop/src/mcp/wait_tool.rs | 248 +++++++++++++++++++++++++--- crates/desktop/src/uia.rs | 15 ++ crates/desktop/src/uia_windows.rs | 13 +- crates/desktop/src/wait.rs | 234 ++++++++++++++++++++++++-- 4 files changed, 468 insertions(+), 42 deletions(-) diff --git a/crates/desktop/src/mcp/wait_tool.rs b/crates/desktop/src/mcp/wait_tool.rs index dddbe56..75e5b05 100644 --- a/crates/desktop/src/mcp/wait_tool.rs +++ b/crates/desktop/src/mcp/wait_tool.rs @@ -13,6 +13,8 @@ //! - **Calls** [`crate::wait::run`] with [`Live`], which reads //! [`crate::uia`] and [`crate::capture`] through [`super::guard`]. +use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; +use std::sync::{mpsc, Arc}; use std::time::{Duration, Instant}; use serde_json::Value; @@ -21,7 +23,7 @@ use crate::capture::{self, CaptureOptions, ShotFormat}; use crate::fingerprint::{self, Fingerprint}; use crate::selector::Selector; use crate::uia::{self, Tree, WindowInfo}; -use crate::wait::{self, Budget, Condition, Env, Outcome, DEFAULT_TOLERANCE}; +use crate::wait::{self, Budget, Condition, Env, Outcome, DEFAULT_TOLERANCE, UNANSWERED}; use super::guard::{refuse_app, refuse_blocked}; use super::tools::{guard_capture, text, window_arg}; @@ -30,9 +32,113 @@ use super::tools::{guard_capture, text, window_arg}; /// is 128 bits, so more pixels only cost. const FINGERPRINT_EDGE: u32 = 256; +/// How far past the wait's deadline a look may run before it is given up on. +/// +/// **No look may outlive the deadline by more than this.** A look is a call +/// into the accessibility layer, which asks the program that owns the window; +/// when that program is closing or not responding the call blocks for as long +/// as the provider takes to give up (measured on Windows: a pinned window that +/// had just closed kept a wait with `timeout_ms: 4000` from returning for more +/// than five minutes). A look therefore runs on a thread of its own, and the +/// wait stops waiting for it at its deadline plus this much. +const LOOK_GRACE: Duration = Duration::from_millis(1000); + +/// Most looks that have been given up on and are still blocked, in this +/// process. At the limit no new look is started: another would only join them. +const MAX_ABANDONED: usize = 4; + +/// Looks given up on that have not returned yet. +static ABANDONED: AtomicUsize = AtomicUsize::new(0); + +/// `work`, run on a thread of its own and waited for at most `limit`. +/// +/// On time it is `work`'s answer. Past `limit` the thread is left to finish +/// by itself (it cannot be stopped from outside: it is inside a call the +/// operating system has not returned from) and the answer is +/// [`UNANSWERED`], counted in `abandoned` until the thread does return. +fn bounded<T: Send + 'static>( + what: &str, + limit: Duration, + abandoned: &'static AtomicUsize, + work: impl FnOnce() -> Result<T, String> + Send + 'static, +) -> Result<T, String> { + let still_blocked = abandoned.load(Ordering::SeqCst); + if still_blocked >= MAX_ABANDONED { + return Err(format!( + "{UNANSWERED}: {still_blocked} earlier looks are still blocked in the \ + accessibility layer, so none was started to {what}" + )); + } + // 0 running, 1 returned in time, 2 given up on. + let state = Arc::new(AtomicU8::new(0)); + let (send, receive) = mpsc::channel(); + let theirs = Arc::clone(&state); + let spawned = std::thread::Builder::new() + .name("wait-look".to_string()) + .spawn(move || { + let answer = work(); + // If the wait has given up on this look, the thread is the one + // left to take it off the count. + if theirs + .compare_exchange(0, 1, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + abandoned.fetch_sub(1, Ordering::SeqCst); + } + let _ = send.send(answer); + }); + if let Err(e) = spawned { + return Err(format!("could not start a look to {what}: {e}")); + } + match receive.recv_timeout(limit) { + Ok(answer) => answer, + Err(_) => { + if state + .compare_exchange(0, 2, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + abandoned.fetch_add(1, Ordering::SeqCst); + Err(format!( + "{UNANSWERED} to {what} within {} ms (a window that is closing or not \ + responding can do this)", + limit.as_millis() + )) + } else { + // It returned in the instant between the timeout and this. + receive + .recv() + .unwrap_or_else(|_| Err(format!("the look to {what} ended without an answer"))) + } + } + } +} + /// The desktop as this process sees it, with a clock that starts now. struct Live { started: Instant, + /// When the wait's `timeout_ms` is up. A look may run to this and + /// [`LOOK_GRACE`] beyond it. + deadline: Instant, +} + +impl Live { + fn new(budget: &Budget) -> Live { + let started = Instant::now(); + Live { + started, + deadline: started + Duration::from_millis(budget.timeout_ms), + } + } + + /// `work` as one look, held to the deadline and the grace after it. + fn within<T: Send + 'static>( + &self, + what: &str, + work: impl FnOnce() -> Result<T, String> + Send + 'static, + ) -> Result<T, String> { + let limit = (self.deadline + LOOK_GRACE).saturating_duration_since(Instant::now()); + bounded(what, limit, &ABANDONED, work) + } } fn text_of(e: impl ToString) -> String { @@ -51,43 +157,60 @@ impl Env for Live { fn windows(&mut self) -> Result<Vec<WindowInfo>, String> { // Not the windows of a blocked application: they are not listed, and a // selector must not be able to find one by its title. - Ok(listed()? + let all = self.within("list the windows", listed)?; + Ok(all .into_iter() .filter(|w| crate::limits::get().blocked(&w.app).is_none()) .collect()) } + fn window_alive(&mut self, id: &str) -> Option<bool> { + uia::window_exists(id) + } + fn focused_window(&mut self) -> Result<WindowInfo, String> { - let window = uia::window_info(None).map_err(text_of)?; + let window = self.within("find the focused window", || { + uia::window_info(None).map_err(text_of) + })?; refuse_blocked(&window)?; Ok(window) } fn tree(&mut self, window: &str, depth: Option<u32>) -> Result<Tree, String> { - let info = uia::window_info(Some(window)).map_err(text_of)?; - refuse_blocked(&info)?; - let tree = uia::tree_now(window, depth).map_err(text_of)?; + let id = window.to_string(); + let (info, tree) = self.within("walk the window", move || { + let info = uia::window_info(Some(&id)).map_err(text_of)?; + // Refused before the window is walked: a blocked application's + // window is never read. + refuse_blocked(&info)?; + let tree = uia::tree_now(&id, depth).map_err(text_of)?; + Ok((info, tree)) + })?; // Walked in this process, and discarded unread when blocked. + refuse_blocked(&info)?; refuse_app(&tree.app)?; Ok(tree) } fn screen(&mut self, window: Option<&str>) -> Result<Fingerprint, String> { - let of = guard_capture(window)?; - let options = CaptureOptions { - max_edge: FINGERPRINT_EDGE, - format: ShotFormat::Png, - ..CaptureOptions::default() - }; - let shot = match &of { - Some(info) => capture::capture_region(info.rect, options), - None => capture::capture_primary(options), - } - .map_err(text_of)?; - let picture = image::load_from_memory(&shot.bytes) - .map_err(|e| format!("could not read the capture back to fingerprint it: {e}"))? - .to_rgba8(); - Ok(fingerprint::of(&picture)) + let window = window.map(str::to_string); + self.within("capture the screen", move || { + let of = guard_capture(window.as_deref())?; + let options = CaptureOptions { + max_edge: FINGERPRINT_EDGE, + format: ShotFormat::Png, + ..CaptureOptions::default() + }; + let shot = match &of { + Some(info) => capture::capture_region(info.rect, options), + None => capture::capture_primary(options), + } + .map_err(text_of)?; + let picture = image::load_from_memory(&shot.bytes) + .map_err(|e| format!("could not read the capture back to fingerprint it: {e}"))? + .to_rgba8(); + Ok(fingerprint::of(&picture)) + }) } } @@ -186,9 +309,7 @@ fn condition_of(args: &Value) -> Result<Condition, String> { pub(super) fn wait_for(args: &Value, stopped: &dyn Fn() -> bool) -> Result<Vec<Value>, String> { let condition = condition_of(args)?; let budget = Budget::new(whole(args, "timeout_ms")?, whole(args, "poll_ms")?); - let mut env = Live { - started: Instant::now(), - }; + let mut env = Live::new(&budget); let outcome = wait::run(&mut env, &condition, &budget, stopped)?; if matches!(outcome, Outcome::Stopped { .. }) { return Err(crate::safety::Refusal::Stopped.message()); @@ -206,6 +327,85 @@ mod tests { use super::*; use serde_json::json; + /// **A look that blocks cannot outlive its limit.** The live hang was a + /// call that did not return for minutes; here the work sleeps far longer + /// than the limit and `bounded` must hand back long before it returns. + #[test] + fn a_look_that_blocks_is_given_up_on_at_its_limit_not_waited_for() { + static COUNT: AtomicUsize = AtomicUsize::new(0); + let started = Instant::now(); + let answer = bounded( + "walk the window", + Duration::from_millis(150), + &COUNT, + || { + std::thread::sleep(Duration::from_secs(5)); + Ok(1) + }, + ); + let took = started.elapsed(); + let why = answer.unwrap_err(); + assert!(why.starts_with(UNANSWERED), "{why}"); + assert!(why.contains("walk the window"), "{why}"); + assert!( + took < Duration::from_millis(150) + LOOK_GRACE, + "a 150 ms limit held the caller for {took:?}" + ); + assert_eq!(COUNT.load(Ordering::SeqCst), 1, "counted while it blocks"); + } + + #[test] + fn a_look_given_up_on_is_taken_off_the_count_when_it_returns() { + static COUNT: AtomicUsize = AtomicUsize::new(0); + let answer = bounded("walk", Duration::from_millis(50), &COUNT, || { + std::thread::sleep(Duration::from_millis(300)); + Ok(()) + }); + assert!(answer.is_err()); + assert_eq!(COUNT.load(Ordering::SeqCst), 1); + let until = Instant::now() + Duration::from_secs(5); + while COUNT.load(Ordering::SeqCst) != 0 { + assert!(Instant::now() < until, "the count was never given back"); + std::thread::sleep(Duration::from_millis(20)); + } + } + + #[test] + fn a_look_that_answers_in_time_gives_its_answer_or_its_error() { + static COUNT: AtomicUsize = AtomicUsize::new(0); + let ok = bounded("list", Duration::from_secs(5), &COUNT, || Ok(vec![1, 2])); + assert_eq!(ok.unwrap(), vec![1, 2]); + let refused: Result<(), String> = bounded("list", Duration::from_secs(5), &COUNT, || { + Err("refused: blocked".to_string()) + }); + assert_eq!(refused.unwrap_err(), "refused: blocked"); + assert_eq!(COUNT.load(Ordering::SeqCst), 0); + } + + #[test] + fn no_new_look_is_started_while_too_many_are_still_blocked() { + static COUNT: AtomicUsize = AtomicUsize::new(MAX_ABANDONED); + let ran = Arc::new(AtomicU8::new(0)); + let flag = Arc::clone(&ran); + let answer = bounded("walk", Duration::from_secs(5), &COUNT, move || { + flag.store(1, Ordering::SeqCst); + Ok(()) + }); + let why = answer.unwrap_err(); + assert!(why.starts_with(UNANSWERED), "{why}"); + assert!(why.contains("still blocked"), "{why}"); + std::thread::sleep(Duration::from_millis(100)); + assert_eq!(ran.load(Ordering::SeqCst), 0, "a look was started anyway"); + } + + #[test] + fn the_live_deadline_is_the_budgets_timeout_from_now() { + let live = Live::new(&Budget::new(Some(4_000), None)); + let left = live.deadline.saturating_duration_since(Instant::now()); + assert!(left <= Duration::from_millis(4_000)); + assert!(left > Duration::from_millis(3_000), "{left:?}"); + } + #[test] fn the_conditions_are_read_from_the_arguments() { let c = condition_of(&json!({"condition": "element_gone", "selector": "role=button;name=Save", "window": "7"})) diff --git a/crates/desktop/src/uia.rs b/crates/desktop/src/uia.rs index 8a69d87..0f241d0 100644 --- a/crates/desktop/src/uia.rs +++ b/crates/desktop/src/uia.rs @@ -633,6 +633,21 @@ pub fn act(window: Option<&str>, id: u32, action: &str) -> Result<Done, DesktopE platform::act(Some(&info.id), &want, depth, action) } +/// Whether the window with this id still exists, when that can be told from +/// the window manager alone, without asking the window anything: `Some(false)` +/// is a window that has been destroyed. Windows only (`IsWindow`); `None` +/// elsewhere, where the window list is the only way to tell. +#[must_use] +pub fn window_exists(window: &str) -> Option<bool> { + #[cfg(windows)] + return platform::window_exists(window); + #[cfg(not(windows))] + { + let _ = window; + None + } +} + /// Bring a window to the front, so keystrokes go to it. pub fn focus_window(window: &str) -> Result<Done, DesktopError> { platform::focus_window(window) diff --git a/crates/desktop/src/uia_windows.rs b/crates/desktop/src/uia_windows.rs index 39088f8..1ae04a5 100644 --- a/crates/desktop/src/uia_windows.rs +++ b/crates/desktop/src/uia_windows.rs @@ -20,7 +20,7 @@ use windows_sys::Win32::System::Threading::{ OpenProcess, QueryFullProcessImageNameW, PROCESS_NAME_WIN32, PROCESS_QUERY_LIMITED_INFORMATION, }; use windows_sys::Win32::UI::WindowsAndMessaging::{ - GetForegroundWindow, IsIconic, SetForegroundWindow, ShowWindow, SW_RESTORE, + GetForegroundWindow, IsIconic, IsWindow, SetForegroundWindow, ShowWindow, SW_RESTORE, }; use super::{find, walk, Done, Element, Role, Tree, Want, WindowInfo}; @@ -118,6 +118,17 @@ fn parse_id(window: &str) -> Result<isize, DesktopError> { }) } +/// Whether the window `window` names still exists (`IsWindow`), asked of the +/// window manager and of nothing else: no accessibility call, which for a +/// window that is closing is the one that can block on the program going away. +/// `None` for something that is not a window id. +pub(super) fn window_exists(window: &str) -> Option<bool> { + let hwnd = parse_id(window).ok()?; + // SAFETY: IsWindow takes any handle value and only answers whether it + // names a window. + Some(unsafe { IsWindow(hwnd as HWND) } != 0) +} + fn foreground_hwnd() -> isize { // SAFETY: an argument-free query of the foreground window. unsafe { GetForegroundWindow() as isize } diff --git a/crates/desktop/src/wait.rs b/crates/desktop/src/wait.rs index d9eeab7..12a520f 100644 --- a/crates/desktop/src/wait.rs +++ b/crates/desktop/src/wait.rs @@ -89,6 +89,14 @@ impl Budget { } } +/// What an [`Env`] call's error begins with when the desktop did not answer in +/// time (a look given up on at its limit, see `mcp::wait_tool`). +/// +/// It is not a failure of the wait: it is a look that saw nothing. The wait +/// goes on to its deadline, or ends there, as `timeout` with this as +/// `last_seen`, instead of ending as an error the moment one look is lost. +pub const UNANSWERED: &str = "the desktop did not answer"; + /// What a wait touches. See the module header. pub trait Env { /// Milliseconds since some fixed moment of this wait's own. @@ -97,6 +105,17 @@ pub trait Env { /// The windows there are to look at: those of blocked applications are not /// among them. fn windows(&mut self) -> Result<Vec<WindowInfo>, String>; + /// Whether the window with this id still exists, when that can be told + /// without asking the window anything (Windows: `IsWindow`): `Some(false)` + /// is a window that has been destroyed, which no later look is made of. + /// `None` when it cannot be told that way. + /// + /// A window that is closing is the one a look can block on: the + /// accessibility layer asks the program that owns it, and that program is + /// going away. So this is asked first and costs no such question. + fn window_alive(&mut self, _id: &str) -> Option<bool> { + None + } /// The window keystrokes go to now. fn focused_window(&mut self) -> Result<WindowInfo, String>; /// A fresh walk of one window. @@ -223,9 +242,12 @@ fn wait_until<E: Env, T>( return Ok(Waited::Stopped(env.now_ms() - start)); } polls += 1; - let last = match check(env)? { - Poll::Ready(found) => return Ok(Waited::Met(found, env.now_ms() - start, polls)), - Poll::Pending(seen) => seen, + let last = match check(env) { + Ok(Poll::Ready(found)) => return Ok(Waited::Met(found, env.now_ms() - start, polls)), + Ok(Poll::Pending(seen)) => seen, + // A look the desktop did not answer is a look that saw nothing. + Err(why) if why.starts_with(UNANSWERED) => why, + Err(why) => return Err(why), }; let elapsed = env.now_ms() - start; if elapsed >= budget.timeout_ms { @@ -272,25 +294,44 @@ fn windows_named(env: &mut impl Env, selector: &Selector) -> Result<Vec<WindowIn } /// The window the wait watches, decided once at the start. +/// +/// A window named by id that is not there is an error for `appears` (nothing +/// can appear in a window that does not exist) and an answer for `gone`: a +/// pinned window that has closed is exactly what its control being gone looks +/// like, and the id of one that closed a moment before the call is the usual +/// case (the click that closed it was the last call). fn target_of( env: &mut impl Env, selector: &Selector, window: Option<&str>, + appears: bool, ) -> Result<Target, String> { if let Some(id) = window { - let all = env.windows()?; - let Some(w) = all.iter().find(|w| w.id == id) else { - return Err(format!( - "no window has the id {id}; take one from list_windows" - )); + let alive = env.window_alive(id); + // Asked of nothing when the window is known to be destroyed. + let listed = if alive == Some(false) { + None + } else { + env.windows()?.into_iter().find(|w| w.id == id) }; - if selector.has_window_part() && !selector.matches_window(w) { - return Err(format!( - "window {id} is \"{}\" ({}), which the selector's window part `{}` does not name", - w.title, - w.app, - selector.render_window_part() - )); + let there = listed.is_some() || alive == Some(true); + if !there { + if appears { + return Err(format!( + "no window has the id {id}; take one from list_windows" + )); + } + return Ok(Target::Pinned(id.to_string())); + } + if let Some(w) = &listed { + if selector.has_window_part() && !selector.matches_window(w) { + return Err(format!( + "window {id} is \"{}\" ({}), which the selector's window part `{}` does not name", + w.title, + w.app, + selector.render_window_part() + )); + } } return Ok(Target::Pinned(id.to_string())); } @@ -304,7 +345,14 @@ fn target_of( fn look(env: &mut impl Env, target: &Target, selector: &Selector) -> Result<Look, String> { let window = match target { - Target::Pinned(id) => env.windows()?.into_iter().find(|w| &w.id == id), + Target::Pinned(id) => { + // Destroyed: nothing to ask. This is the cheap, certain answer + // that ends an `element_gone` wait on a window that closed. + if env.window_alive(id) == Some(false) { + return Ok(Look::NoWindow); + } + env.windows()?.into_iter().find(|w| &w.id == id) + } Target::Named => { let mut named = windows_named(env, selector)?; match named.len() { @@ -325,10 +373,16 @@ fn look(env: &mut impl Env, target: &Target, selector: &Selector) -> Result<Look }; let tree = match env.tree(&window.id, selector.depth) { Ok(tree) => tree, + // Nothing was seen, and asking again would only wait for the same + // answer: let the wait decide (`wait_until`). + Err(why) if why.starts_with(UNANSWERED) => return Err(why), Err(why) => { // A window that closes while it is being walked fails the walk: // that is the window going away, which is an answer, not a // failure. Only a window that is still there makes it an error. + if env.window_alive(&window.id) == Some(false) { + return Ok(Look::NoWindow); + } let still_there = env.windows()?.iter().any(|w| w.id == window.id); if still_there { return Err(why); @@ -474,7 +528,7 @@ pub fn run<E: Env>( appears, } => { selector.require_element()?; - let target = target_of(env, selector, window.as_deref())?; + let target = target_of(env, selector, window.as_deref(), *appears)?; let waited = wait_until(env, budget, stopped, |env| { element_check(env, &target, selector, *appears) })?; @@ -700,6 +754,11 @@ mod tests { truncated: bool, /// From this time on a walk fails, as one does of a closing window. tree_fails_from: Option<u64>, + /// Whether the window exists, by time, for a desktop that can tell. + alive: Option<At<bool>>, + /// A walk that blocks this many virtual ms and is then given up on. + tree_blocks_ms: Option<u64>, + windows_calls: u32, screen: At<u128>, sleeps: Vec<u64>, trees: u32, @@ -714,6 +773,9 @@ mod tests { elements: Box::new(|_| Vec::new()), truncated: false, tree_fails_from: None, + alive: None, + tree_blocks_ms: None, + windows_calls: 0, screen: Box::new(|_| 0), sleeps: Vec::new(), trees: 0, @@ -730,8 +792,12 @@ mod tests { self.now += ms; } fn windows(&mut self) -> Result<Vec<WindowInfo>, String> { + self.windows_calls += 1; Ok((self.windows)(self.now)) } + fn window_alive(&mut self, _: &str) -> Option<bool> { + self.alive.as_ref().map(|alive| alive(self.now)) + } fn focused_window(&mut self) -> Result<WindowInfo, String> { (self.windows)(self.now) .into_iter() @@ -740,6 +806,12 @@ mod tests { } fn tree(&mut self, window: &str, _: Option<u32>) -> Result<Tree, String> { self.trees += 1; + if let Some(blocked) = self.tree_blocks_ms { + self.now += blocked; + return Err(format!( + "{UNANSWERED} to walk the window within {blocked} ms" + )); + } if self.tree_fails_from.is_some_and(|t| self.now >= t) { return Err("no desktop element: Unspecified error".to_string()); } @@ -1143,6 +1215,134 @@ mod tests { assert!(err.contains("Unspecified error"), "{err}"); } + /// **The window the caller pinned is destroyed mid-wait: `element_gone` + /// is found at once.** The look that met a closing window is where the + /// live wait hung: no look is made of a window known to be destroyed. + #[test] + fn a_pinned_window_destroyed_mid_wait_ends_element_gone_without_another_look() { + let mut env = Fake::new(); + env.elements = Box::new(|_| vec![button(1, "Don't save")]); + env.alive = Some(Box::new(|t| t < 300)); + let c = Condition::Element { + selector: sel("role=button;name=Don't save"), + window: Some("1".to_string()), + appears: false, + }; + let out = run(&mut env, &c, &budget(4_000, 250), &never).unwrap(); + match out { + Outcome::Met { + found: Found::ElementGone { why }, + waited_ms, + .. + } => { + assert!(why.contains("window"), "{why}"); + assert_eq!(waited_ms, 500, "found at the first look after it closed"); + } + other => panic!("{other:?}"), + } + let trees_before_it_closed = env.trees; + assert!(trees_before_it_closed <= 2, "{trees_before_it_closed}"); + } + + #[test] + fn a_pinned_window_already_destroyed_is_gone_at_once_and_nothing_can_appear_in_it() { + let mut env = Fake::new(); + env.alive = Some(Box::new(|_| false)); + env.windows = Box::new(|_| vec![]); + let gone_c = Condition::Element { + selector: sel("role=button;name=OK"), + window: Some("1".to_string()), + appears: false, + }; + let out = run(&mut env, &gone_c, &budget(4_000, 250), &never).unwrap(); + assert!( + matches!( + out, + Outcome::Met { + found: Found::ElementGone { .. }, + waited_ms: 0, + polls: 1, + .. + } + ), + "{out:?}" + ); + assert_eq!(env.trees, 0, "no walk of a window that is not there"); + assert_eq!(env.windows_calls, 0, "and not even a listing"); + // The same id, waiting for something to appear in it, is an error. + let appears_c = Condition::Element { + selector: sel("role=button;name=OK"), + window: Some("1".to_string()), + appears: true, + }; + let err = run(&mut env, &appears_c, &budget(4_000, 250), &never).unwrap_err(); + assert!(err.contains("no window has the id"), "{err}"); + } + + /// An id that is not listed, on a desktop that cannot tell, is gone for + /// `element_gone` as well: a closed window is not in the list. + #[test] + fn an_unlisted_pinned_window_is_gone_for_element_gone() { + let mut env = Fake::new(); + env.windows = Box::new(|_| vec![window("2", "app", "Other", true)]); + let c = Condition::Element { + selector: sel("role=button;name=OK"), + window: Some("1".to_string()), + appears: false, + }; + let out = run(&mut env, &c, &budget(4_000, 250), &never).unwrap(); + assert!(matches!( + out, + Outcome::Met { + found: Found::ElementGone { .. }, + .. + } + )); + } + + /// **A look that blocks cannot keep the wait past its deadline.** The + /// desktop gives up on such a look at its limit (`mcp::wait_tool`) and the + /// wait treats it as a look that saw nothing: no second look is started + /// after the deadline, and the reply is a timeout that says why. + #[test] + fn a_look_the_desktop_did_not_answer_is_a_timeout_not_a_hang_or_an_error() { + let mut env = Fake::new(); + env.elements = Box::new(|_| vec![button(1, "OK")]); + // Every walk takes 3 s of virtual time and is given up on. + env.tree_blocks_ms = Some(3_000); + let c = appears("name=OK"); + let out = run(&mut env, &c, &budget(4_000, 250), &never).unwrap(); + match out { + Outcome::TimedOut { + waited_ms, + polls, + last, + .. + } => { + assert!(last.starts_with(UNANSWERED), "{last}"); + // Looks start at 0, 3250 (after its poll interval) and no + // later: the one at 3250 ends at 6250, past the deadline, + // and no third is begun. + assert_eq!(polls, 2, "no look started after the deadline"); + assert_eq!(env.trees, 2); + assert!(waited_ms < 4_000 + 3_000, "{waited_ms}"); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn a_look_that_is_never_answered_does_not_turn_an_element_gone_wait_into_gone() { + let mut env = Fake::new(); + env.elements = Box::new(|_| vec![]); + env.tree_blocks_ms = Some(500); + let out = run(&mut env, &gone("name=OK"), &budget(1_000, 250), &never).unwrap(); + assert!( + matches!(out, Outcome::TimedOut { .. }), + "an unanswered look proves nothing: {out:?}" + ); + } + #[test] fn a_cut_tree_cannot_prove_an_element_gone() { let mut env = Fake::new(); From 074c6c82168ecd7f6eb3552094cb1d5022c2fdca Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:27:26 -0600 Subject: [PATCH 5/9] desktop: set_value says only what its read-back can prove Live test: set_value on Notepad replied "set e2 (Edit) to 109 chars (unverified: it reads back as 109 characters, not the 109 sent)" and, on a Chrome input, "reads back as 0 characters" when the text was there. - A read-back is judged by one pure function (uia/readback.rs) on every platform, with line endings normalised (CRLF, a RichEdit lone CR) so a multi-line value no longer "differs" by length alone. - Same: verified. Empty after a non-empty set: a browser or custom control that does not report its value; unverified, not an error. A real value that is exactly what the field held before, or other text: an error that gives lengths and the first differing position, never the text. - An unconfirmed reply says "tried to set eN (Role) to N chars", never "set". Windows adds when a program brought its window to the front while its value was set (Notepad does, with no incu involved). - set_value's tool description says Notepad's ValuePattern change is not counted as an edit, so type_text or paste_text are the real edits. --- crates/desktop/src/linux.rs | 28 +++-- crates/desktop/src/macos.rs | 33 +++-- crates/desktop/src/mcp/definitions.rs | 12 +- crates/desktop/src/uia.rs | 20 +++ crates/desktop/src/uia/readback.rs | 170 ++++++++++++++++++++++++++ crates/desktop/src/uia_windows.rs | 61 ++++++--- 6 files changed, 285 insertions(+), 39 deletions(-) create mode 100644 crates/desktop/src/uia/readback.rs diff --git a/crates/desktop/src/linux.rs b/crates/desktop/src/linux.rs index 440c0b3..0936f36 100644 --- a/crates/desktop/src/linux.rs +++ b/crates/desktop/src/linux.rs @@ -1226,6 +1226,11 @@ pub(crate) fn set_value( let desktop = desktop()?; let (bus, node, described) = control(&desktop, window, want, max_depth)?; let what = described.label(); + // What it held before (never a password field's), to tell a field that + // ignored the set from one whose read-back is unreliable. + let before = (!described.secret) + .then(|| ask::<String, _>(bus, &node, "org.a11y.atspi.Text", "GetText", &(0i32, -1i32)).ok()) + .flatten(); let set: bool = ask( bus, &node, @@ -1242,21 +1247,22 @@ pub(crate) fn set_value( return Err(DesktopError::Os(format!("{what} refused the new text"))); } let did = described.set_to(value); + let tried = described.tried_to_set(value); if described.secret { - return Ok(Done::unverified(did, "a password field is never read back")); + return Ok(Done::unverified(tried, "a password field is never read back")); } Ok( match ask::<String, _>(bus, &node, "org.a11y.atspi.Text", "GetText", &(0i32, -1i32)) { - Ok(now) if now == value => Done::verified(did), - Ok(now) => Done::unverified( - did, - &format!( - "it reads back as {} characters, not the {} sent", - now.chars().count(), - value.chars().count() - ), - ), - Err(_) => Done::unverified(did, "its text could not be read back"), + Ok(now) => match uia::judge(value, before.as_deref(), &now, false) { + uia::ReadBack::Same => Done::verified(did), + uia::ReadBack::Unverified(why) => Done::unverified(tried, &why), + uia::ReadBack::Differs(why) => { + return Err(DesktopError::Os(format!( + "{what} did not take the value: {why}" + ))) + } + }, + Err(_) => Done::unverified(tried, "its text could not be read back"), }, ) } diff --git a/crates/desktop/src/macos.rs b/crates/desktop/src/macos.rs index 16596f4..d958c68 100644 --- a/crates/desktop/src/macos.rs +++ b/crates/desktop/src/macos.rs @@ -713,26 +713,39 @@ pub(crate) fn set_value( let what = described.label(); let text = cf_string(value).ok_or_else(|| DesktopError::Os("could not hold the value".to_string()))?; + // What it held before (never a secure field's), to tell a field that + // ignored the set from one whose read-back is unreliable. + let before = (!described.secret) + .then(|| text_attribute(&element, "AXValue")) + .flatten(); if !set_attribute(&element, "AXValue", text.0) { return Err(DesktopError::Os(format!( "{what} has no value to set; click_element it and use type_text instead" ))); } let did = described.set_to(value); + let tried = described.tried_to_set(value); if described.secret { - return Ok(Done::unverified(did, "a password field is never read back")); + return Ok(Done::unverified(tried, "a password field is never read back")); } + // `text_attribute` trims and drops an empty value, so the sent text is + // trimmed too and an empty read-back is `None`. Ok(match text_attribute(&element, "AXValue") { - Some(now) if now == value.trim() => Done::verified(did), - Some(now) => Done::unverified( - did, - &format!( - "it reads back as {} characters, not the {} sent", - now.chars().count(), - value.chars().count() - ), + Some(now) => match super::uia::judge(value, before.as_deref(), &now, true) { + super::uia::ReadBack::Same => Done::verified(did), + super::uia::ReadBack::Unverified(why) => Done::unverified(tried, &why), + super::uia::ReadBack::Differs(why) => { + return Err(DesktopError::Os(format!( + "{what} did not take the value: {why}" + ))) + } + }, + None if value.trim().is_empty() => Done::verified(did), + None => Done::unverified( + tried, + "it reads back empty, which a browser or custom control does when it does not \ + report its value, so the text may be there; look at the result", ), - None => Done::unverified(did, "its value could not be read back"), }) } diff --git a/crates/desktop/src/mcp/definitions.rs b/crates/desktop/src/mcp/definitions.rs index 3170e3f..ea208f1 100644 --- a/crates/desktop/src/mcp/definitions.rs +++ b/crates/desktop/src/mcp/definitions.rs @@ -106,8 +106,16 @@ fn base_definitions() -> Vec<Value> { the accessibility interface, by the id ui_tree gave it or by a \ `selector` — no typing, no focus needed, no clipboard. PREFER \ THIS over type_text for a field. The value is read back where \ - allowed and the result says whether it matched (a password \ - field is never read back).", + allowed: it reads back as sent (verified), or the call is an \ + ERROR when it reads back as something else or as it was \ + before, or the reply says `tried to set` and `unverified` \ + when the control cannot tell (a browser's input reads back \ + empty; a password field is never read back). Line endings do \ + not count as a difference. A program may bring its window to \ + the front when its value is set, and some do not count it as \ + an edit (Notepad does not mark the document modified, so \ + closing it discards the text): type_text or paste_text are \ + the real edits.", "inputSchema": { "type": "object", "properties": { diff --git a/crates/desktop/src/uia.rs b/crates/desktop/src/uia.rs index 0f241d0..3be22ce 100644 --- a/crates/desktop/src/uia.rs +++ b/crates/desktop/src/uia.rs @@ -182,6 +182,18 @@ impl Element { value.chars().count() ) } + + /// What `set_value` did when it could not confirm it: + /// `tried to set e49 (Edit) to 17 chars`. Never "set": a reply that says + /// `set` claims a change, and an unconfirmed one must not. + pub(crate) fn tried_to_set(&self, value: &str) -> String { + format!( + "tried to set e{} ({:?}) to {} chars", + self.id, + self.role, + value.chars().count() + ) + } } /// The result of one walk. @@ -653,6 +665,14 @@ pub fn focus_window(window: &str) -> Result<Done, DesktopError> { platform::focus_window(window) } +// What reading a field back after `set_value` can honestly say: `readback.rs`. +mod readback; +#[cfg_attr( + not(any(windows, target_os = "macos", target_os = "linux")), + allow(unused_imports) +)] +pub(crate) use readback::{judge, ReadBack}; + // Acting on a control by selector instead of by id: `resolve.rs`. mod resolve; pub use resolve::{act_on, resolve_in, set_value_on, tree_now, window_for, Resolved, WindowGuard}; diff --git a/crates/desktop/src/uia/readback.rs b/crates/desktop/src/uia/readback.rs new file mode 100644 index 0000000..cf793b2 --- /dev/null +++ b/crates/desktop/src/uia/readback.rs @@ -0,0 +1,170 @@ +//! What reading a field back after `set_value` can honestly say. +//! +//! `set_value` sends text through the accessibility interface and reads the +//! field back. Whether what comes back can be believed depends on the control, +//! and the three answers are kept apart because they mean different things to +//! the agent: +//! +//! - **Same**: the field reads back what was sent. Line endings are not +//! differences: Windows edit controls return `\r\n`, RichEdit (Notepad) `\r`, +//! and the text sent had `\n`. It is `verified`. +//! - **Unverified**: the read-back is not evidence either way. A browser's +//! input reads back as EMPTY after the text was put in it (measured on +//! Chrome: the page showed the text and `ValuePattern` said 0 characters). +//! An empty read-back after a non-empty set is a control that does not +//! report its value, not one that refused it. The reply says the text may be +//! there and was not confirmed, and never claims it was set. +//! - **Differs**: the read-back is a real value and it is not what was sent: +//! either it is exactly what the field held BEFORE (the control ignored the +//! set), or it is other text. That is an error, not a success with a note. +//! +//! Errors are journalled word for word, so none quotes a value: only lengths +//! and the position of the first difference. +//! +//! Pure, so it is tested on every platform; the platforms call [`judge`]. + +/// `text` with every line ending as `\n`: `\r\n` and a lone `\r` (RichEdit's +/// paragraph mark) are both one `\n`. +#[must_use] +pub(crate) fn lines(text: &str) -> String { + text.replace("\r\n", "\n").replace('\r', "\n") +} + +/// What reading the field back after the set means. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ReadBack { + /// Reads back as sent, line endings aside. + Same, + /// The read-back proves nothing; why, in words for the reply. + Unverified(String), + /// A real value that is not what was sent; why, without the values. + Differs(String), +} + +/// Judge `now`, what the field reads back after `sent` was set. `before` is +/// what it read before the set, when that could be read. `trim` is for a +/// platform whose read-back is trimmed (macOS), so the sent text is too. +#[must_use] +pub(crate) fn judge(sent: &str, before: Option<&str>, now: &str, trim: bool) -> ReadBack { + let cut = |s: String| if trim { s.trim().to_string() } else { s }; + let (sent, now) = (cut(lines(sent)), cut(lines(now))); + if sent == now { + return ReadBack::Same; + } + if now.is_empty() { + return ReadBack::Unverified( + "it reads back empty, which a browser or custom control does when it does not \ + report its value, so the text may be there; look at the result" + .to_string(), + ); + } + if let Some(before) = before.map(|b| cut(lines(b))).filter(|b| !b.is_empty()) { + if before == now { + return ReadBack::Differs(format!( + "the control kept its old value: it reads back exactly as it did before the set \ + ({} characters), not as sent ({} characters)", + now.chars().count(), + sent.chars().count() + )); + } + } + let at = sent + .chars() + .zip(now.chars()) + .position(|(a, b)| a != b) + .unwrap_or_else(|| sent.chars().count().min(now.chars().count())); + ReadBack::Differs(format!( + "it reads back differently from what was sent (line endings aside): {} characters \ + sent, {} read back, the first difference at character {}", + sent.chars().count(), + now.chars().count(), + at + 1 + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn line_endings_are_not_differences() { + assert_eq!(lines("a\r\nb\rc\nd"), "a\nb\nc\nd"); + // Notepad: RichEdit reads paragraphs back with a bare CR. + assert_eq!(judge("one\ntwo\nthree", None, "one\rtwo\rthree", false), ReadBack::Same); + // A Windows edit control: CRLF. + assert_eq!(judge("one\ntwo", None, "one\r\ntwo", false), ReadBack::Same); + // And the other way round. + assert_eq!(judge("one\r\ntwo", None, "one\ntwo", false), ReadBack::Same); + } + + #[test] + fn the_same_text_is_verified_and_a_trimmed_platform_trims_what_it_sent() { + assert_eq!(judge("hello", None, "hello", false), ReadBack::Same); + assert_eq!(judge(" hello \n", None, "hello", true), ReadBack::Same); + assert!(matches!( + judge(" hello ", None, "hello", false), + ReadBack::Differs(_) + )); + } + + /// Chrome: the text was put in, the read-back is empty. Not an error, and + /// not a success either. + #[test] + fn an_empty_read_back_after_a_set_is_unverified_not_an_error() { + match judge("hello", Some(""), "", false) { + ReadBack::Unverified(why) => assert!(why.contains("reads back empty"), "{why}"), + other => panic!("{other:?}"), + } + // Even when the field held text before: a browser reads nothing at all. + assert!(matches!( + judge("hello", Some("old text"), "", false), + ReadBack::Unverified(_) + )); + // Setting an empty value reads back empty, and that is the same. + assert_eq!(judge("", Some("old"), "", false), ReadBack::Same); + } + + #[test] + fn a_field_that_still_reads_as_it_did_before_ignored_the_set() { + match judge("new text", Some("old text"), "old text", false) { + ReadBack::Differs(why) => { + assert!(why.contains("kept its old value"), "{why}"); + assert!(!why.contains("old text") && !why.contains("new text"), "{why}"); + } + other => panic!("{other:?}"), + } + // The old value compared the way the new one is: line endings aside. + assert!(matches!( + judge("new", Some("a\r\nb"), "a\nb", false), + ReadBack::Differs(_) + )); + } + + #[test] + fn other_text_is_an_error_that_gives_numbers_and_never_the_text() { + match judge("hello world", Some(""), "hello there!", false) { + ReadBack::Differs(why) => { + assert!(why.contains("11 characters sent"), "{why}"); + assert!(why.contains("12 read back"), "{why}"); + assert!(why.contains("character 7"), "{why}"); + assert!(!why.contains("hello"), "{why}"); + } + other => panic!("{other:?}"), + } + // A prefix: the first difference is where the shorter one ends. + match judge("abc", None, "abcdef", false) { + ReadBack::Differs(why) => assert!(why.contains("character 4"), "{why}"), + other => panic!("{other:?}"), + } + } + + #[test] + fn an_unchanged_empty_field_is_not_taken_for_an_ignored_set() { + // A field that was empty and reads back something else is just other + // text; "kept its old value" needs an old value to have kept. + match judge("hello", Some(""), "x", false) { + ReadBack::Differs(why) => assert!(!why.contains("kept its old value"), "{why}"), + other => panic!("{other:?}"), + } + } +} diff --git a/crates/desktop/src/uia_windows.rs b/crates/desktop/src/uia_windows.rs index 1ae04a5..fbd44f2 100644 --- a/crates/desktop/src/uia_windows.rs +++ b/crates/desktop/src/uia_windows.rs @@ -308,34 +308,63 @@ pub(super) fn set_value( max_depth: u32, value: &str, ) -> Result<Done, DesktopError> { - let (element, described, _) = control(window, want, max_depth)?; + let (element, described, info) = control(window, want, max_depth)?; let what = described.label(); let did = described.set_to(value); + let tried = described.tried_to_set(value); if let Ok(pattern) = element.get_pattern::<UIValuePattern>() { + // What it held before, so a field that ignored the set can be told + // from one whose read-back is just unreliable. Never a password's. + let before = (!described.secret) + .then(|| pattern.get_value().ok()) + .flatten(); + let in_front_before = foreground_hwnd(); pattern .set_value(value) .map_err(|e| DesktopError::Os(format!("could not set {what}: {e}")))?; + // `set_value` is documented as needing no focus. Some programs take it + // themselves when a value is set (Notepad does: measured with a plain + // .NET UIA client, no incu involved), and the reply says so. + let came_to_front = in_front_before != foreground_hwnd() + && info.id.parse::<isize>().ok() == Some(foreground_hwnd()); + let said = |done: Done| { + if came_to_front { + Done { + detail: format!( + "{}; the program brought its window to the front when its value was set", + done.detail + ), + ..done + } + } else { + done + } + }; if described.secret { - return Ok(Done::unverified(did, "a password field is never read back")); + return Ok(said(Done::unverified( + tried, + "a password field is never read back", + ))); } - return Ok(match pattern.get_value() { - Ok(now) if now == value => Done::verified(did), - Ok(now) => Done::unverified( - did, - &format!( - "it reads back as {} characters, not the {} sent", - now.chars().count(), - value.chars().count() - ), - ), - Err(_) => Done::unverified(did, "its value could not be read back"), - }); + return match pattern.get_value() { + Ok(now) => match super::judge(value, before.as_deref(), &now, false) { + super::ReadBack::Same => Ok(said(Done::verified(did))), + super::ReadBack::Unverified(why) => Ok(said(Done::unverified(tried, &why))), + super::ReadBack::Differs(why) => Err(DesktopError::Os(format!( + "{what} did not take the value: {why}" + ))), + }, + Err(_) => Ok(said(Done::unverified( + tried, + "its value could not be read back", + ))), + }; } if let Ok(pattern) = element.get_pattern::<UILegacyIAccessiblePattern>() { if pattern.set_value(value).is_ok() { return Ok(Done::unverified( - did, - "set through the legacy accessibility interface, which cannot be read back", + tried, + "sent through the legacy accessibility interface, which cannot be read back", )); } } From 1530260bb89d5f50a048ef90f82c5d742d46b66c Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:51:17 -0600 Subject: [PATCH 6/9] desktop: selectors resolve past the print cap, and give a browser's page time Live test, Chrome on a long Wikipedia article: links below the 400th element could not be found by selector (resolution shared ui_tree's print cap), and the first look at a fresh page held only the browser's own controls. - A walk made to FIND a control (selector resolution, wait_for, acting on what a selector found) keeps up to tree_max_visited elements (default 4000), not tree_max_elements. Printing needs a cap that fits a reply; finding reports one control. The visit bound already limited time and cost, and is now the only bound. Walk order is unchanged, so a control within the first 400 keeps its id, nth and path: a selector ui_tree printed still means the same control. The cut warning now names the real bound. - A selector that matches nothing in a browser window (chrome, msedge, firefox, brave, vivaldi, opera, chromium) is walked once more after 1.5 s: the first walk is what makes the browser build the page's accessibility tree. Only a miss is retried, once; a match, an ambiguity or another program is not, and the error then says the wait happened. wait_for polls and needs none of it. Smallest safe fix: no new tool, no platform code, and no cost on a hit. - Tests: a control 600 elements past the print cap is found and the print-cap walk cannot see it; selectors printed within the cap resolve to the same control in the larger walk; the retry runs once on a miss, never on a hit, an ambiguity or a non-browser. - cargo fmt over uia/readback.rs, which the previous commit left unformatted. --- crates/desktop/src/linux.rs | 16 +- crates/desktop/src/macos.rs | 13 +- crates/desktop/src/mcp/tools.rs | 5 +- crates/desktop/src/uia.rs | 68 ++++++- crates/desktop/src/uia/readback.rs | 10 +- crates/desktop/src/uia/resolve.rs | 283 ++++++++++++++++++++++++++++- crates/desktop/src/uia_windows.rs | 8 +- docs/COMPUTER_USE.md | 21 ++- 8 files changed, 396 insertions(+), 28 deletions(-) diff --git a/crates/desktop/src/linux.rs b/crates/desktop/src/linux.rs index 0936f36..24d9e08 100644 --- a/crates/desktop/src/linux.rs +++ b/crates/desktop/src/linux.rs @@ -1143,13 +1143,18 @@ pub(crate) fn window_info(window: Option<&str>) -> Result<WindowInfo, DesktopErr Ok(target(&desktop, window)?.info.clone()) } -pub(crate) fn tree(window: Option<&str>, max_depth: u32) -> Result<Tree, DesktopError> { +pub(crate) fn tree( + window: Option<&str>, + max_depth: u32, + keep: usize, +) -> Result<Tree, DesktopError> { let desktop = desktop()?; let listed = target(&desktop, window)?; let (bus, window) = accessible(&desktop, listed)?; let walked = uia::walk( window, max_depth, + keep, &|node: &Node, id, depth| describe(bus, node, id, depth), &|node: &Node| children(bus, node), ); @@ -1229,7 +1234,9 @@ pub(crate) fn set_value( // What it held before (never a password field's), to tell a field that // ignored the set from one whose read-back is unreliable. let before = (!described.secret) - .then(|| ask::<String, _>(bus, &node, "org.a11y.atspi.Text", "GetText", &(0i32, -1i32)).ok()) + .then(|| { + ask::<String, _>(bus, &node, "org.a11y.atspi.Text", "GetText", &(0i32, -1i32)).ok() + }) .flatten(); let set: bool = ask( bus, @@ -1249,7 +1256,10 @@ pub(crate) fn set_value( let did = described.set_to(value); let tried = described.tried_to_set(value); if described.secret { - return Ok(Done::unverified(tried, "a password field is never read back")); + return Ok(Done::unverified( + tried, + "a password field is never read back", + )); } Ok( match ask::<String, _>(bus, &node, "org.a11y.atspi.Text", "GetText", &(0i32, -1i32)) { diff --git a/crates/desktop/src/macos.rs b/crates/desktop/src/macos.rs index d958c68..5777f44 100644 --- a/crates/desktop/src/macos.rs +++ b/crates/desktop/src/macos.rs @@ -679,9 +679,13 @@ pub(crate) fn window_info(window: Option<&str>) -> Result<WindowInfo, DesktopErr Ok(target(window)?.1) } -pub(crate) fn tree(window: Option<&str>, max_depth: u32) -> Result<Tree, DesktopError> { +pub(crate) fn tree( + window: Option<&str>, + max_depth: u32, + keep: usize, +) -> Result<Tree, DesktopError> { let (root, info) = target(window)?; - let walked = walk(&root, max_depth, &describe, &children); + let walked = walk(&root, max_depth, keep, &describe, &children); Ok(Tree { root_name: info.title, window: info.id, @@ -726,7 +730,10 @@ pub(crate) fn set_value( let did = described.set_to(value); let tried = described.tried_to_set(value); if described.secret { - return Ok(Done::unverified(tried, "a password field is never read back")); + return Ok(Done::unverified( + tried, + "a password field is never read back", + )); } // `text_attribute` trims and drops an empty value, so the sent text is // trimmed too and an empty read-back is `None`. diff --git a/crates/desktop/src/mcp/tools.rs b/crates/desktop/src/mcp/tools.rs index ee9636a..d8fd5c4 100644 --- a/crates/desktop/src/mcp/tools.rs +++ b/crates/desktop/src/mcp/tools.rs @@ -624,8 +624,9 @@ fn by_selector(done: String, found: &uia::Resolved) -> String { fn with_cut_note(base: String, found: &uia::Resolved) -> String { if found.truncated { format!( - "{base} (the tree was cut at its element limit, so a second control that fits the \ - selector may lie beyond it)" + "{base} (the walk was cut at its bound of {} nodes visited, so a second control \ + that fits the selector may lie beyond it)", + crate::limits::get().tree_max_visited ) } else { base diff --git a/crates/desktop/src/uia.rs b/crates/desktop/src/uia.rs index 3be22ce..bdebf2a 100644 --- a/crates/desktop/src/uia.rs +++ b/crates/desktop/src/uia.rs @@ -45,11 +45,32 @@ pub const DEFAULT_DEPTH: u32 = super::limits::DEFAULT_TREE_DEPTH; /// admin.json `desktop_limits.tree_max_elements` changes it. pub const MAX_ELEMENTS: usize = super::limits::DEFAULT_TREE_MAX_ELEMENTS as usize; -/// The element cap in force. +/// The element cap in force for what `ui_tree` PRINTS. fn max_elements() -> usize { super::limits::get().tree_max_elements } +/// The element cap in force when a walk is made to FIND a control (a selector, +/// `wait_for`, acting on what a selector found). +/// +/// Printing is bounded by what fits in the agent's context (400 by default); +/// finding is not, since only one control is reported. Sharing the print cap +/// hid everything past the first 400 elements of a long page: a link low on a +/// Wikipedia article could not be selected at all. So finding is bounded by the +/// walk's other bound, the number of nodes VISITED (`tree_max_visited`, 4000 by +/// default), which already caps its time and cost, and never by less than the +/// print cap (an operator who raises that cap is not lowered below it). +/// +/// The walk order is the same, so a control within the first 400 has the same +/// id, `nth` and `path` whichever cap walked: a selector `ui_tree` printed +/// still means the same control. +pub(crate) fn resolve_max_elements() -> usize { + let limits = super::limits::get(); + limits + .tree_max_elements + .max(limits.tree_max_visited as usize) +} + /// A control's kind, in a small closed vocabulary. /// /// NOT the platform's own name. Windows says "Button", macOS says @@ -520,7 +541,7 @@ pub fn window_position_unknown(window: &WindowInfo) -> Option<String> { /// the focused window. pub fn window_tree(window: Option<&str>, depth: u32) -> Result<Tree, DesktopError> { let depth = bounded(depth); - let tree = platform::tree(window, depth)?; + let tree = platform::tree(window, depth, max_elements())?; remember_shown(&tree, depth); Ok(tree) } @@ -617,7 +638,7 @@ pub fn locate(window: Option<&str>, id: u32) -> Result<Option<Element>, DesktopE let (info, depth, want) = wanted(window, id)?; // Not `window_tree`: this walk is a check, and recording it as shown // would let a retry of the same stale id pass against it unseen. - let tree = platform::tree(Some(&info.id), depth)?; + let tree = platform::tree(Some(&info.id), depth, max_elements())?; // A position is what the caller wants from this; one that is not real // would put the click in the window's corner. if let Some(why) = tree.no_positions { @@ -705,7 +726,7 @@ mod platform { pub(super) fn window_info(_: Option<&str>) -> Result<WindowInfo, DesktopError> { Err(NO) } - pub(super) fn tree(_: Option<&str>, _: u32) -> Result<Tree, DesktopError> { + pub(super) fn tree(_: Option<&str>, _: u32, _: usize) -> Result<Tree, DesktopError> { Err(NO) } pub(super) fn set_value( @@ -740,6 +761,8 @@ struct Walk<'a, N> { out: Vec<Element>, next_id: u32, max_visited: u32, + /// Most elements kept: `ui_tree`'s cap, or [`resolve_max_elements`]. + keep: usize, truncated: bool, offscreen: u32, want: Option<&'a Want>, @@ -762,7 +785,7 @@ impl<N: Clone> Walk<'_, N> { if self.done { return; } - if self.out.len() >= max_elements() || self.next_id >= self.max_visited { + if self.out.len() >= self.keep || self.next_id >= self.max_visited { self.truncated = true; self.done = true; return; @@ -806,6 +829,7 @@ impl<N: Clone> Walk<'_, N> { pub(crate) fn walk<N: Clone>( root: &N, max_depth: u32, + keep: usize, describe: &dyn Fn(&N, u32, u32) -> Element, children: &dyn Fn(&N) -> Vec<N>, ) -> Walked { @@ -816,6 +840,7 @@ pub(crate) fn walk<N: Clone>( out: Vec::new(), next_id: 0, max_visited: super::limits::get().tree_max_visited, + keep, truncated: false, offscreen: 0, want: None, @@ -850,6 +875,8 @@ pub(crate) fn find<N: Clone>( out: Vec::new(), next_id: 0, max_visited: super::limits::get().tree_max_visited, + // Ids a selector's walk handed out can lie past what `ui_tree` prints. + keep: resolve_max_elements(), truncated: false, offscreen: 0, want: Some(want), @@ -906,7 +933,7 @@ mod tests { } fn walk_of(root: &Node, depth: u32) -> (Vec<Element>, bool) { - let walked = walk(root, depth, &node, &|n: &Node| n.2.clone()); + let walked = walk(root, depth, MAX_ELEMENTS, &node, &|n: &Node| n.2.clone()); (walked.elements, walked.truncated) } @@ -1060,7 +1087,7 @@ mod tests { ], ); let children = |n: &Node| n.2.clone(); - let walked = walk(&tree, 8, &node, &children); + let walked = walk(&tree, 8, MAX_ELEMENTS, &node, &children); let seen: Vec<(u32, &str)> = walked .elements .iter() @@ -1116,6 +1143,33 @@ mod tests { assert!(truncated); } + /// Finding is bounded by visits, not by what printing is: the resolving cap + /// is never below the print cap, and a control past the print cap keeps the + /// id it would have, found by id too. + #[test] + fn finding_a_control_is_not_held_to_the_print_cap() { + assert!(resolve_max_elements() >= max_elements()); + assert!(resolve_max_elements() > MAX_ELEMENTS); + let leaves = (0..MAX_ELEMENTS + 50) + .map(|_| Node(Role::Button, "b", vec![])) + .collect::<Vec<_>>(); + let tree = Node(Role::Window, "w", leaves); + let children = |n: &Node| n.2.clone(); + let printed = walk(&tree, 4, max_elements(), &node, &children); + let found = walk(&tree, 4, resolve_max_elements(), &node, &children); + assert!(printed.truncated && !found.truncated); + // Every leaf, and the window above them. + assert_eq!(found.elements.len(), MAX_ELEMENTS + 51); + assert_eq!( + printed.elements[..], + found.elements[..MAX_ELEMENTS], + "the larger walk starts exactly as the printed one" + ); + let low = &found.elements[MAX_ELEMENTS + 10]; + let (_, again) = find(&tree, 4, &node, &children, &want(low)).expect("past the cap"); + assert_eq!(again.id, low.id); + } + /// macOS roles map onto the same vocabulary as Windows; a tab is a radio /// button with a subrole there, and an unknown role is `Other`. #[test] diff --git a/crates/desktop/src/uia/readback.rs b/crates/desktop/src/uia/readback.rs index cf793b2..3a17e1d 100644 --- a/crates/desktop/src/uia/readback.rs +++ b/crates/desktop/src/uia/readback.rs @@ -90,7 +90,10 @@ mod tests { fn line_endings_are_not_differences() { assert_eq!(lines("a\r\nb\rc\nd"), "a\nb\nc\nd"); // Notepad: RichEdit reads paragraphs back with a bare CR. - assert_eq!(judge("one\ntwo\nthree", None, "one\rtwo\rthree", false), ReadBack::Same); + assert_eq!( + judge("one\ntwo\nthree", None, "one\rtwo\rthree", false), + ReadBack::Same + ); // A Windows edit control: CRLF. assert_eq!(judge("one\ntwo", None, "one\r\ntwo", false), ReadBack::Same); // And the other way round. @@ -129,7 +132,10 @@ mod tests { match judge("new text", Some("old text"), "old text", false) { ReadBack::Differs(why) => { assert!(why.contains("kept its old value"), "{why}"); - assert!(!why.contains("old text") && !why.contains("new text"), "{why}"); + assert!( + !why.contains("old text") && !why.contains("new text"), + "{why}" + ); } other => panic!("{other:?}"), } diff --git a/crates/desktop/src/uia/resolve.rs b/crates/desktop/src/uia/resolve.rs index 2d5ad0a..1f4b6bf 100644 --- a/crates/desktop/src/uia/resolve.rs +++ b/crates/desktop/src/uia/resolve.rs @@ -30,7 +30,7 @@ //! [`tree_now`]. use super::{bounded, platform, same_control, Done, Element, Tree, Want, WindowInfo}; -use crate::selector::Selector; +use crate::selector::{SelectError, Selector}; use crate::DesktopError; /// Windows listed in an error about a window that was not found. @@ -173,7 +173,41 @@ pub fn window_for( /// own, or the default), for a caller that only looks (`wait_for`). Not /// recorded as shown: an id is not being handed out. pub fn tree_now(window: &str, depth: Option<u32>) -> Result<Tree, DesktopError> { - platform::tree(Some(window), resolve_depth(depth)) + platform::tree( + Some(window), + resolve_depth(depth), + super::resolve_max_elements(), + ) +} + +/// How long a browser is given, once, to build a page's accessibility tree +/// after a selector found nothing in it. +/// +/// A browser does not expose a page until an accessibility client asks (Chrome +/// measured: the first walk of a fresh page held only the browser's own +/// controls, and about 3 s later the page was there). The walk is the ask; this +/// is the pause before the second walk. +const BROWSER_WARMUP: std::time::Duration = std::time::Duration::from_millis(1500); + +/// Programs whose windows hold a web page, by the start of their name +/// (lower-cased): Windows and Linux process names, macOS application names. +const BROWSERS: &[&str] = &[ + "chrome", + "google chrome", + "chromium", + "msedge", + "microsoft edge", + "brave", + "vivaldi", + "opera", + "firefox", +]; + +/// Whether `app` is a web browser, whose page content may not be in its first +/// walk. +fn is_browser(app: &str) -> bool { + let app = app.trim().to_lowercase(); + BROWSERS.iter().any(|b| app.starts_with(b)) } /// The control `selector` names in `tree`, which was walked at `depth`. @@ -189,8 +223,10 @@ pub(crate) fn resolved_from( .map_err(|e| { let cut = if tree.truncated { format!( - " (the tree was cut at {} elements, so it may lie beyond the cut)", - super::max_elements() + " (the walk was cut at its bound, {} elements kept or {} nodes visited, so \ + it may lie beyond)", + tree.elements.len(), + crate::limits::get().tree_max_visited ) } else { String::new() @@ -223,8 +259,49 @@ pub(crate) fn resolved_from( pub fn resolve_in(info: WindowInfo, selector: &Selector) -> Result<Resolved, DesktopError> { selector.require_element().map_err(os)?; let depth = resolve_depth(selector.depth); - let tree = platform::tree(Some(&info.id), depth)?; - resolved_from(info, tree, selector, depth) + let window = info.id.clone(); + resolve_with( + info, + selector, + depth, + &mut || platform::tree(Some(&window), depth, super::resolve_max_elements()), + &mut || std::thread::sleep(BROWSER_WARMUP), + ) +} + +/// [`resolve_in`] with its two outside effects handed in: `look` walks the +/// window, `pause` waits [`BROWSER_WARMUP`]. Separate so a test needs neither a +/// desktop nor a clock. +/// +/// A browser window in which nothing matched is walked a second time after the +/// pause, once: the first walk is what makes the browser build the page's +/// accessibility tree. Only a miss is retried (an ambiguity, a bad `nth`, or a +/// match is the answer already), so a hit costs nothing. +pub(crate) fn resolve_with( + info: WindowInfo, + selector: &Selector, + depth: u32, + look: &mut dyn FnMut() -> Result<Tree, DesktopError>, + pause: &mut dyn FnMut(), +) -> Result<Resolved, DesktopError> { + let mut tree = look()?; + let warm = is_browser(&info.app) + && matches!( + selector.resolve_in(&tree.elements), + Err(SelectError::Missing) + ); + if warm { + pause(); + tree = look()?; + } + resolved_from(info, tree, selector, depth).map_err(|e| match (warm, e) { + (true, DesktopError::Os(message)) => os(format!( + "{message} (a browser builds a page's accessibility tree when first asked, and \ + this waited {} ms for it once; if the control is on the page, ask again)", + BROWSER_WARMUP.as_millis() + )), + (_, e) => e, + }) } /// [`super::set_value`] on a control found by selector. @@ -550,6 +627,7 @@ mod tests { walk( root, depth, + crate::uia::resolve_max_elements(), &|n: &Node, id, d| Element { id, depth: d, @@ -635,4 +713,197 @@ mod tests { "held to the bound every walk is" ); } + + // ---- finding is not bounded by what printing is ---- + + /// A long page: `links` links under one window, named `link0`, `link1`, ... + fn long_page(links: usize) -> Node { + node( + Role::Window, + "Article", + (0..links) + .map(|i| node(Role::Link, &format!("link{i}"), vec![])) + .collect(), + ) + } + + /// **A link low on a long page is found by selector.** Resolution shared + /// `ui_tree`'s 400-element cap, so everything past the 400th element of an + /// article could not be selected (measured on a Wikipedia article). + #[test] + fn a_control_past_the_print_cap_is_found_by_selector() { + let print_cap = crate::limits::get().tree_max_elements; + let low = print_cap + 600; + let page = long_page(low + 50); + let found = walked(&page, 4); + assert!(found.len() > low, "the resolving walk kept {}", found.len()); + let s = sel(&format!("role=link;name=link{low}")); + let hit = s.resolve_in(&found).expect("found past the print cap"); + assert_eq!(hit.name, format!("link{low}")); + // The same walk, cut at what is printed, cannot see it. + let printed = walk( + &page, + 4, + print_cap, + &|n: &Node, id, d| Element { + id, + depth: d, + role: n.role, + name: n.name.clone(), + rect: rect(), + enabled: true, + secret: false, + automation_id: String::new(), + }, + &|n: &Node| n.kids.clone(), + ); + assert!(printed.truncated); + assert_eq!(printed.elements.len(), print_cap); + assert!(s.resolve_in(&printed.elements).is_err()); + // And the resolving walk is bounded: by visits, not unbounded. + let huge = walked( + &long_page(crate::limits::get().tree_max_visited as usize + 500), + 4, + ); + assert!(huge.len() <= crate::limits::get().tree_max_visited as usize); + } + + /// What `ui_tree` printed still means the same control to the resolver: + /// the walk order is the same, the larger cap only keeps more of the tail. + #[test] + fn a_selector_printed_within_the_cap_still_means_the_same_control() { + let print_cap = crate::limits::get().tree_max_elements; + // Names repeat, so the selectors printed for them carry an nth. + let twins = node( + Role::Window, + "Article", + (0..print_cap + 300) + .map(|i| node(Role::Link, &format!("same{}", i % 3), vec![])) + .collect(), + ); + let all_twins = walked(&twins, 4); + let printed_twins = &all_twins[..print_cap]; + let selectors = selectors_for(printed_twins, false, None); + for (e, s) in printed_twins.iter().zip(&selectors) { + let s = Selector::parse(&s.render()).unwrap(); + let got = s.resolve_in(&all_twins).unwrap_or_else(|why| { + panic!("`{}` for {e:?}: {why}", s.render()); + }); + assert_eq!((got.id, got.name.as_str()), (e.id, e.name.as_str())); + } + assert!(selectors.iter().any(|s| s.render().contains("nth="))); + } + + // ---- a browser's first walk ---- + + fn page_tree(with_target: bool) -> Tree { + let mut elements = vec![button(1, "Back")]; + if with_target { + elements.push(Element { + role: Role::Link, + ..button(2, "Honshu") + }); + } + tree_of(elements, false) + } + + /// Run `resolve_with` over `trees` (one per look, the last repeating) and + /// say how many looks and pauses it took. + fn resolve_over( + app: &str, + selector: &str, + trees: Vec<Tree>, + ) -> (Result<Resolved, DesktopError>, usize, usize) { + let looks = std::cell::Cell::new(0usize); + let pauses = std::cell::Cell::new(0usize); + let result = resolve_with( + window("7", app, "Page"), + &sel(selector), + 5, + &mut || { + let at = looks.get().min(trees.len() - 1); + looks.set(looks.get() + 1); + Ok(trees[at].clone()) + }, + &mut || pauses.set(pauses.get() + 1), + ); + (result, looks.get(), pauses.get()) + } + + /// **A browser's first walk of a page may hold only the browser.** Chrome + /// builds the page's accessibility tree once asked, so a selector that finds + /// nothing in a browser is tried once more after a pause. + #[test] + fn a_miss_in_a_browser_waits_once_and_looks_again() { + let (got, looks, pauses) = resolve_over( + "chrome", + "role=link;name=Honshu", + vec![page_tree(false), page_tree(true)], + ); + assert_eq!(got.unwrap().element.name, "Honshu"); + assert_eq!((looks, pauses), (2, 1)); + } + + #[test] + fn a_hit_in_a_browser_costs_no_pause_and_a_second_look() { + let (got, looks, pauses) = resolve_over( + "Google Chrome", + "role=link;name=Honshu", + vec![page_tree(true)], + ); + assert!(got.is_ok()); + assert_eq!((looks, pauses), (1, 0)); + } + + #[test] + fn a_browser_that_still_has_no_match_is_asked_once_not_forever_and_says_so() { + let (got, looks, pauses) = + resolve_over("msedge", "role=link;name=Honshu", vec![page_tree(false)]); + let err = got.unwrap_err().to_string(); + assert_eq!( + (looks, pauses), + (2, 1), + "one retry, however often it misses" + ); + assert!(err.contains("matches no element"), "{err}"); + assert!(err.contains("accessibility tree when first asked"), "{err}"); + } + + #[test] + fn a_miss_in_another_program_is_not_retried() { + let (got, looks, pauses) = + resolve_over("notepad", "role=link;name=Honshu", vec![page_tree(false)]); + let err = got.unwrap_err().to_string(); + assert_eq!((looks, pauses), (1, 0)); + assert!(!err.contains("when first asked"), "{err}"); + } + + #[test] + fn an_ambiguity_in_a_browser_is_an_answer_and_is_not_retried() { + let mut two = page_tree(true); + two.elements.push(Element { + role: Role::Link, + ..button(3, "Honshu") + }); + let (got, looks, pauses) = resolve_over("firefox", "role=link;name=Honshu", vec![two]); + assert!(got.unwrap_err().to_string().contains("matches 2 elements")); + assert_eq!((looks, pauses), (1, 0)); + } + + #[test] + fn browsers_are_told_from_other_programs_by_name() { + for app in [ + "chrome", + "Google Chrome", + "msedge", + "Microsoft Edge", + "firefox", + "brave", + ] { + assert!(is_browser(app), "{app}"); + } + for app in ["notepad", "explorer", "archive", "search", ""] { + assert!(!is_browser(app), "{app}"); + } + } } diff --git a/crates/desktop/src/uia_windows.rs b/crates/desktop/src/uia_windows.rs index fbd44f2..52054bb 100644 --- a/crates/desktop/src/uia_windows.rs +++ b/crates/desktop/src/uia_windows.rs @@ -260,13 +260,17 @@ pub(super) fn window_info(window: Option<&str>) -> Result<WindowInfo, DesktopErr Ok(target(&automation, window)?.1) } -pub(super) fn tree(window: Option<&str>, max_depth: u32) -> Result<Tree, DesktopError> { +pub(super) fn tree( + window: Option<&str>, + max_depth: u32, + keep: usize, +) -> Result<Tree, DesktopError> { let automation = automation()?; let (root, info) = target(&automation, window)?; let walker = automation .create_tree_walker() .map_err(|e| DesktopError::Os(format!("could not walk the tree: {e}")))?; - let walked = walk(&root, max_depth, &describe, &|e: &UIElement| { + let walked = walk(&root, max_depth, keep, &describe, &|e: &UIElement| { children_of(&walker, e) }); Ok(Tree { diff --git a/docs/COMPUTER_USE.md b/docs/COMPUTER_USE.md index 37b99b6..e1afa87 100644 --- a/docs/COMPUTER_USE.md +++ b/docs/COMPUTER_USE.md @@ -280,6 +280,18 @@ ids work exactly as before. | `nth` | which of several matches, from 0, in `ui_tree` order. Out of range is an error | | `depth` | how deep the window is walked to find the control (default 12, held to 4x that). `nth` and `path` count the controls of a walk, so the walk that prints a selector and the one that resolves it must be the same: `ui_tree` writes `depth=N` into every selector when it was asked for a depth other than the default, and the resolver walks as deep as the selector says, else to the default. Not a control part on its own | +*Resolution's own bound.* `ui_tree` stops printing at 400 elements (what fits in a reply). A +selector is resolved against its **own** walk, which is bounded by the number of nodes +visited (`desktop_limits.tree_max_visited`, default 4000: the limit on time and cost every walk +already had) and not by the print cap, so a link far down a long article is found. The walk +order is the same, so a control within the first 400 has the same id, `nth` and `path` in +both. When even the larger bound cuts the walk, an error or result says so and that a second +match may lie beyond it. *Browsers.* A browser exposes a page only after a client asks for it, +so its first walk can hold just its own controls. A selector that matches nothing in a +browser window (chrome, msedge, firefox, brave, vivaldi, opera, chromium) is walked once more +after 1.5 s, and the error then says it waited; a match, an ambiguity or any other program is +never retried. `wait_for` needs none of this, since it polls. + `key=value` pairs joined by `;`, with no spaces around the `;`; `\\`, `\;`, `\n`, `\r` escape a value. Unknown or repeated keys and bad numbers are refused by name. `app`/`window` pick the window (the one named, else `window=<id>`, else the focused one); the other keys pick the @@ -318,9 +330,12 @@ is the one watched, by id, so a change of focus is not mistaken for the control *Bounds.* `timeout_ms` default 5000, held to **60000** (a longer request is held to it and the reply's `notes` says so); `poll_ms` default 250, held to 50 to 2000. The first look is at once; no look starts after the deadline, and the last starts exactly at it, so a condition that -comes true at the deadline is caught; a look under way is not interrupted, so the call can end -later than `timeout_ms` by one look (listing the windows takes a few hundred ms on a busy -Windows desktop, measured). The operator's stop (stop file or `Safety`) is asked between +comes true at the deadline is caught. Each look runs on a thread of its own and is waited for +until the deadline plus 1 s: a look the desktop does not answer by then (a program that +hangs, a window that closed under the walk) counts as a look that saw nothing, and the wait +ends as a `timeout` whose `last_seen` says so, never as `gone`. So the call ends no later +than `timeout_ms` plus that second. A pinned window that no +longer exists is `element_gone` at once. The operator's stop (stop file or `Safety`) is asked between looks and every 100 ms of sleeping, and ends the wait with the stop's refusal. *Reply.* JSON whose `result` is `found` or `timeout` (a timeout is a reply, not a tool error, From df9c139b65e8f716cb7ea064b0e8c0d4c50fa3f2 Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:30:01 -0600 Subject: [PATCH 7/9] desktop: wait_for honours the stop during a blocked look, and one hung window does not block the others Review findings 2 and 3 on eaf0942. - A look is waited for in slices of 100 ms and the operator's stop is asked each time, so a stop during a look at a hung window returns within a slice, not at the deadline plus 1 s. wait_until asks the stop BEFORE the deadline, so the reply is Stopped and not a timeout. - The cap on abandoned looks (4) is now counted per window (the id of the window looked at, or the kind of look when it is of none), so one hung program no longer makes every later wait on an unrelated window time out at once. A higher process-wide net (24) remains. - Tests: a stop 300 ms into a 60 s look returns at about 300 ms; a stop during a blocked look is Stopped when the deadline is already past; a hung window's looks are refused while another window's run; the process-wide net holds and is above the per-window limit. --- crates/desktop/src/mcp/wait_tool.rs | 344 ++++++++++++++++++++++------ crates/desktop/src/wait.rs | 31 +++ 2 files changed, 306 insertions(+), 69 deletions(-) diff --git a/crates/desktop/src/mcp/wait_tool.rs b/crates/desktop/src/mcp/wait_tool.rs index 75e5b05..8d2cc93 100644 --- a/crates/desktop/src/mcp/wait_tool.rs +++ b/crates/desktop/src/mcp/wait_tool.rs @@ -13,8 +13,8 @@ //! - **Calls** [`crate::wait::run`] with [`Live`], which reads //! [`crate::uia`] and [`crate::capture`] through [`super::guard`]. -use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; -use std::sync::{mpsc, Arc}; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{mpsc, Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; use serde_json::Value; @@ -43,36 +43,103 @@ const FINGERPRINT_EDGE: u32 = 256; /// wait stops waiting for it at its deadline plus this much. const LOOK_GRACE: Duration = Duration::from_millis(1000); -/// Most looks that have been given up on and are still blocked, in this -/// process. At the limit no new look is started: another would only join them. -const MAX_ABANDONED: usize = 4; +/// Most looks at ONE window (or one kind of look) that have been given up on +/// and are still blocked. At the limit no new look at it is started: another +/// would only join them. Counted per window, so one hung program does not stop +/// a wait on any other. +const MAX_ABANDONED_PER_KEY: usize = 4; + +/// Most looks given up on and still blocked in the whole process: a safety net +/// above the per-window limit, against a desktop where very many windows hang. +const MAX_ABANDONED_IN_ALL: usize = 24; + +/// How often a look being waited for asks whether the operator stopped the +/// wait. A stop is acted on within this, not at the look's limit. +const STOP_SLICE: Duration = Duration::from_millis(100); + +/// The looks given up on that have not returned yet, by what they looked at +/// (a window's id, or the kind of look when it is not of one window). +struct Stuck { + looks: Mutex<Vec<(String, usize)>>, +} + +impl Stuck { + const fn new() -> Stuck { + Stuck { + looks: Mutex::new(Vec::new()), + } + } + + fn held(&self) -> MutexGuard<'_, Vec<(String, usize)>> { + self.looks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// How many looks at `key` are still blocked. + fn of(&self, key: &str) -> usize { + self.held() + .iter() + .find(|(k, _)| k == key) + .map_or(0, |(_, n)| *n) + } + + /// How many are still blocked, whatever they looked at. + fn all(&self) -> usize { + self.held().iter().map(|(_, n)| *n).sum() + } + + fn add(&self, key: &str) { + let mut held = self.held(); + match held.iter_mut().find(|(k, _)| k == key) { + Some((_, n)) => *n += 1, + None => held.push((key.to_string(), 1)), + } + } + + fn remove(&self, key: &str) { + let mut held = self.held(); + if let Some(at) = held.iter().position(|(k, _)| k == key) { + held[at].1 = held[at].1.saturating_sub(1); + if held[at].1 == 0 { + held.remove(at); + } + } + } +} /// Looks given up on that have not returned yet. -static ABANDONED: AtomicUsize = AtomicUsize::new(0); +static STUCK: Stuck = Stuck::new(); -/// `work`, run on a thread of its own and waited for at most `limit`. +/// `work`, run on a thread of its own and waited for at most `limit`, or until +/// `stopped` says the operator stopped the wait (asked every [`STOP_SLICE`]). /// -/// On time it is `work`'s answer. Past `limit` the thread is left to finish -/// by itself (it cannot be stopped from outside: it is inside a call the -/// operating system has not returned from) and the answer is -/// [`UNANSWERED`], counted in `abandoned` until the thread does return. +/// On time it is `work`'s answer. Past `limit`, or at the stop, the thread is +/// left to finish by itself (it cannot be stopped from outside: it is inside a +/// call the operating system has not returned from) and the answer is +/// [`UNANSWERED`], counted against `key` in `stuck` until the thread does +/// return. The wait that asked ends as stopped, not as unanswered, because it +/// asks the stop first. fn bounded<T: Send + 'static>( what: &str, + key: &str, limit: Duration, - abandoned: &'static AtomicUsize, + stuck: &'static Stuck, + stopped: &dyn Fn() -> bool, work: impl FnOnce() -> Result<T, String> + Send + 'static, ) -> Result<T, String> { - let still_blocked = abandoned.load(Ordering::SeqCst); - if still_blocked >= MAX_ABANDONED { + let (here, everywhere) = (stuck.of(key), stuck.all()); + if here >= MAX_ABANDONED_PER_KEY || everywhere >= MAX_ABANDONED_IN_ALL { return Err(format!( - "{UNANSWERED}: {still_blocked} earlier looks are still blocked in the \ - accessibility layer, so none was started to {what}" + "{UNANSWERED}: {here} earlier looks at this are still blocked in the \ + accessibility layer ({everywhere} in all), so none was started to {what}" )); } // 0 running, 1 returned in time, 2 given up on. let state = Arc::new(AtomicU8::new(0)); let (send, receive) = mpsc::channel(); let theirs = Arc::clone(&state); + let owned_key = key.to_string(); let spawned = std::thread::Builder::new() .name("wait-look".to_string()) .spawn(move || { @@ -83,61 +150,81 @@ fn bounded<T: Send + 'static>( .compare_exchange(0, 1, Ordering::SeqCst, Ordering::SeqCst) .is_err() { - abandoned.fetch_sub(1, Ordering::SeqCst); + stuck.remove(&owned_key); } let _ = send.send(answer); }); if let Err(e) = spawned { return Err(format!("could not start a look to {what}: {e}")); } - match receive.recv_timeout(limit) { - Ok(answer) => answer, - Err(_) => { - if state - .compare_exchange(0, 2, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - { - abandoned.fetch_add(1, Ordering::SeqCst); - Err(format!( - "{UNANSWERED} to {what} within {} ms (a window that is closing or not \ - responding can do this)", - limit.as_millis() - )) - } else { - // It returned in the instant between the timeout and this. - receive - .recv() - .unwrap_or_else(|_| Err(format!("the look to {what} ended without an answer"))) + let end = Instant::now() + limit; + loop { + let left = end.saturating_duration_since(Instant::now()); + match receive.recv_timeout(left.min(STOP_SLICE)) { + Ok(answer) => return answer, + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err(format!("the look to {what} ended without an answer")) } + Err(mpsc::RecvTimeoutError::Timeout) => {} } + let why = if stopped() { + "the wait was stopped".to_string() + } else if Instant::now() >= end { + format!( + "within {} ms (a window that is closing or not responding can do this)", + limit.as_millis() + ) + } else { + continue; + }; + return if state + .compare_exchange(0, 2, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + stuck.add(key); + Err(format!("{UNANSWERED} to {what}: {why}")) + } else { + // It returned in the instant between the timeout and this. + receive + .recv() + .unwrap_or_else(|_| Err(format!("the look to {what} ended without an answer"))) + }; } } +/// What a look at nothing in particular is counted against. +const ANYWHERE: &str = "(the desktop)"; + /// The desktop as this process sees it, with a clock that starts now. -struct Live { +struct Live<'a> { started: Instant, /// When the wait's `timeout_ms` is up. A look may run to this and /// [`LOOK_GRACE`] beyond it. deadline: Instant, + /// The operator's stop, asked while a look is waited for. + stopped: &'a dyn Fn() -> bool, } -impl Live { - fn new(budget: &Budget) -> Live { +impl<'a> Live<'a> { + fn new(budget: &Budget, stopped: &'a dyn Fn() -> bool) -> Live<'a> { let started = Instant::now(); Live { started, deadline: started + Duration::from_millis(budget.timeout_ms), + stopped, } } - /// `work` as one look, held to the deadline and the grace after it. + /// `work` as one look at `key` (a window's id, or [`ANYWHERE`]), held to + /// the deadline and the grace after it, and to the operator's stop. fn within<T: Send + 'static>( &self, what: &str, + key: &str, work: impl FnOnce() -> Result<T, String> + Send + 'static, ) -> Result<T, String> { let limit = (self.deadline + LOOK_GRACE).saturating_duration_since(Instant::now()); - bounded(what, limit, &ABANDONED, work) + bounded(what, key, limit, &STUCK, self.stopped, work) } } @@ -145,7 +232,7 @@ fn text_of(e: impl ToString) -> String { e.to_string() } -impl Env for Live { +impl Env for Live<'_> { fn now_ms(&mut self) -> u64 { u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX) } @@ -157,7 +244,7 @@ impl Env for Live { fn windows(&mut self) -> Result<Vec<WindowInfo>, String> { // Not the windows of a blocked application: they are not listed, and a // selector must not be able to find one by its title. - let all = self.within("list the windows", listed)?; + let all = self.within("list the windows", ANYWHERE, listed)?; Ok(all .into_iter() .filter(|w| crate::limits::get().blocked(&w.app).is_none()) @@ -169,7 +256,7 @@ impl Env for Live { } fn focused_window(&mut self) -> Result<WindowInfo, String> { - let window = self.within("find the focused window", || { + let window = self.within("find the focused window", ANYWHERE, || { uia::window_info(None).map_err(text_of) })?; refuse_blocked(&window)?; @@ -178,7 +265,10 @@ impl Env for Live { fn tree(&mut self, window: &str, depth: Option<u32>) -> Result<Tree, String> { let id = window.to_string(); - let (info, tree) = self.within("walk the window", move || { + // Counted against this window, so a window that hangs stops the looks + // at it and at no other. + let key = window.to_string(); + let (info, tree) = self.within("walk the window", &key, move || { let info = uia::window_info(Some(&id)).map_err(text_of)?; // Refused before the window is walked: a blocked application's // window is never read. @@ -194,7 +284,8 @@ impl Env for Live { fn screen(&mut self, window: Option<&str>) -> Result<Fingerprint, String> { let window = window.map(str::to_string); - self.within("capture the screen", move || { + let key = window.clone().unwrap_or_else(|| ANYWHERE.to_string()); + self.within("capture the screen", &key, move || { let of = guard_capture(window.as_deref())?; let options = CaptureOptions { max_edge: FINGERPRINT_EDGE, @@ -309,7 +400,7 @@ fn condition_of(args: &Value) -> Result<Condition, String> { pub(super) fn wait_for(args: &Value, stopped: &dyn Fn() -> bool) -> Result<Vec<Value>, String> { let condition = condition_of(args)?; let budget = Budget::new(whole(args, "timeout_ms")?, whole(args, "poll_ms")?); - let mut env = Live::new(&budget); + let mut env = Live::new(&budget, stopped); let outcome = wait::run(&mut env, &condition, &budget, stopped)?; if matches!(outcome, Outcome::Stopped { .. }) { return Err(crate::safety::Refusal::Stopped.message()); @@ -332,12 +423,14 @@ mod tests { /// than the limit and `bounded` must hand back long before it returns. #[test] fn a_look_that_blocks_is_given_up_on_at_its_limit_not_waited_for() { - static COUNT: AtomicUsize = AtomicUsize::new(0); + static STUCK: Stuck = Stuck::new(); let started = Instant::now(); let answer = bounded( "walk the window", + "w1", Duration::from_millis(150), - &COUNT, + &STUCK, + &|| false, || { std::thread::sleep(Duration::from_secs(5)); Ok(1) @@ -351,20 +444,59 @@ mod tests { took < Duration::from_millis(150) + LOOK_GRACE, "a 150 ms limit held the caller for {took:?}" ); - assert_eq!(COUNT.load(Ordering::SeqCst), 1, "counted while it blocks"); + assert_eq!(STUCK.of("w1"), 1, "counted while it blocks"); + } + + /// **The operator's stop is acted on within a slice, not at the limit.** A + /// 60 s look at a hung window, stopped at 300 ms, gives the caller back at + /// about 300 ms (and the caller's wait asks the stop before its deadline, + /// so it ends as stopped). + #[test] + fn a_stop_ends_the_wait_for_a_blocked_look_within_a_slice() { + static STUCK: Stuck = Stuck::new(); + let stop_at = Instant::now() + Duration::from_millis(300); + let started = Instant::now(); + let answer: Result<(), String> = bounded( + "walk the window", + "w1", + Duration::from_secs(60), + &STUCK, + &|| Instant::now() >= stop_at, + || { + std::thread::sleep(Duration::from_secs(5)); + Ok(()) + }, + ); + let took = started.elapsed(); + let why = answer.unwrap_err(); + assert!(why.starts_with(UNANSWERED), "{why}"); + assert!(why.contains("stopped"), "{why}"); + assert!( + took >= Duration::from_millis(300) + && took < Duration::from_millis(300) + STOP_SLICE * 5, + "stopped at 300 ms, returned at {took:?}" + ); + assert_eq!(STUCK.of("w1"), 1); } #[test] fn a_look_given_up_on_is_taken_off_the_count_when_it_returns() { - static COUNT: AtomicUsize = AtomicUsize::new(0); - let answer = bounded("walk", Duration::from_millis(50), &COUNT, || { - std::thread::sleep(Duration::from_millis(300)); - Ok(()) - }); + static STUCK: Stuck = Stuck::new(); + let answer = bounded( + "walk", + "w1", + Duration::from_millis(50), + &STUCK, + &|| false, + || { + std::thread::sleep(Duration::from_millis(300)); + Ok(()) + }, + ); assert!(answer.is_err()); - assert_eq!(COUNT.load(Ordering::SeqCst), 1); + assert_eq!(STUCK.of("w1"), 1); let until = Instant::now() + Duration::from_secs(5); - while COUNT.load(Ordering::SeqCst) != 0 { + while STUCK.all() != 0 { assert!(Instant::now() < until, "the count was never given back"); std::thread::sleep(Duration::from_millis(20)); } @@ -372,25 +504,47 @@ mod tests { #[test] fn a_look_that_answers_in_time_gives_its_answer_or_its_error() { - static COUNT: AtomicUsize = AtomicUsize::new(0); - let ok = bounded("list", Duration::from_secs(5), &COUNT, || Ok(vec![1, 2])); + static STUCK: Stuck = Stuck::new(); + let ok = bounded( + "list", + "w1", + Duration::from_secs(5), + &STUCK, + &|| false, + || Ok(vec![1, 2]), + ); assert_eq!(ok.unwrap(), vec![1, 2]); - let refused: Result<(), String> = bounded("list", Duration::from_secs(5), &COUNT, || { - Err("refused: blocked".to_string()) - }); + let refused: Result<(), String> = bounded( + "list", + "w1", + Duration::from_secs(5), + &STUCK, + &|| false, + || Err("refused: blocked".to_string()), + ); assert_eq!(refused.unwrap_err(), "refused: blocked"); - assert_eq!(COUNT.load(Ordering::SeqCst), 0); + assert_eq!(STUCK.all(), 0); } #[test] - fn no_new_look_is_started_while_too_many_are_still_blocked() { - static COUNT: AtomicUsize = AtomicUsize::new(MAX_ABANDONED); + fn no_new_look_is_started_while_too_many_are_still_blocked_at_the_same_window() { + static STUCK: Stuck = Stuck::new(); + for _ in 0..MAX_ABANDONED_PER_KEY { + STUCK.add("hung"); + } let ran = Arc::new(AtomicU8::new(0)); let flag = Arc::clone(&ran); - let answer = bounded("walk", Duration::from_secs(5), &COUNT, move || { - flag.store(1, Ordering::SeqCst); - Ok(()) - }); + let answer = bounded( + "walk", + "hung", + Duration::from_secs(5), + &STUCK, + &|| false, + move || { + flag.store(1, Ordering::SeqCst); + Ok(()) + }, + ); let why = answer.unwrap_err(); assert!(why.starts_with(UNANSWERED), "{why}"); assert!(why.contains("still blocked"), "{why}"); @@ -398,9 +552,61 @@ mod tests { assert_eq!(ran.load(Ordering::SeqCst), 0, "a look was started anyway"); } + /// **One hung program does not stop a wait on any other.** The limit on + /// abandoned looks is per window: with a window's looks all stuck, a look + /// at another window runs and answers. + #[test] + fn a_hung_window_does_not_stop_looks_at_another() { + static STUCK: Stuck = Stuck::new(); + for _ in 0..MAX_ABANDONED_PER_KEY { + STUCK.add("hung"); + } + let other = bounded( + "walk", + "healthy", + Duration::from_secs(5), + &STUCK, + &|| false, + || Ok("walked"), + ); + assert_eq!(other.unwrap(), "walked"); + // The hung one is still refused. + assert!(bounded( + "walk", + "hung", + Duration::from_secs(5), + &STUCK, + &|| false, + || { Ok("walked") } + ) + .is_err()); + } + + /// The process-wide net is above the per-window limit, and holds. + #[test] + fn the_net_over_all_windows_is_higher_than_one_windows_limit_and_holds() { + const { assert!(MAX_ABANDONED_IN_ALL > MAX_ABANDONED_PER_KEY) }; + static STUCK: Stuck = Stuck::new(); + for at in 0..MAX_ABANDONED_IN_ALL { + STUCK.add(&format!("window {at}")); + } + let why = bounded( + "walk", + "fresh", + Duration::from_secs(5), + &STUCK, + &|| false, + || Ok(()), + ) + .unwrap_err(); + assert!(why.starts_with(UNANSWERED), "{why}"); + assert!(why.contains("in all"), "{why}"); + } + #[test] fn the_live_deadline_is_the_budgets_timeout_from_now() { - let live = Live::new(&Budget::new(Some(4_000), None)); + let never = || false; + let live = Live::new(&Budget::new(Some(4_000), None), &never); let left = live.deadline.saturating_duration_since(Instant::now()); assert!(left <= Duration::from_millis(4_000)); assert!(left > Duration::from_millis(3_000), "{left:?}"); diff --git a/crates/desktop/src/wait.rs b/crates/desktop/src/wait.rs index 12a520f..48e8202 100644 --- a/crates/desktop/src/wait.rs +++ b/crates/desktop/src/wait.rs @@ -249,6 +249,12 @@ fn wait_until<E: Env, T>( Err(why) if why.starts_with(UNANSWERED) => why, Err(why) => return Err(why), }; + // The stop is asked BEFORE the deadline: a look that was waited for + // until the deadline (or abandoned because the stop came) must end the + // wait as stopped, not as a timeout the operator did not ask for. + if stopped() { + return Ok(Waited::Stopped(env.now_ms() - start)); + } let elapsed = env.now_ms() - start; if elapsed >= budget.timeout_ms { return Ok(Waited::TimedOut(elapsed, polls, last)); @@ -758,6 +764,8 @@ mod tests { alive: Option<At<bool>>, /// A walk that blocks this many virtual ms and is then given up on. tree_blocks_ms: Option<u64>, + /// Set when a walk blocks: the operator's stop arrives during it. + stop_during_block: Option<std::rc::Rc<std::cell::Cell<bool>>>, windows_calls: u32, screen: At<u128>, sleeps: Vec<u64>, @@ -775,6 +783,7 @@ mod tests { tree_fails_from: None, alive: None, tree_blocks_ms: None, + stop_during_block: None, windows_calls: 0, screen: Box::new(|_| 0), sleeps: Vec::new(), @@ -807,6 +816,9 @@ mod tests { fn tree(&mut self, window: &str, _: Option<u32>) -> Result<Tree, String> { self.trees += 1; if let Some(blocked) = self.tree_blocks_ms { + if let Some(stop) = &self.stop_during_block { + stop.set(true); + } self.now += blocked; return Err(format!( "{UNANSWERED} to walk the window within {blocked} ms" @@ -1300,6 +1312,25 @@ mod tests { )); } + /// **A stop that comes during a blocked look ends the wait as stopped.** + /// The look is given up on (by the stop, or at its limit) and the wait + /// returns to its loop past the deadline: the stop is asked first, so the + /// operator is told it stopped, not that it timed out. + #[test] + fn a_stop_during_a_blocked_look_is_a_stop_not_a_timeout() { + let stop = std::rc::Rc::new(std::cell::Cell::new(false)); + let mut env = Fake::new(); + env.elements = Box::new(|_| vec![button(1, "OK")]); + env.tree_blocks_ms = Some(3_000); + env.stop_during_block = Some(std::rc::Rc::clone(&stop)); + let flag = std::rc::Rc::clone(&stop); + let c = appears("name=OK"); + // The deadline (2 s) is past by the time the look is given up on. + let out = run(&mut env, &c, &budget(2_000, 250), &move || flag.get()).unwrap(); + assert!(matches!(out, Outcome::Stopped { .. }), "{out:?}"); + assert_eq!(env.trees, 1, "no look after the stop"); + } + /// **A look that blocks cannot keep the wait past its deadline.** The /// desktop gives up on such a look at its limit (`mcp::wait_tool`) and the /// wait treats it as a look that saw nothing: no second look is started From 5dc3bd78c798958dfcbe49770d782d71d158279b Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:30:02 -0600 Subject: [PATCH 8/9] desktop: printed selectors resolve over the whole walk, macOS read-back and journal names Review findings 1, 4 and 5 on 1530260, 074c6c8 and b56356a. 1. ui_tree printed selectors from its 400 elements while the resolver sees up to 4000, so "role=link;name=Privacy policy" (unique among the 400) matched two once a twin lay at element 1500. ui_tree now walks with the resolving bound, prints the first 400, and makes every printed selector unique, with its nth, among the whole walk (selectors_for_first). Only the printed elements are remembered as shown, so an id still names only what was printed. The comment on resolve_max_elements said otherwise and is corrected. 4. macOS set_value read "no value" (an empty AXValue or a failed read, which text_attribute cannot tell apart) as verified when the value sent was empty. judge_optional: nothing read is never verified. 5. A name that can be a value is never journalled: Role::name_may_be_content (edit, combo box, static text, slider, other) now governs the names in messages (Element::label), the look-alikes of an ambiguity error and Selector::render_redacted. What ui_tree prints on macOS also leaves a combo box's name out, as it does an edit field's, since it changes as it is used. Tests: the Privacy-policy page (twin at 1500) prints nth=0 and resolves to its own element, and only the printed part is remembered; a combo box and text names do not reach the redacted selector, the examples or a label; nothing read back is unverified for an empty and a non-empty value. --- crates/desktop/src/macos.rs | 14 +-- crates/desktop/src/selector.rs | 86 +++++++++++++++-- crates/desktop/src/uia.rs | 146 ++++++++++++++++++++++++++--- crates/desktop/src/uia/readback.rs | 50 ++++++++++ docs/COMPUTER_USE.md | 12 ++- 5 files changed, 273 insertions(+), 35 deletions(-) diff --git a/crates/desktop/src/macos.rs b/crates/desktop/src/macos.rs index 5777f44..c8944c5 100644 --- a/crates/desktop/src/macos.rs +++ b/crates/desktop/src/macos.rs @@ -737,8 +737,10 @@ pub(crate) fn set_value( } // `text_attribute` trims and drops an empty value, so the sent text is // trimmed too and an empty read-back is `None`. - Ok(match text_attribute(&element, "AXValue") { - Some(now) => match super::uia::judge(value, before.as_deref(), &now, true) { + // `None` is an empty value or a failed read: neither is a proof. + let now = text_attribute(&element, "AXValue"); + Ok( + match super::uia::judge_optional(value, before.as_deref(), now.as_deref(), true) { super::uia::ReadBack::Same => Done::verified(did), super::uia::ReadBack::Unverified(why) => Done::unverified(tried, &why), super::uia::ReadBack::Differs(why) => { @@ -747,13 +749,7 @@ pub(crate) fn set_value( ))) } }, - None if value.trim().is_empty() => Done::verified(did), - None => Done::unverified( - tried, - "it reads back empty, which a browser or custom control does when it does not \ - report its value, so the text may be there; look at the result", - ), - }) + ) } /// The `AX…` action for one of the shared vocabulary's names. Pure. diff --git a/crates/desktop/src/selector.rs b/crates/desktop/src/selector.rs index e95105f..aed926e 100644 --- a/crates/desktop/src/selector.rs +++ b/crates/desktop/src/selector.rs @@ -388,12 +388,14 @@ impl Selector { } /// The selector as a journal may carry it. A control's name is left out - /// when it might be what the user typed: an edit field's (on macOS the - /// name of a text field IS its value), or any control whose role is not - /// given. Everything else is the same as [`Self::render`]. + /// when it might be what the user typed: that of an edit field, a combo + /// box, static text or any role whose name can come from its value + /// ([`Role::name_may_be_content`]: on macOS an unlabelled control is named by + /// its `AXValue`), or of any control whose role is not given. Everything + /// else is the same as [`Self::render`]. #[must_use] pub fn render_redacted(&self) -> String { - let keep_name = self.role.is_some_and(|r| r != Role::Edit); + let keep_name = self.role.is_some_and(|r| !r.name_may_be_content()); let mut parts = Vec::new(); self.push_window_parts(&mut parts); self.push_element_parts(&mut parts, keep_name); @@ -527,7 +529,7 @@ impl Selector { // the same ones (see [`Selector::depth`]). Selector { depth: self.depth, - ..selector_for(elements, i, true) + ..selector_hiding(elements, i, Role::name_may_be_content) } .render_element_part() }) @@ -604,8 +606,23 @@ fn name_text(name: &str) -> Text { /// which changes as it is typed in, so the field's name is not used. #[must_use] pub fn selector_for(elements: &[Element], index: usize, names_are_values: bool) -> Selector { + selector_hiding(elements, index, |role| { + names_are_values && role.holds_typed_text() + }) +} + +/// [`selector_for`] that never writes the name of a control whose role `hide` +/// picks. An example in an error is journalled word for word, so it hides every +/// role whose name may be a value ([`Role::name_may_be_content`]), on every +/// platform; what `ui_tree` prints hides only what changes as it is typed in. +#[must_use] +pub fn selector_hiding( + elements: &[Element], + index: usize, + hide: impl Fn(Role) -> bool, +) -> Selector { let e = &elements[index]; - let name = if (names_are_values && e.role == Role::Edit) || e.name.trim().is_empty() { + let name = if hide(e.role) || e.name.trim().is_empty() { None } else { Some(name_text(&e.name)) @@ -659,7 +676,22 @@ pub fn selectors_for( names_are_values: bool, depth: Option<u32>, ) -> Vec<Selector> { - (0..elements.len()) + selectors_for_first(elements, elements.len(), names_are_values, depth) +} + +/// [`selectors_for`] the first `count` of `elements` only, each made unique +/// among ALL of `elements`. `ui_tree` prints a part of what the resolver +/// walks (see `uia::resolve_max_elements`); a selector made unique among the +/// part would match two controls once the resolver sees the rest of the page, +/// so uniqueness and `nth` are counted over the whole walk. +#[must_use] +pub fn selectors_for_first( + elements: &[Element], + count: usize, + names_are_values: bool, + depth: Option<u32>, +) -> Vec<Selector> { + (0..count.min(elements.len())) .map(|i| Selector { depth, ..selector_for(elements, i, names_are_values) @@ -1179,6 +1211,46 @@ mod tests { assert!(!no_role.render_redacted().contains("hunter2")); } + /// On macOS a combo box with no title is named by its `AXValue`, which is + /// text somebody typed. Every journalled form of a selector hides the name + /// of every role whose name can come from its value. + #[test] + fn a_combo_boxs_name_is_never_journalled_either() { + for role in ["combobox", "text", "edit", "slider", "other"] { + let s = Selector::parse(&format!("role={role};name=typed secret")).unwrap(); + let said = s.render_redacted(); + assert!(!said.contains("typed secret"), "{role}: {said}"); + } + let button = Selector::parse("role=menuitem;name=Open").unwrap(); + assert_eq!(button.render_redacted(), "role=menuitem;name=Open"); + + // The look-alikes of an ambiguity error are journalled too. + let twins = vec![ + el(1, 1, Role::ComboBox, "typed secret"), + el(2, 1, Role::ComboBox, "typed secret"), + el(3, 1, Role::Text, "typed secret"), + el(4, 1, Role::Text, "typed secret"), + ]; + for query in [ + "role=combobox;name=typed secret", + "role=text;name=typed secret", + ] { + let err = Selector::parse(query) + .unwrap() + .resolve_in(&twins) + .unwrap_err(); + let said = err.to_string(); + assert!(!said.contains("typed secret"), "{query}: {said}"); + assert!(said.contains("nth="), "an example still says which: {said}"); + } + // And what `ui_tree` prints on macOS does not name a combo box by + // what is in it (it changes as it is used). + let printed = selector_for(&twins, 0, true).render_element_part(); + assert!(!printed.contains("typed secret"), "{printed}"); + let elsewhere = selector_for(&twins, 0, false).render_element_part(); + assert!(elsewhere.contains("typed secret"), "{elsewhere}"); + } + #[test] fn the_parts_render_separately() { let s = Selector::parse("app=a;window=W;role=edit;nth=1").unwrap(); diff --git a/crates/desktop/src/uia.rs b/crates/desktop/src/uia.rs index bdebf2a..a0ba919 100644 --- a/crates/desktop/src/uia.rs +++ b/crates/desktop/src/uia.rs @@ -25,7 +25,7 @@ use serde::{Deserialize, Serialize}; use super::coords::PhysicalPoint; -use super::selector::{selectors_for, Selector, Text}; +use super::selector::{selectors_for_first, Selector, Text}; use super::{DesktopError, Rect}; /// Whether the platform names a text field by its content, so a selector must @@ -62,8 +62,12 @@ fn max_elements() -> usize { /// print cap (an operator who raises that cap is not lowered below it). /// /// The walk order is the same, so a control within the first 400 has the same -/// id, `nth` and `path` whichever cap walked: a selector `ui_tree` printed -/// still means the same control. +/// id and `path` whichever cap walked. A selector's uniqueness and `nth` are +/// NOT the same, though: another control past the 400th can match what was +/// unique among the printed ones. So `ui_tree` walks with this cap too, prints +/// the first 400 and makes each printed selector unique among the whole walk +/// ([`Tree::printed`], `selector::selectors_for_first`): what it prints is +/// what the resolver resolves. pub(crate) fn resolve_max_elements() -> usize { let limits = super::limits::get(); limits @@ -106,6 +110,26 @@ pub enum Role { } impl Role { + /// Whether the control holds text the user types or picks, so that its + /// name, on a platform that names it by its value (macOS), changes as it + /// is used and is no way to find it. + pub(crate) fn holds_typed_text(self) -> bool { + matches!(self, Role::Edit | Role::ComboBox) + } + + /// Whether the control's NAME may be its content, and so may have been + /// typed. On macOS `describe` names a control by its `AXValue` when it has + /// no title or description, and that is true of any role whose value is + /// content: a text field, an editable combo box, static text, a slider, or + /// one whose role is not known. Messages and journals never carry such a + /// name. + pub(crate) fn name_may_be_content(self) -> bool { + matches!( + self, + Role::Edit | Role::Text | Role::ComboBox | Role::Slider | Role::Other + ) + } + /// Whether clicking this is a normal thing for an agent to want. /// /// Used to filter the noise out of a tree: a window full of anonymous @@ -298,8 +322,11 @@ impl Tree { )); } let stamp = (depth != super::limits::get().tree_depth).then_some(depth); - let selectors = selectors_for(&self.elements, NAMES_ARE_VALUES, stamp); - for (e, selector) in self.elements.iter().zip(&selectors) { + // Printed up to the print cap, but each selector unique among the + // whole walk: the resolver sees all of it (`resolve_max_elements`). + let printed = self.printed(); + let selectors = selectors_for_first(&self.elements, printed.len(), NAMES_ARE_VALUES, stamp); + for (e, selector) in printed.iter().zip(&selectors) { out.push_str(&format!( "{:indent$}[e{}] {:<10} \"{}\" ({},{},{}x{}) sel: {}{}{}{}\n", "", @@ -329,15 +356,22 @@ impl Tree { self.offscreen )); } - if self.truncated { + if self.truncated || printed.len() < self.elements.len() { out.push_str(&format!( "\n… stopped at {} elements. Narrow the request.\n", - max_elements() + printed.len() )); } out } + /// The elements `ui_tree` prints and an id may name: the first + /// [`max_elements`] of the walk. `elements` holds the whole walk, which can + /// be longer (see [`resolve_max_elements`]). + pub fn printed(&self) -> &[Element] { + &self.elements[..self.elements.len().min(max_elements())] + } + /// The element with this id, if the walk produced one. pub fn get(&self, id: u32) -> Option<&Element> { self.elements.iter().find(|e| e.id == id) @@ -468,16 +502,18 @@ impl Done { } impl Element { - /// The name as a message may say it: `None` for an edit or a text field. + /// The name as a message may say it: `None` for an edit field, a combo box, + /// static text, a slider or an unknown role ([`Role::name_may_be_content`]). /// /// Messages go to the operator's journal word for word (an error is /// recorded as it was returned), and on macOS the name of such a field IS /// what is in it (`AXValue`, the fallback of `describe`), which may have /// been typed. So no message names one, on any platform. pub(crate) fn name_for_messages(&self) -> Option<&str> { - match self.role { - Role::Edit | Role::Text => None, - _ => Some(&self.name), + if self.role.name_may_be_content() { + None + } else { + Some(&self.name) } } @@ -541,7 +577,10 @@ pub fn window_position_unknown(window: &WindowInfo) -> Option<String> { /// the focused window. pub fn window_tree(window: Option<&str>, depth: u32) -> Result<Tree, DesktopError> { let depth = bounded(depth); - let tree = platform::tree(window, depth, max_elements())?; + // The whole walk the resolver would make, so a selector printed for one of + // the first elements is unique among all of them; only the first + // `max_elements()` are printed and remembered as shown ([`Tree::printed`]). + let tree = platform::tree(window, depth, resolve_max_elements())?; remember_shown(&tree, depth); Ok(tree) } @@ -578,7 +617,7 @@ fn remember_shown(tree: &Tree, depth: u32) { shown.push(Shown { window: tree.window.clone(), depth, - elements: tree.elements.clone(), + elements: tree.printed().to_vec(), }); } @@ -688,11 +727,16 @@ pub fn focus_window(window: &str) -> Result<Done, DesktopError> { // What reading a field back after `set_value` can honestly say: `readback.rs`. mod readback; +#[cfg(target_os = "macos")] +pub(crate) use readback::judge_optional; +// macOS judges an optional read-back (`judge_optional`); the others a string. +#[cfg(any(windows, target_os = "linux"))] +pub(crate) use readback::judge; #[cfg_attr( not(any(windows, target_os = "macos", target_os = "linux")), allow(unused_imports) )] -pub(crate) use readback::{judge, ReadBack}; +pub(crate) use readback::ReadBack; // Acting on a control by selector instead of by id: `resolve.rs`. mod resolve; @@ -1134,6 +1178,27 @@ mod tests { assert!(!said.contains("\"\""), "{said}"); } + /// A message never names a control whose name can be its typed value. + #[test] + fn a_message_names_no_control_whose_name_may_be_its_value() { + let rect = Rect::from_edges(0, 0, 10, 10); + for role in [ + Role::Edit, + Role::Text, + Role::ComboBox, + Role::Slider, + Role::Other, + ] { + let said = element(5, role, "typed secret", rect).label(); + assert!(!said.contains("typed secret"), "{role:?}: {said}"); + assert!(said.contains("e5"), "{said}"); + } + for role in [Role::Button, Role::MenuItem, Role::Link, Role::TabItem] { + let said = element(5, role, "Save", rect).label(); + assert!(said.contains("\"Save\""), "{role:?}: {said}"); + } + } + /// The walk stops at [`MAX_ELEMENTS`] and says so. #[test] fn a_huge_tree_is_cut_and_reported_as_cut() { @@ -1170,6 +1235,57 @@ mod tests { assert_eq!(again.id, low.id); } + /// **What `ui_tree` prints is what the resolver resolves.** A page with a + /// "Privacy policy" link in its header and another at element 1500: the + /// first is unique among the 400 printed, but the resolver walks all of the + /// page and would find two. The printed selector must carry the `nth` that + /// tells them apart, and must resolve to its own element in the whole walk. + #[test] + fn a_printed_selector_stays_unique_when_a_twin_lies_past_the_print_cap() { + let rect = Rect::from_edges(0, 0, 10, 10); + let mut elements: Vec<Element> = (0..1600) + .map(|i| element(i, Role::Link, &format!("link{i}"), rect)) + .collect(); + elements[3].name = "Privacy policy".to_string(); + elements[1500].name = "Privacy policy".to_string(); + let tree = Tree { + root_name: "Page".to_string(), + window: "long-page".to_string(), + app: "chrome".to_string(), + no_positions: None, + elements, + truncated: false, + offscreen: 0, + }; + assert_eq!(tree.printed().len(), MAX_ELEMENTS); + let text = tree.render(); + assert!(!text.contains("[e1500]"), "only the print cap is printed"); + assert!(text.contains("stopped at 400 elements"), "{text}"); + let line = text + .lines() + .find(|l| l.contains("[e3]")) + .expect("the header link is printed"); + let written = line.split("sel: ").nth(1).expect("a selector").trim(); + assert!(written.contains("nth=0"), "{written}"); + // Pasted back as an agent would, it resolves, over the whole walk, to + // the control it was printed for and not to "matches 2". + let s = Selector::parse(written).expect("it reads back"); + assert_eq!(s.resolve_in(&tree.elements).expect("resolves").id, 3); + // A control with no twin keeps its short selector. + let plain = text.lines().find(|l| l.contains("[e4]")).expect("e4"); + assert!(!plain.contains("nth="), "{plain}"); + // And only the printed part is remembered as shown. + remember_shown(&tree, 12); + let shown = SHOWN + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mine = shown + .iter() + .find(|s| s.window == "long-page") + .expect("remembered"); + assert_eq!(mine.elements.len(), MAX_ELEMENTS); + } + /// macOS roles map onto the same vocabulary as Windows; a tab is a radio /// button with a subrole there, and an unknown role is `Other`. #[test] @@ -1400,7 +1516,7 @@ mod tests { }; assert!( - tree.elements.len() <= MAX_ELEMENTS, + tree.elements.len() <= resolve_max_elements(), "the walk ran past its own bound: {}", tree.elements.len() ); diff --git a/crates/desktop/src/uia/readback.rs b/crates/desktop/src/uia/readback.rs index 3a17e1d..89c7241 100644 --- a/crates/desktop/src/uia/readback.rs +++ b/crates/desktop/src/uia/readback.rs @@ -82,10 +82,60 @@ pub(crate) fn judge(sent: &str, before: Option<&str>, now: &str, trim: bool) -> )) } +/// [`judge`] for a platform whose read-back is `None` both when the value is +/// empty and when it could not be read at all (macOS `text_attribute` trims, +/// and drops an empty value and a failed call alike). Nothing read is nothing +/// proven: never `Same`, even when an empty value was sent, since a failed +/// read cannot be told from an emptied field. +#[cfg(any(target_os = "macos", test))] +#[must_use] +pub(crate) fn judge_optional( + sent: &str, + before: Option<&str>, + now: Option<&str>, + trim: bool, +) -> ReadBack { + match now { + Some(now) => judge(sent, before, now, trim), + None if lines(sent).trim().is_empty() => ReadBack::Unverified( + "its value could not be read back (an empty value and an unreadable one look \ + the same), so it is not known to be empty" + .to_string(), + ), + None => judge(sent, before, "", trim), + } +} + #[cfg(test)] mod tests { use super::*; + /// macOS gives `None` for an empty value AND for a failed read, so neither + /// is "verified": an empty value sent and nothing read back is unverified. + #[test] + fn nothing_read_back_is_never_verified() { + for sent in ["", " ", "\n"] { + match judge_optional(sent, None, None, true) { + ReadBack::Unverified(why) => assert!(why.contains("could not be read"), "{why}"), + other => panic!("{sent:?}: {other:?}"), + } + } + // Text sent and nothing read: the browser case, unverified too. + assert!(matches!( + judge_optional("hello", None, None, true), + ReadBack::Unverified(_) + )); + // A value that was read is judged as before. + assert_eq!( + judge_optional("hello", None, Some("hello"), true), + ReadBack::Same + ); + assert!(matches!( + judge_optional("hello", Some("old"), Some("old"), true), + ReadBack::Differs(_) + )); + } + #[test] fn line_endings_are_not_differences() { assert_eq!(lines("a\r\nb\rc\nd"), "a\nb\nc\nd"); diff --git a/docs/COMPUTER_USE.md b/docs/COMPUTER_USE.md index e1afa87..4d19cc3 100644 --- a/docs/COMPUTER_USE.md +++ b/docs/COMPUTER_USE.md @@ -283,9 +283,10 @@ ids work exactly as before. *Resolution's own bound.* `ui_tree` stops printing at 400 elements (what fits in a reply). A selector is resolved against its **own** walk, which is bounded by the number of nodes visited (`desktop_limits.tree_max_visited`, default 4000: the limit on time and cost every walk -already had) and not by the print cap, so a link far down a long article is found. The walk -order is the same, so a control within the first 400 has the same id, `nth` and `path` in -both. When even the larger bound cuts the walk, an error or result says so and that a second +already had) and not by the print cap, so a link far down a long article is found. `ui_tree` +walks the same way, prints the first 400 and makes each printed selector unique among the +**whole** walk, so a selector it printed (`nth` included, when a twin lies past the 400th +element) resolves to its own control. When even the larger bound cuts the walk, an error or result says so and that a second match may lie beyond it. *Browsers.* A browser exposes a page only after a client asks for it, so its first walk can hold just its own controls. A selector that matches nothing in a browser window (chrome, msedge, firefox, brave, vivaldi, opera, chromium) is walked once more @@ -334,7 +335,10 @@ comes true at the deadline is caught. Each look runs on a thread of its own and until the deadline plus 1 s: a look the desktop does not answer by then (a program that hangs, a window that closed under the walk) counts as a look that saw nothing, and the wait ends as a `timeout` whose `last_seen` says so, never as `gone`. So the call ends no later -than `timeout_ms` plus that second. A pinned window that no +than `timeout_ms` plus that second. The operator's stop is asked every 100 ms while a look is +waited for, and before the deadline, so a stop is `Stopped`, not a timeout. Looks given up on +are counted per window (4 still blocked at one window and no more are started at it; a safety +net of 24 over all windows), so one hung program does not stop a wait on another. A pinned window that no longer exists is `element_gone` at once. The operator's stop (stop file or `Safety`) is asked between looks and every 100 ms of sleeping, and ends the wait with the stop's refusal. From 946c940585dbf0e2ecda1a485df7e7473a6e9c33 Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:47:09 -0600 Subject: [PATCH 9/9] desktop: a look returning as it is given up cannot leak a stuck entry; a stop while picking the window is Stopped Re-review of df9c139. - The wait added a given-up look to the per-window count AFTER settling with it, so the look's thread, returning in between, removed an entry that was not there yet (or another look's), and the later addition was never taken off: four such leaks on the windows-listing key made every element or window wait time out until restart. The wait now counts first, settles second, and takes the count back if the look had in fact returned, so a look's removal always follows the addition it undoes. - A stop during the look that picks the watched window ended the wait as a plain error ("the wait was stopped"); it is now the Stopped outcome, as a stop during the polling is. - Tests: a look made to return exactly after the wait settled, and exactly before, each leave no entry (a test hook in the give-up path); one window's returning look does not take off another's entry; a stop during the window-picking look is Stopped, an unanswered one without a stop is still an error. --- crates/desktop/src/mcp/wait_tool.rs | 124 +++++++++++++++++++++++++++- crates/desktop/src/wait.rs | 46 ++++++++++- 2 files changed, 167 insertions(+), 3 deletions(-) diff --git a/crates/desktop/src/mcp/wait_tool.rs b/crates/desktop/src/mcp/wait_tool.rs index 8d2cc93..a45acc0 100644 --- a/crates/desktop/src/mcp/wait_tool.rs +++ b/crates/desktop/src/mcp/wait_tool.rs @@ -127,6 +127,29 @@ fn bounded<T: Send + 'static>( stuck: &'static Stuck, stopped: &dyn Fn() -> bool, work: impl FnOnce() -> Result<T, String> + Send + 'static, +) -> Result<T, String> { + bounded_with(what, key, limit, stuck, stopped, &|_| {}, work) +} + +/// A point in giving up on a look, where a test may let the look return. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Gap { + /// The wait has decided to give up and has not yet settled with the look. + Before, + /// The look is settled as given up on and counted. + After, +} + +/// [`bounded`], with `gap` called at each [`Gap`]: production passes nothing; +/// a test makes the look return exactly there. +fn bounded_with<T: Send + 'static>( + what: &str, + key: &str, + limit: Duration, + stuck: &'static Stuck, + stopped: &dyn Fn() -> bool, + gap: &dyn Fn(Gap), + work: impl FnOnce() -> Result<T, String> + Send + 'static, ) -> Result<T, String> { let (here, everywhere) = (stuck.of(key), stuck.all()); if here >= MAX_ABANDONED_PER_KEY || everywhere >= MAX_ABANDONED_IN_ALL { @@ -177,14 +200,24 @@ fn bounded<T: Send + 'static>( } else { continue; }; + gap(Gap::Before); + // COUNT FIRST, then settle with the look, and undo the count if the look + // had returned. The look's thread takes itself off the count only + // after it has seen the settlement, so its removal always follows this + // addition: counting after the settlement let a thread returning in + // between remove what was not yet added, and the entry then stayed + // for the life of the process. + stuck.add(key); return if state .compare_exchange(0, 2, Ordering::SeqCst, Ordering::SeqCst) .is_ok() { - stuck.add(key); + gap(Gap::After); Err(format!("{UNANSWERED} to {what}: {why}")) } else { - // It returned in the instant between the timeout and this. + // It returned in the instant between the timeout and this, and + // took no part in the count. + stuck.remove(key); receive .recv() .unwrap_or_else(|_| Err(format!("the look to {what} ended without an answer"))) @@ -502,6 +535,93 @@ mod tests { } } + /// Block (up to 5 s) until `flag` is set, then give the look's thread a + /// moment to finish what it does after `work` returns. + fn until_returned(flag: &std::sync::atomic::AtomicBool) { + let by = Instant::now() + Duration::from_secs(5); + while !flag.load(Ordering::SeqCst) { + assert!(Instant::now() < by, "the look never returned"); + std::thread::sleep(Duration::from_millis(5)); + } + std::thread::sleep(Duration::from_millis(150)); + } + + /// **A look that returns just as it is given up on leaves no entry behind.** + /// The look returns AFTER the wait has settled with it as given up on (its + /// thread then removes itself from the count) but before the wait would, + /// in the old order, have added it: the removal then found nothing, the + /// addition followed, and the entry stayed for good. + #[test] + fn a_look_returning_just_after_it_is_given_up_on_leaves_no_entry() { + static STUCK: Stuck = Stuck::new(); + let done = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (mine, theirs) = (Arc::clone(&done), Arc::clone(&done)); + let answer = bounded_with( + "walk", + "w1", + Duration::from_millis(20), + &STUCK, + &|| false, + &move |at| { + if at == Gap::After { + until_returned(&mine); + } + }, + move || { + std::thread::sleep(Duration::from_millis(100)); + theirs.store(true, Ordering::SeqCst); + Ok(()) + }, + ); + assert!(answer.unwrap_err().starts_with(UNANSWERED)); + assert_eq!(STUCK.all(), 0, "{:?}", STUCK.held()); + } + + /// And a look that returns after the wait decided to give up but before it + /// settled: its answer is returned, and nothing is left counted. + #[test] + fn a_look_returning_just_before_it_is_settled_gives_its_answer_and_no_entry() { + static STUCK: Stuck = Stuck::new(); + let done = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (mine, theirs) = (Arc::clone(&done), Arc::clone(&done)); + let answer = bounded_with( + "walk", + "w1", + Duration::from_millis(20), + &STUCK, + &|| false, + &move |at| { + if at == Gap::Before { + until_returned(&mine); + } + }, + move || { + std::thread::sleep(Duration::from_millis(100)); + theirs.store(true, Ordering::SeqCst); + Ok(7) + }, + ); + assert_eq!(answer.unwrap(), 7); + assert_eq!(STUCK.all(), 0, "{:?}", STUCK.held()); + } + + /// Entries of one window are not taken off by another's look returning. + #[test] + fn a_returning_look_takes_off_only_its_own_windows_entry() { + static STUCK: Stuck = Stuck::new(); + STUCK.add("other"); + let answer: Result<(), String> = bounded( + "walk", + "w1", + Duration::from_secs(5), + &STUCK, + &|| false, + || Ok(()), + ); + assert!(answer.is_ok()); + assert_eq!((STUCK.of("other"), STUCK.of("w1")), (1, 0)); + } + #[test] fn a_look_that_answers_in_time_gives_its_answer_or_its_error() { static STUCK: Stuck = Stuck::new(); diff --git a/crates/desktop/src/wait.rs b/crates/desktop/src/wait.rs index 48e8202..918e082 100644 --- a/crates/desktop/src/wait.rs +++ b/crates/desktop/src/wait.rs @@ -534,7 +534,18 @@ pub fn run<E: Env>( appears, } => { selector.require_element()?; - let target = target_of(env, selector, window.as_deref(), *appears)?; + let began = env.now_ms(); + let target = match target_of(env, selector, window.as_deref(), *appears) { + Ok(target) => target, + // A stop that ended the look which picks the window is the + // stop, the same refusal as one during the polling. + Err(why) if why.starts_with(UNANSWERED) && stopped() => { + return Ok(Outcome::Stopped { + waited_ms: env.now_ms() - began, + }) + } + Err(why) => return Err(why), + }; let waited = wait_until(env, budget, stopped, |env| { element_check(env, &target, selector, *appears) })?; @@ -766,6 +777,8 @@ mod tests { tree_blocks_ms: Option<u64>, /// Set when a walk blocks: the operator's stop arrives during it. stop_during_block: Option<std::rc::Rc<std::cell::Cell<bool>>>, + /// The window list is given up on, with the stop arriving during it. + listing_stopped: Option<std::rc::Rc<std::cell::Cell<bool>>>, windows_calls: u32, screen: At<u128>, sleeps: Vec<u64>, @@ -784,6 +797,7 @@ mod tests { alive: None, tree_blocks_ms: None, stop_during_block: None, + listing_stopped: None, windows_calls: 0, screen: Box::new(|_| 0), sleeps: Vec::new(), @@ -802,6 +816,12 @@ mod tests { } fn windows(&mut self) -> Result<Vec<WindowInfo>, String> { self.windows_calls += 1; + if let Some(stop) = &self.listing_stopped { + stop.set(true); + return Err(format!( + "{UNANSWERED} to list the windows: the wait was stopped" + )); + } Ok((self.windows)(self.now)) } fn window_alive(&mut self, _: &str) -> Option<bool> { @@ -1331,6 +1351,30 @@ mod tests { assert_eq!(env.trees, 1, "no look after the stop"); } + /// A stop during the look that picks the window is the stop, not a plain + /// error; an unanswered look with no stop is still not one. + #[test] + fn a_stop_during_the_window_picking_look_is_stopped_not_an_error() { + let stop = std::rc::Rc::new(std::cell::Cell::new(false)); + let mut env = Fake::new(); + env.listing_stopped = Some(std::rc::Rc::clone(&stop)); + let flag = std::rc::Rc::clone(&stop); + // Pinned to a window id: picking it lists the windows. + let c = Condition::Element { + selector: sel("name=OK"), + window: Some("1".to_string()), + appears: true, + }; + let out = run(&mut env, &c, &budget(5_000, 250), &move || flag.get()).unwrap(); + assert!(matches!(out, Outcome::Stopped { .. }), "{out:?}"); + + // The same look unanswered, with no stop: an error, as before. + let mut env = Fake::new(); + env.listing_stopped = Some(std::rc::Rc::new(std::cell::Cell::new(false))); + let err = run(&mut env, &c, &budget(5_000, 250), &never).unwrap_err(); + assert!(err.starts_with(UNANSWERED), "{err}"); + } + /// **A look that blocks cannot keep the wait past its deadline.** The /// desktop gives up on such a look at its limit (`mcp::wait_tool`) and the /// wait treats it as a look that saw nothing: no second look is started