diff --git a/src/desktop/mcp.rs b/src/desktop/mcp.rs index 088a6df..2928770 100644 --- a/src/desktop/mcp.rs +++ b/src/desktop/mcp.rs @@ -132,6 +132,36 @@ fn into(focused: &Option) -> String { } } +/// Why a pointer action at `at` is refused, when no display in `screens` +/// holds it. The decision [`refuse_point`] makes, apart from reading the +/// displays. +fn off_every_screen(at: PhysicalPoint, screens: &[super::Rect]) -> Option { + if screens.iter().any(|screen| screen.contains(at.x, at.y)) { + return None; + } + Some(format!( + "refused: {},{} is on no display, so the pointer would be pushed to a screen edge and \ + act on whatever is there; an element marked off-screen in ui_tree can be read, not \ + clicked — scroll its window to bring it into view and call ui_tree again", + at.x, at.y + )) +} + +/// Every display's rectangle, in the physical pixels pointer actions take. +/// Refuses, as [`blocked_windows`] does, when they cannot be read: a point +/// that cannot be checked is not acted on. +/// +/// Called by [`refuse_point`] and by `ui_tree` (to mark off-screen elements). +fn screen_rects() -> Result, String> { + let info = super::screen_info().map_err(|e| { + format!( + "refused: the displays could not be read ({e}), so this cannot be checked to be on \ + one" + ) + })?; + Ok(info.monitors.into_iter().map(|m| m.rect).collect()) +} + /// The blocked windows on screen. A desktop whose windows cannot be listed /// is refused rather than taken to have none: the check would otherwise /// pass exactly when it can see nothing. @@ -154,7 +184,19 @@ fn blocked_windows() -> Result, String> { /// window is refused even where another window covers it, since stacking can /// change between this check and the click, and a click that falls through /// to a password manager is the one that must never happen. +/// +/// And a point on no display at all: the OS clamps such a pointer to the +/// nearest screen edge, so the action would land on whatever is there, not +/// on what was aimed at — an element a page keeps 16,000 px off-screen for +/// screen readers (#52), shown by `ui_tree` as `off-screen`, or a coordinate +/// past the edge. +/// +/// Called before every pointer action: `click`, `click_element`, `scroll` +/// (over an element or at the cursor) and `drag` (both ends). fn refuse_point(at: PhysicalPoint) -> Result<(), String> { + if let Some(why) = off_every_screen(at, &screen_rects()?) { + return Err(why); + } match blocked_windows()? .iter() .find(|w| w.rect.contains(at.x, at.y)) @@ -733,7 +775,13 @@ impl Server { .map_err(|e| e.to_string())?; // Walked in this process, and discarded unread when blocked. refuse_app(&tree.app)?; - Ok(vec![text(format!("{}\n{UNTRUSTED}", tree.render()))]) + // With no displays readable the tree is still shown, unmarked: + // reading it is harmless, and every action re-checks its point. + let screens = screen_rects().unwrap_or_default(); + Ok(vec![text(format!( + "{}\n{UNTRUSTED}", + tree.render_on(&screens) + ))]) } "click_element" => { @@ -1617,6 +1665,23 @@ mod tests { assert!(!text.contains('\n'), "a multi-line config breaks the flag"); } + #[test] + fn a_point_on_no_display_is_refused_and_one_on_any_display_is_not() { + let screens = [ + crate::desktop::Rect::from_edges(-1920, 0, 0, 1080), + crate::desktop::Rect::from_edges(0, 0, 2560, 1440), + ]; + let at = |x, y| PhysicalPoint { x, y }; + for (x, y) in [(0, 0), (-1920, 500), (2559, 1439), (-1, 1079)] { + assert_eq!(off_every_screen(at(x, y), &screens), None, "{x},{y}"); + } + for (x, y) in [(0, -16079), (2560, 0), (-1921, 10), (-500, 1100)] { + let why = off_every_screen(at(x, y), &screens) + .unwrap_or_else(|| panic!("{x},{y} was not refused")); + assert!(why.contains("on no display"), "{why}"); + } + } + #[test] fn an_image_block_is_base64_and_names_its_type() { let png = image(&[0x89, b'P', b'N', b'G'], ShotFormat::Png); @@ -2050,6 +2115,25 @@ mod tests { ); } + /// **A CLICK ON NO DISPLAY IS REFUSED** (#52): sent, it would be + /// clamped to a screen edge and click whatever is there. Through the + /// real protocol path, on a running (not stopped) desktop, with this + /// machine's real displays. (The pointer is not compared before and + /// after: on a desktop a person is using, that measures the person.) + #[test] + fn a_click_on_no_display_is_refused() { + let mut s = server(); + let reply = call( + &mut s, + r#"{"jsonrpc":"2.0","id":9,"method":"tools/call","params":{"name":"click","arguments":{"x":0,"y":-16079}}}"#, + ); + assert_eq!(reply["result"]["isError"], true, "{reply}"); + let why = reply["result"]["content"][0]["text"] + .as_str() + .expect("a reason"); + assert!(why.contains("0,-16079 is on no display"), "{why}"); + } + #[test] fn an_unknown_tool_is_a_tool_error_too() { let mut s = server(); diff --git a/src/desktop/uia.rs b/src/desktop/uia.rs index f0b5e5e..35fd4ca 100644 --- a/src/desktop/uia.rs +++ b/src/desktop/uia.rs @@ -147,6 +147,16 @@ impl Element { self.rect.x == i32::MIN || self.rect.y == i32::MIN } + /// Whether the control lies wholly outside every display in `screens`: + /// at a real place, unlike [`Self::is_offscreen`], but one no pointer can + /// reach. Pages keep text for screen readers this way (1x1, thousands of + /// pixels off, #52). It is kept in the tree, for its text, and marked so + /// it is not mistaken for something to click. `false` when `screens` is + /// empty: with no displays known, nothing is judged. + pub fn beyond_screens(&self, screens: &[super::Rect]) -> bool { + !screens.is_empty() && !screens.iter().any(|screen| screen.overlaps(self.rect)) + } + /// What `set_value` did, for its reply: `set e49 (Edit) to 17 chars`. /// Not the label: an edit field's name is often empty, and `set e49 ""` /// reads as though the value set were empty. The length, never the @@ -196,6 +206,16 @@ fn is_zero(n: &u32) -> bool { impl Tree { /// Render as the indented text an agent reads. pub fn render(&self) -> String { + self.render_on(&[]) + } + + /// [`Self::render`], with each element that lies outside every display in + /// `screens` marked `off-screen` ([`Element::beyond_screens`]): its text + /// is still there to read, but a pointer cannot reach it, and an action + /// aimed at it by id is refused (`mcp.rs` `refuse_point`). + /// + /// Called by `mcp.rs`'s `ui_tree` with the displays `screen_info` reports. + pub fn render_on(&self, screens: &[super::Rect]) -> String { let mut out = String::new(); out.push_str(&format!("Window \"{}\"", self.root_name)); if !self.window.is_empty() { @@ -210,7 +230,7 @@ impl Tree { } for e in &self.elements { out.push_str(&format!( - "{:indent$}[e{}] {:<10} \"{}\" ({},{},{}x{}){}{}\n", + "{:indent$}[e{}] {:<10} \"{}\" ({},{},{}x{}){}{}{}\n", "", e.id, format!("{:?}", e.role), @@ -221,6 +241,11 @@ impl Tree { e.rect.height, if e.enabled { "" } else { " disabled" }, if e.secret { " password" } else { "" }, + if e.beyond_screens(screens) { + " off-screen" + } else { + "" + }, indent = (e.depth as usize).min(12) * 2, )); } @@ -1108,6 +1133,72 @@ mod tests { ); } + /// **AN ELEMENT OFF EVERY SCREEN IS KEPT, AND MARKED** (#52, decided in + /// #56): a page's screen-reader text at y=-16079 keeps its name for the + /// model to read, but says `off-screen`, so it is not taken for something + /// to click. One past the edge of the left-hand display, or overlapping a + /// display by a pixel, is on a screen and is not marked; with no displays + /// known, nothing is. + #[test] + fn an_element_outside_every_display_is_kept_and_marked_off_screen() { + let screens = [ + Rect::from_edges(-1920, 0, 0, 1080), + Rect::from_edges(0, 0, 2560, 1440), + ]; + let tree = Tree { + root_name: "Page".to_string(), + window: String::new(), + app: String::new(), + no_positions: None, + elements: vec![ + element( + 1, + Role::Text, + "press ? for shortcuts", + Rect { + x: 0, + y: -16079, + width: 1, + height: 1, + }, + ), + element( + 2, + Role::Button, + "Left screen", + Rect::from_edges(-1900, 10, -1800, 40), + ), + element(3, Role::Button, "Edge", Rect::from_edges(2559, 0, 2600, 20)), + element( + 4, + Role::Button, + "Past the right", + Rect::from_edges(2600, 0, 2700, 20), + ), + ], + truncated: false, + offscreen: 0, + }; + let text = tree.render_on(&screens); + let line = |name: &str| { + text.lines() + .find(|l| l.contains(name)) + .unwrap_or_else(|| panic!("{name} was dropped: {text}")) + .to_string() + }; + assert!( + line("press ? for shortcuts").ends_with(" off-screen"), + "{text}" + ); + assert!(line("Past the right").ends_with(" off-screen"), "{text}"); + assert!(!line("Left screen").contains("off-screen"), "{text}"); + assert!(!line("Edge").contains("off-screen"), "{text}"); + assert!( + !tree.render().contains("off-screen"), + "without displays nothing is judged" + ); + } + #[test] fn lookup_by_id_finds_only_what_the_walk_returned() { let tree = Tree {