Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 85 additions & 1 deletion src/desktop/mcp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,36 @@ fn into(focused: &Option<WindowInfo>) -> String {
}
}

/// Why a pointer action at `at` is refused, when no display in `screens`
/// holds it. The decision [`refuse_point`] makes, apart from reading the
/// displays.
fn off_every_screen(at: PhysicalPoint, screens: &[super::Rect]) -> Option<String> {
if screens.iter().any(|screen| screen.contains(at.x, at.y)) {
return None;
}
Some(format!(
"refused: {},{} is on no display, so the pointer would be pushed to a screen edge and \
act on whatever is there; an element marked off-screen in ui_tree can be read, not \
clicked — scroll its window to bring it into view and call ui_tree again",
at.x, at.y
))
}

/// Every display's rectangle, in the physical pixels pointer actions take.
/// Refuses, as [`blocked_windows`] does, when they cannot be read: a point
/// that cannot be checked is not acted on.
///
/// Called by [`refuse_point`] and by `ui_tree` (to mark off-screen elements).
fn screen_rects() -> Result<Vec<super::Rect>, String> {
let info = super::screen_info().map_err(|e| {
format!(
"refused: the displays could not be read ({e}), so this cannot be checked to be on \
one"
)
})?;
Ok(info.monitors.into_iter().map(|m| m.rect).collect())
}

/// The blocked windows on screen. A desktop whose windows cannot be listed
/// is refused rather than taken to have none: the check would otherwise
/// pass exactly when it can see nothing.
Expand All @@ -154,7 +184,19 @@ fn blocked_windows() -> Result<Vec<WindowInfo>, String> {
/// window is refused even where another window covers it, since stacking can
/// change between this check and the click, and a click that falls through
/// to a password manager is the one that must never happen.
///
/// And a point on no display at all: the OS clamps such a pointer to the
/// nearest screen edge, so the action would land on whatever is there, not
/// on what was aimed at — an element a page keeps 16,000 px off-screen for
/// screen readers (#52), shown by `ui_tree` as `off-screen`, or a coordinate
/// past the edge.
///
/// Called before every pointer action: `click`, `click_element`, `scroll`
/// (over an element or at the cursor) and `drag` (both ends).
fn refuse_point(at: PhysicalPoint) -> Result<(), String> {
if let Some(why) = off_every_screen(at, &screen_rects()?) {
return Err(why);
}
match blocked_windows()?
.iter()
.find(|w| w.rect.contains(at.x, at.y))
Expand Down Expand Up @@ -733,7 +775,13 @@ impl Server {
.map_err(|e| e.to_string())?;
// Walked in this process, and discarded unread when blocked.
refuse_app(&tree.app)?;
Ok(vec![text(format!("{}\n{UNTRUSTED}", tree.render()))])
// With no displays readable the tree is still shown, unmarked:
// reading it is harmless, and every action re-checks its point.
let screens = screen_rects().unwrap_or_default();
Ok(vec![text(format!(
"{}\n{UNTRUSTED}",
tree.render_on(&screens)
))])
}

"click_element" => {
Expand Down Expand Up @@ -1617,6 +1665,23 @@ mod tests {
assert!(!text.contains('\n'), "a multi-line config breaks the flag");
}

#[test]
fn a_point_on_no_display_is_refused_and_one_on_any_display_is_not() {
let screens = [
crate::desktop::Rect::from_edges(-1920, 0, 0, 1080),
crate::desktop::Rect::from_edges(0, 0, 2560, 1440),
];
let at = |x, y| PhysicalPoint { x, y };
for (x, y) in [(0, 0), (-1920, 500), (2559, 1439), (-1, 1079)] {
assert_eq!(off_every_screen(at(x, y), &screens), None, "{x},{y}");
}
for (x, y) in [(0, -16079), (2560, 0), (-1921, 10), (-500, 1100)] {
let why = off_every_screen(at(x, y), &screens)
.unwrap_or_else(|| panic!("{x},{y} was not refused"));
assert!(why.contains("on no display"), "{why}");
}
}

#[test]
fn an_image_block_is_base64_and_names_its_type() {
let png = image(&[0x89, b'P', b'N', b'G'], ShotFormat::Png);
Expand Down Expand Up @@ -2050,6 +2115,25 @@ mod tests {
);
}

/// **A CLICK ON NO DISPLAY IS REFUSED** (#52): sent, it would be
/// clamped to a screen edge and click whatever is there. Through the
/// real protocol path, on a running (not stopped) desktop, with this
/// machine's real displays. (The pointer is not compared before and
/// after: on a desktop a person is using, that measures the person.)
#[test]
fn a_click_on_no_display_is_refused() {
let mut s = server();
let reply = call(
&mut s,
r#"{"jsonrpc":"2.0","id":9,"method":"tools/call","params":{"name":"click","arguments":{"x":0,"y":-16079}}}"#,
);
assert_eq!(reply["result"]["isError"], true, "{reply}");
let why = reply["result"]["content"][0]["text"]
.as_str()
.expect("a reason");
assert!(why.contains("0,-16079 is on no display"), "{why}");
}

#[test]
fn an_unknown_tool_is_a_tool_error_too() {
let mut s = server();
Expand Down
93 changes: 92 additions & 1 deletion src/desktop/uia.rs
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,16 @@ impl Element {
self.rect.x == i32::MIN || self.rect.y == i32::MIN
}

/// Whether the control lies wholly outside every display in `screens`:
/// at a real place, unlike [`Self::is_offscreen`], but one no pointer can
/// reach. Pages keep text for screen readers this way (1x1, thousands of
/// pixels off, #52). It is kept in the tree, for its text, and marked so
/// it is not mistaken for something to click. `false` when `screens` is
/// empty: with no displays known, nothing is judged.
pub fn beyond_screens(&self, screens: &[super::Rect]) -> bool {
!screens.is_empty() && !screens.iter().any(|screen| screen.overlaps(self.rect))
}

/// What `set_value` did, for its reply: `set e49 (Edit) to 17 chars`.
/// Not the label: an edit field's name is often empty, and `set e49 ""`
/// reads as though the value set were empty. The length, never the
Expand Down Expand Up @@ -196,6 +206,16 @@ fn is_zero(n: &u32) -> bool {
impl Tree {
/// Render as the indented text an agent reads.
pub fn render(&self) -> String {
self.render_on(&[])
}

/// [`Self::render`], with each element that lies outside every display in
/// `screens` marked `off-screen` ([`Element::beyond_screens`]): its text
/// is still there to read, but a pointer cannot reach it, and an action
/// aimed at it by id is refused (`mcp.rs` `refuse_point`).
///
/// Called by `mcp.rs`'s `ui_tree` with the displays `screen_info` reports.
pub fn render_on(&self, screens: &[super::Rect]) -> String {
let mut out = String::new();
out.push_str(&format!("Window \"{}\"", self.root_name));
if !self.window.is_empty() {
Expand All @@ -210,7 +230,7 @@ impl Tree {
}
for e in &self.elements {
out.push_str(&format!(
"{:indent$}[e{}] {:<10} \"{}\" ({},{},{}x{}){}{}\n",
"{:indent$}[e{}] {:<10} \"{}\" ({},{},{}x{}){}{}{}\n",
"",
e.id,
format!("{:?}", e.role),
Expand All @@ -221,6 +241,11 @@ impl Tree {
e.rect.height,
if e.enabled { "" } else { " disabled" },
if e.secret { " password" } else { "" },
if e.beyond_screens(screens) {
" off-screen"
} else {
""
},
indent = (e.depth as usize).min(12) * 2,
));
}
Expand Down Expand Up @@ -1108,6 +1133,72 @@ mod tests {
);
}

/// **AN ELEMENT OFF EVERY SCREEN IS KEPT, AND MARKED** (#52, decided in
/// #56): a page's screen-reader text at y=-16079 keeps its name for the
/// model to read, but says `off-screen`, so it is not taken for something
/// to click. One past the edge of the left-hand display, or overlapping a
/// display by a pixel, is on a screen and is not marked; with no displays
/// known, nothing is.
#[test]
fn an_element_outside_every_display_is_kept_and_marked_off_screen() {
let screens = [
Rect::from_edges(-1920, 0, 0, 1080),
Rect::from_edges(0, 0, 2560, 1440),
];
let tree = Tree {
root_name: "Page".to_string(),
window: String::new(),
app: String::new(),
no_positions: None,
elements: vec![
element(
1,
Role::Text,
"press ? for shortcuts",
Rect {
x: 0,
y: -16079,
width: 1,
height: 1,
},
),
element(
2,
Role::Button,
"Left screen",
Rect::from_edges(-1900, 10, -1800, 40),
),
element(3, Role::Button, "Edge", Rect::from_edges(2559, 0, 2600, 20)),
element(
4,
Role::Button,
"Past the right",
Rect::from_edges(2600, 0, 2700, 20),
),
],
truncated: false,
offscreen: 0,
};
let text = tree.render_on(&screens);
let line = |name: &str| {
text.lines()
.find(|l| l.contains(name))
.unwrap_or_else(|| panic!("{name} was dropped: {text}"))
.to_string()
};
assert!(
line("press ? for shortcuts").ends_with(" off-screen"),
"{text}"
);
assert!(line("Past the right").ends_with(" off-screen"), "{text}");
assert!(!line("Left screen").contains("off-screen"), "{text}");
assert!(!line("Edge").contains("off-screen"), "{text}");
assert!(
!tree.render().contains("off-screen"),
"without displays nothing is judged"
);
}

#[test]
fn lookup_by_id_finds_only_what_the_walk_returned() {
let tree = Tree {
Expand Down
Loading