feat: hints mode

Cargo.toml

@@ -20,6 +20,8 @@ x11 = ["dep:x11rb", "dep:xkbcommon"]
 anyhow = "1.0"
 clap = { version = "4", features = ["derive"] }
 font8x8 = "0.3"
+image = { version = "0.25", default-features = false }
+imageproc = { version = "0.26", default-features = false }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 toml = "1.1"
@@ -31,8 +33,14 @@ wayland-protocols-wlr = { version = "0.3", features = ["client"], optional = tru
 tempfile = { version = "3", optional = true }
 x11rb = { version = "0.13", features = ["xkb", "xtest"], optional = true }
 xkbcommon = { version = "0.8", optional = true }
+atspi = { version = "0.30", features = ["connection", "proxies", "zbus"] }
+futures-executor = "0.3"
+futures-util = { version = "0.3", default-features = false, features = ["alloc"] }
 
 [target.'cfg(target_os = "macos")'.dependencies]
 objc2 = "0.6"
 objc2-foundation = { version = "0.3", features = ["NSDate", "NSGeometry", "NSString", "NSValue"] }
-objc2-app-kit = { version = "0.3", features = ["NSApplication", "NSColor", "NSEvent", "NSPanel", "NSResponder", "NSScreen", "NSView", "NSWindow"] }
+objc2-app-kit = { version = "0.3", features = ["NSApplication", "NSColor", "NSEvent", "NSPanel", "NSResponder", "NSRunningApplication", "NSScreen", "NSView", "NSWindow", "NSWorkspace"] }
+objc2-application-services = { version = "0.3", default-features = false, features = ["std", "libc", "AXUIElement", "AXValue", "AXAttributeConstants", "AXRoleConstants", "AXError", "HIServices"] }
+objc2-core-graphics = { version = "0.3", default-features = false, features = ["std", "CGDirectDisplay", "CGImage", "CGDataProvider", "CGError"] }
+objc2-core-foundation = { version = "0.3", default-features = false, features = ["std", "CFString", "CFArray", "CFData", "CFBase", "CFCGTypes", "CFNumber"] }

README.md

@@ -96,6 +96,7 @@ Released binaries are not codesigned with a Developer ID, so after every upgrade
 | Tab | Search macros / quick-save position |
 | `b` | Switch to bisect mode |
 | `n` | Switch back to normal mode (from bisect) |
+| `c` | Switch to hint mode |
 | `v` | Switch to free mode |
 
 All keys are configurable (see Configuration below).
@@ -105,6 +106,7 @@ All keys are configurable (see Configuration below).
 | Flag | Effect |
 |------|--------|
 | `--bisect` | Start the overlay directly in bisect mode |
+| `--hint` | Start the overlay directly in hint mode |
 | `--free` | Start the overlay directly in free mode (cursor starts at current mouse position) |
 | `--free-center` | Start the overlay directly in free mode with the cursor at the center of the screen |
 | `--allow-multiple` | Skip the single-instance lock |
@@ -121,6 +123,27 @@ An alternative grid mode that recursively subdivides. Instead of two-key combos
 - Subdivision stops automatically once a cell would fall below `min_cell_size` pixels — the region is highlighted and you can act or back out
 - Press `n` to switch back to normal mode
 
+### Hint mode
+
+An alternative mode that labels likely clickable UI targets directly, instead of asking you to navigate a grid. Type the label shown on a target, then perform an action.
+
+- Enter hint mode: press `c` from normal, bisect, or free mode, or launch with `--hint`
+- Space / Enter / `m` / Delete / Insert act (click / double-click / triple-click / right-click / middle-click) on the selected target and exit
+- `/` starts a drag from the selected target; select another hint and press Space to drag
+- Backspace edits the typed hint label, or returns to the previous mode when no label is typed
+- Escape closes the overlay
+
+Detection is pluggable behind a `HintDetector` trait, selected by `hint.detector`:
+
+- **`atspi`** (Linux, built in): reads the desktop [AT-SPI](https://docs.rs/atspi) accessibility tree over D-Bus and labels interactive elements using their roles and bounds. It is usually the most accurate detector. GTK, Qt, and Gecko apps tend to work well; Chromium/Electron may need accessibility enabled; games and canvas-heavy apps may expose little or nothing. On Wayland, exact placement requires compositor support; Hyprland and Sway are supported. On X11, coordinates are already absolute. Only the primary monitor is handled.
+- **`cv`** (always available): a pure-Rust computer-vision pass over a screenshot — downscale → Canny edges → morphological closing → connected components → resolution-relative size/shape filtering with a per-box quality score. No model, no network, no services to talk to. CV sees visual *structure*, not semantic clickability, so it will still occasionally label a border, separator, or decorative shape, and miss a low-contrast control. Tune `edge_low_threshold`/`edge_high_threshold` (raise to cut clutter, lower to catch faint controls).
+- **`auto`** (default): tries AT-SPI first and falls back to CV when needed. If an app exposes window chrome but not content, CV labels the unhandled content area.
+
+```toml
+[hint]
+detector = "auto"   # or "atspi" / "cv"
+```
+
 ### Free mode
 
 An alternative mode for direct keyboard-driven cursor movement. Instead of selecting a grid cell, you steer the cursor continuously with movement keys, then act.
@@ -156,6 +179,7 @@ All fields are optional. Missing fields use defaults.
 
 ```toml
 font_size = 2  # Glyph scale multiplier for the 8x8 bitmap font: 1=8px, 2=16px, 3=24px
+hint_font_size = 1  # Optional override for element hint labels; defaults to 1 for dense target screens
 sub_hint_font_size = 2  # Optional override for sub-grid hint glyphs; defaults to font_size when omitted
 panel_font_size = 2  # Optional override for macro/search popup panels; defaults to sub_hint_font_size, then font_size
 
@@ -171,12 +195,22 @@ rows = 2
 cols = 2
 min_cell_size = 16  # Stop subdividing once a cell would be smaller than this, in pixels
 
+[hint]
+detector = "auto"  # "auto" (default) | "atspi" (Linux) | "cv"
+alphabet = ["a", "s", "d", "f", "j", "k", "l", ";", "g", "h", "q", "w", "e", "r", "t", "y", "u", "i", "o", "p"]  # Keys used to build hint labels
+max_label_len = 3
+auto_click = false  # If true, click immediately when a typed label uniquely identifies a target
+downscale_longest = 1280  # CV: longest screen-capture side fed to the detector; 0 = no downscale
+edge_low_threshold = 50  # CV: Canny low threshold (raise to cut clutter, lower to catch faint controls)
+edge_high_threshold = 150  # CV: Canny high threshold
+
 [macros]
 playback_speed = 1.0  # 1.0 = recorded speed, 2.0 = twice as fast, 0.0 or negative = instant (no waits)
 
 [keys]
 normal = "n"
 bisect = "b"
+hint = "c"
 free_mode = "v"
 click = "space"
 double_click = "enter"
@@ -211,6 +245,10 @@ rec_bg = "#f44336ff"             # Material Design Red (urgent, visible everywhe
 border = "#00e676ff"             # Material Green (fresh, visible)
 border_dragging = "#e91e63ff"    # Material Pink (strong attention grabber)
 crosshair = "#ffaa00ff"          # Amber crosshair in free mode
+hint_chip_bg = "#2e1e1eee"       # Hint label background
+hint_text = "#00ccffff"          # Untyped hint label text
+hint_text_typed = "#50ff50ff"    # Typed hint label text
+hint_dim = "#66666688"           # Dimmed non-matching hints
 
 [free]
 left = "h"
@@ -245,6 +283,14 @@ max_speed = 500
 - `rows` and `cols` set the grid shape used at every subdivision level (default 2x2).
 - `min_cell_size` is the pixel floor at which subdivision stops. The glyph scale auto-shrinks to fit the cell, so lowering this lets you reach tiny regions.
 
+### Hint mode
+
+- `alphabet` sets the characters used for generated hint labels. Defaults to the same set as `[grid].hints`; override to change which keys label elements.
+- `max_label_len` caps generated label length. Higher values allow more targets, but require more keystrokes.
+- `auto_click` clicks as soon as the typed label uniquely matches one target. Leave it `false` if you want to choose the action after selecting.
+- `downscale_longest` controls CV detector speed/precision. Lower values are faster but less precise; `0` disables downscaling.
+- `edge_low_threshold` / `edge_high_threshold` tune Canny edge detection. Lower values find more targets and more false positives; higher values are stricter.
+
 ### Free mode
 
 - `left`, `down`, `up`, `right` set the movement keys (default: `h`, `j`, `k`, `l`).

src/app.rs

@@ -1,5 +1,6 @@
 use crate::backend::Backend;
 use crate::config::config;
+use crate::hint::build_hint_mode;
 use crate::input::InputState;
 use crate::macro_store::MacroStore;
 use crate::mode::{Mode, ModeTransition};
@@ -8,6 +9,7 @@ use crate::mode::{Mode, ModeTransition};
 pub enum InitialMode {
     Normal,
     Bisect,
+    Hint,
     Free,
     FreeCenter,
 }
@@ -26,6 +28,11 @@ pub fn run<B: Backend>(backend: &mut B, initial: InitialMode) -> anyhow::Result<
         InitialMode::Bisect => Mode::Bisect {
             region: (0, 0, w, h),
         },
+        InitialMode::Hint => Mode::Hint {
+            elements: build_hint_mode(backend, w, h)?.into(),
+            typed: Vec::new(),
+            target: None,
+        },
         InitialMode::Free => {
             let (x, y) = backend.mouse_pos()?;
             Mode::Free {
@@ -41,7 +48,10 @@ pub fn run<B: Backend>(backend: &mut B, initial: InitialMode) -> anyhow::Result<
         },
     };
 
-    if !matches!(initial, InitialMode::Free | InitialMode::FreeCenter) {
+    if !matches!(
+        initial,
+        InitialMode::Free | InitialMode::FreeCenter | InitialMode::Hint
+    ) {
         backend.move_mouse(w / 2, h / 2)?;
     }
 

src/backend/macos.rs

@@ -173,6 +173,18 @@ unsafe extern "C" {
         intent: u32,
     ) -> CGImageRef;
     fn CGImageRelease(img: CGImageRef);
+
+    fn CGBitmapContextCreate(
+        data: *mut c_void,
+        width: usize,
+        height: usize,
+        bits_per_component: usize,
+        bytes_per_row: usize,
+        space: CGColorSpaceRef,
+        bitmap_info: u32,
+    ) -> *mut c_void;
+    fn CGContextDrawImage(c: *mut c_void, rect: CGRect, image: CGImageRef);
+    fn CGContextRelease(c: *mut c_void);
 }
 
 #[link(name = "ApplicationServices", kind = "framework")]
@@ -615,6 +627,74 @@ impl Backend for MacosBackend {
         Ok((pt.x as u32, pt.y as u32))
     }
 
+    fn capture_screen(&mut self) -> Result<super::Capture> {
+        use objc2_core_graphics::CGImage;
+
+        let display = objc2_core_graphics::CGMainDisplayID();
+        // CGDisplayCreateImage is marked deprecated in favour of
+        // ScreenCaptureKit, but it still functions on macOS 14/15 and SCK is
+        // async-only.
+        // Revisit when SCK exposes a synchronous shim.
+        #[allow(deprecated)]
+        let image = objc2_core_graphics::CGDisplayCreateImage(display).ok_or_else(|| {
+            anyhow!("CGDisplayCreateImage returned nil (screen recording permission?)")
+        })?;
+        let width = CGImage::width(Some(&image));
+        let height = CGImage::height(Some(&image));
+        if width == 0 || height == 0 {
+            return Err(anyhow!("captured image has zero dimensions"));
+        }
+
+        // Reading bytes straight from `CGImage::data_provider` is unreliable.
+        // On Retina/IOSurface-backed images the buffer may not be in the
+        // declared bitmap format, and the bits CV gets back are effectively
+        // noise. Drawing the image into our own BGRA-premultiplied bitmap
+        // context normalises the format and gives a tight buffer to keep.
+        let row_bytes = width * 4;
+        let mut bgra: Vec<u8> = vec![0; row_bytes * height];
+        let raw_image: *mut c_void = objc2_core_foundation::CFRetained::into_raw(image)
+            .as_ptr()
+            .cast();
+        let ctx = unsafe {
+            let cs = CGColorSpaceCreateDeviceRGB();
+            if cs.is_null() {
+                CGImageRelease(raw_image);
+                return Err(anyhow!("CGColorSpaceCreateDeviceRGB failed"));
+            }
+            let ctx = CGBitmapContextCreate(
+                bgra.as_mut_ptr().cast(),
+                width,
+                height,
+                8,
+                row_bytes,
+                cs,
+                K_CGIMAGE_ALPHA_PREMULTIPLIED_FIRST | K_CGBITMAP_BYTE_ORDER32_LITTLE,
+            );
+            CGColorSpaceRelease(cs);
+            if ctx.is_null() {
+                CGImageRelease(raw_image);
+                return Err(anyhow!("CGBitmapContextCreate failed"));
+            }
+            let rect = CGRect {
+                origin: CGPoint { x: 0.0, y: 0.0 },
+                size: CGSize {
+                    width: width as f64,
+                    height: height as f64,
+                },
+            };
+            CGContextDrawImage(ctx, rect, raw_image);
+            CGImageRelease(raw_image);
+            ctx
+        };
+        unsafe { CGContextRelease(ctx) };
+
+        Ok(super::Capture {
+            bgra,
+            w: width as u32,
+            h: height as u32,
+        })
+    }
+
     fn move_mouse(&mut self, x: u32, y: u32) -> Result<()> {
         self.move_cursor(x, y);
         Ok(())

src/backend/mod.rs

@@ -5,6 +5,7 @@ use anyhow::Result;
 pub enum KeyEvent {
     Normal,
     Bisect,
+    Hint,
     FreeMode,
     Char(char),
     Click,
@@ -22,10 +23,21 @@ pub enum KeyEvent {
     ScrollRight,
 }
 
+/// A captured desktop frame. Pixels are tightly packed BGRA (byte order
+/// B, G, R, A; 8 bits per channel), matching the buffer layout `render.rs`
+/// produces. `w`/`h` are in capture (physical) pixels, which may exceed the
+/// overlay's logical size on scaled outputs — callers must rescale.
+pub struct Capture {
+    pub bgra: Vec<u8>,
+    pub w: u32,
+    pub h: u32,
+}
+
 /// Platform backend — one implementation per OS/display-server.
 ///
-/// `render.rs` produces a raw ARGB pixel buffer that every backend receives
-/// unchanged via `present()`. All other methods are input/pointer control.
+/// `render.rs` produces a raw ARGB8888 pixel buffer (BGRA byte order, the same
+/// layout as [`Capture`]) that every backend receives unchanged via
+/// `present()`. All other methods are input/pointer control.
 pub trait Backend {
     /// Screen dimensions in pixels.
     fn screen_size(&self) -> (u32, u32);
@@ -36,6 +48,11 @@ pub trait Backend {
     /// Query the current mouse pointer position.
     fn mouse_pos(&mut self) -> Result<(u32, u32)>;
 
+    /// Capture the current desktop as a BGRA pixel buffer (see [`Capture`]).
+    fn capture_screen(&mut self) -> Result<Capture> {
+        anyhow::bail!("screen capture is not implemented for this backend")
+    }
+
     /// Move the mouse pointer to an absolute position.
     fn move_mouse(&mut self, x: u32, y: u32) -> Result<()>;