From 04510981de994dc1dbede852a8962252863fd358 Mon Sep 17 00:00:00 2001 From: ploMP4 Date: Mon, 15 Jun 2026 17:56:04 +0300 Subject: [PATCH 1/4] feat(linux): hints mode --- Cargo.lock | 1654 ++++++++++++++++++++++++++----- Cargo.toml | 5 + README.md | 46 + src/app.rs | 12 +- src/backend/mod.rs | 21 +- src/backend/wayland.rs | 255 ++++- src/backend/x11.rs | 11 +- src/cli.rs | 8 + src/config.rs | 78 ++ src/hint/atspi.rs | 985 ++++++++++++++++++ src/hint/compositor/hyprland.rs | 140 +++ src/hint/compositor/mod.rs | 44 + src/hint/compositor/sway.rs | 173 ++++ src/hint/cv.rs | 374 +++++++ src/hint/detect.rs | 363 +++++++ src/hint/mod.rs | 424 ++++++++ src/main.rs | 1 + src/mode/bisect.rs | 3 + src/mode/free.rs | 1 + src/mode/hint.rs | 128 +++ src/mode/mod.rs | 16 +- src/mode/normal.rs | 21 +- src/render.rs | 249 ++++- 23 files changed, 4774 insertions(+), 238 deletions(-) create mode 100644 src/hint/atspi.rs create mode 100644 src/hint/compositor/hyprland.rs create mode 100644 src/hint/compositor/mod.rs create mode 100644 src/hint/compositor/sway.rs create mode 100644 src/hint/cv.rs create mode 100644 src/hint/detect.rs create mode 100644 src/hint/mod.rs create mode 100644 src/mode/hint.rs diff --git a/Cargo.lock b/Cargo.lock index 6de242e..a893fd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -25,9 +25,9 @@ checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -38,7 +38,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys", ] [[package]] @@ -49,20 +49,223 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys", ] [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys", +] + +[[package]] +name = "async-lock" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atspi" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf601cccedfffec598ec2db1f9d6745885458bccc0e8916d7023f017c94b3d0" +dependencies = [ + "atspi-common", + "atspi-connection", + "atspi-proxies", + "zbus", +] + +[[package]] +name = "atspi-common" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a79bed3f5b408ce3152f36e07327a845e6ed5d7e2821a89264037dbcc11daf" +dependencies = [ + "enumflags2", + "serde", + "static_assertions", + "zbus", + "zbus-lockstep", + "zbus-lockstep-macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "atspi-connection" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fab8e4f574f5a7d3af280b38eff25fb6f47a537dac9ae39ce152f52b19fb10b" +dependencies = [ + "atspi-common", + "atspi-proxies", + "futures-lite", + "zbus", +] + +[[package]] +name = "atspi-proxies" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53403acd3ab2fdb5914f6558da22e540fc07656fce5510f8c02be0e6ef68413e" +dependencies = [ + "atspi-common", + "serde", + "zbus", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "block2" @@ -73,20 +276,52 @@ dependencies = [ "objc2", ] +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "cc" -version = "1.2.27" +version = "1.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ + "find-msvc-tools", "shlex", ] [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -96,9 +331,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "clap" -version = "4.5.60" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -106,9 +341,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -118,9 +353,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -140,6 +375,21 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "dispatch2" version = "0.3.1" @@ -156,6 +406,39 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "endi" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -164,19 +447,52 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", ] [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "font8x8" @@ -184,33 +500,219 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "gethostname" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix 1.0.7", + "rustix", "windows-link", ] [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", - "r-efi", - "wasi", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "glam" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "333928d5eb103c5d4050533cec0384302db6be8ef7d3cebd30ec6a35350353da" + +[[package]] +name = "glam" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3abb554f8ee44336b72d522e0a7fe86a29e09f839a36022fa869a7dfe941a54b" + +[[package]] +name = "glam" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4126c0479ccf7e8664c36a2d719f5f2c140fbb4f9090008098d2c291fa5b3f16" + +[[package]] +name = "glam" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01732b97afd8508eee3333a541b9f7610f454bb818669e66e90f5f57c93a776" + +[[package]] +name = "glam" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525a3e490ba77b8e326fb67d4b44b4bd2f920f44d4cc73ccec50adc68e3bee34" + +[[package]] +name = "glam" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8509e6791516e81c1a630d0bd7fbac36d2fa8712a9da8662e716b52d5051ca" + +[[package]] +name = "glam" +version = "0.20.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43e957e744be03f5801a55472f593d43fabdebf25a4585db250f04d86b1675f" + +[[package]] +name = "glam" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "518faa5064866338b013ff9b2350dc318e14cc4fcd6cb8206d7e7c9886c98815" + +[[package]] +name = "glam" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f597d56c1bd55a811a1be189459e8fad2bbc272616375602443bdfb37fa774" + +[[package]] +name = "glam" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e4afd9ad95555081e109fe1d21f2a30c691b5f0919c67dfa690a2e1eb6bd51c" + +[[package]] +name = "glam" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5418c17512bdf42730f9032c74e1ae39afc408745ebb2acf72fbc4691c17945" + +[[package]] +name = "glam" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "151665d9be52f9bb40fc7966565d39666f2d1e69233571b71b87791c7e0528b3" + +[[package]] +name = "glam" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e05e7e6723e3455f4818c7b26e855439f7546cf617ef669d1adedb8669e5cb9" + +[[package]] +name = "glam" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "779ae4bf7e8421cf91c0b3b64e7e8b40b862fba4d393f59150042de7c4965a94" + +[[package]] +name = "glam" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8babf46d4c1c9d92deac9f7be466f76dfc4482b6452fc5024b5e8daf6ffeb3ee" + +[[package]] +name = "glam" +version = "0.30.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19fc433e8437a212d1b6f1e68c7824af3aed907da60afa994e7f542d18d12aa9" + +[[package]] +name = "glam" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556f6b2ea90b8d15a74e0e7bb41671c9bdf38cd9f78c284d750b9ce58a2b5be7" + +[[package]] +name = "glam" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f70749695b063ecbf6b62949ccccde2e733ec3ecbbd71d467dca4e5c6c97cca0" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", ] [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -218,14 +720,62 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", +] + +[[package]] +name = "imageproc" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645329c490783f3ea465d2b6c7c08286fece97f15e714fd533b6c70a3ead2252" +dependencies = [ + "approx", + "getrandom 0.3.4", + "image", + "itertools", + "nalgebra", + "num", + "rand", + "rand_distr", +] + [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -234,35 +784,77 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.184" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] -name = "linux-raw-sys" -version = "0.4.15" +name = "libm" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] [[package]] name = "memchr" -version = "2.7.5" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" @@ -273,11 +865,63 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "nalgebra" +version = "0.34.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df76ea0ff5c7e6b88689085804d6132ded0ddb9de5ca5b8aeb9eeadc0508a70a" +dependencies = [ + "approx", + "glam 0.14.0", + "glam 0.15.2", + "glam 0.16.0", + "glam 0.17.3", + "glam 0.18.0", + "glam 0.19.0", + "glam 0.20.5", + "glam 0.21.3", + "glam 0.22.0", + "glam 0.23.0", + "glam 0.24.2", + "glam 0.25.0", + "glam 0.27.0", + "glam 0.28.0", + "glam 0.29.3", + "glam 0.30.10", + "glam 0.31.1", + "glam 0.32.1", + "matrixmultiply", + "num-complex", + "num-rational", + "num-traits", + "simba", + "typenum", +] + [[package]] name = "nix" -version = "0.31.2" +version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ "bitflags", "cfg-if", @@ -285,6 +929,79 @@ dependencies = [ "libc", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "objc2" version = "0.6.4" @@ -439,9 +1156,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -449,35 +1166,123 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" + [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" dependencies = [ "memchr", + "serde", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -489,31 +1294,90 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] -name = "rustix" -version = "0.38.44" +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "rand_chacha", + "rand_core", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rustix" -version = "1.0.7" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "linux-raw-sys", + "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.228" @@ -546,9 +1410,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -558,33 +1422,84 @@ dependencies = [ ] [[package]] -name = "serde_spanned" -version = "1.1.0" +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simba" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98" +checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95" dependencies = [ - "serde_core", + "approx", + "num-complex", + "num-traits", + "paste", + "wide", ] [[package]] -name = "shlex" -version = "1.3.0" +name = "slab" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "stochos" version = "0.6.2" dependencies = [ "anyhow", + "atspi", "clap", "font8x8", + "futures-executor", + "futures-util", + "image", + "imageproc", "nix", "objc2", "objc2-app-kit", @@ -618,22 +1533,22 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.4.2", "once_cell", - "rustix 1.0.7", - "windows-sys 0.60.2", + "rustix", + "windows-sys", ] [[package]] name = "toml" -version = "1.1.0+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8195ca05e4eb728f4ba94f3e3291661320af739c4e43779cbdfae82ab239fcc" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ "indexmap", "serde_core", @@ -646,33 +1561,99 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.0+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + [[package]] name = "toml_parser" -version = "1.1.0+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.1.0+spec-1.1.0" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uds_windows" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed" +checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" +dependencies = [ + "memoffset", + "tempfile", + "windows-sys", +] [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "utf8parse" @@ -681,44 +1662,144 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] -name = "wasi" -version = "0.14.4+wasi-0.2.4" +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a5f4a424faf49c3c2c344f166f0662341d470ea185e939657aaff130f0ec4a" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "wit-bindgen", + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", ] [[package]] name = "wayland-backend" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe770181423e5fc79d3e2a7f4410b7799d5aab1de4372853de3c6aa13ca24121" +checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d" dependencies = [ "cc", "downcast-rs", - "rustix 0.38.44", + "rustix", "smallvec", "wayland-sys", ] [[package]] name = "wayland-client" -version = "0.31.10" +version = "0.31.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978fa7c67b0847dbd6a9f350ca2569174974cd4082737054dbb7fbb79d7d9a61" +checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" dependencies = [ "bitflags", - "rustix 0.38.44", + "rustix", "wayland-backend", "wayland-scanner", ] [[package]] name = "wayland-protocols" -version = "0.32.8" +version = "0.32.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "779075454e1e9a521794fed15886323ea0feda3f8b0fc1390f5398141310422a" +checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f" dependencies = [ "bitflags", "wayland-backend", @@ -728,9 +1809,9 @@ dependencies = [ [[package]] name = "wayland-protocols-wlr" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cb6cdc73399c0e06504c437fe3cf886f25568dd5454473d565085b36d6a8bbf" +checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" dependencies = [ "bitflags", "wayland-backend", @@ -741,9 +1822,9 @@ dependencies = [ [[package]] name = "wayland-scanner" -version = "0.31.6" +version = "0.31.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "896fdafd5d28145fce7958917d69f2fd44469b1d4e861cb5961bcbeebc6d1484" +checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a" dependencies = [ "proc-macro2", "quick-xml", @@ -752,213 +1833,334 @@ dependencies = [ [[package]] name = "wayland-sys" -version = "0.31.6" +version = "0.31.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbcebb399c77d5aa9fa5db874806ee7b4eba4e73650948e8f93963f128896615" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" dependencies = [ "pkg-config", ] [[package]] -name = "windows-link" -version = "0.2.1" +name = "wide" +version = "0.7.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] [[package]] -name = "windows-sys" -version = "0.59.0" +name = "windows-link" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-sys" -version = "0.60.2" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-targets 0.53.2", + "windows-link", ] [[package]] -name = "windows-targets" -version = "0.52.6" +name = "winnow" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "memchr", ] [[package]] -name = "windows-targets" -version = "0.53.2" +name = "wit-bindgen" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" dependencies = [ - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "wit-bindgen-rust-macro", ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" +name = "wit-bindgen" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" +name = "wit-bindgen-core" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] [[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" +name = "wit-bindgen-rust" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] [[package]] -name = "windows_i686_gnu" -version = "0.52.6" +name = "wit-bindgen-rust-macro" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] [[package]] -name = "windows_i686_gnu" -version = "0.53.0" +name = "wit-component" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] [[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" +name = "wit-parser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] [[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" +name = "x11rb" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix", + "x11rb-protocol", +] [[package]] -name = "windows_i686_msvc" -version = "0.52.6" +name = "x11rb-protocol" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" [[package]] -name = "windows_i686_msvc" -version = "0.53.0" +name = "xkbcommon" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +checksum = "8d66ca9352cbd4eecbbc40871d8a11b4ac8107cfc528a6e14d7c19c69d0e1ac9" +dependencies = [ + "libc", + "memmap2", + "xkeysym", +] [[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" +name = "xkeysym" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" [[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" +name = "zbus" +version = "5.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +checksum = "eee682d202a77e4a9f3b2c2bdf48a7b28af5c08c34ddf66f98c93e5e39464285" +dependencies = [ + "async-broadcast", + "async-executor", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener", + "futures-core", + "futures-lite", + "hex", + "libc", + "ordered-stream", + "rustix", + "serde", + "serde_repr", + "tracing", + "uds_windows", + "uuid", + "windows-sys", + "winnow", + "zbus_macros", + "zbus_names", + "zvariant", +] [[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" +name = "zbus-lockstep" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +checksum = "6998de05217a084b7578728a9443d04ea4cd80f2a0839b8d78770b76ccd45863" +dependencies = [ + "zbus_xml", + "zvariant", +] [[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" +name = "zbus-lockstep-macros" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +checksum = "10da05367f3a7b7553c8cdf8fa91aee6b64afebe32b51c95177957efc47ca3a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "zbus-lockstep", + "zbus_xml", + "zvariant", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" +name = "zbus_macros" +version = "5.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +checksum = "adf1bd45a81a103745b1757754762a26e8cd01e4532e4d6c8ec431624b80d1d6" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", + "zbus_names", + "zvariant", + "zvariant_utils", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" +name = "zbus_names" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +checksum = "7074f3e50b894eac91750142016d30d0a89be8e67dbfd9704fb875825760e52d" +dependencies = [ + "serde", + "winnow", + "zvariant", +] [[package]] -name = "winnow" -version = "1.0.0" +name = "zbus_xml" +version = "5.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +checksum = "a8067892e940ed1727dea64690378601603b31d62dfde019a5335fbb7c0e0ed9" +dependencies = [ + "quick-xml", + "serde", + "zbus_names", + "zvariant", +] [[package]] -name = "wit-bindgen" -version = "0.45.1" +name = "zerocopy" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] [[package]] -name = "x11rb" -version = "0.13.2" +name = "zerocopy-derive" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ - "gethostname", - "rustix 1.0.7", - "x11rb-protocol", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "x11rb-protocol" -version = "0.13.2" +name = "zmij" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] -name = "xkbcommon" -version = "0.8.0" +name = "zvariant" +version = "5.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d66ca9352cbd4eecbbc40871d8a11b4ac8107cfc528a6e14d7c19c69d0e1ac9" +checksum = "a192a0bde63360d77a7523c833d4b4ce6070a927e2c53246e4c540b1a3e27be0" dependencies = [ - "libc", - "memmap2", - "xkeysym", + "endi", + "enumflags2", + "serde", + "winnow", + "zvariant_derive", + "zvariant_utils", ] [[package]] -name = "xkeysym" -version = "0.2.1" +name = "zvariant_derive" +version = "5.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" +checksum = "90bc6cde9c01c511074be97f7ccb6c19d0da89e3f8662e812e999dcfd4638737" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", + "zvariant_utils", +] [[package]] -name = "zmij" -version = "1.0.21" +name = "zvariant_utils" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "1e8535915cfa75547e559d8c68e8139909a4aeee076831e4ef7fc59d8172c4d6" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "syn", + "winnow", +] diff --git a/Cargo.toml b/Cargo.toml index 27a0845..80d3a86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ x11 = ["dep:x11rb", "dep:xkbcommon"] anyhow = "1.0" clap = { version = "4", features = ["derive"] } font8x8 = "0.3" +image = { version = "0.25", default-features = false } +imageproc = { version = "0.26", default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" toml = "1.1" @@ -31,6 +33,9 @@ wayland-protocols-wlr = { version = "0.3", features = ["client"], optional = tru tempfile = { version = "3", optional = true } x11rb = { version = "0.13", features = ["xkb", "xtest"], optional = true } xkbcommon = { version = "0.8", optional = true } +atspi = { version = "0.30", features = ["connection", "proxies", "zbus"] } +futures-executor = "0.3" +futures-util = { version = "0.3", default-features = false, features = ["alloc"] } [target.'cfg(target_os = "macos")'.dependencies] objc2 = "0.6" diff --git a/README.md b/README.md index b8dc55c..5a4bdcd 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ Released binaries are not codesigned with a Developer ID, so after every upgrade | Tab | Search macros / quick-save position | | `b` | Switch to bisect mode | | `n` | Switch back to normal mode (from bisect) | +| `c` | Switch to hint mode | | `v` | Switch to free mode | All keys are configurable (see Configuration below). @@ -105,6 +106,7 @@ All keys are configurable (see Configuration below). | Flag | Effect | |------|--------| | `--bisect` | Start the overlay directly in bisect mode | +| `--hint` | Start the overlay directly in hint mode | | `--free` | Start the overlay directly in free mode (cursor starts at current mouse position) | | `--free-center` | Start the overlay directly in free mode with the cursor at the center of the screen | | `--allow-multiple` | Skip the single-instance lock | @@ -121,6 +123,27 @@ An alternative grid mode that recursively subdivides. Instead of two-key combos - Subdivision stops automatically once a cell would fall below `min_cell_size` pixels — the region is highlighted and you can act or back out - Press `n` to switch back to normal mode +### Hint mode + +An alternative mode that labels likely clickable UI targets directly, instead of asking you to navigate a grid. Type the label shown on a target, then perform an action. + +- Enter hint mode: press `c` from normal, bisect, or free mode, or launch with `--hint` +- Space / Enter / `m` / Delete / Insert act (click / double-click / triple-click / right-click / middle-click) on the selected target and exit +- `/` starts a drag from the selected target; select another hint and press Space to drag +- Backspace edits the typed hint label, or returns to the previous mode when no label is typed +- Escape closes the overlay + +Detection is pluggable behind a `HintDetector` trait, selected by `hint.detector`: + +- **`atspi`** (Linux, built in): reads the desktop [AT-SPI](https://docs.rs/atspi) accessibility tree over D-Bus and uses each interactive element's *own* role and bounding box. This is the most accurate source where it works — exact boxes, real semantics (button, link, menu item, table row…), no pixel guesswork, essentially no false positives. Each application's whole tree is fetched in a single bulk call (with all apps queried concurrently), so detection stays fast even on browser pages with thousands of nodes; children that toolkits materialize lazily (GTK tree and list views) are pulled in with batched follow-up calls, and an app that hasn't built its tree yet is nudged into doing so (its first hint launch may be sparse, the next one is complete). Coverage depends on the app: GTK/Qt/Gecko expose rich trees, Electron/Chromium need accessibility enabled, and games/native canvases expose nothing. On Wayland a client can't know its own window's on-screen position, so AT-SPI coordinates come in window-relative; stochos corrects them using the compositor's real window geometry and hints the active window, where a click is going. Geometry comes from a per-compositor provider (`src/hint/compositor/`, one file each — currently **Hyprland** and **Sway** via their IPC sockets; adding a compositor means adding one provider file). On X11 coordinates are already absolute and every visible window is hinted. Caveats: on Wayland compositors without a provider the semantic coordinates may be offset (the CV detector remains position-correct everywhere), and only the primary monitor is handled. +- **`cv`** (always available): a pure-Rust computer-vision pass over a screenshot — downscale → Canny edges → morphological closing → connected components → resolution-relative size/shape filtering with a per-box quality score. No model, no network, no services to talk to. CV sees visual *structure*, not semantic clickability, so it will still occasionally label a border, separator, or decorative shape, and miss a low-contrast control. Tune `edge_low_threshold`/`edge_high_threshold` (raise to cut clutter, lower to catch faint controls). +- **`auto`** (default): AT-SPI first; when it is unavailable, not compiled in, or sees almost nothing, CV takes over entirely. And because many apps expose their *chrome* but not their *content* (a file manager's icon view, a browser with content accessibility disabled, an embedded canvas), the cascade also checks the active window for a large contiguous region without semantic targets and lets CV label just that region. Semantic hints stay authoritative everywhere they exist, which is what keeps false positives near zero. + +```toml +[hint] +detector = "auto" # or "atspi" / "cv" +``` + ### Free mode An alternative mode for direct keyboard-driven cursor movement. Instead of selecting a grid cell, you steer the cursor continuously with movement keys, then act. @@ -156,6 +179,7 @@ All fields are optional. Missing fields use defaults. ```toml font_size = 2 # Glyph scale multiplier for the 8x8 bitmap font: 1=8px, 2=16px, 3=24px +hint_font_size = 1 # Optional override for element hint labels; defaults to 1 for dense target screens sub_hint_font_size = 2 # Optional override for sub-grid hint glyphs; defaults to font_size when omitted panel_font_size = 2 # Optional override for macro/search popup panels; defaults to sub_hint_font_size, then font_size @@ -171,12 +195,22 @@ rows = 2 cols = 2 min_cell_size = 16 # Stop subdividing once a cell would be smaller than this, in pixels +[hint] +detector = "auto" # "auto" (default) | "atspi" (Linux) | "cv" +alphabet = ["a", "s", "d", "f", "j", "k", "l", ";", "g", "h", "q", "w", "e", "r", "t", "y", "u", "i", "o", "p"] # Keys used to build hint labels +max_label_len = 3 +auto_click = false # If true, click immediately when a typed label uniquely identifies a target +downscale_longest = 1280 # CV: longest screen-capture side fed to the detector; 0 = no downscale +edge_low_threshold = 50 # CV: Canny low threshold (raise to cut clutter, lower to catch faint controls) +edge_high_threshold = 150 # CV: Canny high threshold + [macros] playback_speed = 1.0 # 1.0 = recorded speed, 2.0 = twice as fast, 0.0 or negative = instant (no waits) [keys] normal = "n" bisect = "b" +hint = "c" free_mode = "v" click = "space" double_click = "enter" @@ -211,6 +245,10 @@ rec_bg = "#f44336ff" # Material Design Red (urgent, visible everywhe border = "#00e676ff" # Material Green (fresh, visible) border_dragging = "#e91e63ff" # Material Pink (strong attention grabber) crosshair = "#ffaa00ff" # Amber crosshair in free mode +hint_chip_bg = "#2e1e1eee" # Hint label background +hint_text = "#00ccffff" # Untyped hint label text +hint_text_typed = "#50ff50ff" # Typed hint label text +hint_dim = "#66666688" # Dimmed non-matching hints [free] left = "h" @@ -245,6 +283,14 @@ max_speed = 500 - `rows` and `cols` set the grid shape used at every subdivision level (default 2x2). - `min_cell_size` is the pixel floor at which subdivision stops. The glyph scale auto-shrinks to fit the cell, so lowering this lets you reach tiny regions. +### Hint mode + +- `alphabet` sets the characters used for generated hint labels. Defaults to the same set as `[grid].hints`; override to change which keys label elements. +- `max_label_len` caps generated label length. Higher values allow more targets, but require more keystrokes. +- `auto_click` clicks as soon as the typed label uniquely matches one target. Leave it `false` if you want to choose the action after selecting. +- `downscale_longest` controls CV detector speed/precision. Lower values are faster but less precise; `0` disables downscaling. +- `edge_low_threshold` / `edge_high_threshold` tune Canny edge detection. Lower values find more targets and more false positives; higher values are stricter. + ### Free mode - `left`, `down`, `up`, `right` set the movement keys (default: `h`, `j`, `k`, `l`). diff --git a/src/app.rs b/src/app.rs index 83f8230..ca873bf 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,5 +1,6 @@ use crate::backend::Backend; use crate::config::config; +use crate::hint::build_hint_mode; use crate::input::InputState; use crate::macro_store::MacroStore; use crate::mode::{Mode, ModeTransition}; @@ -8,6 +9,7 @@ use crate::mode::{Mode, ModeTransition}; pub enum InitialMode { Normal, Bisect, + Hint, Free, FreeCenter, } @@ -26,6 +28,11 @@ pub fn run(backend: &mut B, initial: InitialMode) -> anyhow::Result< InitialMode::Bisect => Mode::Bisect { region: (0, 0, w, h), }, + InitialMode::Hint => Mode::Hint { + elements: build_hint_mode(backend, w, h)?.into(), + typed: Vec::new(), + target: None, + }, InitialMode::Free => { let (x, y) = backend.mouse_pos()?; Mode::Free { @@ -41,7 +48,10 @@ pub fn run(backend: &mut B, initial: InitialMode) -> anyhow::Result< }, }; - if !matches!(initial, InitialMode::Free | InitialMode::FreeCenter) { + if !matches!( + initial, + InitialMode::Free | InitialMode::FreeCenter | InitialMode::Hint + ) { backend.move_mouse(w / 2, h / 2)?; } diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 545ff4e..f57b863 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -5,6 +5,7 @@ use anyhow::Result; pub enum KeyEvent { Normal, Bisect, + Hint, FreeMode, Char(char), Click, @@ -22,10 +23,21 @@ pub enum KeyEvent { ScrollRight, } +/// A captured desktop frame. Pixels are tightly packed BGRA (byte order +/// B, G, R, A; 8 bits per channel), matching the buffer layout `render.rs` +/// produces. `w`/`h` are in capture (physical) pixels, which may exceed the +/// overlay's logical size on scaled outputs — callers must rescale. +pub struct Capture { + pub bgra: Vec, + pub w: u32, + pub h: u32, +} + /// Platform backend — one implementation per OS/display-server. /// -/// `render.rs` produces a raw ARGB pixel buffer that every backend receives -/// unchanged via `present()`. All other methods are input/pointer control. +/// `render.rs` produces a raw ARGB8888 pixel buffer (BGRA byte order, the same +/// layout as [`Capture`]) that every backend receives unchanged via +/// `present()`. All other methods are input/pointer control. pub trait Backend { /// Screen dimensions in pixels. fn screen_size(&self) -> (u32, u32); @@ -36,6 +48,11 @@ pub trait Backend { /// Query the current mouse pointer position. fn mouse_pos(&mut self) -> Result<(u32, u32)>; + /// Capture the current desktop as a BGRA pixel buffer (see [`Capture`]). + fn capture_screen(&mut self) -> Result { + anyhow::bail!("screen capture is not implemented for this backend") + } + /// Move the mouse pointer to an absolute position. fn move_mouse(&mut self, x: u32, y: u32) -> Result<()>; diff --git a/src/backend/wayland.rs b/src/backend/wayland.rs index 02a4b59..958ed2f 100644 --- a/src/backend/wayland.rs +++ b/src/backend/wayland.rs @@ -1,4 +1,4 @@ -use std::io::Write; +use std::io::{Read, Seek, SeekFrom, Write}; use std::os::fd::AsFd; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -17,12 +17,13 @@ use wayland_client::{ }; use wayland_protocols_wlr::{ layer_shell::v1::client::{zwlr_layer_shell_v1, zwlr_layer_surface_v1}, + screencopy::v1::client::{zwlr_screencopy_frame_v1, zwlr_screencopy_manager_v1}, virtual_pointer::v1::client::{zwlr_virtual_pointer_manager_v1, zwlr_virtual_pointer_v1}, }; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; -use super::{Backend, KeyEvent}; +use super::{Backend, Capture, KeyEvent}; use crate::config::{config, Key}; const BTN_LEFT: u32 = 0x110; @@ -62,6 +63,7 @@ impl WaylandBackend { shm: None, surface: None, layer_shell: None, + screencopy_manager: None, layer_surface: None, seat: None, vp_manager: None, @@ -76,6 +78,7 @@ impl WaylandBackend { current_output: None, held: None, pointer_pos: None, + screencopy: None, }; eq.roundtrip(&mut state).context("initial roundtrip")?; @@ -182,6 +185,100 @@ impl Backend for WaylandBackend { Ok(self.state.pointer_pos.unwrap_or((w / 2, h / 2))) } + fn capture_screen(&mut self) -> Result { + self.present_empty()?; + let manager = self + .state + .screencopy_manager + .as_ref() + .context("zwlr_screencopy_manager_v1 not available")?; + let output = self + .state + .current_output + .as_ref() + .context("no Wayland output is associated with the overlay")?; + + self.state.screencopy = Some(ScreencopyState::default()); + let frame = manager.capture_output(0, output, &self.qh, ()); + + // Wait until the compositor has advertised its buffer formats. v3 ends + // the list with `buffer_done`; v1/v2 send a single `buffer` event with no + // terminator, so a non-empty list is also enough to proceed. + self.wait_for_screencopy( + |capture| capture.done.is_some() || capture.buffer_done || !capture.buffers.is_empty(), + "waiting for screencopy buffer parameters", + )?; + + let screencopy = self + .state + .screencopy + .as_ref() + .context("screencopy state missing")?; + if screencopy.buffers.is_empty() { + bail!("screencopy failed before sending buffer parameters"); + } + let params = screencopy + .buffers + .iter() + .copied() + .find(|params| is_supported_screencopy_format(params.format)) + .with_context(|| { + let offered: Vec<_> = screencopy.buffers.iter().map(|b| b.format).collect(); + format!("screencopy offered no supported wl_shm format (got {offered:?})") + })?; + let size = params.stride as usize * params.height as usize; + let mut file = tempfile::tempfile().context("create screencopy shm tempfile")?; + file.set_len(size as u64) + .context("resize screencopy shm tempfile")?; + + let shm = self.state.shm.as_ref().context("wl_shm not available")?; + let pool = shm.create_pool(file.as_fd(), size as i32, &self.qh, ()); + let buffer = pool.create_buffer( + 0, + params.width as i32, + params.height as i32, + params.stride as i32, + params.format, + &self.qh, + (), + ); + frame.copy(&buffer); + self.state + .conn + .flush() + .context("flush screencopy request")?; + + self.wait_for_screencopy( + |capture| capture.done.is_some(), + "waiting for screencopy frame", + )?; + + let capture = self + .state + .screencopy + .take() + .context("screencopy finished without state")?; + let done = capture + .done + .context("screencopy finished without a result")?; + let capture_flags = capture.flags; + frame.destroy(); + buffer.destroy(); + pool.destroy(); + if let ScreencopyDone::Failed = done { + bail!("screencopy frame failed"); + } + + file.seek(SeekFrom::Start(0)) + .context("rewind screencopy shm file")?; + let mut data = vec![0u8; size]; + file.read_exact(&mut data) + .context("read screencopy shm buffer")?; + let bgra = convert_screencopy_buffer(&data, params, capture_flags)?; + let (w, h) = (params.width, params.height); + Ok(Capture { bgra, w, h }) + } + fn move_mouse(&mut self, x: u32, y: u32) -> Result<()> { if let Some(vp) = &self.state.vp { vp.motion_absolute(timestamp(), x, y, self.state.screen_w, self.state.screen_h); @@ -422,6 +519,19 @@ impl Backend for WaylandBackend { } impl WaylandBackend { + fn wait_for_screencopy( + &mut self, + ready: impl Fn(&ScreencopyState) -> bool, + context: &'static str, + ) -> Result<()> { + while !self.state.screencopy.as_ref().is_some_and(&ready) { + self.eq + .blocking_dispatch(&mut self.state) + .with_context(|| format!("dispatch wayland events while {context}"))?; + } + Ok(()) + } + fn wait_for_events(&mut self) -> Result<()> { self.eq.flush().context("flush wayland events")?; @@ -461,6 +571,7 @@ struct WaylandState { shm: Option, surface: Option, layer_shell: Option, + screencopy_manager: Option, layer_surface: Option, seat: Option, vp_manager: Option, @@ -476,8 +587,37 @@ struct WaylandState { current_output: Option, held: Option, pointer_pos: Option<(u32, u32)>, + screencopy: Option, +} + +#[derive(Clone, Copy)] +struct ScreencopyBufferParams { + format: wl_shm::Format, + width: u32, + height: u32, + stride: u32, +} + +#[derive(Clone, Copy)] +enum ScreencopyDone { + Ready, + Failed, +} + +#[derive(Default)] +struct ScreencopyState { + /// All `buffer` formats the compositor advertised, in arrival order; we pick + /// the first one we can convert. + buffers: Vec, + /// Set on the v3 `buffer_done` event that terminates the format list. + buffer_done: bool, + flags: u32, + done: Option, } +/// `zwlr_screencopy_frame_v1` flag bit indicating the buffer is stored bottom-up. +const SCREENCOPY_Y_INVERT: u32 = 1; + impl WaylandState { /// If a key is held, how long until we should fire the next synthesized /// repeat. `None` means nothing is held (caller can block indefinitely). @@ -620,6 +760,9 @@ impl Dispatch for WaylandState { "zwlr_layer_shell_v1" => { state.layer_shell = Some(registry.bind(name, version.min(4), qh, ())); } + "zwlr_screencopy_manager_v1" => { + state.screencopy_manager = Some(registry.bind(name, version.min(3), qh, ())); + } "zwlr_virtual_pointer_manager_v1" => { state.vp_manager = Some(registry.bind(name, version.min(2), qh, ())); } @@ -761,6 +904,49 @@ impl Dispatch for WaylandState { } } +impl Dispatch for WaylandState { + fn event( + state: &mut Self, + _: &zwlr_screencopy_frame_v1::ZwlrScreencopyFrameV1, + event: zwlr_screencopy_frame_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + let Some(capture) = state.screencopy.as_mut() else { + return; + }; + match event { + zwlr_screencopy_frame_v1::Event::Buffer { + format: WEnum::Value(format), + width, + height, + stride, + } => { + capture.buffers.push(ScreencopyBufferParams { + format, + width, + height, + stride, + }); + } + zwlr_screencopy_frame_v1::Event::BufferDone => { + capture.buffer_done = true; + } + zwlr_screencopy_frame_v1::Event::Flags { flags } => { + capture.flags = flags.into(); + } + zwlr_screencopy_frame_v1::Event::Ready { .. } => { + capture.done = Some(ScreencopyDone::Ready); + } + zwlr_screencopy_frame_v1::Event::Failed => { + capture.done = Some(ScreencopyDone::Failed); + } + _ => {} + } + } +} + delegate_noop!(WaylandState: ignore wl_compositor::WlCompositor); delegate_noop!(WaylandState: ignore wl_region::WlRegion); delegate_noop!(WaylandState: ignore wl_output::WlOutput); @@ -794,6 +980,7 @@ impl Dispatch for WaylandState { } } delegate_noop!(WaylandState: ignore zwlr_layer_shell_v1::ZwlrLayerShellV1); +delegate_noop!(WaylandState: ignore zwlr_screencopy_manager_v1::ZwlrScreencopyManagerV1); delegate_noop!(WaylandState: ignore zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1); delegate_noop!(WaylandState: ignore zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1); @@ -885,3 +1072,65 @@ pub(super) fn keycode_to_key(kc: u32, xkb_state: Option<&xkb::State>) -> Option< } Some(Key::Char(ch)) } + +fn is_supported_screencopy_format(format: wl_shm::Format) -> bool { + matches!( + format, + wl_shm::Format::Argb8888 + | wl_shm::Format::Xrgb8888 + | wl_shm::Format::Abgr8888 + | wl_shm::Format::Xbgr8888 + ) +} + +fn convert_screencopy_buffer( + data: &[u8], + params: ScreencopyBufferParams, + flags: u32, +) -> Result> { + let width = params.width as usize; + let height = params.height as usize; + let stride = params.stride as usize; + let row_bytes = width * 4; + if stride < row_bytes || data.len() < stride * height { + bail!("screencopy buffer is smaller than declared dimensions"); + } + + // `swap_rb` selects the BGR-order source formats; `opaque` forces alpha to + // 0xFF for the X-variants that carry no real alpha. Decoded once here rather + // than re-matched per row. + let (swap_rb, opaque) = match params.format { + wl_shm::Format::Argb8888 => (false, false), + wl_shm::Format::Xrgb8888 => (false, true), + wl_shm::Format::Abgr8888 => (true, false), + wl_shm::Format::Xbgr8888 => (true, true), + other => bail!("unsupported screencopy wl_shm format: {other:?}"), + }; + + let mut out = vec![0u8; row_bytes * height]; + for y in 0..height { + let dst_y = if flags & SCREENCOPY_Y_INVERT != 0 { + height - 1 - y + } else { + y + }; + let src = &data[y * stride..y * stride + row_bytes]; + let dst = &mut out[dst_y * row_bytes..(dst_y + 1) * row_bytes]; + if swap_rb { + for (src_px, dst_px) in src.chunks_exact(4).zip(dst.chunks_exact_mut(4)) { + dst_px[0] = src_px[2]; + dst_px[1] = src_px[1]; + dst_px[2] = src_px[0]; + dst_px[3] = if opaque { 0xFF } else { src_px[3] }; + } + } else { + dst.copy_from_slice(src); + if opaque { + for px in dst.chunks_exact_mut(4) { + px[3] = 0xFF; + } + } + } + } + Ok(out) +} diff --git a/src/backend/x11.rs b/src/backend/x11.rs index fabfea9..b181c43 100644 --- a/src/backend/x11.rs +++ b/src/backend/x11.rs @@ -12,7 +12,7 @@ use x11rb::wrapper::ConnectionExt as _; use x11rb::CURRENT_TIME; use xkbcommon::xkb; -use super::{Backend, KeyEvent}; +use super::{Backend, Capture, KeyEvent}; use crate::config::{config, Key}; const BTN_LEFT: u8 = 1; @@ -230,6 +230,15 @@ impl Backend for X11Backend { Ok((reply.root_x as u32, reply.root_y as u32)) } + fn capture_screen(&mut self) -> Result { + let bgra = capture_root(&self.conn, self.root, self.screen_w, self.screen_h)?; + Ok(Capture { + bgra, + w: self.screen_w, + h: self.screen_h, + }) + } + fn move_mouse(&mut self, x: u32, y: u32) -> Result<()> { self.warp_and_sync(x, y) } diff --git a/src/cli.rs b/src/cli.rs index 841845d..6cf6aeb 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -17,6 +17,10 @@ pub struct Args { #[arg(long, group = "mode")] pub free_center: bool, + /// Start in hint mode (clickable-element labels) + #[arg(long, group = "mode")] + pub hint: bool, + /// Allow multiple concurrent instances #[arg(long)] pub allow_multiple: bool, @@ -41,6 +45,10 @@ impl Args { return InitialMode::FreeCenter; } + if self.hint { + return InitialMode::Hint; + } + InitialMode::Normal } } diff --git a/src/config.rs b/src/config.rs index 27d61e0..bd156b2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -210,6 +210,7 @@ impl Default for GridConfig { pub struct KeyBindings { pub normal: Key, pub bisect: Key, + pub hint: Key, pub free_mode: Key, pub click: Key, pub double_click: Key, @@ -231,6 +232,7 @@ impl Default for KeyBindings { Self { normal: Key::Char('n'), bisect: Key::Char('b'), + hint: Key::Char('c'), free_mode: Key::Char('v'), click: Key::Space, double_click: Key::Enter, @@ -259,6 +261,9 @@ impl KeyBindings { if key == self.bisect { return Some(KeyEvent::Bisect); } + if key == self.hint { + return Some(KeyEvent::Hint); + } if key == self.free_mode { return Some(KeyEvent::FreeMode); } @@ -348,6 +353,15 @@ pub struct Colors { #[serde(with = "from_hex")] pub crosshair: [u8; 4], + + #[serde(with = "from_hex")] + pub hint_chip_bg: [u8; 4], + #[serde(with = "from_hex")] + pub hint_text: [u8; 4], + #[serde(with = "from_hex")] + pub hint_text_typed: [u8; 4], + #[serde(with = "from_hex")] + pub hint_dim: [u8; 4], } mod from_hex { use serde::{Deserialize, Deserializer, Serializer}; @@ -402,6 +416,10 @@ impl Default for Colors { border: [0x00, 0xA5, 0xFF, 0xFF], //amber border_dragging: [0xFF, 0x00, 0xFF, 0xFF], //magenta crosshair: [0x00, 0xA5, 0xFF, 0xFF], //amber + hint_chip_bg: [0x2E, 0x1E, 0x1E, 0xEE], + hint_text: [0x00, 0xCC, 0xFF, 0xFF], + hint_text_typed: [0x50, 0xFF, 0x50, 0xFF], + hint_dim: [0x66, 0x66, 0x66, 0x88], } } } @@ -475,6 +493,54 @@ impl Default for FreeConfig { } } +/// Which element-detection backend hint mode uses. `Auto` (the default) tries +/// the semantic AT-SPI detector and falls back to pure CV when AT-SPI is +/// unavailable or sees almost nothing; on platforms without AT-SPI it uses +/// CV, while selecting `Atspi` explicitly reports an error. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum HintDetectorKind { + #[default] + Auto, + Cv, + Atspi, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(default)] +pub struct HintConfig { + /// Which detector to use: "auto" (default), "cv", or "atspi". + pub detector: HintDetectorKind, + pub alphabet: Vec, + pub max_label_len: usize, + pub auto_click: bool, + /// Longest screen-capture side fed to the CV detector; the result is + /// scaled back to screen coordinates. 0 disables downscaling. Bounds both + /// runtime and the absolute size of filtered features. + pub downscale_longest: u32, + /// Canny edge thresholds for the CV detector. Raise them for fewer, more + /// confident hints on noisy screens; lower them to catch faint controls. + pub edge_low_threshold: u32, + pub edge_high_threshold: u32, +} + +impl Default for HintConfig { + fn default() -> Self { + Self { + detector: HintDetectorKind::default(), + alphabet: vec![ + 'a', 's', 'd', 'f', 'j', 'k', 'l', ';', 'g', 'h', 'q', 'w', 'e', 'r', 't', 'y', + 'u', 'i', 'o', 'p', + ], + max_label_len: 3, + auto_click: false, + downscale_longest: 1280, + edge_low_threshold: 50, + edge_high_threshold: 150, + } + } +} + #[derive(Debug, Serialize, Deserialize)] #[serde(default)] pub struct Config { @@ -484,7 +550,9 @@ pub struct Config { pub colors: Colors, pub macros: MacrosConfig, pub free: FreeConfig, + pub hint: HintConfig, pub font_size: u32, + pub hint_font_size: Option, pub sub_hint_font_size: Option, pub panel_font_size: Option, } @@ -498,7 +566,9 @@ impl Default for Config { colors: Colors::default(), macros: MacrosConfig::default(), free: FreeConfig::default(), + hint: HintConfig::default(), font_size: 2, + hint_font_size: None, sub_hint_font_size: None, panel_font_size: None, } @@ -532,6 +602,10 @@ impl Config { clamp_scale(self.font_size) } + pub fn hint_font_size(&self) -> u32 { + self.hint_font_size.map_or(1, clamp_scale) + } + pub fn sub_hint_font_size(&self) -> u32 { self.sub_hint_font_size .map_or_else(|| self.font_size(), clamp_scale) @@ -555,6 +629,10 @@ impl Config { &self.grid.hints } + pub fn hint_alphabet(&self) -> &[char] { + &self.hint.alphabet + } + pub fn cols(&self) -> u32 { self.grid.hints.len() as u32 } diff --git a/src/hint/atspi.rs b/src/hint/atspi.rs new file mode 100644 index 0000000..9664ddc --- /dev/null +++ b/src/hint/atspi.rs @@ -0,0 +1,985 @@ +//! AT-SPI accessibility detector (Linux): semantic click targets with exact +//! boxes from the desktop accessibility tree. Whole trees come from one +//! `Cache.GetItems` per app (fetched concurrently) rather than per-node D-Bus +//! roundtrips; only actionable visible nodes get a `GetExtents`. On Wayland, +//! AT-SPI coordinates arrive pinned to (0,0) and are corrected with [`compositor`] +//! window geometry, hinting only the active window; without it (X11) coordinates +//! are already absolute and used unshifted. Async (zbus) collection runs behind +//! a caller-side timeout so a wedged accessibility bus cannot hold the overlay. + +use std::{ + collections::{HashMap, HashSet}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, + }, + thread, + time::Duration, +}; + +use anyhow::{Context, Result}; +use atspi::proxy::accessible::AccessibleProxy; +use atspi::proxy::cache::CacheProxy; +use atspi::proxy::component::ComponentProxy; +use atspi::zbus::{fdo::DBusProxy, names::BusName}; +use atspi::{AccessibilityConnection, CoordType, Role, State, StateSet}; +use futures_util::future::join_all; + +use super::compositor::{self, CompositorSnapshot, WindowRect}; +use super::detect::{DetectorOutput, HintCandidate, HintDetector}; +use crate::backend::Backend; + +static ACCESSIBILITY_ENABLE_REQUESTED: AtomicBool = AtomicBool::new(false); + +/// Upper bound on returned targets; matches the shared ranking/dedup cap. +const MAX_TARGETS: usize = 1024; + +const COLLECT_TIMEOUT: Duration = Duration::from_secs(2); + +/// Cycle guard for parent-chain walks over cache data we don't control. +const MAX_ANCESTOR_DEPTH: usize = 128; + +pub struct AtspiHintDetector; + +impl HintDetector for AtspiHintDetector { + fn name(&self) -> &'static str { + "atspi" + } + + fn detect(&self, backend: &mut dyn Backend) -> Result { + let (screen_w, screen_h) = backend.screen_size(); + let (candidates, focus_rect) = collect_candidates_with_timeout(screen_w, screen_h)?; + // Boxes are already in screen coordinates, so the shared remap is a no-op. + Ok(DetectorOutput { + candidates, + capture_w: screen_w, + capture_h: screen_h, + focus_rect, + }) + } +} + +/// Roles a user would plausibly want to click; containers are walked for +/// their children but never labelled themselves. +fn is_actionable(role: Role) -> bool { + matches!( + role, + Role::Button + | Role::PushButtonMenu + | Role::ToggleButton + | Role::Link + | Role::Menu + | Role::MenuItem + | Role::CheckMenuItem + | Role::RadioMenuItem + | Role::CheckBox + | Role::RadioButton + | Role::ComboBox + | Role::Entry + | Role::PasswordText + | Role::PageTab + | Role::ListItem + | Role::TreeItem + | Role::TableRow + | Role::TableCell + | Role::TableColumnHeader + | Role::Slider + | Role::SpinButton + | Role::Icon + ) +} + +/// A top-level window node, the anchor for position correction. +fn is_window(role: Role) -> bool { + matches!(role, Role::Frame | Role::Window | Role::Dialog) +} + +struct Node { + path: String, + parent: String, + role: Role, + states: StateSet, + name: String, + /// Child count the node claims to have, `-1` when unknown; a mismatch with + /// the children actually present marks a lazily-materialized subtree. + children: i32, +} + +/// Candidates plus the compositor's active-window rectangle, which the `auto` +/// cascade uses to spot regions the accessibility tree left empty. +type Collected = (Vec, Option<(u32, u32, u32, u32)>); + +fn collect_candidates_with_timeout(screen_w: u32, screen_h: u32) -> Result { + let (tx, rx) = mpsc::sync_channel(1); + thread::spawn(move || { + let result = futures_executor::block_on(collect_candidates(screen_w, screen_h)); + let _ = tx.send(result); + }); + match rx.recv_timeout(COLLECT_TIMEOUT) { + Ok(result) => result, + Err(mpsc::RecvTimeoutError::Timeout) => { + anyhow::bail!("AT-SPI collection timed out after {COLLECT_TIMEOUT:?}") + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + anyhow::bail!("AT-SPI collection worker exited before sending a result") + } + } +} + +async fn collect_candidates(screen_w: u32, screen_h: u32) -> Result { + // Best-effort, once per process: ask the session to enable accessibility + // so toolkits publish their trees. + if !ACCESSIBILITY_ENABLE_REQUESTED.swap(true, Ordering::Relaxed) { + let _ = atspi::connection::set_session_accessibility(true).await; + } + + let a11y = AccessibilityConnection::new() + .await + .context("connect to the AT-SPI accessibility bus (is it running?)")?; + let root = a11y + .root_accessible_on_registry() + .await + .context("read the AT-SPI registry root")?; + + let compositor = compositor::snapshot(); + + // (destination, app root object path) per application. + let apps: Vec<(String, String)> = root + .get_children() + .await + .context("list applications on the accessibility bus")? + .iter() + .filter_map(|app| { + let dest = app.name_as_str()?.to_owned(); + Some((dest, app.path_as_str().to_owned())) + }) + .collect(); + + // Tree fetch and pid resolution are independent D-Bus batches, so overlap + // them instead of paying two serial round-trips at hint-mode entry. + let (trees, pids) = futures_util::future::join( + join_all( + apps.iter() + .map(|(dest, root_path)| fetch_app_tree(&a11y, dest, root_path)), + ), + app_pids(&a11y, &apps, &compositor), + ) + .await; + + let mut out = Vec::new(); + for (((dest, _), nodes), pid) in apps.iter().zip(trees).zip(pids) { + let Some(mut nodes) = nodes else { continue }; + // Match the app's top-level frames first: they live in the raw cache, + // so this needs no tree completion. An app whose frames are all + // skipped (not the active window) contributes nothing, so skip the + // expensive lazy-child expansion entirely. + let offsets = frame_offsets(&a11y, dest, &nodes, pid, &compositor).await; + if offsets.values().all(Option::is_none) { + continue; + } + complete_tree(&a11y, dest, &mut nodes).await; + collect_targets(&a11y, dest, &nodes, &offsets, screen_w, screen_h, &mut out).await; + if out.len() >= MAX_TARGETS { + out.truncate(MAX_TARGETS); + break; + } + } + let focus_rect = compositor + .active_window() + .and_then(|w| clamp_to_screen(w.x, w.y, w.w, w.h, screen_w, screen_h)); + Ok((out, focus_rect)) +} + +/// The pid behind each application's bus connection, from the a11y bus +/// daemon. The compositor reports the same pid on the app's windows, making +/// it the primary window-matching key; only needed when there is compositor +/// geometry to match against. Sandboxed apps reach the bus through a proxy +/// process, so a pid here may match no window — matching then falls back to +/// size and title. +async fn app_pids( + a11y: &AccessibilityConnection, + apps: &[(String, String)], + compositor: &CompositorSnapshot, +) -> Vec> { + if compositor.windows.is_empty() { + return vec![None; apps.len()]; + } + let Ok(dbus) = DBusProxy::new(a11y.connection()).await else { + return vec![None; apps.len()]; + }; + join_all(apps.iter().map(|(dest, _)| { + let dbus = &dbus; + async move { + let name = BusName::try_from(dest.as_str()).ok()?; + let pid = dbus.get_connection_unix_process_id(name).await.ok()?; + i32::try_from(pid).ok() + } + })) + .await +} + +/// At or below this many cache items, assume the app has not built its tree. +const STUB_TREE_MAX_ITEMS: usize = 10; + +/// Fetch an application's entire tree in one `Cache.GetItems` call (legacy +/// wire format as fallback). Toolkits build their tree lazily, on the first +/// sign of a client *walking* them — a cache read is not such a sign — so a +/// stub-sized answer gets one live `get_children` poke on the root and one +/// re-read; by the next hint launch the cache is fully populated. +async fn fetch_app_tree( + a11y: &AccessibilityConnection, + dest: &str, + root_path: &str, +) -> Option> { + let cache = CacheProxy::builder(a11y.connection()) + .destination(dest.to_owned()) + .ok()? + .build() + .await + .ok()?; + let items = cache_nodes(&cache).await?; + if items.len() > STUB_TREE_MAX_ITEMS { + return Some(items); + } + let _ = children_of(a11y, dest, root_path).await; + match cache_nodes(&cache).await { + Some(retried) if retried.len() > items.len() => Some(retried), + _ => Some(items), + } +} + +async fn cache_nodes(cache: &CacheProxy<'_>) -> Option> { + match cache.get_items().await { + Ok(items) => Some( + items + .into_iter() + .map(|item| Node { + path: item.object.path_as_str().to_owned(), + parent: item.parent.path_as_str().to_owned(), + role: item.role, + states: item.states, + name: item.name, + children: item.children, + }) + .collect(), + ), + // Signature error: the app serves the legacy item layout. + Err(_) => Some( + cache + .get_legacy_items() + .await + .ok()? + .into_iter() + .map(|item| Node { + path: item.object.path_as_str().to_owned(), + parent: item.parent.path_as_str().to_owned(), + role: item.role, + states: item.states, + name: item.name, + children: item.children.len() as i32, + }) + .collect(), + ), + } +} + +const MAX_EXPANSION_NODES: usize = 2000; +const MAX_EXPANSION_ROUNDS: usize = 16; + +/// Fill in the parts of the tree the cache only knows *about*: GTK tree and +/// list views materialize items on demand, so the cache holds the container +/// but not its children. Expand exactly the visible containers whose child +/// count mismatches, each at most once, with batched live calls. +async fn complete_tree(a11y: &AccessibilityConnection, dest: &str, nodes: &mut Vec) { + let mut expanded: HashSet = HashSet::new(); + let mut added = 0usize; + for _ in 0..MAX_EXPANSION_ROUNDS { + let mut cached_children: HashMap<&str, i32> = HashMap::new(); + for node in nodes.iter() { + *cached_children.entry(node.parent.as_str()).or_default() += 1; + } + let pending: Vec = nodes + .iter() + .filter(|n| { + n.states.contains(State::Showing) + && !expanded.contains(&n.path) + && (n.children < 0 + || n.children > cached_children.get(n.path.as_str()).copied().unwrap_or(0)) + }) + .map(|n| n.path.clone()) + .collect(); + if pending.is_empty() || added >= MAX_EXPANSION_NODES { + return; + } + + let known: HashSet<&str> = nodes.iter().map(|n| n.path.as_str()).collect(); + let child_lists = join_all(pending.iter().map(|path| children_of(a11y, dest, path))).await; + // (parent path, child path) + let mut new_refs: Vec<(String, String)> = Vec::new(); + for (parent, children) in pending.iter().zip(child_lists) { + for child in children { + if !known.contains(child.as_str()) { + new_refs.push((parent.clone(), child)); + } + } + } + expanded.extend(pending); + new_refs.truncate(MAX_EXPANSION_NODES.saturating_sub(added)); + + let described = join_all(new_refs.iter().map(|(_, path)| describe(a11y, dest, path))).await; + for ((parent, path), description) in new_refs.into_iter().zip(described) { + let Some((role, states)) = description else { + continue; + }; + nodes.push(Node { + path, + parent, + role, + states, + name: String::new(), + children: -1, + }); + added += 1; + } + } +} + +/// A node's live children, restricted to the same application. +async fn children_of(a11y: &AccessibilityConnection, dest: &str, path: &str) -> Vec { + let Ok(accessible) = accessible_at(a11y, dest, path).await else { + return Vec::new(); + }; + let Ok(children) = accessible.get_children().await else { + return Vec::new(); + }; + children + .iter() + .filter(|child| child.name_as_str() == Some(dest)) + .map(|child| child.path_as_str().to_owned()) + .collect() +} + +async fn describe( + a11y: &AccessibilityConnection, + dest: &str, + path: &str, +) -> Option<(Role, StateSet)> { + let accessible = accessible_at(a11y, dest, path).await.ok()?; + let role = accessible.get_role().await.ok()?; + let states = accessible.get_state().await.ok()?; + Some((role, states)) +} + +async fn accessible_at<'c>( + a11y: &'c AccessibilityConnection, + dest: &str, + path: &str, +) -> Result> { + Ok(AccessibleProxy::builder(a11y.connection()) + .destination(dest.to_owned())? + .path(path.to_owned())? + .build() + .await?) +} + +/// The screen-coordinate offset for each showing top-level frame: `Some` when +/// the frame should be hinted, `None` when skipped (a non-active window). Both +/// outcomes are recorded so a node's nearest frame ancestor decides whether it +/// is hinted, even when an outer frame would match. +async fn frame_offsets( + a11y: &AccessibilityConnection, + dest: &str, + nodes: &[Node], + pid: Option, + compositor: &CompositorSnapshot, +) -> HashMap> { + let mut offsets = HashMap::new(); + for frame in nodes + .iter() + .filter(|n| is_window(n.role) && n.states.contains(State::Showing)) + { + let offset = match extents(a11y, dest, &frame.path).await { + Ok(ext) => { + // Cache items often carry an empty name for frames; the title + // is needed as the window-matching tiebreak, so fetch it live. + let name = if frame.name.is_empty() && !compositor.windows.is_empty() { + live_name(a11y, dest, &frame.path).await.unwrap_or_default() + } else { + frame.name.clone() + }; + frame_offset(ext, &name, pid, compositor) + } + Err(_) => None, + }; + offsets.insert(frame.path.clone(), offset); + } + offsets +} + +async fn collect_targets( + a11y: &AccessibilityConnection, + dest: &str, + nodes: &[Node], + offsets: &HashMap>, + screen_w: u32, + screen_h: u32, + out: &mut Vec, +) { + let index: HashMap<&str, &Node> = nodes.iter().map(|n| (n.path.as_str(), n)).collect(); + let rows_with_child_targets = table_rows_with_actionable_descendants(nodes, &index); + let targets: Vec<(&Node, (i32, i32))> = nodes + .iter() + .filter(|n| is_actionable(n.role) && n.states.contains(State::Showing)) + .filter(|n| n.role != Role::TableRow || !rows_with_child_targets.contains(n.path.as_str())) + .filter_map(|n| owning_frame_offset(n, &index, offsets).map(|off| (n, off))) + .take(MAX_TARGETS.saturating_sub(out.len())) + .collect(); + + let boxes = join_all(targets.iter().map(|(n, _)| extents(a11y, dest, &n.path))).await; + let mut unnamed_cells_by_table: HashMap> = HashMap::new(); + for ((node, (dx, dy)), ext) in targets.iter().zip(boxes) { + let Ok((x, y, w, h)) = ext else { continue }; + let Some(bbox) = clamp_to_screen(x + dx, y + dy, w, h, screen_w, screen_h) else { + continue; + }; + push_table_cell_candidate(node, bbox, &mut unnamed_cells_by_table, out); + } + for cells in unnamed_cells_by_table.into_values() { + for bbox in merge_cells_into_rows(cells) { + out.push(HintCandidate { bbox, score: 1.0 }); + } + } +} + +fn table_rows_with_actionable_descendants( + nodes: &[Node], + index: &HashMap<&str, &Node>, +) -> HashSet { + let mut rows = HashSet::new(); + for node in nodes { + if node.role == Role::TableRow + || node.role == Role::Icon + || !node.states.contains(State::Showing) + || !is_actionable(node.role) + { + continue; + } + + let mut current = node; + for _ in 0..MAX_ANCESTOR_DEPTH { + let Some(parent) = index.get(current.parent.as_str()) else { + break; + }; + if parent.role == Role::TableRow { + rows.insert(parent.path.clone()); + break; + } + current = parent; + } + } + rows +} + +/// Union unnamed table cells that share a vertical band into one box per row. +fn merge_cells_into_rows(mut cells: Vec<(u32, u32, u32, u32)>) -> Vec<(u32, u32, u32, u32)> { + cells.sort_by_key(|&(_, y, _, h)| (y, h)); + let mut rows: Vec<(u32, u32, u32, u32)> = Vec::new(); + for cell in cells { + match rows.last_mut() { + Some(row) if vertical_overlap_ratio(*row, cell) >= 0.5 => *row = union_box(*row, cell), + _ => rows.push(cell), + } + } + rows +} + +fn push_table_cell_candidate( + node: &Node, + bbox: (u32, u32, u32, u32), + unnamed_cells_by_table: &mut HashMap>, + out: &mut Vec, +) { + if node.role == Role::TableCell && node.name.trim().is_empty() { + unnamed_cells_by_table + .entry(node.parent.clone()) + .or_default() + .push(bbox); + } else { + out.push(HintCandidate { bbox, score: 1.0 }); + } +} + +/// Overlap height as a fraction of the shorter box's height. +fn vertical_overlap_ratio(a: (u32, u32, u32, u32), b: (u32, u32, u32, u32)) -> f32 { + let top = a.1.max(b.1); + let bottom = (a.1 + a.3).min(b.1 + b.3); + let overlap = bottom.saturating_sub(top); + overlap as f32 / a.3.min(b.3).max(1) as f32 +} + +fn union_box(a: (u32, u32, u32, u32), b: (u32, u32, u32, u32)) -> (u32, u32, u32, u32) { + let x0 = a.0.min(b.0); + let y0 = a.1.min(b.1); + let x1 = (a.0 + a.2).max(b.0 + b.2); + let y1 = (a.1 + a.3).max(b.1 + b.3); + (x0, y0, x1 - x0, y1 - y0) +} + +/// Walk the parent chain to the owning window frame and return its offset. +fn owning_frame_offset( + node: &Node, + index: &HashMap<&str, &Node>, + offsets: &HashMap>, +) -> Option<(i32, i32)> { + let mut current = node; + for _ in 0..MAX_ANCESTOR_DEPTH { + if let Some(offset) = offsets.get(current.path.as_str()) { + return *offset; + } + current = index.get(current.parent.as_str())?; + } + None +} + +/// Offset that turns a frame's subtree coordinates into screen coordinates, +/// `None` when the frame should not be hinted. Without compositor info (X11, +/// unqueryable Wayland compositors) every window is kept unshifted; with it, +/// only the active window is kept — skipping unmatched frames keeps +/// mispositioned hints off the screen. +fn frame_offset( + extents: (i32, i32, i32, i32), + name: &str, + pid: Option, + compositor: &CompositorSnapshot, +) -> Option<(i32, i32)> { + if compositor.windows.is_empty() { + return Some((0, 0)); + } + let (fx, fy, fw, fh) = extents; + let index = match_window(fw, fh, name, pid, &compositor.windows)?; + if compositor.active.is_some_and(|active| active != index) { + return None; + } + let window = &compositor.windows[index]; + Some((window.x - fx, window.y - fy)) +} + +/// Best compositor window for an AT-SPI frame. The application's pid is the +/// primary key; size within a small tolerance and longest title-prefix +/// overlap only disambiguate between one app's windows. When no window +/// carries the pid (sandboxed apps connect through a proxy process), the +/// size+title heuristic runs over all windows, and a size match is then +/// required — the only safety against shifting by an unrelated window. Within +/// a pid group it is merely preferred, since decoration handling can skew the +/// reported size. +fn match_window( + fw: i32, + fh: i32, + name: &str, + pid: Option, + windows: &[WindowRect], +) -> Option { + const TOLERANCE: i32 = 8; + let same_pid: Vec = windows + .iter() + .enumerate() + .filter(|(_, w)| pid.is_some() && w.pid == pid) + .map(|(index, _)| index) + .collect(); + if let [only] = same_pid[..] { + return Some(only); + } + let (pool, require_size) = match same_pid.is_empty() { + true => ((0..windows.len()).collect(), true), + false => (same_pid, false), + }; + let name = name.to_lowercase(); + let mut best: Option<(usize, (bool, usize))> = None; + for index in pool { + let window = &windows[index]; + let size_ok = (window.w - fw).abs() <= TOLERANCE && (window.h - fh).abs() <= TOLERANCE; + if require_size && !size_ok { + continue; + } + let title = window.title.to_lowercase(); + let title_prefix = title + .chars() + .zip(name.chars()) + .take_while(|(x, y)| x == y) + .count(); + let score = (size_ok, title_prefix); + if best.is_none_or(|(_, best_score)| score > best_score) { + best = Some((index, score)); + } + } + best.map(|(index, _)| index) +} + +/// A node's live `name` property (for window frames: the title). +async fn live_name(a11y: &AccessibilityConnection, dest: &str, path: &str) -> Option { + accessible_at(a11y, dest, path) + .await + .ok()? + .name() + .await + .ok() +} + +async fn extents( + a11y: &AccessibilityConnection, + dest: &str, + path: &str, +) -> Result<(i32, i32, i32, i32)> { + let component = ComponentProxy::builder(a11y.connection()) + .destination(dest.to_owned())? + .path(path.to_owned())? + .build() + .await?; + Ok(component.get_extents(CoordType::Screen).await?) +} + +/// Clamp a box to the screen; `None` for empty or fully offscreen boxes. +fn clamp_to_screen( + x: i32, + y: i32, + w: i32, + h: i32, + screen_w: u32, + screen_h: u32, +) -> Option<(u32, u32, u32, u32)> { + if w <= 0 || h <= 0 { + return None; + } + let x0 = x.max(0); + let y0 = y.max(0); + let x1 = (x + w).min(screen_w as i32); + let y1 = (y + h).min(screen_h as i32); + if x1 <= x0 || y1 <= y0 { + return None; + } + Some((x0 as u32, y0 as u32, (x1 - x0) as u32, (y1 - y0) as u32)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn window(x: i32, w: i32, h: i32, title: &str, pid: Option) -> WindowRect { + WindowRect { + x, + y: 28, + w, + h, + title: title.into(), + pid, + } + } + + #[test] + fn match_window_without_pid_requires_size_and_prefers_title() { + let windows = vec![ + window(4, 1912, 1048, "github — zen browser", None), + window(1000, 900, 1048, "notes", None), + ]; + // Exact size match picks the right window even with a partial title. + assert_eq!( + match_window(1912, 1048, "github — zen browser", None, &windows), + Some(0) + ); + // No size within tolerance → no match. + assert!(match_window(300, 200, "whatever", None, &windows).is_none()); + } + + #[test] + fn pid_disambiguates_equal_size_tiled_windows() { + // Two tiled halves: identical size, and the AT-SPI frame name shares + // no prefix with either title — the heuristic alone is a coin toss. + let windows = vec![ + window(0, 956, 1048, "inbox", Some(100)), + window(964, 956, 1048, "scratch", Some(200)), + ]; + assert_eq!(match_window(956, 1048, "", Some(200), &windows), Some(1)); + assert_eq!(match_window(956, 1048, "", Some(100), &windows), Some(0)); + } + + #[test] + fn pid_match_survives_a_decoration_size_mismatch() { + // Server-side decorations: the compositor's height includes a 30px + // titlebar the AT-SPI frame knows nothing about. The size gate would + // reject this window; the pid keeps it. + let windows = vec![ + window(4, 1912, 1078, "editor", Some(100)), + window(1000, 900, 1048, "notes", Some(200)), + ]; + assert_eq!( + match_window(1912, 1048, "editor", Some(100), &windows), + Some(0) + ); + // Without the pid, the same frame finds no window at all. + assert_eq!(match_window(1912, 1048, "editor", None, &windows), None); + } + + #[test] + fn within_a_pid_group_size_then_title_break_the_tie() { + // One app, two windows: size narrows first, title settles the rest. + let windows = vec![ + window(0, 956, 1048, "main", Some(100)), + window(964, 700, 500, "palette", Some(100)), + window(0, 800, 600, "other app", Some(200)), + ]; + assert_eq!( + match_window(700, 500, "palette", Some(100), &windows), + Some(1) + ); + assert_eq!( + match_window(956, 1048, "main", Some(100), &windows), + Some(0) + ); + } + + #[test] + fn unmatched_pid_falls_back_to_the_size_heuristic() { + // A sandboxed app's bus pid matches no compositor window; the size + // gate still finds the right one. + let windows = vec![ + window(4, 1912, 1048, "browser", Some(100)), + window(1000, 900, 1048, "notes", Some(200)), + ]; + assert_eq!( + match_window(900, 1048, "notes", Some(99999), &windows), + Some(1) + ); + } + + #[test] + fn clamp_drops_offscreen_and_empty() { + assert_eq!( + clamp_to_screen(10, 20, 30, 40, 1920, 1080), + Some((10, 20, 30, 40)) + ); + // Straddling the left edge: clamp to the on-screen part. + assert_eq!( + clamp_to_screen(-20, 10, 50, 50, 1920, 1080), + Some((0, 10, 30, 50)) + ); + // Fully off the left edge, or off the bottom-right → dropped. + assert!(clamp_to_screen(-100, 0, 50, 50, 1920, 1080).is_none()); + assert!(clamp_to_screen(5000, 5000, 50, 50, 1920, 1080).is_none()); + // Zero area → dropped. + assert!(clamp_to_screen(0, 0, 0, 10, 1920, 1080).is_none()); + } + + fn snapshot(active: Option) -> CompositorSnapshot { + CompositorSnapshot { + windows: vec![ + window(4, 1912, 1048, "browser", Some(100)), + window(1000, 900, 1048, "notes", Some(200)), + ], + active, + } + } + + #[test] + fn frame_offset_keeps_active_window_and_skips_the_rest() { + let snap = snapshot(Some(0)); + // The frame matching the active window gets the position correction: + // AT-SPI reports it pinned at (0,0), Hyprland knows it sits at (4,28). + assert_eq!( + frame_offset((0, 0, 1912, 1048), "browser", Some(100), &snap), + Some((4, 28)) + ); + // A different window is skipped entirely. + assert_eq!( + frame_offset((0, 0, 900, 1048), "notes", Some(200), &snap), + None + ); + // An unmatchable frame (popup, odd size) is skipped, not mispositioned. + assert_eq!(frame_offset((0, 0, 300, 200), "menu", None, &snap), None); + } + + #[test] + fn frame_offset_without_compositor_info_uses_raw_coordinates() { + let snap = CompositorSnapshot { + windows: Vec::new(), + active: None, + }; + // X11 / unknown compositor: every frame kept, coordinates unshifted. + assert_eq!( + frame_offset((40, 60, 800, 600), "any", None, &snap), + Some((0, 0)) + ); + } + + fn node(path: &str, parent: &str, role: Role) -> Node { + Node { + path: path.into(), + parent: parent.into(), + role, + states: StateSet::new(State::Showing), + name: String::new(), + children: 0, + } + } + + fn named_node(path: &str, parent: &str, role: Role, name: &str) -> Node { + Node { + name: name.into(), + ..node(path, parent, role) + } + } + + #[test] + fn owning_frame_offset_walks_to_the_right_frame() { + let nodes = [ + node("/app", "/", Role::Application), + node("/frame_a", "/app", Role::Frame), + node("/panel", "/frame_a", Role::Panel), + node("/button", "/panel", Role::Button), + node("/frame_b", "/app", Role::Frame), + node("/other_button", "/frame_b", Role::Button), + node("/orphan", "/nowhere", Role::Button), + ]; + let index: HashMap<&str, &Node> = nodes.iter().map(|n| (n.path.as_str(), n)).collect(); + let offsets: HashMap> = [ + ("/frame_a".to_owned(), Some((10, 20))), + ("/frame_b".to_owned(), None), + ] + .into(); + + // Deep child resolves through intermediate containers to its frame. + assert_eq!( + owning_frame_offset(&nodes[3], &index, &offsets), + Some((10, 20)) + ); + // A child of a skipped frame is skipped with it. + assert_eq!(owning_frame_offset(&nodes[5], &index, &offsets), None); + // A node with a broken parent chain finds no frame. + assert_eq!(owning_frame_offset(&nodes[6], &index, &offsets), None); + } + + #[test] + fn owning_frame_offset_survives_parent_cycles() { + let nodes = [ + node("/a", "/b", Role::Button), + node("/b", "/a", Role::Panel), + ]; + let index: HashMap<&str, &Node> = nodes.iter().map(|n| (n.path.as_str(), n)).collect(); + let offsets: HashMap> = HashMap::new(); + assert_eq!(owning_frame_offset(&nodes[0], &index, &offsets), None); + } + + #[test] + fn table_row_with_actionable_children_is_not_a_target() { + let nodes = [ + node("/table", "/frame", Role::Table), + named_node("/row", "/table", Role::TableRow, "src initial commit"), + named_node("/file", "/row", Role::Link, "src"), + named_node("/commit", "/row", Role::Link, "initial commit"), + ]; + let index: HashMap<&str, &Node> = nodes.iter().map(|n| (n.path.as_str(), n)).collect(); + + let rows = table_rows_with_actionable_descendants(&nodes, &index); + assert!(rows.contains("/row")); + } + + #[test] + fn table_row_without_actionable_children_remains_a_target() { + let nodes = [ + node("/table", "/frame", Role::Table), + named_node("/row", "/table", Role::TableRow, "Documents"), + node("/panel", "/row", Role::Panel), + ]; + let index: HashMap<&str, &Node> = nodes.iter().map(|n| (n.path.as_str(), n)).collect(); + + let rows = table_rows_with_actionable_descendants(&nodes, &index); + assert!(!rows.contains("/row")); + } + + #[test] + fn table_row_with_only_icon_child_remains_a_target() { + let nodes = [ + node("/table", "/frame", Role::Table), + named_node("/row", "/table", Role::TableRow, "src"), + named_node("/icon", "/row", Role::Icon, "Directory"), + ]; + let index: HashMap<&str, &Node> = nodes.iter().map(|n| (n.path.as_str(), n)).collect(); + + let rows = table_rows_with_actionable_descendants(&nodes, &index); + assert!(!rows.contains("/row")); + } + + #[test] + fn table_cells_merge_into_one_box_per_row() { + // Thunar-style sidebar: each row is several cells (zero-width padding + // cells are already dropped by clamping before merge). + let cells = vec![ + // Row 1 at y=100. + (2, 100, 12, 24), + (16, 100, 24, 24), + (42, 100, 130, 24), + // Row 2 at y=126, slightly different heights. + (2, 126, 12, 27), + (16, 127, 24, 24), + (42, 126, 130, 27), + ]; + let rows = merge_cells_into_rows(cells); + assert_eq!(rows, vec![(2, 100, 170, 24), (2, 126, 170, 27)]); + } + + #[test] + fn named_table_cells_keep_their_own_click_target() { + let file_cell = named_node("/file", "/row", Role::TableCell, "src"); + let commit_cell = named_node("/commit", "/row", Role::TableCell, "initial commit"); + let mut unnamed = HashMap::new(); + let mut out = Vec::new(); + + push_table_cell_candidate(&file_cell, (10, 100, 200, 24), &mut unnamed, &mut out); + push_table_cell_candidate(&commit_cell, (600, 100, 300, 24), &mut unnamed, &mut out); + + assert!(unnamed.is_empty()); + assert_eq!(out.len(), 2); + assert_eq!(out[0].bbox, (10, 100, 200, 24)); + assert_eq!(out[1].bbox, (600, 100, 300, 24)); + } + + #[test] + fn unnamed_table_cells_still_merge_by_row() { + let icon = node("/icon", "/row", Role::TableCell); + let text = node("/text", "/row", Role::TableCell); + let mut unnamed = HashMap::new(); + let mut out = Vec::new(); + + push_table_cell_candidate(&icon, (2, 100, 12, 24), &mut unnamed, &mut out); + push_table_cell_candidate(&text, (16, 100, 130, 24), &mut unnamed, &mut out); + + assert!(out.is_empty()); + let rows = merge_cells_into_rows(unnamed.remove("/row").unwrap()); + assert_eq!(rows, vec![(2, 100, 144, 24)]); + } + + #[test] + fn single_cells_pass_through_unmerged() { + let rows = merge_cells_into_rows(vec![(5, 10, 100, 20)]); + assert_eq!(rows, vec![(5, 10, 100, 20)]); + assert!(merge_cells_into_rows(Vec::new()).is_empty()); + } + + /// Exercises the full async collection against the live accessibility bus. + /// Ignored by default since it needs a running AT-SPI bus and GUI apps: + /// cargo test -- --ignored --nocapture walks_accessibility_tree + #[test] + #[ignore = "requires a running AT-SPI bus and GUI apps"] + fn walks_accessibility_tree() { + let start = std::time::Instant::now(); + let (candidates, _) = futures_executor::block_on(collect_candidates(1920, 1080)) + .expect("accessibility collection should succeed"); + eprintln!( + "atspi collect: {} candidates in {:?}", + candidates.len(), + start.elapsed() + ); + for c in &candidates { + eprintln!(" candidate bbox={:?}", c.bbox); + } + } +} diff --git a/src/hint/compositor/hyprland.rs b/src/hint/compositor/hyprland.rs new file mode 100644 index 0000000..8951f57 --- /dev/null +++ b/src/hint/compositor/hyprland.rs @@ -0,0 +1,140 @@ +//! Hyprland provider: window geometry over its Unix IPC socket. + +use std::io::{Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; + +use super::{CompositorSnapshot, WindowRect}; + +pub fn snapshot() -> Option { + std::env::var_os("HYPRLAND_INSTANCE_SIGNATURE")?; + let clients = clients_from_json(&ipc_json("clients")?, true); + // `clients` and `activewindow` are separate IPC calls, so volatile fields + // (the title, above all) can change in between; the stable address is what + // identifies the active window in the list. + let active = ipc_json("activewindow") + .map(|json| clients_from_json(&json, false)) + .and_then(|mut active| (!active.is_empty()).then(|| active.remove(0))) + .and_then(|active| active_index(&clients, &active)); + Some(CompositorSnapshot { + windows: clients.into_iter().map(window_rect).collect(), + active, + }) +} + +fn clients_from_json(json: &str, require_mapped: bool) -> Vec { + // `clients` returns an array, `activewindow` a single object. + let clients: Vec = serde_json::from_str(json) + .or_else(|_| serde_json::from_str::(json).map(|client| vec![client])) + .unwrap_or_default(); + clients + .into_iter() + .filter(|c| (!require_mapped || c.mapped) && !c.hidden && c.size.0 > 0 && c.size.1 > 0) + .collect() +} + +fn active_index(clients: &[HyprClient], active: &HyprClient) -> Option { + clients + .iter() + .position(|c| match (&c.address, &active.address) { + (Some(a), Some(b)) => a == b, + _ => c.at == active.at && c.size == active.size && c.title == active.title, + }) +} + +fn window_rect(c: HyprClient) -> WindowRect { + WindowRect { + x: c.at.0, + y: c.at.1, + w: c.size.0, + h: c.size.1, + title: c.title, + pid: c.pid.filter(|&pid| pid > 0), + } +} + +fn ipc_json(command: &str) -> Option { + let socket = socket_path()?; + let mut stream = UnixStream::connect(socket).ok()?; + // `j/` requests JSON; the suffix is the query name, as in hyprctl. + stream.write_all(format!("j/{command}").as_bytes()).ok()?; + let mut out = String::new(); + stream.read_to_string(&mut out).ok()?; + (!out.is_empty()).then_some(out) +} + +fn socket_path() -> Option { + let sig = PathBuf::from(std::env::var_os("HYPRLAND_INSTANCE_SIGNATURE")?); + let mut candidates = Vec::with_capacity(2); + if let Some(runtime) = std::env::var_os("XDG_RUNTIME_DIR") { + candidates.push(PathBuf::from(runtime).join("hypr").join(&sig)); + } + candidates.push(PathBuf::from("/tmp/hypr").join(&sig)); + candidates + .into_iter() + .map(|dir| dir.join(".socket.sock")) + .find(|path| path.exists()) +} + +#[derive(serde::Deserialize)] +struct HyprClient { + at: (i32, i32), + size: (i32, i32), + #[serde(default)] + address: Option, + #[serde(default)] + pid: Option, + #[serde(default)] + title: String, + #[serde(default)] + mapped: bool, + #[serde(default)] + hidden: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_clients_and_activewindow_payloads() { + let clients = r#"[ + {"at": [4, 28], "size": [1912, 1048], "title": "browser", "mapped": true, "pid": 4242, "address": "0xabc"}, + {"at": [0, 0], "size": [800, 600], "title": "hidden", "mapped": true, "hidden": true}, + {"at": [0, 0], "size": [800, 600], "title": "unmapped"} + ]"#; + let clients = clients_from_json(clients, true); + assert_eq!(clients.len(), 1); + let window = window_rect(clients.into_iter().next().unwrap()); + assert_eq!( + (window.x, window.y, window.w, window.h), + (4, 28, 1912, 1048) + ); + assert_eq!(window.title, "browser"); + assert_eq!(window.pid, Some(4242)); + + let active = r#"{"at": [966, 28], "size": [950, 1048], "title": "term"}"#; + let clients = clients_from_json(active, false); + assert_eq!(clients.len(), 1); + assert_eq!(clients[0].title, "term"); + } + + #[test] + fn active_is_matched_by_address_despite_a_title_change() { + // `clients` and `activewindow` are separate IPC calls; the browser's + // title changed in between, but the address pins down the window. + let clients = clients_from_json( + r#"[ + {"at": [4, 28], "size": [956, 1048], "title": "old title", "mapped": true, "address": "0xa"}, + {"at": [964, 28], "size": [956, 1048], "title": "notes", "mapped": true, "address": "0xb"} + ]"#, + true, + ); + let active = clients_from_json( + r#"{"at": [4, 28], "size": [956, 1048], "title": "new title", "address": "0xa"}"#, + false, + ) + .remove(0); + assert_eq!(active_index(&clients, &active), Some(0)); + } +} diff --git a/src/hint/compositor/mod.rs b/src/hint/compositor/mod.rs new file mode 100644 index 0000000..ae94464 --- /dev/null +++ b/src/hint/compositor/mod.rs @@ -0,0 +1,44 @@ +//! Real window geometry from the compositor, for the AT-SPI detector. +//! +//! On Wayland a client cannot know its window's on-screen position (no standard +//! protocol exposes it), so AT-SPI coordinates need shifting by it. Each +//! compositor with its own IPC gets a provider file here, returning `Some` with +//! absolute-screen rectangles when detected, else `None`. With no provider, +//! [`snapshot`] is empty and coordinates are used unshifted (correct on X11). + +mod hyprland; +mod sway; + +/// A compositor-reported window rectangle, in absolute screen coordinates. +pub struct WindowRect { + pub x: i32, + pub y: i32, + pub w: i32, + pub h: i32, + pub title: String, + /// Owning process, when the compositor reports one; the primary key for + /// matching a window to its AT-SPI application. + pub pid: Option, +} + +#[derive(Default)] +pub struct CompositorSnapshot { + /// Every mapped, visible window. + pub windows: Vec, + /// Index into `windows` of the focused window, when the compositor + /// reports one. + pub active: Option, +} + +impl CompositorSnapshot { + pub fn active_window(&self) -> Option<&WindowRect> { + self.windows.get(self.active?) + } +} + +/// Window geometry from the first provider whose compositor is running. +pub fn snapshot() -> CompositorSnapshot { + hyprland::snapshot() + .or_else(sway::snapshot) + .unwrap_or_default() +} diff --git a/src/hint/compositor/sway.rs b/src/hint/compositor/sway.rs new file mode 100644 index 0000000..9e03113 --- /dev/null +++ b/src/hint/compositor/sway.rs @@ -0,0 +1,173 @@ +//! Sway provider: window geometry over the i3 IPC socket (`$SWAYSOCK`). + +use std::io::{Read, Write}; +use std::os::unix::net::UnixStream; + +use super::{CompositorSnapshot, WindowRect}; + +pub fn snapshot() -> Option { + let socket = std::env::var_os("SWAYSOCK")?; + let tree = get_tree(&socket)?; + let mut snapshot = CompositorSnapshot::default(); + collect_windows(&tree, &mut snapshot); + Some(snapshot) +} + +/// One i3-IPC `GET_TREE` request/reply. The wire format is `"i3-ipc"` + +/// payload length (u32 LE) + message type (u32 LE) + JSON payload. +fn get_tree(socket: &std::ffi::OsStr) -> Option { + const MAGIC: &[u8] = b"i3-ipc"; + const GET_TREE: u32 = 4; + + let mut stream = UnixStream::connect(socket).ok()?; + let mut request = Vec::with_capacity(14); + request.extend_from_slice(MAGIC); + request.extend_from_slice(&0u32.to_le_bytes()); + request.extend_from_slice(&GET_TREE.to_le_bytes()); + stream.write_all(&request).ok()?; + + let mut header = [0u8; 14]; + stream.read_exact(&mut header).ok()?; + if &header[..6] != MAGIC { + return None; + } + let len = u32::from_le_bytes(header[6..10].try_into().ok()?) as usize; + let mut payload = vec![0u8; len]; + stream.read_exact(&mut payload).ok()?; + serde_json::from_slice(&payload).ok() +} + +/// Walk the layout tree collecting visible application windows. A window node +/// carries a `pid`; its on-screen content rect is the node `rect` shifted by +/// `window_rect` (which excludes borders and title bar). +fn collect_windows(node: &serde_json::Value, out: &mut CompositorSnapshot) { + if node.get("pid").is_some_and(|pid| !pid.is_null()) + && node.get("visible").and_then(|v| v.as_bool()) == Some(true) + { + if let Some(window) = window_rect_of(node) { + if node.get("focused").and_then(|v| v.as_bool()) == Some(true) { + out.active = Some(out.windows.len()); + } + out.windows.push(window); + } + } + for key in ["nodes", "floating_nodes"] { + for child in node + .get(key) + .and_then(|v| v.as_array()) + .into_iter() + .flatten() + { + collect_windows(child, out); + } + } +} + +fn window_rect_of(node: &serde_json::Value) -> Option { + let field = |rect: &serde_json::Value, key: &str| rect.get(key)?.as_i64().map(|v| v as i32); + let rect = node.get("rect")?; + let content = node.get("window_rect")?; + let (w, h) = (field(content, "width")?, field(content, "height")?); + if w <= 0 || h <= 0 { + return None; + } + Some(WindowRect { + x: field(rect, "x")? + field(content, "x")?, + y: field(rect, "y")? + field(content, "y")?, + w, + h, + title: node + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_owned(), + pid: node + .get("pid") + .and_then(|v| v.as_i64()) + .and_then(|pid| i32::try_from(pid).ok()) + .filter(|&pid| pid > 0), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn collects_visible_windows_from_a_tree() { + let tree: serde_json::Value = serde_json::from_str( + r#"{ + "type": "root", "nodes": [{ + "type": "output", "nodes": [{ + "type": "workspace", "nodes": [ + { + "type": "con", "pid": 100, "visible": true, "focused": false, + "name": "editor", + "rect": {"x": 0, "y": 23, "width": 960, "height": 1057}, + "window_rect": {"x": 2, "y": 2, "width": 956, "height": 1053}, + "nodes": [] + }, + { + "type": "con", "pid": 200, "visible": true, "focused": true, + "name": "browser", + "rect": {"x": 960, "y": 23, "width": 960, "height": 1057}, + "window_rect": {"x": 0, "y": 0, "width": 960, "height": 1057}, + "nodes": [] + }, + { + "type": "con", "pid": 300, "visible": false, "focused": false, + "name": "scratchpad", + "rect": {"x": 0, "y": 0, "width": 800, "height": 600}, + "window_rect": {"x": 0, "y": 0, "width": 800, "height": 600}, + "nodes": [] + } + ], + "floating_nodes": [{ + "type": "floating_con", "pid": 400, "visible": true, "focused": false, + "name": "popup", + "rect": {"x": 500, "y": 300, "width": 404, "height": 304}, + "window_rect": {"x": 2, "y": 2, "width": 400, "height": 300}, + "nodes": [] + }] + }] + }] + }"#, + ) + .unwrap(); + + let mut snapshot = CompositorSnapshot::default(); + collect_windows(&tree, &mut snapshot); + + let summary: Vec<_> = snapshot + .windows + .iter() + .map(|w| (w.title.as_str(), w.x, w.y, w.w, w.h, w.pid)) + .collect(); + assert_eq!( + summary, + vec![ + ("editor", 2, 25, 956, 1053, Some(100)), + ("browser", 960, 23, 960, 1057, Some(200)), + ("popup", 502, 302, 400, 300, Some(400)), + ] + ); + let active = snapshot.active_window().expect("focused window"); + assert_eq!(active.title, "browser"); + assert_eq!((active.x, active.y), (960, 23)); + } + + #[test] + fn container_without_pid_is_not_a_window() { + let tree: serde_json::Value = serde_json::from_str( + r#"{"type": "workspace", "visible": true, + "rect": {"x": 0, "y": 0, "width": 1920, "height": 1080}, + "window_rect": {"x": 0, "y": 0, "width": 1920, "height": 1080}, + "nodes": []}"#, + ) + .unwrap(); + let mut snapshot = CompositorSnapshot::default(); + collect_windows(&tree, &mut snapshot); + assert!(snapshot.windows.is_empty()); + assert!(snapshot.active.is_none()); + } +} diff --git a/src/hint/cv.rs b/src/hint/cv.rs new file mode 100644 index 0000000..1b89990 --- /dev/null +++ b/src/hint/cv.rs @@ -0,0 +1,374 @@ +//! Pure computer-vision detector (always-available fallback; `image` + +//! `imageproc` only). Pipeline: downscale → grayscale → Canny → morphological +//! close → connected components → shape filtering/scoring → drop icon-in-hit-area +//! nesting. Sees visual structure, not semantic clickability. + +use anyhow::{bail, Context}; +use image::{GrayImage, Luma}; +use imageproc::{ + distance_transform::Norm, + edges::canny, + morphology::close, + region_labelling::{connected_components, Connectivity}, +}; +use std::collections::HashMap; + +use super::detect::{DetectorOutput, HintCandidate, HintDetector}; +use crate::{backend::Backend, config::config}; + +#[derive(Default)] +pub struct CvHintDetector; + +impl HintDetector for CvHintDetector { + fn name(&self) -> &'static str { + "cv" + } + + fn detect(&self, backend: &mut dyn Backend) -> anyhow::Result { + let capture = backend.capture_screen().context( + "hint mode requires screen capture; the CV detector cannot run on this backend", + )?; + let candidates = detect_candidates_cv(&capture.bgra, capture.w, capture.h)?; + Ok(DetectorOutput { + candidates, + capture_w: capture.w, + capture_h: capture.h, + focus_rect: None, + }) + } +} + +/// A component in the downscaled working image: its box plus how many mask +/// pixels it actually occupies (used for the solidity score). +struct Component { + bbox: (u32, u32, u32, u32), + pixels: u32, +} + +fn detect_candidates_cv( + bgra: &[u8], + capture_w: u32, + capture_h: u32, +) -> anyhow::Result> { + if capture_w == 0 || capture_h == 0 { + bail!("cannot run hint CV detector on an empty capture"); + } + if bgra.len() < (capture_w as usize * capture_h as usize * 4) { + bail!("screen capture buffer is smaller than declared dimensions"); + } + + let cfg = &config().hint; + let longest = capture_w.max(capture_h); + let scale = if cfg.downscale_longest > 0 && longest > cfg.downscale_longest { + cfg.downscale_longest as f32 / longest as f32 + } else { + 1.0 + }; + let ww = ((capture_w as f32 * scale).round() as u32).max(1); + let wh = ((capture_h as f32 * scale).round() as u32).max(1); + + let gray = grayscale_resized(bgra, capture_w as usize, capture_h as usize, ww, wh); + let edges = canny( + &gray, + cfg.edge_low_threshold as f32, + cfg.edge_high_threshold as f32, + ); + // Closing (dilate + erode) reconnects the broken strokes of one element + // without inflating boxes or bridging neighbours. + const CLOSE_RADIUS: u8 = 2; + let closed = close(&edges, Norm::LInf, CLOSE_RADIUS); + + let components = connected_component_boxes(&closed); + + // Shape filters, relative to the working image so behaviour matches at + // 1080p and 4K: minimum side (sub-glyph noise), maximum side/area + // fractions (bars, panels, banners), maximum elongation (separators, + // scrollbars, underlines). + const MIN_SIZE: u32 = 8; + const MAX_SIZE_FRAC: f32 = 0.6; + const MAX_AREA_FRAC: f32 = 0.25; + const MAX_ASPECT: f32 = 12.0; + + let work_area = (ww as f32) * (wh as f32); + let max_w = (MAX_SIZE_FRAC * ww as f32).max(MIN_SIZE as f32); + let max_h = (MAX_SIZE_FRAC * wh as f32).max(MIN_SIZE as f32); + let max_area = (MAX_AREA_FRAC * work_area).max(1.0); + + let kept: Vec<(Component, f32)> = components + .into_iter() + .filter_map(|component| { + let (_, _, w, h) = component.bbox; + if w < MIN_SIZE || h < MIN_SIZE { + return None; + } + if (w as f32) > max_w || (h as f32) > max_h { + return None; + } + let area = (w as f32) * (h as f32); + if area > max_area { + return None; + } + let aspect = (w.max(h) as f32) / (w.min(h).max(1) as f32); + if aspect > MAX_ASPECT { + return None; + } + // Coherent controls fill their box; gradient/text speckle scores low. + let solidity = (component.pixels as f32 / area).clamp(0.0, 1.0); + Some((component, solidity)) + }) + .collect(); + + let kept = drop_icon_inner_boxes(kept); + + let inv_scale = 1.0 / scale; + Ok(kept + .into_iter() + .map(|(component, solidity)| { + let bbox = scale_box_to_capture(component.bbox, inv_scale, capture_w, capture_h); + HintCandidate { + bbox, + score: solidity, + } + }) + .collect()) +} + +/// Map a working-image box back to clamped capture-pixel coordinates. +fn scale_box_to_capture( + (x, y, w, h): (u32, u32, u32, u32), + inv_scale: f32, + capture_w: u32, + capture_h: u32, +) -> (u32, u32, u32, u32) { + let nx = ((x as f32 * inv_scale).round() as u32).min(capture_w.saturating_sub(1)); + let ny = ((y as f32 * inv_scale).round() as u32).min(capture_h.saturating_sub(1)); + let nw = ((w as f32 * inv_scale).round() as u32) + .max(1) + .min(capture_w - nx); + let nh = ((h as f32 * inv_scale).round() as u32) + .max(1) + .min(capture_h - ny); + (nx, ny, nw, nh) +} + +/// Drop a box sitting concentrically inside a small, near-square box — an +/// icon glyph inside its own hit area, where both labels would point at the +/// same place. Deliberately narrow so container children are preserved. +/// Thresholds are in working-image pixels. +fn drop_icon_inner_boxes(mut items: Vec<(Component, f32)>) -> Vec<(Component, f32)> { + const MAX_PARENT_SIDE: u32 = 48; + const SQUARE_TOLERANCE: u32 = 6; + const CONCENTRIC_TOLERANCE: u32 = 6; + + let boxes: Vec<(u32, u32, u32, u32)> = items.iter().map(|(c, _)| c.bbox).collect(); + let mut drop = vec![false; boxes.len()]; + + for (i, &child) in boxes.iter().enumerate() { + let smallest_parent = boxes + .iter() + .enumerate() + .filter(|(j, _)| *j != i && !drop[*j]) + .filter(|(_, parent)| contains(**parent, child)) + .min_by_key(|(_, parent)| parent.2 * parent.3) + .map(|(_, parent)| *parent); + + let Some(parent) = smallest_parent else { + continue; + }; + let near_square = parent.2.abs_diff(parent.3) <= SQUARE_TOLERANCE; + let small = parent.2 <= MAX_PARENT_SIDE && parent.3 <= MAX_PARENT_SIDE; + let child_center = (child.0 + child.2 / 2, child.1 + child.3 / 2); + let parent_center = (parent.0 + parent.2 / 2, parent.1 + parent.3 / 2); + let concentric = child_center.0.abs_diff(parent_center.0) <= CONCENTRIC_TOLERANCE + && child_center.1.abs_diff(parent_center.1) <= CONCENTRIC_TOLERANCE; + if small && near_square && concentric { + drop[i] = true; + } + } + + let mut keep = drop.into_iter(); + items.retain(|_| !keep.next().unwrap_or(false)); + items +} + +fn contains(parent: (u32, u32, u32, u32), child: (u32, u32, u32, u32)) -> bool { + parent.0 <= child.0 + && parent.1 <= child.1 + && parent.0 + parent.2 >= child.0 + child.2 + && parent.1 + parent.3 >= child.1 + child.3 +} + +/// Nearest-neighbour downscale of a BGRA capture straight into grayscale — +/// edges survive it, and it is far cheaper than a filtered resize. +fn grayscale_resized(bgra: &[u8], src_w: usize, src_h: usize, dst_w: u32, dst_h: u32) -> GrayImage { + let dst_w_usize = dst_w as usize; + let dst_h_usize = dst_h as usize; + let mut out = vec![0u8; dst_w_usize * dst_h_usize]; + for y in 0..dst_h_usize { + let src_y = (y * src_h / dst_h_usize).min(src_h - 1); + for x in 0..dst_w_usize { + let src_x = (x * src_w / dst_w_usize).min(src_w - 1); + let off = (src_y * src_w + src_x) * 4; + let b = bgra[off] as u32; + let g = bgra[off + 1] as u32; + let r = bgra[off + 2] as u32; + out[y * dst_w_usize + x] = ((29 * b + 150 * g + 77 * r) / 256) as u8; + } + } + GrayImage::from_vec(dst_w, dst_h, out).expect("grayscale buffer size matches image dimensions") +} + +/// Connected components of a binary mask, with occupied-pixel counts for the +/// solidity score. +fn connected_component_boxes(mask: &GrayImage) -> Vec { + let labels = connected_components(mask, Connectivity::Eight, Luma([0u8])); + // (min_x, min_y, max_x, max_y, pixel_count) + let mut rects: HashMap = HashMap::new(); + + for (x, y, label) in labels.enumerate_pixels() { + let label = label[0]; + if label == 0 { + continue; + } + rects + .entry(label) + .and_modify(|rect| { + rect.0 = rect.0.min(x); + rect.1 = rect.1.min(y); + rect.2 = rect.2.max(x); + rect.3 = rect.3.max(y); + rect.4 += 1; + }) + .or_insert((x, y, x, y, 1)); + } + + rects + .into_values() + .map(|(min_x, min_y, max_x, max_y, pixels)| Component { + bbox: (min_x, min_y, max_x - min_x + 1, max_y - min_y + 1), + pixels, + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn solidity_is_pixel_fill_ratio() { + // A hollow 4x4 ring: 12 of 16 box pixels filled, so solidity = 12/16. + let mut img = GrayImage::new(4, 4); + for (x, y, p) in img.enumerate_pixels_mut() { + if x == 0 || x == 3 || y == 0 || y == 3 { + *p = Luma([255]); + } + } + let comps = connected_component_boxes(&img); + assert_eq!(comps.len(), 1); + assert_eq!(comps[0].pixels, 12); + assert_eq!(comps[0].bbox, (0, 0, 4, 4)); + let (_, _, w, h) = comps[0].bbox; + let solidity = comps[0].pixels as f32 / (w * h) as f32; + assert!((solidity - 12.0 / 16.0).abs() < 1e-6); + } + + #[test] + fn icon_inner_box_is_dropped_but_container_children_survive() { + let comp = |bbox: (u32, u32, u32, u32)| { + ( + Component { + bbox, + pixels: bbox.2 * bbox.3, + }, + 1.0, + ) + }; + // Small near-square icon (outer ring) with a concentric inner glyph. + let icon_outer = comp((100, 100, 24, 24)); + let icon_inner = comp((106, 106, 12, 12)); + // A large container with two real buttons inside it. + let container = comp((0, 0, 400, 120)); + let button_a = comp((10, 20, 80, 30)); + let button_b = comp((120, 20, 80, 30)); + + let kept = + drop_icon_inner_boxes(vec![icon_outer, icon_inner, container, button_a, button_b]); + let boxes: Vec<_> = kept.iter().map(|(c, _)| c.bbox).collect(); + + // The concentric inner glyph is gone. + assert!(!boxes.contains(&(106, 106, 12, 12))); + // Everything else — including the container's children — is preserved. + assert!(boxes.contains(&(100, 100, 24, 24))); + assert!(boxes.contains(&(0, 0, 400, 120))); + assert!(boxes.contains(&(10, 20, 80, 30))); + assert!(boxes.contains(&(120, 20, 80, 30))); + } +} + +/// Dev tool, not a test of invariants: run the CV detector over a real +/// screenshot and print every candidate box, to study behaviour on actual UIs: +/// grim -t ppm /tmp/shot.ppm +/// STOCHOS_CV_IMAGE=/tmp/shot.ppm cargo test --release -- --ignored --nocapture cv_boxes +#[cfg(test)] +mod probe { + use super::*; + + /// Minimal binary PPM (P6, maxval 255) reader → BGRA buffer. + fn read_ppm(path: &str) -> (Vec, u32, u32) { + let data = std::fs::read(path).expect("read image file"); + let mut fields = Vec::new(); + let mut pos = 0; + while fields.len() < 4 { + // Skip whitespace and comments between header fields. + while pos < data.len() && (data[pos].is_ascii_whitespace() || data[pos] == b'#') { + if data[pos] == b'#' { + while pos < data.len() && data[pos] != b'\n' { + pos += 1; + } + } + pos += 1; + } + let start = pos; + while pos < data.len() && !data[pos].is_ascii_whitespace() { + pos += 1; + } + fields.push(std::str::from_utf8(&data[start..pos]).unwrap().to_owned()); + } + pos += 1; // single whitespace after maxval + assert_eq!(fields[0], "P6", "expected binary PPM (grim -t ppm)"); + let (w, h): (u32, u32) = (fields[1].parse().unwrap(), fields[2].parse().unwrap()); + let rgb = &data[pos..pos + (w * h * 3) as usize]; + let mut bgra = vec![255u8; (w * h * 4) as usize]; + for i in 0..(w * h) as usize { + bgra[i * 4] = rgb[i * 3 + 2]; + bgra[i * 4 + 1] = rgb[i * 3 + 1]; + bgra[i * 4 + 2] = rgb[i * 3]; + } + (bgra, w, h) + } + + #[test] + #[ignore = "set STOCHOS_CV_IMAGE to a P6 .ppm screenshot"] + fn cv_boxes_from_screenshot() { + let path = std::env::var("STOCHOS_CV_IMAGE").expect("set STOCHOS_CV_IMAGE"); + std::env::set_var( + "XDG_CONFIG_HOME", + std::env::temp_dir().join("stochos-cv-probe"), + ); + crate::config::init(); + let (bgra, w, h) = read_ppm(&path); + let start = std::time::Instant::now(); + let candidates = detect_candidates_cv(&bgra, w, h).expect("cv detection"); + eprintln!( + "cv: {} candidates in {:?} on {w}x{h}", + candidates.len(), + start.elapsed() + ); + let mut sorted = candidates; + sorted.sort_by_key(|c| (c.bbox.1, c.bbox.0)); + for c in &sorted { + eprintln!(" bbox={:?} score={:.2}", c.bbox, c.score); + } + } +} diff --git a/src/hint/detect.rs b/src/hint/detect.rs new file mode 100644 index 0000000..9d4f162 --- /dev/null +++ b/src/hint/detect.rs @@ -0,0 +1,363 @@ +//! The detector seam. A [`HintDetector`] turns the current screen into a list +//! of candidate click targets in *capture-pixel* coordinates; the shared code +//! in [`super`] handles everything after that. + +use crate::backend::Backend; +use crate::config::{config, HintDetectorKind}; + +#[derive(Clone, Copy, Debug)] +pub struct HintCandidate { + /// Box in capture-pixel coordinates: `(x, y, w, h)`. + pub bbox: (u32, u32, u32, u32), + /// Detector-provided quality, roughly 0..1; blended with the shared + /// size/center preferences during ranking. + pub score: f32, +} + +/// Detector output plus the pixel dimensions of the capture the boxes are +/// expressed in; the caller rescales to the overlay's logical space. +pub struct DetectorOutput { + pub candidates: Vec, + pub capture_w: u32, + pub capture_h: u32, + /// The window the candidates are scoped to (screen coordinates), when the + /// detector knows it; lets `auto` keep CV supplements in the active window. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub focus_rect: Option<(u32, u32, u32, u32)>, +} + +pub trait HintDetector { + fn name(&self) -> &'static str; + fn detect(&self, backend: &mut dyn Backend) -> anyhow::Result; +} + +/// Pick the detector named by `hint.detector`. Selecting `atspi` on a +/// platform without it is an error rather than a silent fall back to CV; +/// `auto` uses whatever the platform offers. +pub fn select_detector() -> anyhow::Result> { + match config().hint.detector { + HintDetectorKind::Auto => Ok(Box::new(AutoDetector)), + HintDetectorKind::Cv => Ok(Box::new(super::cv::CvHintDetector)), + HintDetectorKind::Atspi => atspi_detector(), + } +} + +fn atspi_detector() -> anyhow::Result> { + #[cfg(target_os = "linux")] + { + Ok(Box::new(super::atspi::AtspiHintDetector)) + } + #[cfg(not(target_os = "linux"))] + { + anyhow::bail!("hint.detector = \"atspi\" is only available on Linux") + } +} + +/// Fewest AT-SPI targets the cascade accepts without falling back to CV; +/// fewer means the accessibility tree is absent or a stub. +#[cfg(target_os = "linux")] +const CASCADE_MIN_ATSPI_TARGETS: usize = 5; + +/// The default detector: prefer semantic AT-SPI targets, but supplement them +/// with CV for small visual controls a browser may omit from accessibility. +struct AutoDetector; + +impl HintDetector for AutoDetector { + fn name(&self) -> &'static str { + "auto" + } + + fn detect(&self, backend: &mut dyn Backend) -> anyhow::Result { + #[cfg(target_os = "linux")] + match super::atspi::AtspiHintDetector.detect(backend) { + Ok(output) if output.candidates.len() >= CASCADE_MIN_ATSPI_TARGETS => { + return Ok(supplement_with_cv(backend, output)); + } + Ok(output) => eprintln!( + "hint: atspi found only {} targets, falling back to cv", + output.candidates.len() + ), + Err(e) => eprintln!("hint: atspi detection failed, falling back to cv: {e:#}"), + } + super::cv::CvHintDetector.detect(backend) + } +} + +/// Add visual candidates that are not already covered by a precise semantic box. +#[cfg(target_os = "linux")] +fn supplement_with_cv(backend: &mut dyn Backend, mut semantic: DetectorOutput) -> DetectorOutput { + let Some(window) = semantic.focus_rect else { + return semantic; + }; + let Ok(cv) = super::cv::CvHintDetector.detect(backend) else { + return semantic; + }; + let cv_candidates = super::remap_to_screen( + cv.candidates, + cv.capture_w, + cv.capture_h, + semantic.capture_w, + semantic.capture_h, + ); + let raw_visual: Vec = cv_candidates + .into_iter() + .filter(|c| contains_point(window, center(c.bbox))) + .collect(); + let semantic_boxes: Vec<(u32, u32, u32, u32)> = + semantic.candidates.iter().map(|c| c.bbox).collect(); + let visual: Vec = raw_visual + .iter() + .copied() + .filter(|c| { + !semantic_boxes + .iter() + .any(|&b| precise_semantic_box_covers(b, c.bbox)) + }) + .collect(); + + merge_visual_supplements(&mut semantic.candidates, visual, &raw_visual); + semantic +} + +#[cfg(target_os = "linux")] +fn merge_visual_supplements( + semantic: &mut Vec, + visual: Vec, + raw_visual: &[HintCandidate], +) { + let semantic_snapshot = semantic.clone(); + let mut remove_semantic = vec![false; semantic_snapshot.len()]; + let mut drop_visual = vec![false; visual.len()]; + let mut replacements = Vec::new(); + + for (semantic_idx, semantic_candidate) in semantic_snapshot.iter().enumerate() { + let text_visual: Vec = visual + .iter() + .enumerate() + .filter(|(_, v)| broad_semantic_box_contains(semantic_candidate.bbox, v.bbox)) + .filter(|(_, v)| !is_icon_like(v.bbox)) + .map(|(idx, _)| idx) + .collect(); + let anchor = leftmost_anchor_icon(semantic_candidate.bbox, raw_visual, &semantic_snapshot); + + if text_visual.is_empty() && anchor.is_none() { + continue; + } + + remove_semantic[semantic_idx] = true; + for (idx, candidate) in semantic_snapshot.iter().enumerate() { + if idx != semantic_idx && is_left_anchor_icon(semantic_candidate.bbox, candidate.bbox) { + remove_semantic[idx] = true; + } + } + for (idx, candidate) in visual.iter().enumerate() { + if broad_semantic_box_contains(semantic_candidate.bbox, candidate.bbox) + && is_icon_like(candidate.bbox) + { + drop_visual[idx] = true; + } + } + + if text_visual.is_empty() { + if let Some(icon) = anchor { + replacements.push(text_target_after_icon(semantic_candidate.bbox, icon)); + } + } + } + + let mut remove_semantic = remove_semantic.into_iter(); + semantic.retain(|_| !remove_semantic.next().unwrap_or(false)); + semantic.extend( + visual + .into_iter() + .enumerate() + .filter_map(|(idx, candidate)| (!drop_visual[idx]).then_some(candidate)), + ); + semantic.extend(replacements); +} + +#[cfg(target_os = "linux")] +fn contains_point((x, y, w, h): (u32, u32, u32, u32), (px, py): (u32, u32)) -> bool { + px >= x && px < x + w && py >= y && py < y + h +} + +#[cfg(target_os = "linux")] +fn center((x, y, w, h): (u32, u32, u32, u32)) -> (u32, u32) { + (x + w / 2, y + h / 2) +} + +#[cfg(target_os = "linux")] +fn precise_semantic_box_covers( + semantic: (u32, u32, u32, u32), + visual: (u32, u32, u32, u32), +) -> bool { + contains_point(semantic, center(visual)) && !is_broad_horizontal_container(semantic, visual) +} + +#[cfg(target_os = "linux")] +fn broad_semantic_box_contains( + semantic: (u32, u32, u32, u32), + visual: (u32, u32, u32, u32), +) -> bool { + contains_point(semantic, center(visual)) && is_broad_horizontal_container(semantic, visual) +} + +#[cfg(target_os = "linux")] +fn is_broad_horizontal_container( + semantic: (u32, u32, u32, u32), + visual: (u32, u32, u32, u32), +) -> bool { + let (_, _, semantic_w, semantic_h) = semantic; + let (_, _, visual_w, visual_h) = visual; + u64::from(semantic_w) >= u64::from(visual_w.max(1)) * 3 + && u64::from(semantic_h) <= u64::from(visual_h.max(1)) * 3 +} + +#[cfg(target_os = "linux")] +fn leftmost_anchor_icon( + semantic: (u32, u32, u32, u32), + raw_visual: &[HintCandidate], + semantic_snapshot: &[HintCandidate], +) -> Option<(u32, u32, u32, u32)> { + raw_visual + .iter() + .map(|c| c.bbox) + .chain(semantic_snapshot.iter().map(|c| c.bbox)) + .filter(|&bbox| is_left_anchor_icon(semantic, bbox)) + .min_by_key(|&(x, _, _, _)| x) +} + +#[cfg(target_os = "linux")] +fn is_left_anchor_icon(semantic: (u32, u32, u32, u32), icon: (u32, u32, u32, u32)) -> bool { + if !broad_semantic_box_contains(semantic, icon) || !is_icon_like(icon) { + return false; + } + let (x, _, w, _) = semantic; + center(icon).0 <= x + w / 3 +} + +#[cfg(target_os = "linux")] +fn is_icon_like((_, _, w, h): (u32, u32, u32, u32)) -> bool { + let short = w.min(h).max(1); + let long = w.max(h); + long <= 32 && long <= short * 2 +} + +#[cfg(target_os = "linux")] +fn text_target_after_icon( + semantic: (u32, u32, u32, u32), + icon: (u32, u32, u32, u32), +) -> HintCandidate { + let (sx, sy, sw, sh) = semantic; + let (ix, _, iw, ih) = icon; + let right = sx + sw; + let x = (ix + iw + 6).clamp(sx, right.saturating_sub(1)); + let w = sh.max(ih).clamp(16, 40).min(right - x); + HintCandidate { + bbox: (x, sy, w.max(1), sh.max(1)), + score: 1.0, + } +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + #[test] + fn cv_supplement_never_doubles_a_semantic_target() { + let semantic = (500u32, 500u32, 40u32, 40u32); + assert!(contains_point(semantic, (520, 520))); + assert!(!contains_point(semantic, (560, 520))); + } + + #[test] + fn cv_supplement_stays_inside_focus_window() { + let window = (100, 100, 800, 600); + assert!(contains_point(window, (405, 305))); + assert!(!contains_point(window, (15, 305))); + } + + #[test] + fn broad_semantic_row_does_not_cover_precise_visual_target() { + let semantic_row = (10, 100, 900, 24); + let file_link = (20, 102, 120, 20); + + assert!(!precise_semantic_box_covers(semantic_row, file_link)); + assert!(broad_semantic_box_contains(semantic_row, file_link)); + } + + #[test] + fn tight_semantic_box_covers_matching_visual_target() { + let semantic_link = (20, 100, 140, 24); + let visual_link = (25, 102, 120, 20); + + assert!(precise_semantic_box_covers(semantic_link, visual_link)); + assert!(!broad_semantic_box_contains(semantic_link, visual_link)); + } + + #[test] + fn broad_semantic_with_text_visual_drops_icon_anchor() { + let mut semantic = vec![ + HintCandidate { + bbox: (10, 100, 900, 24), + score: 1.0, + }, + HintCandidate { + bbox: (22, 104, 16, 16), + score: 1.0, + }, + ]; + let visual = vec![HintCandidate { + bbox: (48, 105, 80, 14), + score: 1.0, + }]; + let raw_visual = [HintCandidate { + bbox: (22, 104, 16, 16), + score: 1.0, + }]; + + merge_visual_supplements(&mut semantic, visual, &raw_visual); + + let boxes: Vec<_> = semantic.iter().map(|c| c.bbox).collect(); + assert_eq!(boxes, vec![(48, 105, 80, 14)]); + } + + #[test] + fn broad_semantic_with_only_icon_anchor_clicks_text_side() { + let mut semantic = vec![ + HintCandidate { + bbox: (10, 100, 900, 24), + score: 1.0, + }, + HintCandidate { + bbox: (22, 104, 16, 16), + score: 1.0, + }, + ]; + let raw_visual = [HintCandidate { + bbox: (22, 104, 16, 16), + score: 1.0, + }]; + + merge_visual_supplements(&mut semantic, Vec::new(), &raw_visual); + + let boxes: Vec<_> = semantic.iter().map(|c| c.bbox).collect(); + assert_eq!(boxes, vec![(44, 100, 24, 24)]); + } + + #[test] + fn icon_without_broad_parent_stays_available() { + let mut semantic = vec![HintCandidate { + bbox: (22, 104, 16, 16), + score: 1.0, + }]; + let raw_visual = [HintCandidate { + bbox: (22, 104, 16, 16), + score: 1.0, + }]; + + merge_visual_supplements(&mut semantic, Vec::new(), &raw_visual); + + assert_eq!(semantic[0].bbox, (22, 104, 16, 16)); + } +} diff --git a/src/hint/mod.rs b/src/hint/mod.rs new file mode 100644 index 0000000..2a4d9e7 --- /dev/null +++ b/src/hint/mod.rs @@ -0,0 +1,424 @@ +//! Hint mode: detect likely click targets on screen, label them with +//! prefix-free key sequences, and move the cursor to a chosen target. +//! +//! Detection is pluggable behind the [`HintDetector`] trait, with two sources: +//! the AT-SPI accessibility tree ([`atspi`], Linux-only) and a pure-CV +//! screenshot scan ([`cv`]); the default `auto` detector cascades from the +//! first to the second. Everything downstream of a detector (remapping, +//! scoring, deduplication, label assignment) is shared and lives here. + +#[cfg(target_os = "linux")] +mod atspi; +#[cfg(target_os = "linux")] +mod compositor; +mod cv; +mod detect; + +pub use detect::{select_detector, HintCandidate, HintDetector}; + +use crate::{backend::Backend, config::config}; + +#[derive(Clone, Debug)] +pub struct HintElement { + pub cx: u32, + pub cy: u32, + pub bbox: (u32, u32, u32, u32), + pub label: String, +} + +/// Prefix test over `label`'s chars without allocating a `Vec` (run for +/// every element on every keystroke). +pub fn label_starts_with(label: &str, prefix: &[char]) -> bool { + let mut chars = label.chars(); + prefix.iter().all(|&c| chars.next() == Some(c)) +} + +/// Build hint mode using the detector selected in config (`hint.detector`). +pub fn build_hint_mode( + backend: &mut B, + screen_w: u32, + screen_h: u32, +) -> anyhow::Result> { + let detector = select_detector()?; + build_hint_mode_with_detector(backend, screen_w, screen_h, detector.as_ref()) +} + +pub fn build_hint_mode_with_detector( + backend: &mut B, + screen_w: u32, + screen_h: u32, + detector: &dyn HintDetector, +) -> anyhow::Result> { + use anyhow::Context; + let output = detector + .detect(backend) + .with_context(|| format!("hint {} detection failed", detector.name()))?; + let candidates = remap_to_screen( + output.candidates, + output.capture_w, + output.capture_h, + screen_w, + screen_h, + ); + Ok(assign_hint_elements(candidates, screen_w, screen_h)) +} + +/// Rescale candidate boxes from capture pixels into the overlay's logical +/// space: on HiDPI/fractional-scaled Wayland outputs the screencopy buffer is +/// physical pixels while the pointer and overlay are logical. A no-op when +/// the sizes already match. +fn remap_to_screen( + mut candidates: Vec, + capture_w: u32, + capture_h: u32, + screen_w: u32, + screen_h: u32, +) -> Vec { + if capture_w == 0 + || capture_h == 0 + || screen_w == 0 + || screen_h == 0 + || (capture_w == screen_w && capture_h == screen_h) + { + return candidates; + } + let sx = screen_w as f32 / capture_w as f32; + let sy = screen_h as f32 / capture_h as f32; + for candidate in &mut candidates { + let (x, y, w, h) = candidate.bbox; + let nx = ((x as f32 * sx).round() as u32).min(screen_w - 1); + let ny = ((y as f32 * sy).round() as u32).min(screen_h - 1); + let nw = ((w as f32 * sx).round() as u32).max(1).min(screen_w - nx); + let nh = ((h as f32 * sy).round() as u32).max(1).min(screen_h - ny); + candidate.bbox = (nx, ny, nw, nh); + } + candidates +} + +fn assign_hint_elements( + mut candidates: Vec, + screen_w: u32, + screen_h: u32, +) -> Vec { + let cfg = config(); + let mut alphabet = cfg.hint_alphabet().to_vec(); + // A repeated alphabet key would generate two identical labels and make + // one element permanently unreachable. + let mut seen = std::collections::HashSet::new(); + alphabet.retain(|ch| seen.insert(*ch)); + if alphabet.is_empty() { + return Vec::new(); + } + + for candidate in &mut candidates { + let (x, y, w, h) = candidate.bbox; + candidate.score = + W_DETECTOR * candidate.score + importance_for(x, y, w, h, screen_w, screen_h); + } + candidates.sort_by(|a, b| b.score.total_cmp(&a.score)); + // Bounds the O(n^2) dedup on pathologically cluttered screens. + const MAX_DEDUP_CANDIDATES: usize = 1024; + candidates.truncate(MAX_DEDUP_CANDIDATES); + candidates = dedup_candidates(candidates); + + let max = alphabet + .len() + .saturating_pow(cfg.hint.max_label_len.max(1) as u32) + .max(1); + candidates.truncate(max); + + let labels = assign_labels(candidates.len(), &alphabet, cfg.hint.max_label_len.max(1)); + candidates + .into_iter() + .zip(labels) + .map(|(candidate, label)| { + let (x, y, w, h) = candidate.bbox; + HintElement { + cx: x + w / 2, + cy: y + h / 2, + bbox: candidate.bbox, + label, + } + }) + .collect() +} + +/// Ranking weights: detector quality, then a preference for mid-sized +/// targets, then a mild center bias. They only decide which targets get the +/// shortest labels, so they are implementation detail, not configuration. +const W_DETECTOR: f32 = 0.6; +const W_SIZE: f32 = 0.5; +const W_CENTER: f32 = 0.2; + +/// Resolution-independent ranking bonus for mid-sized, roughly centered targets. +fn importance_for(x: u32, y: u32, w: u32, h: u32, screen_w: u32, screen_h: u32) -> f32 { + let area = (w as f32) * (h as f32); + let screen_area = (screen_w as f32 * screen_h as f32).max(1.0); + let area_ratio = area / screen_area; + let size_pref = if area_ratio < 0.00005 { + 0.0 + } else if area_ratio > 0.05 { + 0.2 + } else { + 1.0 - (area_ratio - 0.004).abs().min(0.004) / 0.004 + }; + + let cx = x as f32 + w as f32 / 2.0; + let cy = y as f32 + h as f32 / 2.0; + let dx = (cx / screen_w.max(1) as f32 - 0.5).abs(); + let dy = (cy / screen_h.max(1) as f32 - 0.5).abs(); + let center_bias = 1.0 - (dx * dx + dy * dy).sqrt().min(1.0); + + W_SIZE * size_pref + W_CENTER * center_bias +} + +/// Greedy non-maximum suppression over score-sorted candidates. Conservative +/// on purpose: distinct controls can legitimately overlap heavily (tab + +/// close button, row + child action), so only collapse boxes that are +/// effectively the same target. +fn dedup_candidates(candidates: Vec) -> Vec { + let mut kept: Vec = Vec::new(); + 'outer: for candidate in candidates { + for existing in &kept { + if is_duplicate(candidate.bbox, existing.bbox) { + continue 'outer; + } + } + kept.push(candidate); + } + kept +} + +fn is_duplicate(a: (u32, u32, u32, u32), b: (u32, u32, u32, u32)) -> bool { + let overlap = iou(a, b); + if overlap > 0.9 { + return true; + } + if overlap <= 0.7 { + return false; + } + + // High-but-not-near-total overlap also needs matching centers and sizes, + // so adjacent/nested controls survive. + let acx = a.0 as i64 + a.2 as i64 / 2; + let acy = a.1 as i64 + a.3 as i64 / 2; + let bcx = b.0 as i64 + b.2 as i64 / 2; + let bcy = b.1 as i64 + b.3 as i64 / 2; + let center_close = (acx - bcx).abs() <= 4 && (acy - bcy).abs() <= 4; + let width_ratio = a.2.min(b.2) as f32 / a.2.max(b.2).max(1) as f32; + let height_ratio = a.3.min(b.3) as f32 / a.3.max(b.3).max(1) as f32; + center_close && width_ratio >= 0.8 && height_ratio >= 0.8 +} + +fn iou(a: (u32, u32, u32, u32), b: (u32, u32, u32, u32)) -> f32 { + let ax2 = a.0 + a.2; + let ay2 = a.1 + a.3; + let bx2 = b.0 + b.2; + let by2 = b.1 + b.3; + let ix = ax2.min(bx2).saturating_sub(a.0.max(b.0)); + let iy = ay2.min(by2).saturating_sub(a.1.max(b.1)); + let intersection = ix * iy; + if intersection == 0 { + return 0.0; + } + let union = a.2 * a.3 + b.2 * b.3 - intersection; + intersection as f32 / union.max(1) as f32 +} + +pub fn assign_labels(n: usize, alphabet: &[char], max_len: usize) -> Vec { + if n == 0 || alphabet.is_empty() || max_len == 0 { + return Vec::new(); + } + let k = alphabet.len(); + if k == 1 { + return labels_of_len(alphabet, 1).take(n.min(1)).collect(); + } + let mut len = 1usize; + while len < max_len && k.saturating_pow(len as u32) < n { + len += 1; + } + if len == 1 { + return labels_of_len(alphabet, 1).take(n.min(k)).collect(); + } + + let capacity = k.saturating_pow(len as u32); + let short_count = + ((capacity.saturating_sub(n)) / (k - 1)).min(k.saturating_pow((len - 1) as u32)); + // Short labels are reserved prefixes; a full-length label is kept only if it + // starts with none of them. + let short: Vec = labels_of_len(alphabet, len - 1).take(short_count).collect(); + let mut long: Vec = Vec::new(); + for label in labels_of_len(alphabet, len) { + if short.iter().any(|prefix| label.starts_with(prefix)) { + continue; + } + long.push(label); + if short.len() + long.len() == n { + break; + } + } + short.into_iter().chain(long).collect() +} + +fn labels_of_len(alphabet: &[char], len: usize) -> impl Iterator + '_ { + let total = alphabet.len().saturating_pow(len as u32); + (0..total).map(move |mut idx| { + let mut chars = vec![alphabet[0]; len]; + for pos in (0..len).rev() { + chars[pos] = alphabet[idx % alphabet.len()]; + idx /= alphabet.len(); + } + chars.into_iter().collect() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn is_prefix_free(labels: &[String]) -> bool { + for (i, a) in labels.iter().enumerate() { + for (j, b) in labels.iter().enumerate() { + if i != j && b.starts_with(a.as_str()) { + return false; + } + } + } + true + } + + fn assert_label_invariants(n: usize, alphabet: &[char], max_len: usize) { + let labels = assign_labels(n, alphabet, max_len); + assert_eq!(labels.len(), n, "label count for n={n}"); + + let unique: std::collections::HashSet<_> = labels.iter().collect(); + assert_eq!(unique.len(), labels.len(), "labels not unique for n={n}"); + + assert!(is_prefix_free(&labels), "labels not prefix-free for n={n}"); + + assert!( + labels.iter().all(|l| l.chars().count() <= max_len), + "label exceeds max_len for n={n}" + ); + + // Shorter labels (more important) must precede longer ones. + let lengths: Vec = labels.iter().map(|l| l.chars().count()).collect(); + assert!( + lengths.windows(2).all(|w| w[0] <= w[1]), + "labels not length-ordered for n={n}: {lengths:?}" + ); + } + + #[test] + fn labels_are_well_formed_across_counts() { + let alphabet: Vec = "asdfjkl".chars().collect(); + for n in 0..=alphabet.len().pow(3) { + assert_label_invariants(n, &alphabet, 3); + } + } + + #[test] + fn single_key_alphabet_yields_one_label() { + let alphabet = ['a']; + assert_eq!(assign_labels(5, &alphabet, 3), vec!["a".to_string()]); + } + + #[test] + fn empty_inputs_yield_no_labels() { + assert!(assign_labels(0, &['a', 'b'], 3).is_empty()); + assert!(assign_labels(5, &[], 3).is_empty()); + assert!(assign_labels(5, &['a', 'b'], 0).is_empty()); + } + + #[test] + fn iou_matches_known_cases() { + // Identical boxes overlap fully. + assert!((iou((0, 0, 10, 10), (0, 0, 10, 10)) - 1.0).abs() < 1e-6); + // Disjoint boxes do not overlap. + assert_eq!(iou((0, 0, 10, 10), (20, 20, 10, 10)), 0.0); + // Half-overlapping boxes: intersection 50, union 150. + assert!((iou((0, 0, 10, 10), (5, 0, 10, 10)) - (50.0 / 150.0)).abs() < 1e-6); + } + + #[test] + fn dedup_drops_near_duplicate_boxes() { + // Two nearly identical boxes (IoU > 0.6) collapse to the higher-scored + // one; a disjoint box survives. + let candidates = vec![ + HintCandidate { + bbox: (0, 0, 40, 40), + score: 1.0, + }, + HintCandidate { + bbox: (2, 2, 40, 40), + score: 0.5, + }, + HintCandidate { + bbox: (200, 200, 30, 30), + score: 0.4, + }, + ]; + let kept = dedup_candidates(candidates); + assert_eq!(kept.len(), 2); + assert_eq!(kept[0].bbox, (0, 0, 40, 40)); + assert_eq!(kept[1].bbox, (200, 200, 30, 30)); + } + + #[test] + fn dedup_keeps_overlapping_distinct_targets() { + let candidates = vec![ + HintCandidate { + bbox: (0, 0, 100, 30), + score: 1.0, + }, + HintCandidate { + bbox: (18, 0, 100, 30), + score: 0.9, + }, + HintCandidate { + bbox: (70, 4, 20, 20), + score: 0.8, + }, + ]; + let kept = dedup_candidates(candidates); + assert_eq!(kept.len(), 3); + } + + #[test] + fn dedup_drops_same_center_same_size_boxes() { + let candidates = vec![ + HintCandidate { + bbox: (10, 10, 40, 20), + score: 1.0, + }, + HintCandidate { + bbox: (11, 10, 40, 20), + score: 0.9, + }, + ]; + let kept = dedup_candidates(candidates); + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].bbox, (10, 10, 40, 20)); + } + + #[test] + fn remap_scales_into_logical_space() { + let candidates = vec![HintCandidate { + bbox: (100, 200, 40, 20), + score: 0.5, + }]; + // Capture is 2x the logical screen (HiDPI). + let out = remap_to_screen(candidates, 3840, 2160, 1920, 1080); + assert_eq!(out[0].bbox, (50, 100, 20, 10)); + } + + #[test] + fn remap_is_noop_when_sizes_match() { + let candidates = vec![HintCandidate { + bbox: (10, 20, 30, 40), + score: 0.5, + }]; + let out = remap_to_screen(candidates, 1920, 1080, 1920, 1080); + assert_eq!(out[0].bbox, (10, 20, 30, 40)); + } +} diff --git a/src/main.rs b/src/main.rs index 46eb64e..62b4d44 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ mod app; mod backend; mod cli; mod config; +mod hint; mod input; mod macro_store; mod mode; diff --git a/src/mode/bisect.rs b/src/mode/bisect.rs index ce1e592..f48009f 100644 --- a/src/mode/bisect.rs +++ b/src/mode/bisect.rs @@ -12,6 +12,8 @@ fn subcell_size(region: (u32, u32, u32, u32), rows: u32, cols: u32) -> (u32, u32 } pub(super) fn handle_key( + width: u32, + height: u32, key: &KeyEvent, backend: &mut B, region: (u32, u32, u32, u32), @@ -96,6 +98,7 @@ pub(super) fn handle_key( y: cy, speed: config().free.base_speed.max(1), })), + KeyEvent::Hint => Ok(super::hint::enter(backend, width, height)), _ => Ok(ModeTransition::Stay), } } diff --git a/src/mode/free.rs b/src/mode/free.rs index 2b0c99c..f440399 100644 --- a/src/mode/free.rs +++ b/src/mode/free.rs @@ -53,6 +53,7 @@ pub(super) fn handle_key( backend.scroll_right()?; Ok(ModeTransition::Redraw) } + KeyEvent::Hint => Ok(super::hint::enter(backend, width, height)), KeyEvent::Char(ch) => handle_char(width, height, *ch, backend, x, y, speed), _ => Ok(ModeTransition::Stay), } diff --git a/src/mode/hint.rs b/src/mode/hint.rs new file mode 100644 index 0000000..f819342 --- /dev/null +++ b/src/mode/hint.rs @@ -0,0 +1,128 @@ +use std::rc::Rc; + +use crate::{ + backend::{Backend, KeyEvent}, + config::config, + hint::{build_hint_mode, label_starts_with, HintElement}, + mode::{Mode, ModeTransition}, + render::render_hints, +}; + +/// Build and enter hint mode, degrading gracefully when screen capture is +/// unavailable (e.g. a compositor without `zwlr_screencopy_v1`). A failed +/// capture must not tear down the whole app mid-session, so we report it and +/// stay in the current mode rather than propagating the error up the run loop. +pub(super) fn enter(backend: &mut B, width: u32, height: u32) -> ModeTransition { + match build_hint_mode(backend, width, height) { + Ok(elements) => ModeTransition::Enter(Mode::Hint { + elements: elements.into(), + typed: Vec::new(), + target: None, + }), + Err(e) => { + eprintln!("hint mode unavailable: {e:#}"); + ModeTransition::Stay + } + } +} + +pub(super) fn handle_key( + key: &KeyEvent, + backend: &mut B, + elements: &Rc<[HintElement]>, + typed: &[char], + target: Option<(u32, u32)>, +) -> anyhow::Result { + match key { + KeyEvent::Close => Ok(ModeTransition::Exit), + KeyEvent::Undo => { + if typed.is_empty() { + Ok(ModeTransition::Back) + } else { + let mut next_typed = typed.to_vec(); + next_typed.pop(); + Ok(ModeTransition::Replace(Mode::Hint { + elements: Rc::clone(elements), + typed: next_typed, + target: None, + })) + } + } + KeyEvent::Click => action(backend, target, Backend::click), + KeyEvent::DoubleClick => action(backend, target, Backend::double_click), + KeyEvent::TripleClick => action(backend, target, Backend::triple_click), + KeyEvent::RightClick => action(backend, target, Backend::right_click), + KeyEvent::MiddleClick => action(backend, target, Backend::middle_click), + KeyEvent::Char('/') => { + if let Some((x, y)) = target { + Ok(ModeTransition::Enter(Mode::Normal { + input_state: crate::input::InputState::First, + target: None, + drag_origin: Some((x, y)), + })) + } else { + Ok(ModeTransition::Stay) + } + } + KeyEvent::Char(ch) if config().hint_alphabet().contains(ch) => { + let mut next_typed = typed.to_vec(); + next_typed.push(*ch); + let matches: Vec<_> = elements + .iter() + .filter(|element| label_starts_with(&element.label, &next_typed)) + .collect(); + if matches.is_empty() { + return Ok(ModeTransition::Stay); + } + + if let Some(element) = matches + .iter() + .copied() + .find(|element| element.label.chars().eq(next_typed.iter().copied())) + { + backend.move_mouse(element.cx, element.cy)?; + if config().hint.auto_click { + backend.click(element.cx, element.cy)?; + return Ok(ModeTransition::Exit); + } + return Ok(ModeTransition::Replace(Mode::Hint { + elements: Rc::clone(elements), + typed: next_typed, + target: Some((element.cx, element.cy)), + })); + } + + Ok(ModeTransition::Replace(Mode::Hint { + elements: Rc::clone(elements), + typed: next_typed, + target: None, + })) + } + _ => Ok(ModeTransition::Stay), + } +} + +fn action( + backend: &mut B, + target: Option<(u32, u32)>, + f: fn(&mut B, u32, u32) -> anyhow::Result<()>, +) -> anyhow::Result { + if let Some((x, y)) = target { + f(backend, x, y)?; + Ok(ModeTransition::Exit) + } else { + Ok(ModeTransition::Stay) + } +} + +pub(super) fn draw( + backend: &mut B, + pixels: &mut [u8], + width: u32, + height: u32, + elements: &[HintElement], + typed: &[char], +) -> anyhow::Result<()> { + render_hints(pixels, width, height, elements, typed); + backend.present(pixels, width, height) +} diff --git a/src/mode/mod.rs b/src/mode/mod.rs index f99b10a..933862b 100644 --- a/src/mode/mod.rs +++ b/src/mode/mod.rs @@ -1,5 +1,6 @@ mod bisect; mod free; +mod hint; mod macro_bind_key; mod macro_name; mod macro_replay; @@ -81,6 +82,11 @@ pub enum Mode { y: u32, speed: u32, }, + Hint { + elements: std::rc::Rc<[crate::hint::HintElement]>, + typed: Vec, + target: Option<(u32, u32)>, + }, } impl Mode { @@ -127,7 +133,7 @@ impl Mode { *target, *drag_origin, ), - Mode::Bisect { region } => bisect::handle_key(key, backend, *region), + Mode::Bisect { region } => bisect::handle_key(width, height, key, backend, *region), Mode::MacroRecording { input_state, target, @@ -162,6 +168,11 @@ impl Mode { Mode::Free { x, y, speed } => { free::handle_key(width, height, key, backend, *x, *y, *speed) } + Mode::Hint { + elements, + typed, + target, + } => hint::handle_key(key, backend, elements, typed, *target), } } @@ -216,6 +227,9 @@ impl Mode { Mode::Free { x, y, speed } => { free::draw(backend, pixels, width, height, *x, *y, *speed) } + Mode::Hint { + elements, typed, .. + } => hint::draw(backend, pixels, width, height, elements, typed), } } } diff --git a/src/mode/normal.rs b/src/mode/normal.rs index 422560a..f5d1cd6 100644 --- a/src/mode/normal.rs +++ b/src/mode/normal.rs @@ -113,9 +113,7 @@ pub(super) fn handle_key( let (x, y) = if let Some((x, y)) = target { (x, y) } else { - let (x, y) = (width / 2, height / 2); - backend.move_mouse(x, y)?; - (x, y) + backend.mouse_pos()? }; Ok(ModeTransition::Enter(Mode::Free { @@ -124,6 +122,23 @@ pub(super) fn handle_key( speed: config().free.base_speed.max(1), })) } + KeyEvent::Hint => { + let cfg = config(); + if collides_with_hint(cfg.keys.hint, input_state, cfg.hints(), cfg.sub_hints()) { + if let Key::Char(ch) = cfg.keys.hint { + return handle_key( + width, + height, + &KeyEvent::Char(ch), + backend, + input_state, + target, + drag_origin, + ); + } + } + Ok(super::hint::enter(backend, width, height)) + } KeyEvent::Char(ch) if config().hints().contains(ch) || (matches!(input_state, InputState::SubFirst { .. }) diff --git a/src/render.rs b/src/render.rs index 9342a02..18a1fbf 100644 --- a/src/render.rs +++ b/src/render.rs @@ -1,4 +1,8 @@ -use crate::{config::config, input::InputState}; +use crate::{ + config::config, + hint::{label_starts_with, HintElement}, + input::InputState, +}; use font8x8::UnicodeFonts; fn line_height(scale: u32) -> u32 { @@ -274,6 +278,215 @@ pub fn render_free(buf: &mut [u8], w: u32, h: u32, x: u32, y: u32, speed: u32) { c.draw_text(8 + pad, 12, bytes, colors.text_white, scale); } +pub fn render_hints(buf: &mut [u8], w: u32, h: u32, elements: &[HintElement], typed: &[char]) { + let cfg = config(); + let colors = &cfg.colors; + let scale = cfg.hint_font_size(); + let cw = char_width(scale); + let chip_h = 8 * scale + 6; + let mut c = Canvas { buf, w }; + c.clear(); + + if elements.is_empty() { + c.draw_text( + 8, + 8, + b"no clickable elements found - esc to exit", + colors.hint_text, + scale, + ); + return; + } + + // Place every chip so chips never cover each other's letters. `elements` is + // sorted by importance, so the most important targets keep their natural spot + // and overlapping neighbours (e.g. a browser tab and its close button) are + // nudged to the nearest free position instead of stacking unreadably. + let placements = place_chips(elements, w, h, chip_h, cw); + + // Layered draw: dimmed (non-matching) chips first, then the matching ones on + // top, so a label you can still type is never hidden under a dimmed + // neighbour. Reverse order within each layer keeps important chips on top. + for matching in [false, true] { + for i in (0..elements.len()).rev() { + if label_starts_with(&elements[i].label, typed) != matching { + continue; + } + let (x, y, chip_w) = placements[i]; + draw_hint_chip( + &mut c, + colors, + scale, + cw, + chip_h, + &elements[i], + x, + y, + chip_w, + typed, + matching, + ); + } + } + + // Crosshair on the fully-typed target, drawn last so it sits above all chips. + if !typed.is_empty() { + for el in elements { + if el.label.chars().eq(typed.iter().copied()) { + draw_crosshair(&mut c, colors, el.cx, el.cy); + } + } + } +} + +/// Greedily assign each element a chip rectangle `(x, y, chip_w)` that does not +/// overlap an already-placed chip, searching positions near the element. Earlier +/// (more important) elements are placed first and win contested spots. +fn place_chips( + elements: &[HintElement], + w: u32, + h: u32, + chip_h: u32, + cw: u32, +) -> Vec<(u32, u32, u32)> { + let mut placed: Vec<(u32, u32, u32, u32)> = Vec::with_capacity(elements.len()); + let mut out = Vec::with_capacity(elements.len()); + for el in elements { + let chip_w = (el.label.chars().count() as u32 * cw + 8).max(18); + let rect = find_free_chip(el.bbox, chip_w, chip_h, w, h, &placed); + placed.push(rect); + out.push((rect.0, rect.1, chip_w)); + } + out +} + +/// Search positions anchored around an element for one whose chip rect clears all +/// `placed` rects (plus a small gap). Falls back to the preferred spot, accepting +/// overlap, when the area is too crowded to find a free slot. +fn find_free_chip( + bbox: (u32, u32, u32, u32), + chip_w: u32, + chip_h: u32, + w: u32, + h: u32, + placed: &[(u32, u32, u32, u32)], +) -> (u32, u32, u32, u32) { + let (bx, by, bw, bh) = bbox; + let clamp_x = |x: i64| x.clamp(0, w.saturating_sub(chip_w) as i64) as u32; + let clamp_y = |y: i64| y.clamp(0, h.saturating_sub(chip_h) as i64) as u32; + let left = clamp_x(bx as i64); + let right = clamp_x(bx as i64 + bw as i64 - chip_w as i64); + let on = clamp_y(by as i64 - chip_h as i64 / 2); + let above = clamp_y(by as i64 - chip_h as i64); + let below = clamp_y(by as i64 + bh as i64); + + let mut candidates: Vec<(u32, u32)> = vec![ + (left, on), + (left, above), + (right, above), + (right, on), + (left, below), + (right, below), + ]; + // Then fan out vertically, then horizontally, staying near the element. + for k in 1..=12i64 { + let dy = k * (chip_h as i64 + 2); + candidates.push((left, clamp_y(by as i64 - chip_h as i64 / 2 + dy))); + candidates.push((left, clamp_y(by as i64 - chip_h as i64 / 2 - dy))); + } + for k in 1..=12i64 { + let dx = k * (chip_w as i64 + 2); + candidates.push((clamp_x(bx as i64 + dx), on)); + candidates.push((clamp_x(bx as i64 - dx), on)); + } + + for (x, y) in candidates { + let rect = (x, y, chip_w, chip_h); + if !placed.iter().any(|p| rects_overlap(*p, rect, 2)) { + return rect; + } + } + (left, on, chip_w, chip_h) +} + +/// Axis-aligned overlap test with a `gap` margin so chips keep a little air +/// between them rather than sitting flush. +fn rects_overlap(a: (u32, u32, u32, u32), b: (u32, u32, u32, u32), gap: u32) -> bool { + let ax1 = a.0.saturating_sub(gap); + let ay1 = a.1.saturating_sub(gap); + let ax2 = a.0 + a.2 + gap; + let ay2 = a.1 + a.3 + gap; + ax1 < b.0 + b.2 && b.0 < ax2 && ay1 < b.1 + b.3 && b.1 < ay2 +} + +#[allow(clippy::too_many_arguments)] +fn draw_hint_chip( + c: &mut Canvas<'_>, + colors: &crate::config::Colors, + scale: u32, + cw: u32, + chip_h: u32, + el: &HintElement, + x: u32, + y: u32, + chip_w: u32, + typed: &[char], + matching: bool, +) { + let typed_full = matching && !typed.is_empty() && el.label.chars().eq(typed.iter().copied()); + let text_color = if typed_full { + colors.hint_text_typed + } else if matching { + colors.hint_text + } else { + colors.hint_dim + }; + let bg = if matching { + colors.hint_chip_bg + } else { + [ + colors.hint_chip_bg[0], + colors.hint_chip_bg[1], + colors.hint_chip_bg[2], + colors.hint_chip_bg[3] / 3, + ] + }; + c.fill_rect(x, y, chip_w, chip_h, bg); + // Border only on active chips — keeps dimmed neighbours visually quiet. + if matching && chip_w > 2 && chip_h > 2 { + c.fill_rect(x, y, chip_w, 1, colors.border); + c.fill_rect(x, y + chip_h - 1, chip_w, 1, colors.border); + c.fill_rect(x, y, 1, chip_h, colors.border); + c.fill_rect(x + chip_w - 1, y, 1, chip_h, colors.border); + } + for (idx, ch) in el.label.chars().enumerate() { + let color = if matching && idx < typed.len() { + colors.hint_text_typed + } else { + text_color + }; + c.draw_glyph(x + 4 + idx as u32 * cw, y + 3, ch, color, scale); + } +} + +fn draw_crosshair(c: &mut Canvas<'_>, colors: &crate::config::Colors, cx: u32, cy: u32) { + let arm = 8; + c.fill_rect( + cx.saturating_sub(arm), + cy, + arm * 2 + 1, + 1, + colors.hint_text_typed, + ); + c.fill_rect( + cx, + cy.saturating_sub(arm), + 1, + arm * 2 + 1, + colors.hint_text_typed, + ); +} + pub fn render_macro_bind_key(buf: &mut [u8], w: u32, h: u32) { let colors = &config().colors; let mut p = Panel::new(buf, w, h, 6); @@ -581,3 +794,37 @@ fn render_sub_grid( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn coincident_elements_get_non_overlapping_chips() { + // Three targets sharing the same tiny box (e.g. stacked controls). Their + // chips must end up at distinct, non-overlapping positions. + let el = |label: &str| HintElement { + cx: 110, + cy: 110, + bbox: (100, 100, 20, 20), + label: label.to_string(), + }; + let elements = vec![el("a"), el("s"), el("d")]; + let chip_h = 14; + let placements = place_chips(&elements, 1920, 1080, chip_h, 16); + let rects: Vec<(u32, u32, u32, u32)> = placements + .iter() + .map(|&(x, y, cw)| (x, y, cw, chip_h)) + .collect(); + for i in 0..rects.len() { + for j in (i + 1)..rects.len() { + assert!( + !rects_overlap(rects[i], rects[j], 0), + "chips {i} {:?} and {j} {:?} overlap", + rects[i], + rects[j] + ); + } + } + } +} From b320b4f53d96105e5cb9203fda966d2a9e7055f3 Mon Sep 17 00:00:00 2001 From: Fanis Tharropoulos Date: Fri, 3 Jul 2026 15:44:53 +0300 Subject: [PATCH 2/4] feat(macos): hints mode --- Cargo.lock | 14 ++ Cargo.toml | 5 +- src/backend/macos.rs | 80 ++++++++ src/hint/ax.rs | 445 +++++++++++++++++++++++++++++++++++++++++++ src/hint/detect.rs | 100 +++++++--- src/hint/mod.rs | 2 + 6 files changed, 622 insertions(+), 24 deletions(-) create mode 100644 src/hint/ax.rs diff --git a/Cargo.lock b/Cargo.lock index a893fd4..afdab6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1032,6 +1032,17 @@ dependencies = [ "objc2-quartz-core", ] +[[package]] +name = "objc2-application-services" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69282c2b5bc58fba07cb9de2113619532eb551e98efe3d8d695509ef45fbd53b" +dependencies = [ + "bitflags", + "libc", + "objc2-core-foundation", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -1503,6 +1514,9 @@ dependencies = [ "nix", "objc2", "objc2-app-kit", + "objc2-application-services", + "objc2-core-foundation", + "objc2-core-graphics", "objc2-foundation", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 80d3a86..56bd63b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,4 +40,7 @@ futures-util = { version = "0.3", default-features = false, features = ["alloc"] [target.'cfg(target_os = "macos")'.dependencies] objc2 = "0.6" objc2-foundation = { version = "0.3", features = ["NSDate", "NSGeometry", "NSString", "NSValue"] } -objc2-app-kit = { version = "0.3", features = ["NSApplication", "NSColor", "NSEvent", "NSPanel", "NSResponder", "NSScreen", "NSView", "NSWindow"] } +objc2-app-kit = { version = "0.3", features = ["NSApplication", "NSColor", "NSEvent", "NSPanel", "NSResponder", "NSRunningApplication", "NSScreen", "NSView", "NSWindow", "NSWorkspace"] } +objc2-application-services = { version = "0.3", default-features = false, features = ["std", "libc", "AXUIElement", "AXValue", "AXAttributeConstants", "AXRoleConstants", "AXError", "HIServices"] } +objc2-core-graphics = { version = "0.3", default-features = false, features = ["std", "CGDirectDisplay", "CGImage", "CGDataProvider", "CGError"] } +objc2-core-foundation = { version = "0.3", default-features = false, features = ["std", "CFString", "CFArray", "CFData", "CFBase", "CFCGTypes", "CFNumber"] } diff --git a/src/backend/macos.rs b/src/backend/macos.rs index 723c76c..c396b70 100644 --- a/src/backend/macos.rs +++ b/src/backend/macos.rs @@ -173,6 +173,18 @@ unsafe extern "C" { intent: u32, ) -> CGImageRef; fn CGImageRelease(img: CGImageRef); + + fn CGBitmapContextCreate( + data: *mut c_void, + width: usize, + height: usize, + bits_per_component: usize, + bytes_per_row: usize, + space: CGColorSpaceRef, + bitmap_info: u32, + ) -> *mut c_void; + fn CGContextDrawImage(c: *mut c_void, rect: CGRect, image: CGImageRef); + fn CGContextRelease(c: *mut c_void); } #[link(name = "ApplicationServices", kind = "framework")] @@ -615,6 +627,74 @@ impl Backend for MacosBackend { Ok((pt.x as u32, pt.y as u32)) } + fn capture_screen(&mut self) -> Result { + use objc2_core_graphics::CGImage; + + let display = objc2_core_graphics::CGMainDisplayID(); + // CGDisplayCreateImage is marked deprecated in favour of + // ScreenCaptureKit, but it still functions on macOS 14/15 and SCK is + // async-only. + // Revisit when SCK exposes a synchronous shim. + #[allow(deprecated)] + let image = objc2_core_graphics::CGDisplayCreateImage(display).ok_or_else(|| { + anyhow!("CGDisplayCreateImage returned nil (screen recording permission?)") + })?; + let width = CGImage::width(Some(&image)); + let height = CGImage::height(Some(&image)); + if width == 0 || height == 0 { + return Err(anyhow!("captured image has zero dimensions")); + } + + // Reading bytes straight from `CGImage::data_provider` is unreliable. + // On Retina/IOSurface-backed images the buffer may not be in the + // declared bitmap format, and the bits CV gets back are effectively + // noise. Drawing the image into our own BGRA-premultiplied bitmap + // context normalises the format and gives a tight buffer to keep. + let row_bytes = width * 4; + let mut bgra: Vec = vec![0; row_bytes * height]; + let raw_image: *mut c_void = objc2_core_foundation::CFRetained::into_raw(image) + .as_ptr() + .cast(); + let ctx = unsafe { + let cs = CGColorSpaceCreateDeviceRGB(); + if cs.is_null() { + CGImageRelease(raw_image); + return Err(anyhow!("CGColorSpaceCreateDeviceRGB failed")); + } + let ctx = CGBitmapContextCreate( + bgra.as_mut_ptr().cast(), + width, + height, + 8, + row_bytes, + cs, + K_CGIMAGE_ALPHA_PREMULTIPLIED_FIRST | K_CGBITMAP_BYTE_ORDER32_LITTLE, + ); + CGColorSpaceRelease(cs); + if ctx.is_null() { + CGImageRelease(raw_image); + return Err(anyhow!("CGBitmapContextCreate failed")); + } + let rect = CGRect { + origin: CGPoint { x: 0.0, y: 0.0 }, + size: CGSize { + width: width as f64, + height: height as f64, + }, + }; + CGContextDrawImage(ctx, rect, raw_image); + CGImageRelease(raw_image); + ctx + }; + unsafe { CGContextRelease(ctx) }; + + Ok(super::Capture { + bgra, + w: width as u32, + h: height as u32, + }) + } + fn move_mouse(&mut self, x: u32, y: u32) -> Result<()> { self.move_cursor(x, y); Ok(()) diff --git a/src/hint/ax.rs b/src/hint/ax.rs new file mode 100644 index 0000000..a181f1d --- /dev/null +++ b/src/hint/ax.rs @@ -0,0 +1,445 @@ +//! macos accessibility detector: walks the AXUIElement tree of the frontmost +//! app and emits a [`HintCandidate`] per actionable control, mirroring +//! [`super::atspi`] on linux. +//! +//! a node is a candidate if its `AXRole` is in [`HINT_ROLES`] or it advertises +//! an action in [`ACTIONABLE_ACTIONS`]. the action path is what surfaces +//! Electron and ARIA web widgets that show up as `AXGroup`/`AXImage`. +//! +//! `AXPosition`/`AXSize` are already screen-space, so the shared remap in +//! [`super`] is a no-op. + +use std::ffi::c_void; +use std::ptr::NonNull; +use std::time::{Duration, Instant}; + +use anyhow::{anyhow, Result}; + +use objc2_app_kit::NSWorkspace; +use objc2_application_services::{AXError, AXUIElement, AXValue, AXValueType}; +use objc2_core_foundation::{ + kCFBooleanTrue, CFArray, CFRetained, CFString, CFType, CGPoint, CGSize, Type, +}; + +use super::detect::{DetectorOutput, HintCandidate, HintDetector}; +use crate::backend::Backend; + +pub struct AxHintDetector; + +impl HintDetector for AxHintDetector { + fn name(&self) -> &'static str { + "ax" + } + + fn detect(&self, backend: &mut dyn Backend) -> Result { + let (screen_w, screen_h) = backend.screen_size(); + let (candidates, focus_rect) = collect_candidates(screen_w, screen_h)?; + Ok(DetectorOutput { + candidates, + capture_w: screen_w, + capture_h: screen_h, + focus_rect, + }) + } +} + +/// upper bound on returned targets. generous because Electron pages routinely +/// blow past 1k actionable elements and we rank-and-cap downstream. +const MAX_TARGETS: usize = 8192; + +/// wall-clock cap on the whole walk. even with `set_messaging_timeout` a deep +/// tree of slow children can drag on, so we bail and let the caller fall back +/// to cv. +const COLLECT_BUDGET: Duration = Duration::from_secs(2); + +/// per-element ax message timeout. the 6 s default lets one frozen Electron +/// app ruin hint mode. 0.4 s is generous for healthy trees. +const AX_MESSAGING_TIMEOUT_SECS: f32 = 0.4; + +/// cycle guard for tree walks, matching the at-spi counterpart. +const MAX_TREE_DEPTH: usize = 100; + +/// roles that are intrinsically clickable. anything outside this list can +/// still be a hint via [`ACTIONABLE_ACTIONS`]. +const HINT_ROLES: &[&str] = &[ + "AXButton", + "AXLink", + "AXMenuItem", + "AXMenuBarItem", + "AXCheckBox", + "AXRadioButton", + "AXPopUpButton", + "AXMenuButton", + "AXTab", + "AXSearchField", + "AXTextField", + "AXTextArea", + "AXComboBox", + "AXDisclosureTriangle", + "AXIncrementor", + "AXSlider", + "AXSwitch", + "AXRow", + "AXOutlineRow", + "AXCell", + "AXToolbarButton", + "AXSegmentedControl", + "AXStepper", +]; + +/// actions that mean "does something when clicked". an element advertising +/// any of these is a hint regardless of role, which is what makes Electron +/// and ARIA web widgets work. +const ACTIONABLE_ACTIONS: &[&str] = &["AXPress", "AXShowMenu", "AXPick", "AXOpen"]; + +/// structural roles that never carry an action and are only walked for their +/// children. probing `AXActions` on each is one mach ipc per node, a +/// measurable slice of walk time on busy trees, so skip the probe. +const SKIP_ACTION_PROBE_ROLES: &[&str] = &[ + "AXWindow", + "AXSheet", + "AXDrawer", + "AXSplitter", + "AXSplitGroup", + "AXScrollBar", + "AXScrollArea", + "AXLayoutArea", + "AXLayoutItem", + "AXToolbar", + "AXMenuBar", + "AXMenu", + "AXTable", + "AXOutline", + "AXList", + "AXBrowser", + "AXMatte", + "AXRuler", + "AXRulerMarker", + "AXUnknown", +]; + +fn collect_candidates( + screen_w: u32, + screen_h: u32, +) -> Result<(Vec, Option<(u32, u32, u32, u32)>)> { + let pid = frontmost_pid()?; + // SAFETY: pid is a valid pid_t (32-bit signed int) from NSWorkspace. + let app = unsafe { AXUIElement::new_application(pid) }; + let _ = unsafe { app.set_messaging_timeout(AX_MESSAGING_TIMEOUT_SECS) }; + wake_renderer_accessibility(&app); + + let roots = collect_roots(&app); + // first root's rect scopes the cv fallback. it's the focused/main window + // when we have one, or the first window in the all-windows fallback. + let focus_rect = roots + .first() + .and_then(|root| rect_of(root)) + .and_then(|r| clamp_rect(r, screen_w, screen_h)); + + let mut candidates = Vec::new(); + let deadline = Instant::now() + COLLECT_BUDGET; + for root in &roots { + if candidates.len() >= MAX_TARGETS || Instant::now() >= deadline { + break; + } + walk(root, 0, &mut candidates, screen_w, screen_h, deadline); + } + + Ok((candidates, focus_rect)) +} + +fn frontmost_pid() -> Result { + let workspace = NSWorkspace::sharedWorkspace(); + let app = workspace + .frontmostApplication() + .ok_or_else(|| anyhow!("NSWorkspace has no frontmost application"))?; + Ok(app.processIdentifier()) +} + +/// roots to walk, in priority order. the focused/main window is preferred +/// because it scopes cv fallback nicely. the all-windows fallback catches +/// apps that don't expose a focused window while the cursor hovers content. +fn collect_roots(app: &AXUIElement) -> Vec> { + let mut roots = Vec::new(); + for attr in ["AXFocusedWindow", "AXMainWindow"] { + if let Some(value) = copy_attribute(app, &ax_str(attr)) { + if let Ok(window) = value.downcast::() { + roots.push(window); + break; + } + } + } + if roots.is_empty() { + if let Some(value) = copy_attribute(app, &ax_str("AXWindows")) { + if let Some(arr) = cf_array(value) { + for child in arr.iter() { + if let Some(window) = child.downcast_ref::() { + roots.push(window.retain()); + } + } + } + } + } + if roots.is_empty() { + roots.push(app.retain()); + } + roots +} + +fn walk( + element: &AXUIElement, + depth: usize, + out: &mut Vec, + screen_w: u32, + screen_h: u32, + deadline: Instant, +) { + if depth >= MAX_TREE_DEPTH || out.len() >= MAX_TARGETS || Instant::now() >= deadline { + return; + } + + // fast path is the role match. slow path is `AXActions`, one extra ipc + // per element. it's what makes web content work, but we skip it for + // structural roles where no real control lives. + let role = role_of(element); + let role_str = role.as_deref().unwrap_or(""); + let role_match = HINT_ROLES.contains(&role_str); + let skip_action_probe = SKIP_ACTION_PROBE_ROLES.contains(&role_str); + let is_hint = role_match || (!skip_action_probe && has_actionable_action(element)); + if debug() { + let indent = " ".repeat(depth.min(40)); + eprintln!( + "{:>3} {}{}{}", + depth, + indent, + role.as_deref().unwrap_or("?"), + if is_hint { " *" } else { "" } + ); + } + if is_hint { + if let Some(rect) = rect_of(element) { + if let Some(bbox) = clamp_rect(rect, screen_w, screen_h) { + out.push(HintCandidate { bbox, score: 1.0 }); + } + } + } + + let Some(children) = children_of(element) else { + return; + }; + for child in children.iter() { + if out.len() >= MAX_TARGETS || Instant::now() >= deadline { + break; + } + if let Some(child_el) = child.downcast_ref::() { + walk(child_el, depth + 1, out, screen_w, screen_h, deadline); + } + } +} + +fn debug() -> bool { + // cache the env lookup: the walk runs this on every node and `var_os` is + // a syscall each call. + use std::sync::OnceLock; + static FLAG: OnceLock = OnceLock::new(); + *FLAG.get_or_init(|| std::env::var_os("STOCHOS_AX_DEBUG").is_some()) +} + +/// nudge Chromium/Electron into building its renderer accessibility tree. +/// Chrome gates that tree behind an explicit consumer signal, without which +/// only the native window chrome is exposed. two attributes cover it: +/// +/// - `AXManualAccessibility=true`: Chrome's own opt-in, honoured by Slack, +/// Discord, VS Code, Notion. +/// - `AXEnhancedUserInterface=true`: broader AppKit signal that VoiceOver +/// sets. some apps expose extra structure when they see it. +/// +/// both calls are advisory and ignore errors. we then give the renderer a +/// brief moment to publish, paid only on the first walk. +fn wake_renderer_accessibility(app: &AXUIElement) { + let Some(boolean_true) = (unsafe { kCFBooleanTrue.as_ref() }) else { + return; + }; + let truthy: &CFType = boolean_true.as_ref(); + for attr in ["AXManualAccessibility", "AXEnhancedUserInterface"] { + let name = ax_str(attr); + let _ = unsafe { app.set_attribute_value(&name, truthy) }; + } + // first call enables the tree, so wait for it to populate. skipped after + // that: the toggle persists on the app element until quit. + use std::sync::atomic::{AtomicBool, Ordering}; + static FIRST_WAKE: AtomicBool = AtomicBool::new(true); + if FIRST_WAKE.swap(false, Ordering::Relaxed) { + std::thread::sleep(Duration::from_millis(120)); + } +} + +fn role_of(element: &AXUIElement) -> Option { + let value = copy_attribute(element, &ax_str("AXRole"))?; + Some(value.downcast_ref::()?.to_string()) +} + +fn children_of(element: &AXUIElement) -> Option>> { + let value = copy_attribute(element, &ax_str("AXChildren"))?; + cf_array(value) +} + +/// true if any action this element advertises is in [`ACTIONABLE_ACTIONS`]. +fn has_actionable_action(element: &AXUIElement) -> bool { + let mut raw: *const CFArray = std::ptr::null(); + let Some(out_ptr) = NonNull::new(&mut raw as *mut *const CFArray) else { + return false; + }; + let err = unsafe { element.copy_action_names(out_ptr) }; + if err != AXError::Success { + return false; + } + let Some(nn) = NonNull::new(raw as *mut CFArray) else { + return false; + }; + let arr = unsafe { CFRetained::from_raw(nn) }; + // SAFETY: CFArray of CFString action names; both subclass CFType. + let arr = unsafe { CFRetained::cast_unchecked::>(arr) }; + for entry in arr.iter() { + if let Some(name) = entry.downcast_ref::() { + let s = name.to_string(); + if ACTIONABLE_ACTIONS.contains(&s.as_str()) { + return true; + } + } + } + false +} + +/// downcast a `CFType` into a `CFArray`. the default `CFArray` +/// parameterization is `Opaque`, and reinterpreting as `CFType` is a +/// zero-cost marker swap since both are zero-sized. +fn cf_array(value: CFRetained) -> Option>> { + let arr = value.downcast::().ok()?; + Some(unsafe { CFRetained::cast_unchecked::>(arr) }) +} + +/// read AXPosition + AXSize and convert to a screen-pixel `(x, y, w, h)`. +fn rect_of(element: &AXUIElement) -> Option<(i32, i32, i32, i32)> { + let pos_value = copy_attribute(element, &ax_str("AXPosition"))?; + let size_value = copy_attribute(element, &ax_str("AXSize"))?; + let pos_ax = pos_value.downcast_ref::()?; + let size_ax = size_value.downcast_ref::()?; + let point = read_point(pos_ax)?; + let size = read_size(size_ax)?; + let w = size.width.round() as i32; + let h = size.height.round() as i32; + if w <= 0 || h <= 0 { + return None; + } + Some((point.x.round() as i32, point.y.round() as i32, w, h)) +} + +/// clip a signed screen-rect into the on-screen `u32` box used downstream. +/// off-screen elements are dropped rather than wrapped to the corner. +fn clamp_rect( + (x, y, w, h): (i32, i32, i32, i32), + screen_w: u32, + screen_h: u32, +) -> Option<(u32, u32, u32, u32)> { + let sw = screen_w as i32; + let sh = screen_h as i32; + let x2 = (x + w).min(sw); + let y2 = (y + h).min(sh); + let cx = x.max(0); + let cy = y.max(0); + let cw = (x2 - cx).max(0); + let ch = (y2 - cy).max(0); + if cw <= 0 || ch <= 0 { + return None; + } + Some((cx as u32, cy as u32, cw as u32, ch as u32)) +} + +fn read_point(value: &AXValue) -> Option { + let mut out = CGPoint { x: 0.0, y: 0.0 }; + let ptr = NonNull::from(&mut out).cast::(); + if unsafe { value.value(AXValueType::CGPoint, ptr) } { + Some(out) + } else { + None + } +} + +fn read_size(value: &AXValue) -> Option { + let mut out = CGSize { + width: 0.0, + height: 0.0, + }; + let ptr = NonNull::from(&mut out).cast::(); + if unsafe { value.value(AXValueType::CGSize, ptr) } { + Some(out) + } else { + None + } +} + +/// safe wrapper over `AXUIElement::copy_attribute_value`. returns `None` on +/// any error (unsupported attribute, timeout, invalid element), which is what +/// every call site wants anyway. +fn copy_attribute(element: &AXUIElement, name: &CFString) -> Option> { + let mut raw: *const CFType = std::ptr::null(); + let err = unsafe { + element.copy_attribute_value(name, NonNull::new(&mut raw as *mut *const CFType)?) + }; + if err != AXError::Success { + return None; + } + let nn = NonNull::new(raw as *mut CFType)?; + Some(unsafe { CFRetained::from_raw(nn) }) +} + +/// construct an attribute-name `CFString`. we allocate once per call, which +/// is cheap next to the round-trip ax rpc. static caching would need a `Sync` +/// `CFString`, which the `objc2-core-foundation` wrapper rules out. +fn ax_str(name: &str) -> CFRetained { + CFString::from_str(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clamp_keeps_on_screen_rect() { + assert_eq!( + clamp_rect((10, 20, 30, 40), 1920, 1080), + Some((10, 20, 30, 40)) + ); + } + + #[test] + fn clamp_clips_overflowing_rect() { + assert_eq!( + clamp_rect((1900, 1070, 100, 100), 1920, 1080), + Some((1900, 1070, 20, 10)) + ); + } + + #[test] + fn clamp_drops_offscreen_rect() { + assert_eq!(clamp_rect((-200, -50, 100, 30), 1920, 1080), None); + assert_eq!(clamp_rect((2000, 1100, 100, 30), 1920, 1080), None); + } + + #[test] + fn role_whitelist_covers_common_controls() { + for role in ["AXButton", "AXLink", "AXMenuItem", "AXCheckBox"] { + assert!(HINT_ROLES.contains(&role), "missing role: {role}"); + } + for role in ["AXWindow", "AXGroup", "AXScrollArea"] { + assert!(!HINT_ROLES.contains(&role), "unexpected role: {role}"); + } + } + + #[test] + fn action_list_includes_press_and_show_menu() { + assert!(ACTIONABLE_ACTIONS.contains(&"AXPress")); + assert!(ACTIONABLE_ACTIONS.contains(&"AXShowMenu")); + } +} diff --git a/src/hint/detect.rs b/src/hint/detect.rs index 9d4f162..9710fd8 100644 --- a/src/hint/detect.rs +++ b/src/hint/detect.rs @@ -22,7 +22,6 @@ pub struct DetectorOutput { pub capture_h: u32, /// The window the candidates are scoped to (screen coordinates), when the /// detector knows it; lets `auto` keep CV supplements in the active window. - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] pub focus_rect: Option<(u32, u32, u32, u32)>, } @@ -55,7 +54,6 @@ fn atspi_detector() -> anyhow::Result> { /// Fewest AT-SPI targets the cascade accepts without falling back to CV; /// fewer means the accessibility tree is absent or a stub. -#[cfg(target_os = "linux")] const CASCADE_MIN_ATSPI_TARGETS: usize = 5; /// The default detector: prefer semantic AT-SPI targets, but supplement them @@ -69,22 +67,88 @@ impl HintDetector for AutoDetector { fn detect(&self, backend: &mut dyn Backend) -> anyhow::Result { #[cfg(target_os = "linux")] - match super::atspi::AtspiHintDetector.detect(backend) { - Ok(output) if output.candidates.len() >= CASCADE_MIN_ATSPI_TARGETS => { - return Ok(supplement_with_cv(backend, output)); + { + match super::atspi::AtspiHintDetector.detect(backend) { + Ok(output) if output.candidates.len() >= CASCADE_MIN_ATSPI_TARGETS => { + return Ok(supplement_with_cv(backend, output)); + } + Ok(output) => eprintln!( + "hint: atspi found only {} targets, falling back to cv", + output.candidates.len() + ), + Err(e) => eprintln!("hint: atspi detection failed, falling back to cv: {e:#}"), } - Ok(output) => eprintln!( - "hint: atspi found only {} targets, falling back to cv", - output.candidates.len() - ), - Err(e) => eprintln!("hint: atspi detection failed, falling back to cv: {e:#}"), + super::cv::CvHintDetector.detect(backend) } - super::cv::CvHintDetector.detect(backend) + + // On macOS the AX tree is often thin (Electron/Firefox), so when the + // semantic pass is too sparse to use directly we still keep its focus + // rect to scope the CV fallback to the active window rather than + // painting hints across the whole desktop. + #[cfg(target_os = "macos")] + { + let mut fallback_focus: Option<(u32, u32, u32, u32)> = None; + match super::ax::AxHintDetector.detect(backend) { + Ok(output) if output.candidates.len() >= CASCADE_MIN_ATSPI_TARGETS => { + return Ok(supplement_with_cv(backend, output)); + } + Ok(output) => { + eprintln!( + "hint: ax found only {} targets, falling back to cv", + output.candidates.len() + ); + fallback_focus = output.focus_rect; + } + Err(e) => eprintln!("hint: ax detection failed, falling back to cv: {e:#}"), + } + let cv_output = super::cv::CvHintDetector.detect(backend)?; + Ok(scope_cv_output( + cv_output, + fallback_focus, + backend.screen_size(), + )) + } + } +} + +/// Clip a pure-CV pass to the active-window rect that the semantic detector +/// did expose, even when its candidate list was too sparse to use directly. +/// Capture coordinates are physical pixels; focus rects are screen-logical, so +/// we filter in logical space after the shared remap. +#[cfg(target_os = "macos")] +fn scope_cv_output( + output: DetectorOutput, + focus_rect: Option<(u32, u32, u32, u32)>, + screen: (u32, u32), +) -> DetectorOutput { + let Some(rect) = focus_rect else { + return output; + }; + let (screen_w, screen_h) = screen; + let logical = super::remap_to_screen( + output.candidates, + output.capture_w, + output.capture_h, + screen_w, + screen_h, + ); + let kept: Vec = logical + .into_iter() + .filter(|c| { + let cx = c.bbox.0 + c.bbox.2 / 2; + let cy = c.bbox.1 + c.bbox.3 / 2; + cx >= rect.0 && cx < rect.0 + rect.2 && cy >= rect.1 && cy < rect.1 + rect.3 + }) + .collect(); + DetectorOutput { + candidates: kept, + capture_w: screen_w, + capture_h: screen_h, + focus_rect: Some(rect), } } /// Add visual candidates that are not already covered by a precise semantic box. -#[cfg(target_os = "linux")] fn supplement_with_cv(backend: &mut dyn Backend, mut semantic: DetectorOutput) -> DetectorOutput { let Some(window) = semantic.focus_rect else { return semantic; @@ -119,7 +183,6 @@ fn supplement_with_cv(backend: &mut dyn Backend, mut semantic: DetectorOutput) - semantic } -#[cfg(target_os = "linux")] fn merge_visual_supplements( semantic: &mut Vec, visual: Vec, @@ -176,17 +239,14 @@ fn merge_visual_supplements( semantic.extend(replacements); } -#[cfg(target_os = "linux")] fn contains_point((x, y, w, h): (u32, u32, u32, u32), (px, py): (u32, u32)) -> bool { px >= x && px < x + w && py >= y && py < y + h } -#[cfg(target_os = "linux")] fn center((x, y, w, h): (u32, u32, u32, u32)) -> (u32, u32) { (x + w / 2, y + h / 2) } -#[cfg(target_os = "linux")] fn precise_semantic_box_covers( semantic: (u32, u32, u32, u32), visual: (u32, u32, u32, u32), @@ -194,7 +254,6 @@ fn precise_semantic_box_covers( contains_point(semantic, center(visual)) && !is_broad_horizontal_container(semantic, visual) } -#[cfg(target_os = "linux")] fn broad_semantic_box_contains( semantic: (u32, u32, u32, u32), visual: (u32, u32, u32, u32), @@ -202,7 +261,6 @@ fn broad_semantic_box_contains( contains_point(semantic, center(visual)) && is_broad_horizontal_container(semantic, visual) } -#[cfg(target_os = "linux")] fn is_broad_horizontal_container( semantic: (u32, u32, u32, u32), visual: (u32, u32, u32, u32), @@ -213,7 +271,6 @@ fn is_broad_horizontal_container( && u64::from(semantic_h) <= u64::from(visual_h.max(1)) * 3 } -#[cfg(target_os = "linux")] fn leftmost_anchor_icon( semantic: (u32, u32, u32, u32), raw_visual: &[HintCandidate], @@ -227,7 +284,6 @@ fn leftmost_anchor_icon( .min_by_key(|&(x, _, _, _)| x) } -#[cfg(target_os = "linux")] fn is_left_anchor_icon(semantic: (u32, u32, u32, u32), icon: (u32, u32, u32, u32)) -> bool { if !broad_semantic_box_contains(semantic, icon) || !is_icon_like(icon) { return false; @@ -236,14 +292,12 @@ fn is_left_anchor_icon(semantic: (u32, u32, u32, u32), icon: (u32, u32, u32, u32 center(icon).0 <= x + w / 3 } -#[cfg(target_os = "linux")] fn is_icon_like((_, _, w, h): (u32, u32, u32, u32)) -> bool { let short = w.min(h).max(1); let long = w.max(h); long <= 32 && long <= short * 2 } -#[cfg(target_os = "linux")] fn text_target_after_icon( semantic: (u32, u32, u32, u32), icon: (u32, u32, u32, u32), @@ -259,7 +313,7 @@ fn text_target_after_icon( } } -#[cfg(all(test, target_os = "linux"))] +#[cfg(test)] mod tests { use super::*; diff --git a/src/hint/mod.rs b/src/hint/mod.rs index 2a4d9e7..3bcdab3 100644 --- a/src/hint/mod.rs +++ b/src/hint/mod.rs @@ -9,6 +9,8 @@ #[cfg(target_os = "linux")] mod atspi; +#[cfg(target_os = "macos")] +mod ax; #[cfg(target_os = "linux")] mod compositor; mod cv; From 7d906b899910351165c1b7199e4a2840448cdb4b Mon Sep 17 00:00:00 2001 From: Konstantinos Artopoulos Date: Mon, 6 Jul 2026 18:31:07 +0300 Subject: [PATCH 3/4] feat(hints): allow transition to free mode from hint mode --- src/mode/hint.rs | 13 +++++++++++++ src/mode/normal.rs | 1 + 2 files changed, 14 insertions(+) diff --git a/src/mode/hint.rs b/src/mode/hint.rs index f819342..98ba022 100644 --- a/src/mode/hint.rs +++ b/src/mode/hint.rs @@ -98,6 +98,19 @@ pub(super) fn handle_key( target: None, })) } + KeyEvent::FreeMode => { + let (x, y) = if let Some((x, y)) = target { + (x, y) + } else { + backend.mouse_pos()? + }; + + Ok(ModeTransition::Enter(Mode::Free { + x, + y, + speed: config().free.base_speed.max(1), + })) + } _ => Ok(ModeTransition::Stay), } } diff --git a/src/mode/normal.rs b/src/mode/normal.rs index f5d1cd6..aedb07d 100644 --- a/src/mode/normal.rs +++ b/src/mode/normal.rs @@ -281,6 +281,7 @@ pub(super) fn draw( ) -> anyhow::Result<()> { draw_grid(pixels, width, height, input_state, dragging, false, backend) } + fn collides_with_hint( key: Key, input_state: &InputState, From 707e1cbe3f02dadd7ce3f63f79ca29a09f9ca9cd Mon Sep 17 00:00:00 2001 From: Konstantinos Artopoulos Date: Mon, 6 Jul 2026 18:33:19 +0300 Subject: [PATCH 4/4] docs: unclaudify hints readme section --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5a4bdcd..fef8fa3 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,9 @@ An alternative mode that labels likely clickable UI targets directly, instead of Detection is pluggable behind a `HintDetector` trait, selected by `hint.detector`: -- **`atspi`** (Linux, built in): reads the desktop [AT-SPI](https://docs.rs/atspi) accessibility tree over D-Bus and uses each interactive element's *own* role and bounding box. This is the most accurate source where it works — exact boxes, real semantics (button, link, menu item, table row…), no pixel guesswork, essentially no false positives. Each application's whole tree is fetched in a single bulk call (with all apps queried concurrently), so detection stays fast even on browser pages with thousands of nodes; children that toolkits materialize lazily (GTK tree and list views) are pulled in with batched follow-up calls, and an app that hasn't built its tree yet is nudged into doing so (its first hint launch may be sparse, the next one is complete). Coverage depends on the app: GTK/Qt/Gecko expose rich trees, Electron/Chromium need accessibility enabled, and games/native canvases expose nothing. On Wayland a client can't know its own window's on-screen position, so AT-SPI coordinates come in window-relative; stochos corrects them using the compositor's real window geometry and hints the active window, where a click is going. Geometry comes from a per-compositor provider (`src/hint/compositor/`, one file each — currently **Hyprland** and **Sway** via their IPC sockets; adding a compositor means adding one provider file). On X11 coordinates are already absolute and every visible window is hinted. Caveats: on Wayland compositors without a provider the semantic coordinates may be offset (the CV detector remains position-correct everywhere), and only the primary monitor is handled. +- **`atspi`** (Linux, built in): reads the desktop [AT-SPI](https://docs.rs/atspi) accessibility tree over D-Bus and labels interactive elements using their roles and bounds. It is usually the most accurate detector. GTK, Qt, and Gecko apps tend to work well; Chromium/Electron may need accessibility enabled; games and canvas-heavy apps may expose little or nothing. On Wayland, exact placement requires compositor support; Hyprland and Sway are supported. On X11, coordinates are already absolute. Only the primary monitor is handled. - **`cv`** (always available): a pure-Rust computer-vision pass over a screenshot — downscale → Canny edges → morphological closing → connected components → resolution-relative size/shape filtering with a per-box quality score. No model, no network, no services to talk to. CV sees visual *structure*, not semantic clickability, so it will still occasionally label a border, separator, or decorative shape, and miss a low-contrast control. Tune `edge_low_threshold`/`edge_high_threshold` (raise to cut clutter, lower to catch faint controls). -- **`auto`** (default): AT-SPI first; when it is unavailable, not compiled in, or sees almost nothing, CV takes over entirely. And because many apps expose their *chrome* but not their *content* (a file manager's icon view, a browser with content accessibility disabled, an embedded canvas), the cascade also checks the active window for a large contiguous region without semantic targets and lets CV label just that region. Semantic hints stay authoritative everywhere they exist, which is what keeps false positives near zero. +- **`auto`** (default): tries AT-SPI first and falls back to CV when needed. If an app exposes window chrome but not content, CV labels the unhandled content area. ```toml [hint]