diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cff9fd83..fc26698f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,11 @@ jobs: - name: cargo fmt run: cargo fmt --all -- --check + - name: check Java formatting + uses: axel-op/googlejavaformat-action@v4 + with: + args: "--aosp --set-exit-if-changed" + cargo-deny: runs-on: ubuntu-22.04 steps: @@ -81,3 +86,18 @@ jobs: - name: cargo test -p accesskit_windows if: matrix.os == 'windows-2019' run: cargo test -p accesskit_windows + + check-android-dex: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + - uses: android-actions/setup-android@v2 + - run: sdkmanager "platforms;android-30" + - run: sdkmanager "build-tools;33.0.2" + - run: cp platforms/android/classes.dex platforms/android/classes.dex.orig + - run: ./platforms/android/build-dex.sh + - run: cmp platforms/android/classes.dex.orig platforms/android/classes.dex diff --git a/Cargo.lock b/Cargo.lock index 16aa3d38..1b059700 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,18 @@ dependencies = [ "serde", ] +[[package]] +name = "accesskit_android" +version = "0.1.0" +dependencies = [ + "accesskit", + "accesskit_consumer", + "jni", + "log", + "once_cell", + "paste", +] + [[package]] name = "accesskit_atspi_common" version = "0.10.1" @@ -100,6 +112,7 @@ name = "accesskit_winit" version = "0.23.1" dependencies = [ "accesskit", + "accesskit_android", "accesskit_macos", "accesskit_unix", "accesskit_windows", diff --git a/Cargo.toml b/Cargo.toml index dd1bae5e..3c4613ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "common", "consumer", + "platforms/android", "platforms/atspi-common", "platforms/macos", "platforms/unix", diff --git a/README.md b/README.md index 813fc3b9..07919cc4 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ The current released platform adapters are all at rough feature parity. They don The following platform adapters are currently available: +* [Android adapter](https://crates.io/crates/accesskit_android): This adapter implements the Java-based Android accessibility API. * [macOS adapter](https://crates.io/crates/accesskit_macos): This adapter implements the NSAccessibility protocols in the AppKit framework. * [Unix adapter](https://crates.io/crates/accesskit_unix): This adapter implements the AT-SPI D-Bus interfaces, using [zbus](https://github.com/dbus2/zbus), a pure-Rust implementation of D-Bus. * [Windows adapter](https://crates.io/crates/accesskit_windows): This adapter implements UI Automation, the current Windows accessibility API. diff --git a/common/README.md b/common/README.md index a6264ee0..b1fb3666 100644 --- a/common/README.md +++ b/common/README.md @@ -7,6 +7,7 @@ To use AccessKit in your application or toolkit, you will also need a platform a * [accesskit_windows](https://crates.io/crates/accesskit_windows): exposes an AccessKit tree on Windows using the UI Automation API * [accesskit_macos](https://crates.io/crates/accesskit_macos): exposes an AccessKit tree on MacOS through the Cocoa `NSAccessibility` protocol * [accesskit_unix](https://crates.io/crates/accesskit_unix): exposes an AccessKit tree on Linux and Unix systems through the AT-SPI protocol +* [accesskit_android](https://crates.io/crates/accesskit_android): exposes an AccessKit tree on Android through the Java-based Android accessibility API * [accesskit_winit](https://crates.io/crates/accesskit_winit): wraps other platform adapters for use with the [winit](https://crates.io/crates/winit) windowing library Some platform adapters include simple examples. diff --git a/common/src/lib.rs b/common/src/lib.rs index 93b8e167..e023c715 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -1382,6 +1382,20 @@ impl From for FrozenNode { } } +impl From<&FrozenNode> for Node { + fn from(node: &FrozenNode) -> Self { + Self { + role: node.role, + actions: node.actions, + flags: node.flags, + properties: Properties { + indices: node.properties.indices, + values: node.properties.values.to_vec(), + }, + } + } +} + impl FrozenNode { #[inline] pub fn role(&self) -> Role { diff --git a/consumer/src/node.rs b/consumer/src/node.rs index 2a14a998..780285a3 100644 --- a/consumer/src/node.rs +++ b/consumer/src/node.rs @@ -43,7 +43,7 @@ pub struct Node<'a> { } impl<'a> Node<'a> { - pub(crate) fn data(&self) -> &NodeData { + pub fn data(&self) -> &NodeData { &self.state.data } @@ -446,9 +446,7 @@ impl<'a> Node<'a> { && self.is_selected().is_none() } - // The future of the `Action` enum is undecided, so keep the following - // function private for now. - fn supports_action(&self, action: Action) -> bool { + pub fn supports_action(&self, action: Action) -> bool { self.data().supports_action(action) } diff --git a/consumer/src/text.rs b/consumer/src/text.rs index 06863f99..77d38989 100644 --- a/consumer/src/text.rs +++ b/consumer/src/text.rs @@ -192,6 +192,10 @@ pub struct Position<'a> { } impl<'a> Position<'a> { + pub fn to_raw(self) -> WeakPosition { + self.inner.downgrade() + } + pub fn inner_node(&self) -> &Node { &self.inner.node } @@ -223,6 +227,14 @@ impl<'a> Position<'a> { self.is_document_end() || self.inner.is_paragraph_end() } + pub fn is_paragraph_separator(&self) -> bool { + if self.is_document_end() { + return false; + } + let next = self.forward_to_character_end(); + !next.is_document_end() && next.is_paragraph_end() + } + pub fn is_page_start(&self) -> bool { self.is_document_start() } @@ -292,6 +304,20 @@ impl<'a> Position<'a> { lines_before_current } + pub fn biased_to_start(&self) -> Self { + Self { + root_node: self.root_node, + inner: self.inner.biased_to_start(&self.root_node), + } + } + + pub fn biased_to_end(&self) -> Self { + Self { + root_node: self.root_node, + inner: self.inner.biased_to_end(&self.root_node), + } + } + pub fn forward_to_character_start(&self) -> Self { let pos = self.inner.biased_to_start(&self.root_node); Self { @@ -905,6 +931,16 @@ impl<'a> Node<'a> { }) } + pub fn text_selection_anchor(&self) -> Option { + self.data().text_selection().map(|selection| { + let anchor = InnerPosition::clamped_upgrade(self.tree_state, selection.anchor).unwrap(); + Position { + root_node: *self, + inner: anchor, + } + }) + } + pub fn text_selection_focus(&self) -> Option { self.data().text_selection().map(|selection| { let focus = InnerPosition::clamped_upgrade(self.tree_state, selection.focus).unwrap(); diff --git a/consumer/src/tree.rs b/consumer/src/tree.rs index 5047aa1e..c45ad7fd 100644 --- a/consumer/src/tree.rs +++ b/consumer/src/tree.rs @@ -205,6 +205,14 @@ impl State { self.is_host_focused } + pub fn focus_id_in_tree(&self) -> NodeId { + self.focus + } + + pub fn focus_in_tree(&self) -> Node<'_> { + self.node_by_id(self.focus_id_in_tree()).unwrap() + } + pub fn focus_id(&self) -> Option { self.is_host_focused.then_some(self.focus) } diff --git a/platforms/android/.gitignore b/platforms/android/.gitignore new file mode 100644 index 00000000..6b468b62 --- /dev/null +++ b/platforms/android/.gitignore @@ -0,0 +1 @@ +*.class diff --git a/platforms/android/Cargo.toml b/platforms/android/Cargo.toml new file mode 100644 index 00000000..9192d749 --- /dev/null +++ b/platforms/android/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "accesskit_android" +version = "0.1.0" +authors.workspace = true +license.workspace = true +description = "AccessKit UI accessibility infrastructure: Android adapter" +categories.workspace = true +keywords = ["gui", "ui", "accessibility"] +repository.workspace = true +readme = "README.md" +edition.workspace = true +rust-version.workspace = true + +[features] +embedded-dex = [] + +[dependencies] +accesskit = { version = "0.17.1", path = "../../common" } +accesskit_consumer = { version = "0.26.0", path = "../../consumer" } +jni = "0.21.1" +log = "0.4.17" +once_cell = "1.17.1" +paste = "1.0.12" diff --git a/platforms/android/README.md b/platforms/android/README.md new file mode 100644 index 00000000..2526b146 --- /dev/null +++ b/platforms/android/README.md @@ -0,0 +1,10 @@ +# AccessKit Android adapter + +This is the Android adapter for [AccessKit](https://accesskit.dev/). + +This adapter is implemented in two layers: + +* The `Adapter` struct is the core low-level adapter. It provides maximum flexibility in the application threading model, the interface between Java and native code, and the implementation of action callbacks, at the expense of requiring its caller to provide glue code. +* The `InjectingAdapter` struct injects accessibility into an arbitrary Android view without requiring the view class to be modified, at the expense of depending on a specific Java class and providing less flexibility in the aspects listed above. + +The most convenient way to use `InjectingAdapter` is to embed a precompiled `.dex` file containing the associated Java class and its inner classes into the native code. This approach requires the `embedded-dex` Cargo feature. diff --git a/platforms/android/build-dex.sh b/platforms/android/build-dex.sh new file mode 100755 index 00000000..71b46cef --- /dev/null +++ b/platforms/android/build-dex.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e -u -o pipefail +cd `dirname $0` +ANDROID_JAR=$ANDROID_HOME/platforms/android-30/android.jar +find java -name '*.class' -delete +javac --source 8 --target 8 --boot-class-path $ANDROID_JAR -Xlint:deprecation `find java -name '*.java'` +$ANDROID_HOME/build-tools/33.0.2/d8 --classpath $ANDROID_JAR --output . `find java -name '*.class'` diff --git a/platforms/android/classes.dex b/platforms/android/classes.dex new file mode 100644 index 00000000..8685c5d7 Binary files /dev/null and b/platforms/android/classes.dex differ diff --git a/platforms/android/java/dev/accesskit/android/Delegate.java b/platforms/android/java/dev/accesskit/android/Delegate.java new file mode 100644 index 00000000..3abca62c --- /dev/null +++ b/platforms/android/java/dev/accesskit/android/Delegate.java @@ -0,0 +1,391 @@ +// Copyright 2025 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +// Derived from the Flutter engine. +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE.chromium file. + +package dev.accesskit.android; + +import android.os.Bundle; +import android.view.MotionEvent; +import android.view.View; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; +import android.view.accessibility.AccessibilityNodeProvider; + +public final class Delegate extends View.AccessibilityDelegate implements View.OnHoverListener { + private final long adapterHandle; + private int accessibilityFocus = AccessibilityNodeProvider.HOST_VIEW_ID; + private int hoverId = AccessibilityNodeProvider.HOST_VIEW_ID; + + private Delegate(long adapterHandle) { + super(); + this.adapterHandle = adapterHandle; + } + + public static void inject(final View host, final long adapterHandle) { + host.post( + new Runnable() { + @Override + public void run() { + if (host.getAccessibilityDelegate() != null) { + throw new IllegalStateException( + "host already has an accessibility delegate"); + } + Delegate delegate = new Delegate(adapterHandle); + host.setAccessibilityDelegate(delegate); + host.setOnHoverListener(delegate); + } + }); + } + + public static void remove(final View host) { + host.post( + new Runnable() { + @Override + public void run() { + View.AccessibilityDelegate delegate = host.getAccessibilityDelegate(); + if (delegate != null && delegate instanceof Delegate) { + host.setAccessibilityDelegate(null); + host.setOnHoverListener(null); + } + } + }); + } + + private static AccessibilityEvent newEvent(View host, int virtualViewId, int type) { + AccessibilityEvent e = AccessibilityEvent.obtain(type); + e.setPackageName(host.getContext().getPackageName()); + if (virtualViewId == AccessibilityNodeProvider.HOST_VIEW_ID) { + e.setSource(host); + } else { + e.setSource(host, virtualViewId); + } + return e; + } + + private static void sendCompletedEvent(View host, AccessibilityEvent e) { + host.getParent().requestSendAccessibilityEvent(host, e); + } + + private static void sendEventInternal(View host, int virtualViewId, int type) { + AccessibilityEvent e = newEvent(host, virtualViewId, type); + if (type == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) { + e.setContentChangeTypes(AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE); + } + sendCompletedEvent(host, e); + } + + public static void sendEvent(final View host, final int virtualViewId, final int type) { + host.post( + new Runnable() { + @Override + public void run() { + sendEventInternal(host, virtualViewId, type); + } + }); + } + + private static void sendTextChangedInternal( + View host, int virtualViewId, String oldValue, String newValue) { + int i; + for (i = 0; i < oldValue.length() && i < newValue.length(); ++i) { + if (oldValue.charAt(i) != newValue.charAt(i)) { + break; + } + } + if (i >= oldValue.length() && i >= newValue.length()) { + return; // Text did not change + } + AccessibilityEvent e = + newEvent(host, virtualViewId, AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED); + e.setBeforeText(oldValue); + e.getText().add(newValue); + int firstDifference = i; + e.setFromIndex(firstDifference); + int oldIndex = oldValue.length() - 1; + int newIndex = newValue.length() - 1; + while (oldIndex >= firstDifference && newIndex >= firstDifference) { + if (oldValue.charAt(oldIndex) != newValue.charAt(newIndex)) { + break; + } + --oldIndex; + --newIndex; + } + e.setRemovedCount(oldIndex - firstDifference + 1); + e.setAddedCount(newIndex - firstDifference + 1); + sendCompletedEvent(host, e); + } + + public static void sendTextChanged( + final View host, + final int virtualViewId, + final String oldValue, + final String newValue) { + host.post( + new Runnable() { + @Override + public void run() { + sendTextChangedInternal(host, virtualViewId, oldValue, newValue); + } + }); + } + + private static void sendTextSelectionChangedInternal( + View host, int virtualViewId, String text, int start, int end) { + AccessibilityEvent e = + newEvent(host, virtualViewId, AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED); + e.getText().add(text); + e.setFromIndex(start); + e.setToIndex(end); + e.setItemCount(text.length()); + sendCompletedEvent(host, e); + } + + public static void sendTextSelectionChanged( + final View host, + final int virtualViewId, + final String text, + final int start, + final int end) { + host.post( + new Runnable() { + @Override + public void run() { + sendTextSelectionChangedInternal(host, virtualViewId, text, start, end); + } + }); + } + + private static void sendTextTraversedInternal( + View host, + int virtualViewId, + int granularity, + boolean forward, + int segmentStart, + int segmentEnd) { + AccessibilityEvent e = + newEvent( + host, + virtualViewId, + AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY); + e.setMovementGranularity(granularity); + e.setAction( + forward + ? AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY + : AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY); + e.setFromIndex(segmentStart); + e.setToIndex(segmentEnd); + sendCompletedEvent(host, e); + } + + public static void sendTextTraversed( + final View host, + final int virtualViewId, + final int granularity, + final boolean forward, + final int segmentStart, + final int segmentEnd) { + host.post( + new Runnable() { + @Override + public void run() { + sendTextTraversedInternal( + host, + virtualViewId, + granularity, + forward, + segmentStart, + segmentEnd); + } + }); + } + + private static native boolean populateNodeInfo( + long adapterHandle, + View host, + int screenX, + int screenY, + int virtualViewId, + AccessibilityNodeInfo nodeInfo); + + private static native int getInputFocus(long adapterHandle); + + private static native int getVirtualViewAtPoint(long adapterHandle, float x, float y); + + private static native boolean performAction(long adapterHandle, int virtualViewId, int action); + + private static native boolean setTextSelection( + long adapterHandle, View host, int virtualViewId, int anchor, int focus); + + private static native boolean collapseTextSelection( + long adapterHandle, View host, int virtualViewId); + + private static native boolean traverseText( + long adapterHandle, + View host, + int virtualViewId, + int granularity, + boolean forward, + boolean extendSelection); + + @Override + public AccessibilityNodeProvider getAccessibilityNodeProvider(final View host) { + return new AccessibilityNodeProvider() { + @Override + public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) { + int[] location = new int[2]; + host.getLocationOnScreen(location); + AccessibilityNodeInfo nodeInfo; + if (virtualViewId == HOST_VIEW_ID) { + nodeInfo = AccessibilityNodeInfo.obtain(host); + } else { + nodeInfo = AccessibilityNodeInfo.obtain(host, virtualViewId); + } + nodeInfo.setPackageName(host.getContext().getPackageName()); + nodeInfo.setVisibleToUser(true); + if (!populateNodeInfo( + adapterHandle, host, location[0], location[1], virtualViewId, nodeInfo)) { + nodeInfo.recycle(); + return null; + } + if (virtualViewId == accessibilityFocus) { + nodeInfo.setAccessibilityFocused(true); + nodeInfo.addAction(AccessibilityAction.ACTION_CLEAR_ACCESSIBILITY_FOCUS); + } else { + nodeInfo.setAccessibilityFocused(false); + nodeInfo.addAction(AccessibilityAction.ACTION_ACCESSIBILITY_FOCUS); + } + return nodeInfo; + } + + @Override + public boolean performAction(int virtualViewId, int action, Bundle arguments) { + switch (action) { + case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS: + accessibilityFocus = virtualViewId; + host.invalidate(); + sendEventInternal( + host, + virtualViewId, + AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED); + return true; + case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS: + if (accessibilityFocus == virtualViewId) { + accessibilityFocus = AccessibilityNodeProvider.HOST_VIEW_ID; + } + host.invalidate(); + sendEventInternal( + host, + virtualViewId, + AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); + return true; + case AccessibilityNodeInfo.ACTION_SET_SELECTION: + if (!(arguments != null + && arguments.containsKey( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT) + && arguments.containsKey( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT))) { + return Delegate.collapseTextSelection( + adapterHandle, host, virtualViewId); + } + int anchor = + arguments.getInt( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT); + int focus = + arguments.getInt( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT); + return Delegate.setTextSelection( + adapterHandle, host, virtualViewId, anchor, focus); + case AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY: + case AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY: + int granularity = + arguments.getInt( + AccessibilityNodeInfo + .ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT); + boolean forward = + (action + == AccessibilityNodeInfo + .ACTION_NEXT_AT_MOVEMENT_GRANULARITY); + boolean extendSelection = + arguments.getBoolean( + AccessibilityNodeInfo + .ACTION_ARGUMENT_EXTEND_SELECTION_BOOLEAN); + return Delegate.traverseText( + adapterHandle, + host, + virtualViewId, + granularity, + forward, + extendSelection); + } + if (!Delegate.performAction(adapterHandle, virtualViewId, action)) { + return false; + } + switch (action) { + case AccessibilityNodeInfo.ACTION_CLICK: + sendEventInternal( + host, virtualViewId, AccessibilityEvent.TYPE_VIEW_CLICKED); + break; + } + return true; + } + + @Override + public AccessibilityNodeInfo findFocus(int focusType) { + switch (focusType) { + case AccessibilityNodeInfo.FOCUS_ACCESSIBILITY: + { + AccessibilityNodeInfo result = + createAccessibilityNodeInfo(accessibilityFocus); + if (result != null && result.isAccessibilityFocused()) { + return result; + } + break; + } + case AccessibilityNodeInfo.FOCUS_INPUT: + { + AccessibilityNodeInfo result = + createAccessibilityNodeInfo(getInputFocus(adapterHandle)); + if (result != null && result.isFocused()) { + return result; + } + break; + } + } + return null; + } + }; + } + + @Override + public boolean onHover(View v, MotionEvent event) { + switch (event.getAction()) { + case MotionEvent.ACTION_HOVER_ENTER: + case MotionEvent.ACTION_HOVER_MOVE: + int newId = getVirtualViewAtPoint(adapterHandle, event.getX(), event.getY()); + if (newId != hoverId) { + if (newId != AccessibilityNodeProvider.HOST_VIEW_ID) { + sendEventInternal(v, newId, AccessibilityEvent.TYPE_VIEW_HOVER_ENTER); + } + if (hoverId != AccessibilityNodeProvider.HOST_VIEW_ID) { + sendEventInternal(v, hoverId, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT); + } + hoverId = newId; + } + break; + case MotionEvent.ACTION_HOVER_EXIT: + if (hoverId != AccessibilityNodeProvider.HOST_VIEW_ID) { + sendEventInternal(v, hoverId, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT); + hoverId = AccessibilityNodeProvider.HOST_VIEW_ID; + } + break; + } + return true; + } +} diff --git a/platforms/android/src/adapter.rs b/platforms/android/src/adapter.rs new file mode 100644 index 00000000..a80a3679 --- /dev/null +++ b/platforms/android/src/adapter.rs @@ -0,0 +1,691 @@ +// Copyright 2025 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +// Derived from the Flutter engine. +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE.chromium file. + +use accesskit::{ + Action, ActionData, ActionHandler, ActionRequest, ActivationHandler, Node as NodeData, NodeId, + Point, Role, TextSelection, Tree as TreeData, TreeUpdate, +}; +use accesskit_consumer::{FilterResult, Node, TextPosition, Tree, TreeChangeHandler}; +use jni::{ + errors::Result, + objects::{JClass, JObject}, + sys::{jfloat, jint}, + JNIEnv, +}; + +use crate::{filters::filter, node::NodeWrapper, util::*}; + +fn send_event( + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + virtual_view_id: jint, + event_type: jint, +) { + env.call_static_method( + callback_class, + "sendEvent", + "(Landroid/view/View;II)V", + &[host.into(), virtual_view_id.into(), event_type.into()], + ) + .unwrap(); +} + +fn send_window_content_changed(env: &mut JNIEnv, callback_class: &JClass, host: &JObject) { + send_event( + env, + callback_class, + host, + HOST_VIEW_ID, + EVENT_WINDOW_CONTENT_CHANGED, + ); +} + +fn send_focus_event_if_applicable( + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + node_id_map: &mut NodeIdMap, + node: &Node, +) { + if node.is_root() && node.role() == Role::Window { + return; + } + let id = node_id_map.get_or_create_java_id(node); + send_event(env, callback_class, host, id, EVENT_VIEW_FOCUSED); +} + +struct AdapterChangeHandler<'a, 'b, 'c, 'd> { + env: &'a mut JNIEnv<'b>, + callback_class: &'a JClass<'c>, + host: &'a JObject<'d>, + node_id_map: &'a mut NodeIdMap, + sent_window_content_changed: bool, +} + +impl<'a, 'b, 'c, 'd> AdapterChangeHandler<'a, 'b, 'c, 'd> { + fn new( + env: &'a mut JNIEnv<'b>, + callback_class: &'a JClass<'c>, + host: &'a JObject<'d>, + node_id_map: &'a mut NodeIdMap, + ) -> Self { + Self { + env, + callback_class, + host, + node_id_map, + sent_window_content_changed: false, + } + } +} + +impl AdapterChangeHandler<'_, '_, '_, '_> { + fn send_window_content_changed_if_needed(&mut self) { + if self.sent_window_content_changed { + return; + } + send_window_content_changed(self.env, self.callback_class, self.host); + self.sent_window_content_changed = true; + } +} + +impl TreeChangeHandler for AdapterChangeHandler<'_, '_, '_, '_> { + fn node_added(&mut self, _node: &Node) { + self.send_window_content_changed_if_needed(); + // TODO: live regions? + } + + fn node_updated(&mut self, old_node: &Node, new_node: &Node) { + self.send_window_content_changed_if_needed(); + if filter(new_node) != FilterResult::Include { + return; + } + let old_wrapper = NodeWrapper(old_node); + let new_wrapper = NodeWrapper(new_node); + let old_text = old_wrapper.text(); + let new_text = new_wrapper.text(); + if old_text != new_text { + let id = self.node_id_map.get_or_create_java_id(new_node); + let old_text = self + .env + .new_string(old_text.unwrap_or_else(String::new)) + .unwrap(); + let new_text = self + .env + .new_string(new_text.clone().unwrap_or_else(String::new)) + .unwrap(); + self.env + .call_static_method( + self.callback_class, + "sendTextChanged", + "(Landroid/view/View;ILjava/lang/String;Ljava/lang/String;)V", + &[ + self.host.into(), + id.into(), + (&old_text).into(), + (&new_text).into(), + ], + ) + .unwrap(); + } + if old_node.raw_text_selection() != new_node.raw_text_selection() + || (new_node.raw_text_selection().is_some() + && old_node.is_focused() != new_node.is_focused()) + { + if let Some((start, end)) = new_wrapper.text_selection() { + if let Some(text) = new_text { + let id = self.node_id_map.get_or_create_java_id(new_node); + let text = self.env.new_string(text).unwrap(); + self.env + .call_static_method( + self.callback_class, + "sendTextSelectionChanged", + "(Landroid/view/View;ILjava/lang/String;II)V", + &[ + self.host.into(), + id.into(), + (&text).into(), + (start as jint).into(), + (end as jint).into(), + ], + ) + .unwrap(); + } + } + } + // TODO: other events + } + + fn focus_moved(&mut self, _old_node: Option<&Node>, new_node: Option<&Node>) { + if let Some(new_node) = new_node { + send_focus_event_if_applicable( + self.env, + self.callback_class, + self.host, + self.node_id_map, + new_node, + ); + } + } + + fn node_removed(&mut self, _node: &Node) { + self.send_window_content_changed_if_needed(); + // TODO: other events? + } +} + +const PLACEHOLDER_ROOT_ID: NodeId = NodeId(0); + +#[derive(Debug, Default)] +enum State { + #[default] + Inactive, + Placeholder(Tree), + Active(Tree), +} + +impl State { + fn get_or_init_tree( + &mut self, + activation_handler: &mut H, + ) -> &Tree { + match self { + Self::Inactive => { + *self = match activation_handler.request_initial_tree() { + Some(initial_state) => Self::Active(Tree::new(initial_state, true)), + None => { + let placeholder_update = TreeUpdate { + nodes: vec![(PLACEHOLDER_ROOT_ID, NodeData::new(Role::Window))], + tree: Some(TreeData::new(PLACEHOLDER_ROOT_ID)), + focus: PLACEHOLDER_ROOT_ID, + }; + Self::Placeholder(Tree::new(placeholder_update, true)) + } + }; + self.get_or_init_tree(activation_handler) + } + Self::Placeholder(tree) => tree, + Self::Active(tree) => tree, + } + } + + fn get_full_tree(&mut self) -> Option<&mut Tree> { + match self { + Self::Inactive => None, + Self::Placeholder(_) => None, + Self::Active(tree) => Some(tree), + } + } +} + +fn update_tree( + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + node_id_map: &mut NodeIdMap, + tree: &mut Tree, + update: TreeUpdate, +) { + let mut handler = AdapterChangeHandler::new(env, callback_class, host, node_id_map); + tree.update_and_process_changes(update, &mut handler); +} + +/// Low-level AccessKit adapter for Android. +/// +/// This layer provides maximum flexibility in the application threading +/// model, the interface between Java and native code, and the implementation +/// of action callbacks, at the expense of requiring its caller to provide +/// glue code. For a higher-level implementation built on this type, see +/// [`InjectingAdapter`]. +/// +/// Several of this type's functions have a `callback_class` parameter. +/// The reference implementation of the duck-typed contract for this Java class +/// is `dev.accesskit.android.Delegate`, the source code for which is in the +/// `java` directory of this crate. The methods that are called from native +/// code are all marked `public static`, and so far, all of them that are +/// called by this type (rather than [`InjectingAdapter`]) are for sending +/// events. Other implementations may differ by, for example, sending those +/// events synchronously rather than posting them to the UI thread for +/// asynchronous handling. +/// +/// Several of this type's functions have a `host` parameter. This is always +/// a Java object whose class must derive from `android.view.View`. +/// +/// [`InjectingAdapter`]: crate::InjectingAdapter +#[derive(Debug, Default)] +pub struct Adapter { + node_id_map: NodeIdMap, + state: State, +} + +impl Adapter { + /// If and only if the tree has been initialized, call the provided function + /// and apply the resulting update. Note: If the caller's implementation of + /// [`ActivationHandler::request_initial_tree`] initially returned `None`, + /// the [`TreeUpdate`] returned by the provided function must contain + /// a full tree. + pub fn update_if_active( + &mut self, + update_factory: impl FnOnce() -> TreeUpdate, + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + ) { + match &mut self.state { + State::Inactive => (), + State::Placeholder(_) => { + let tree = Tree::new(update_factory(), true); + send_window_content_changed(env, callback_class, host); + let state = tree.state(); + if let Some(focus) = state.focus() { + send_focus_event_if_applicable( + env, + callback_class, + host, + &mut self.node_id_map, + &focus, + ); + } + self.state = State::Active(tree); + } + State::Active(tree) => { + update_tree( + env, + callback_class, + host, + &mut self.node_id_map, + tree, + update_factory(), + ); + } + } + } + + #[allow(clippy::too_many_arguments)] + pub fn populate_node_info( + &mut self, + activation_handler: &mut H, + env: &mut JNIEnv, + host: &JObject, + host_screen_x: jint, + host_screen_y: jint, + virtual_view_id: jint, + jni_node: &JObject, + ) -> Result { + let tree = self.state.get_or_init_tree(activation_handler); + let tree_state = tree.state(); + let node = if virtual_view_id == HOST_VIEW_ID { + tree_state.root() + } else { + let Some(accesskit_id) = self.node_id_map.get_accesskit_id(virtual_view_id) else { + return Ok(false); + }; + let Some(node) = tree_state.node_by_id(accesskit_id) else { + return Ok(false); + }; + node + }; + + let wrapper = NodeWrapper(&node); + wrapper.populate_node_info( + env, + host, + host_screen_x, + host_screen_y, + &mut self.node_id_map, + jni_node, + )?; + Ok(true) + } + + pub fn input_focus( + &mut self, + activation_handler: &mut H, + ) -> jint { + let tree = self.state.get_or_init_tree(activation_handler); + let tree_state = tree.state(); + let node = tree_state.focus_in_tree(); + self.node_id_map.get_or_create_java_id(&node) + } + + pub fn virtual_view_at_point( + &mut self, + activation_handler: &mut H, + x: jfloat, + y: jfloat, + ) -> jint { + let tree = self.state.get_or_init_tree(activation_handler); + let tree_state = tree.state(); + let root = tree_state.root(); + let point = Point::new(x.into(), y.into()); + let point = root.transform().inverse() * point; + let node = root.node_at_point(point, &filter).unwrap_or(root); + self.node_id_map.get_or_create_java_id(&node) + } + + pub fn perform_action( + &mut self, + action_handler: &mut H, + virtual_view_id: jint, + action: jint, + ) -> bool { + let Some(tree) = self.state.get_full_tree() else { + return false; + }; + let tree_state = tree.state(); + let target = if virtual_view_id == HOST_VIEW_ID { + tree_state.root_id() + } else { + let Some(accesskit_id) = self.node_id_map.get_accesskit_id(virtual_view_id) else { + return false; + }; + accesskit_id + }; + let request = match action { + ACTION_CLICK => ActionRequest { + action: { + let node = tree_state.node_by_id(target).unwrap(); + if node.is_focusable() && !node.is_focused() && !node.is_clickable() { + Action::Focus + } else { + Action::Click + } + }, + target, + data: None, + }, + ACTION_FOCUS => ActionRequest { + action: Action::Focus, + target, + data: None, + }, + _ => { + return false; + } + }; + action_handler.do_action(request); + true + } + + fn set_text_selection_common( + &mut self, + action_handler: &mut H, + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + virtual_view_id: jint, + selection_factory: F, + ) -> Option + where + for<'a> F: FnOnce(&'a Node<'a>) -> Option<(TextPosition<'a>, TextPosition<'a>, Extra)>, + { + let tree = self.state.get_full_tree()?; + let tree_state = tree.state(); + let node = if virtual_view_id == HOST_VIEW_ID { + tree_state.root() + } else { + let id = self.node_id_map.get_accesskit_id(virtual_view_id)?; + tree_state.node_by_id(id).unwrap() + }; + let target = node.id(); + // TalkBack expects the text selection change to take effect + // immediately, so we optimistically update the node. + // But don't be *too* optimistic. + if !node.supports_action(Action::SetTextSelection) { + return None; + } + let (anchor, focus, extra) = selection_factory(&node)?; + let selection = TextSelection { + anchor: anchor.to_raw(), + focus: focus.to_raw(), + }; + let mut new_node = NodeData::from(node.data()); + new_node.set_text_selection(selection); + let update = TreeUpdate { + nodes: vec![(node.id(), new_node)], + tree: None, + focus: tree_state.focus_id_in_tree(), + }; + update_tree( + env, + callback_class, + host, + &mut self.node_id_map, + tree, + update, + ); + let request = ActionRequest { + target, + action: Action::SetTextSelection, + data: Some(ActionData::SetTextSelection(selection)), + }; + action_handler.do_action(request); + Some(extra) + } + + #[allow(clippy::too_many_arguments)] + pub fn set_text_selection( + &mut self, + action_handler: &mut H, + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + virtual_view_id: jint, + anchor: jint, + focus: jint, + ) -> bool { + self.set_text_selection_common( + action_handler, + env, + callback_class, + host, + virtual_view_id, + |node| { + let anchor = usize::try_from(anchor).ok()?; + let anchor = node.text_position_from_global_utf16_index(anchor)?; + let focus = usize::try_from(focus).ok()?; + let focus = node.text_position_from_global_utf16_index(focus)?; + Some((anchor, focus, ())) + }, + ) + .is_some() + } + + pub fn collapse_text_selection( + &mut self, + action_handler: &mut H, + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + virtual_view_id: jint, + ) -> bool { + self.set_text_selection_common( + action_handler, + env, + callback_class, + host, + virtual_view_id, + |node| node.text_selection_focus().map(|pos| (pos, pos, ())), + ) + .is_some() + } + + #[allow(clippy::too_many_arguments)] + pub fn traverse_text( + &mut self, + action_handler: &mut H, + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + virtual_view_id: jint, + granularity: jint, + forward: bool, + extend_selection: bool, + ) -> bool { + let Some((segment_start, segment_end)) = self.set_text_selection_common( + action_handler, + env, + callback_class, + host, + virtual_view_id, + |node| { + let current = node.text_selection_focus().unwrap_or_else(|| { + let range = node.document_range(); + if forward { + range.start() + } else { + range.end() + } + }); + if (forward && current.is_document_end()) + || (!forward && current.is_document_start()) + { + return None; + } + let current = if forward { + current.biased_to_start() + } else { + current.biased_to_end() + }; + let (segment_start, segment_end) = match granularity { + MOVEMENT_GRANULARITY_CHARACTER => { + if forward { + (current, current.forward_to_character_end()) + } else { + (current.backward_to_character_start(), current) + } + } + MOVEMENT_GRANULARITY_WORD => { + if forward { + let start = if current.is_word_start() { + current + } else { + let start = current.forward_to_word_start(); + if start.is_document_end() { + return None; + } + start + }; + (start, start.forward_to_word_end()) + } else { + let end = if current.is_line_end() || current.is_word_start() { + current + } else { + let end = current.backward_to_word_start().biased_to_end(); + if end.is_document_start() { + return None; + } + end + }; + (end.backward_to_word_start(), end) + } + } + MOVEMENT_GRANULARITY_LINE => { + if forward { + let start = if current.is_line_start() { + current + } else { + let start = current.forward_to_line_start(); + if start.is_document_end() { + return None; + } + start + }; + (start, start.forward_to_line_end()) + } else { + let end = if current.is_line_end() { + current + } else { + let end = current.backward_to_line_start().biased_to_end(); + if end.is_document_start() { + return None; + } + end + }; + (end.backward_to_line_start(), end) + } + } + MOVEMENT_GRANULARITY_PARAGRAPH => { + if forward { + let mut start = current; + while start.is_paragraph_separator() { + start = start.forward_to_paragraph_start(); + } + if start.is_document_end() { + return None; + } + let mut end = start.forward_to_paragraph_end(); + let prev = end.backward_to_character_start(); + if prev.is_paragraph_separator() { + end = prev; + } + (start, end) + } else { + let mut end = current; + while !end.is_document_start() + && end.backward_to_character_start().is_paragraph_separator() + { + end = end.backward_to_character_start(); + } + if end.is_document_start() { + return None; + } + (end.backward_to_paragraph_start(), end) + } + } + _ => { + return None; + } + }; + if segment_start == segment_end { + return None; + } + let focus = if forward { segment_end } else { segment_start }; + let anchor = if extend_selection { + node.text_selection_anchor().unwrap_or({ + if forward { + segment_start + } else { + segment_end + } + }) + } else { + focus + }; + Some(( + anchor, + focus, + ( + segment_start.to_global_utf16_index(), + segment_end.to_global_utf16_index(), + ), + )) + }, + ) else { + return false; + }; + env.call_static_method( + callback_class, + "sendTextTraversed", + "(Landroid/view/View;IIZII)V", + &[ + host.into(), + virtual_view_id.into(), + granularity.into(), + forward.into(), + (segment_start as jint).into(), + (segment_end as jint).into(), + ], + ) + .unwrap(); + true + } +} diff --git a/platforms/android/src/filters.rs b/platforms/android/src/filters.rs new file mode 100644 index 00000000..40b64ff2 --- /dev/null +++ b/platforms/android/src/filters.rs @@ -0,0 +1,6 @@ +// Copyright 2025 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +pub(crate) use accesskit_consumer::common_filter as filter; diff --git a/platforms/android/src/inject.rs b/platforms/android/src/inject.rs new file mode 100644 index 00000000..ebe399be --- /dev/null +++ b/platforms/android/src/inject.rs @@ -0,0 +1,478 @@ +// Copyright 2025 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +// Derived from jni-rs +// Copyright 2016 Prevoty, Inc. and jni-rs contributors +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +use accesskit::{ActionHandler, ActivationHandler, TreeUpdate}; +use jni::{ + errors::Result, + objects::{GlobalRef, JClass, JObject, WeakRef}, + sys::{jboolean, jfloat, jint, jlong, JNI_FALSE, JNI_TRUE}, + JNIEnv, JavaVM, NativeMethod, +}; +use log::debug; +use once_cell::sync::OnceCell; +use std::{ + collections::BTreeMap, + ffi::c_void, + fmt::{Debug, Formatter}, + sync::{ + atomic::{AtomicI64, Ordering}, + Arc, Mutex, Weak, + }, +}; + +use crate::{adapter::Adapter, util::*}; + +struct InnerInjectingAdapter { + adapter: Adapter, + activation_handler: Box, + action_handler: Box, +} + +impl Debug for InnerInjectingAdapter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InnerInjectingAdapter") + .field("adapter", &self.adapter) + .field("activation_handler", &"ActivationHandler") + .field("action_handler", &"ActionHandler") + .finish() + } +} + +impl InnerInjectingAdapter { + fn populate_node_info( + &mut self, + env: &mut JNIEnv, + host: &JObject, + host_screen_x: jint, + host_screen_y: jint, + virtual_view_id: jint, + jni_node: &JObject, + ) -> Result { + self.adapter.populate_node_info( + &mut *self.activation_handler, + env, + host, + host_screen_x, + host_screen_y, + virtual_view_id, + jni_node, + ) + } + + fn input_focus(&mut self) -> jint { + self.adapter.input_focus(&mut *self.activation_handler) + } + + fn virtual_view_at_point(&mut self, x: jfloat, y: jfloat) -> jint { + self.adapter + .virtual_view_at_point(&mut *self.activation_handler, x, y) + } + + fn perform_action(&mut self, virtual_view_id: jint, action: jint) -> bool { + self.adapter + .perform_action(&mut *self.action_handler, virtual_view_id, action) + } + + fn set_text_selection( + &mut self, + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + virtual_view_id: jint, + anchor: jint, + focus: jint, + ) -> bool { + self.adapter.set_text_selection( + &mut *self.action_handler, + env, + callback_class, + host, + virtual_view_id, + anchor, + focus, + ) + } + + fn collapse_text_selection( + &mut self, + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + virtual_view_id: jint, + ) -> bool { + self.adapter.collapse_text_selection( + &mut *self.action_handler, + env, + callback_class, + host, + virtual_view_id, + ) + } + + #[allow(clippy::too_many_arguments)] + fn traverse_text( + &mut self, + env: &mut JNIEnv, + callback_class: &JClass, + host: &JObject, + virtual_view_id: jint, + granularity: jint, + forward: bool, + extend_selection: bool, + ) -> bool { + self.adapter.traverse_text( + &mut *self.action_handler, + env, + callback_class, + host, + virtual_view_id, + granularity, + forward, + extend_selection, + ) + } +} + +static NEXT_HANDLE: AtomicI64 = AtomicI64::new(0); +static HANDLE_MAP: Mutex>>> = + Mutex::new(BTreeMap::new()); + +fn inner_adapter_from_handle(handle: jlong) -> Option>> { + let handle_map_guard = HANDLE_MAP.lock().unwrap(); + handle_map_guard.get(&handle).and_then(Weak::upgrade) +} + +extern "system" fn populate_node_info( + mut env: JNIEnv, + _class: JClass, + adapter_handle: jlong, + host: JObject, + host_screen_x: jint, + host_screen_y: jint, + virtual_view_id: jint, + node_info: JObject, +) -> jboolean { + let Some(inner_adapter) = inner_adapter_from_handle(adapter_handle) else { + return JNI_FALSE; + }; + let mut inner_adapter = inner_adapter.lock().unwrap(); + if inner_adapter + .populate_node_info( + &mut env, + &host, + host_screen_x, + host_screen_y, + virtual_view_id, + &node_info, + ) + .unwrap() + { + JNI_TRUE + } else { + JNI_FALSE + } +} + +extern "system" fn get_input_focus(_env: JNIEnv, _class: JClass, adapter_handle: jlong) -> jint { + let Some(inner_adapter) = inner_adapter_from_handle(adapter_handle) else { + return HOST_VIEW_ID; + }; + let mut inner_adapter = inner_adapter.lock().unwrap(); + inner_adapter.input_focus() +} + +extern "system" fn get_virtual_view_at_point( + _env: JNIEnv, + _class: JClass, + adapter_handle: jlong, + x: jfloat, + y: jfloat, +) -> jint { + let Some(inner_adapter) = inner_adapter_from_handle(adapter_handle) else { + return HOST_VIEW_ID; + }; + let mut inner_adapter = inner_adapter.lock().unwrap(); + inner_adapter.virtual_view_at_point(x, y) +} + +extern "system" fn perform_action( + _env: JNIEnv, + _class: JClass, + adapter_handle: jlong, + virtual_view_id: jint, + action: jint, +) -> jboolean { + let Some(inner_adapter) = inner_adapter_from_handle(adapter_handle) else { + return JNI_FALSE; + }; + let mut inner_adapter = inner_adapter.lock().unwrap(); + if inner_adapter.perform_action(virtual_view_id, action) { + JNI_TRUE + } else { + JNI_FALSE + } +} + +extern "system" fn set_text_selection( + mut env: JNIEnv, + class: JClass, + adapter_handle: jlong, + host: JObject, + virtual_view_id: jint, + anchor: jint, + focus: jint, +) -> jboolean { + let Some(inner_adapter) = inner_adapter_from_handle(adapter_handle) else { + return JNI_FALSE; + }; + let mut inner_adapter = inner_adapter.lock().unwrap(); + if inner_adapter.set_text_selection(&mut env, &class, &host, virtual_view_id, anchor, focus) { + JNI_TRUE + } else { + JNI_FALSE + } +} + +extern "system" fn collapse_text_selection( + mut env: JNIEnv, + class: JClass, + adapter_handle: jlong, + host: JObject, + virtual_view_id: jint, +) -> jboolean { + let Some(inner_adapter) = inner_adapter_from_handle(adapter_handle) else { + return JNI_FALSE; + }; + let mut inner_adapter = inner_adapter.lock().unwrap(); + if inner_adapter.collapse_text_selection(&mut env, &class, &host, virtual_view_id) { + JNI_TRUE + } else { + JNI_FALSE + } +} + +extern "system" fn traverse_text( + mut env: JNIEnv, + class: JClass, + adapter_handle: jlong, + host: JObject, + virtual_view_id: jint, + granularity: jint, + forward: jboolean, + extend_selection: jboolean, +) -> jboolean { + let Some(inner_adapter) = inner_adapter_from_handle(adapter_handle) else { + return JNI_FALSE; + }; + let mut inner_adapter = inner_adapter.lock().unwrap(); + if inner_adapter.traverse_text( + &mut env, + &class, + &host, + virtual_view_id, + granularity, + forward == JNI_TRUE, + extend_selection == JNI_TRUE, + ) { + JNI_TRUE + } else { + JNI_FALSE + } +} + +fn delegate_class(env: &mut JNIEnv) -> Result<&'static JClass<'static>> { + static CLASS: OnceCell = OnceCell::new(); + let global = CLASS.get_or_try_init(|| { + #[cfg(feature = "embedded-dex")] + let class = { + let dex_class_loader_class = env.find_class("dalvik/system/InMemoryDexClassLoader")?; + let dex_bytes = include_bytes!("../classes.dex"); + let dex_buffer = unsafe { + env.new_direct_byte_buffer(dex_bytes.as_ptr() as *mut u8, dex_bytes.len()) + }?; + let dex_class_loader = env.new_object( + &dex_class_loader_class, + "(Ljava/nio/ByteBuffer;Ljava/lang/ClassLoader;)V", + &[(&dex_buffer).into(), (&JObject::null()).into()], + )?; + let class_name = env.new_string("dev.accesskit.android.Delegate")?; + let class_obj = env + .call_method( + &dex_class_loader, + "loadClass", + "(Ljava/lang/String;)Ljava/lang/Class;", + &[(&class_name).into()], + )? + .l()?; + JClass::from(class_obj) + }; + #[cfg(not(feature = "embedded-dex"))] + let class = env.find_class("dev/accesskit/android/Delegate")?; + env.register_native_methods( + &class, + &[ + NativeMethod { + name: "populateNodeInfo".into(), + sig: "(JLandroid/view/View;IIILandroid/view/accessibility/AccessibilityNodeInfo;)Z" + .into(), + fn_ptr: populate_node_info as *mut c_void, + }, + NativeMethod { + name: "getInputFocus".into(), + sig: "(J)I".into(), + fn_ptr: get_input_focus as *mut c_void, + }, + NativeMethod { + name: "getVirtualViewAtPoint".into(), + sig: "(JFF)I".into(), + fn_ptr: get_virtual_view_at_point as *mut c_void, + }, + NativeMethod { + name: "performAction".into(), + sig: "(JII)Z".into(), + fn_ptr: perform_action as *mut c_void, + }, + NativeMethod { + name: "setTextSelection".into(), + sig: "(JLandroid/view/View;III)Z".into(), + fn_ptr: set_text_selection as *mut c_void, + }, + NativeMethod { + name: "collapseTextSelection".into(), + sig: "(JLandroid/view/View;I)Z".into(), + fn_ptr: collapse_text_selection as *mut c_void, + }, + NativeMethod { + name: "traverseText".into(), + sig: "(JLandroid/view/View;IIZZ)Z".into(), + fn_ptr: traverse_text as *mut c_void, + }, + ], + )?; + env.new_global_ref(class) + })?; + Ok(global.as_obj().into()) +} + +/// High-level AccessKit Android adapter that injects itself into an Android +/// view without requiring the view class to be modified for accessibility. +/// +/// This depends on the Java `dev.accesskit.android.Delegate` class, the source +/// code for which is in this crate's `java` directory. If the `embedded-dex` +/// feature is enabled, then that class is loaded from a prebuilt `.dex` file +/// that this crate embeds. Otherwise, it's simply assumed that the class +/// is in the application package. None of this type's public functions +/// make assumptions about whether they're called from the Android UI thread. +/// As such, some requests are posted to the UI thread and handled +/// asynchronously. +pub struct InjectingAdapter { + vm: JavaVM, + delegate_class: &'static JClass<'static>, + host: WeakRef, + handle: jlong, + inner: Arc>, +} + +impl Debug for InjectingAdapter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InnerInjectingAdapter") + .field("vm", &self.vm) + .field("delegate_class", &self.delegate_class) + .field("host", &"WeakRef") + .field("handle", &self.handle) + .field("inner", &self.inner) + .finish() + } +} + +impl InjectingAdapter { + pub fn new( + env: &mut JNIEnv, + host_view: &JObject, + activation_handler: impl 'static + ActivationHandler + Send, + action_handler: impl 'static + ActionHandler + Send, + ) -> Result { + let inner = Arc::new(Mutex::new(InnerInjectingAdapter { + adapter: Adapter::default(), + activation_handler: Box::new(activation_handler), + action_handler: Box::new(action_handler), + })); + let handle = NEXT_HANDLE.fetch_add(1, Ordering::Relaxed); + HANDLE_MAP + .lock() + .unwrap() + .insert(handle, Arc::downgrade(&inner)); + let delegate_class = delegate_class(env)?; + env.call_static_method( + delegate_class, + "inject", + "(Landroid/view/View;J)V", + &[host_view.into(), handle.into()], + )?; + Ok(Self { + vm: env.get_java_vm()?, + delegate_class, + host: env.new_weak_ref(host_view)?.unwrap(), + handle, + inner, + }) + } + + /// If and only if the tree has been initialized, call the provided function + /// and apply the resulting update. Note: If the caller's implementation of + /// [`ActivationHandler::request_initial_tree`] initially returned `None`, + /// the [`TreeUpdate`] returned by the provided function must contain + /// a full tree. + pub fn update_if_active(&mut self, update_factory: impl FnOnce() -> TreeUpdate) { + let mut env = self.vm.get_env().unwrap(); + let Some(host) = self.host.upgrade_local(&env).unwrap() else { + return; + }; + self.inner.lock().unwrap().adapter.update_if_active( + update_factory, + &mut env, + self.delegate_class, + &host, + ); + } +} + +impl Drop for InjectingAdapter { + fn drop(&mut self) { + fn drop_impl(env: &mut JNIEnv, host: &WeakRef) -> Result<()> { + let Some(host) = host.upgrade_local(env)? else { + return Ok(()); + }; + let delegate_class = delegate_class(env)?; + env.call_static_method( + delegate_class, + "remove", + "(Landroid/view/View;)V", + &[(&host).into()], + )?; + Ok(()) + } + + let res = match self.vm.get_env() { + Ok(mut env) => drop_impl(&mut env, &self.host), + Err(_) => self + .vm + .attach_current_thread() + .and_then(|mut env| drop_impl(&mut env, &self.host)), + }; + + if let Err(err) = res { + debug!("error dropping InjectingAdapter: {:#?}", err); + } + + HANDLE_MAP.lock().unwrap().remove(&self.handle); + } +} diff --git a/platforms/android/src/lib.rs b/platforms/android/src/lib.rs new file mode 100644 index 00000000..0e9af77f --- /dev/null +++ b/platforms/android/src/lib.rs @@ -0,0 +1,16 @@ +// Copyright 2025 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +mod filters; +mod node; +mod util; + +mod adapter; +pub use adapter::Adapter; + +mod inject; +pub use inject::InjectingAdapter; + +pub use jni; diff --git a/platforms/android/src/node.rs b/platforms/android/src/node.rs new file mode 100644 index 00000000..be68aefa --- /dev/null +++ b/platforms/android/src/node.rs @@ -0,0 +1,294 @@ +// Copyright 2025 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +// Derived from Chromium's accessibility abstraction. +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE.chromium file. + +use accesskit::{Live, Role, Toggled}; +use accesskit_consumer::Node; +use jni::{errors::Result, objects::JObject, sys::jint, JNIEnv}; + +use crate::{filters::filter, util::*}; + +pub(crate) struct NodeWrapper<'a>(pub(crate) &'a Node<'a>); + +impl NodeWrapper<'_> { + fn is_editable(&self) -> bool { + self.0.is_text_input() && !self.0.is_read_only() + } + + fn is_enabled(&self) -> bool { + !self.0.is_disabled() + } + + fn is_focusable(&self) -> bool { + self.0.is_focusable() + } + + fn is_focused(&self) -> bool { + self.0.is_focused() + } + + fn is_password(&self) -> bool { + self.0.role() == Role::PasswordInput + } + + fn is_checkable(&self) -> bool { + self.0.toggled().is_some() + } + + fn is_checked(&self) -> bool { + match self.0.toggled().unwrap() { + Toggled::False => false, + Toggled::True => true, + Toggled::Mixed => true, + } + } + + fn is_selected(&self) -> bool { + match self.0.role() { + // https://www.w3.org/TR/core-aam-1.1/#mapping_state-property_table + // SelectionItem.IsSelected is set according to the True or False + // value of aria-checked for 'radio' and 'menuitemradio' roles. + Role::RadioButton | Role::MenuItemRadio => self.0.toggled() == Some(Toggled::True), + // https://www.w3.org/TR/wai-aria-1.1/#aria-selected + // SelectionItem.IsSelected is set according to the True or False + // value of aria-selected. + _ => self.0.is_selected().unwrap_or(false), + } + } + + fn content_description(&self) -> Option { + self.0.label() + } + + pub(crate) fn text(&self) -> Option { + self.0.value() + } + + pub(crate) fn text_selection(&self) -> Option<(usize, usize)> { + if !self.is_focused() { + return None; + } + self.0.text_selection().map(|range| { + ( + range.start().to_global_utf16_index(), + range.end().to_global_utf16_index(), + ) + }) + } + + fn class_name(&self) -> &str { + match self.0.role() { + Role::TextInput + | Role::MultilineTextInput + | Role::SearchInput + | Role::EmailInput + | Role::NumberInput + | Role::PasswordInput + | Role::PhoneNumberInput + | Role::UrlInput => "android.widget.EditText", + Role::Slider => "android.widget.SeekBar", + Role::ColorWell + | Role::ComboBox + | Role::EditableComboBox + | Role::DateInput + | Role::DateTimeInput + | Role::WeekInput + | Role::MonthInput + | Role::TimeInput => "android.widget.Spinner", + Role::Button => { + if self.0.supports_toggle() { + "android.widget.ToggleButton" + } else { + "android.widget.Button" + } + } + Role::PdfActionableHighlight => "android.widget.Button", + Role::CheckBox => "android.widget.CheckBox", + Role::RadioButton => "android.widget.RadioButton", + Role::RadioGroup => "android.widget.RadioGroup", + Role::Switch => "android.widget.ToggleButton", + Role::Canvas | Role::Image | Role::SvgRoot => "android.widget.ImageView", + Role::Meter | Role::ProgressIndicator => "android.widget.ProgressBar", + Role::TabList => "android.widget.TabWidget", + Role::Grid | Role::Table | Role::TreeGrid => "android.widget.GridView", + Role::DescriptionList | Role::List | Role::ListBox => "android.widget.ListView", + Role::Dialog => "android.app.Dialog", + Role::RootWebArea => "android.webkit.WebView", + Role::MenuItem | Role::MenuItemCheckBox | Role::MenuItemRadio => { + "android.view.MenuItem" + } + Role::Label => "android.widget.TextView", + _ => "android.view.View", + } + } + + pub(crate) fn populate_node_info( + &self, + env: &mut JNIEnv, + host: &JObject, + host_screen_x: jint, + host_screen_y: jint, + id_map: &mut NodeIdMap, + jni_node: &JObject, + ) -> Result<()> { + for child in self.0.filtered_children(&filter) { + env.call_method( + jni_node, + "addChild", + "(Landroid/view/View;I)V", + &[host.into(), id_map.get_or_create_java_id(&child).into()], + )?; + } + if let Some(parent) = self.0.filtered_parent(&filter) { + if parent.is_root() { + env.call_method( + jni_node, + "setParent", + "(Landroid/view/View;)V", + &[host.into()], + )?; + } else { + env.call_method( + jni_node, + "setParent", + "(Landroid/view/View;I)V", + &[host.into(), id_map.get_or_create_java_id(&parent).into()], + )?; + } + } + + if let Some(rect) = self.0.bounding_box() { + let android_rect_class = env.find_class("android/graphics/Rect")?; + let android_rect = env.new_object( + &android_rect_class, + "(IIII)V", + &[ + ((rect.x0 as jint) + host_screen_x).into(), + ((rect.y0 as jint) + host_screen_y).into(), + ((rect.x1 as jint) + host_screen_x).into(), + ((rect.y1 as jint) + host_screen_y).into(), + ], + )?; + env.call_method( + jni_node, + "setBoundsInScreen", + "(Landroid/graphics/Rect;)V", + &[(&android_rect).into()], + )?; + } + + if self.is_checkable() { + env.call_method(jni_node, "setCheckable", "(Z)V", &[true.into()])?; + env.call_method(jni_node, "setChecked", "(Z)V", &[self.is_checked().into()])?; + } + env.call_method( + jni_node, + "setEditable", + "(Z)V", + &[self.is_editable().into()], + )?; + env.call_method(jni_node, "setEnabled", "(Z)V", &[self.is_enabled().into()])?; + env.call_method( + jni_node, + "setFocusable", + "(Z)V", + &[self.is_focusable().into()], + )?; + env.call_method(jni_node, "setFocused", "(Z)V", &[self.is_focused().into()])?; + env.call_method( + jni_node, + "setPassword", + "(Z)V", + &[self.is_password().into()], + )?; + env.call_method( + jni_node, + "setSelected", + "(Z)V", + &[self.is_selected().into()], + )?; + if let Some(desc) = self.content_description() { + let desc = env.new_string(desc)?; + env.call_method( + jni_node, + "setContentDescription", + "(Ljava/lang/CharSequence;)V", + &[(&desc).into()], + )?; + } + + if let Some(text) = self.text() { + let text = env.new_string(text)?; + env.call_method( + jni_node, + "setText", + "(Ljava/lang/CharSequence;)V", + &[(&text).into()], + )?; + } + if let Some((start, end)) = self.text_selection() { + env.call_method( + jni_node, + "setTextSelection", + "(II)V", + &[(start as jint).into(), (end as jint).into()], + )?; + } + + let class_name = env.new_string(self.class_name())?; + env.call_method( + jni_node, + "setClassName", + "(Ljava/lang/CharSequence;)V", + &[(&class_name).into()], + )?; + + fn add_action(env: &mut JNIEnv, jni_node: &JObject, action: jint) -> Result<()> { + // Note: We're using the deprecated addAction signature. + // But this one is much easier to call from JNI since it uses + // a simple integer constant. Revisit if Android ever gets strict + // about prohibiting deprecated methods for applications targeting + // newer SDKs. + env.call_method(jni_node, "addAction", "(I)V", &[action.into()])?; + Ok(()) + } + + let can_focus = self.0.is_focusable() && !self.0.is_focused(); + if self.0.is_clickable() || can_focus { + add_action(env, jni_node, ACTION_CLICK)?; + } + if can_focus { + add_action(env, jni_node, ACTION_FOCUS)?; + } + if self.0.supports_text_ranges() { + add_action(env, jni_node, ACTION_SET_SELECTION)?; + add_action(env, jni_node, ACTION_NEXT_AT_MOVEMENT_GRANULARITY)?; + add_action(env, jni_node, ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY)?; + env.call_method( + jni_node, + "setMovementGranularities", + "(I)V", + &[(MOVEMENT_GRANULARITY_CHARACTER + | MOVEMENT_GRANULARITY_WORD + | MOVEMENT_GRANULARITY_LINE + | MOVEMENT_GRANULARITY_PARAGRAPH) + .into()], + )?; + } + + let live = match self.0.live() { + Live::Off => LIVE_REGION_NONE, + Live::Polite => LIVE_REGION_POLITE, + Live::Assertive => LIVE_REGION_ASSERTIVE, + }; + env.call_method(jni_node, "setLiveRegion", "(I)V", &[live.into()])?; + + Ok(()) + } +} diff --git a/platforms/android/src/util.rs b/platforms/android/src/util.rs new file mode 100644 index 00000000..b20b78fe --- /dev/null +++ b/platforms/android/src/util.rs @@ -0,0 +1,53 @@ +// Copyright 2025 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file) or the MIT license (found in +// the LICENSE-MIT file), at your option. + +use accesskit::NodeId; +use accesskit_consumer::Node; +use jni::sys::jint; +use std::collections::HashMap; + +pub(crate) const ACTION_FOCUS: jint = 1 << 0; +pub(crate) const ACTION_CLICK: jint = 1 << 4; +pub(crate) const ACTION_NEXT_AT_MOVEMENT_GRANULARITY: jint = 1 << 8; +pub(crate) const ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY: jint = 1 << 9; +pub(crate) const ACTION_SET_SELECTION: jint = 1 << 17; +pub(crate) const EVENT_VIEW_FOCUSED: jint = 1 << 3; +pub(crate) const EVENT_WINDOW_CONTENT_CHANGED: jint = 1 << 11; +pub(crate) const HOST_VIEW_ID: jint = -1; +pub(crate) const LIVE_REGION_NONE: jint = 0; +pub(crate) const LIVE_REGION_POLITE: jint = 1; +pub(crate) const LIVE_REGION_ASSERTIVE: jint = 2; +pub(crate) const MOVEMENT_GRANULARITY_CHARACTER: jint = 1 << 0; +pub(crate) const MOVEMENT_GRANULARITY_WORD: jint = 1 << 1; +pub(crate) const MOVEMENT_GRANULARITY_LINE: jint = 1 << 2; +pub(crate) const MOVEMENT_GRANULARITY_PARAGRAPH: jint = 1 << 3; + +#[derive(Debug, Default)] +pub(crate) struct NodeIdMap { + java_to_accesskit: HashMap, + accesskit_to_java: HashMap, + next_java_id: jint, +} + +impl NodeIdMap { + pub(crate) fn get_accesskit_id(&self, java_id: jint) -> Option { + self.java_to_accesskit.get(&java_id).copied() + } + + pub(crate) fn get_or_create_java_id(&mut self, node: &Node) -> jint { + if node.is_root() { + return HOST_VIEW_ID; + } + let accesskit_id = node.id(); + if let Some(id) = self.accesskit_to_java.get(&accesskit_id) { + return *id; + } + let java_id = self.next_java_id; + self.next_java_id += 1; + self.accesskit_to_java.insert(accesskit_id, java_id); + self.java_to_accesskit.insert(java_id, accesskit_id); + java_id + } +} diff --git a/platforms/winit/Cargo.toml b/platforms/winit/Cargo.toml index d3310fe0..35f3886d 100644 --- a/platforms/winit/Cargo.toml +++ b/platforms/winit/Cargo.toml @@ -20,7 +20,7 @@ tokio = ["accesskit_unix/tokio"] [dependencies] accesskit = { version = "0.17.1", path = "../../common" } -winit = { version = "0.30", default-features = false } +winit = { version = "0.30.5", default-features = false } rwh_05 = { package = "raw-window-handle", version = "0.5", features = ["std"], optional = true } rwh_06 = { package = "raw-window-handle", version = "0.6.2", features = ["std"], optional = true } @@ -33,7 +33,10 @@ accesskit_macos = { version = "0.18.1", path = "../macos" } [target.'cfg(any(target_os = "linux", target_os = "dragonfly", target_os = "freebsd", target_os = "openbsd", target_os = "netbsd"))'.dependencies] accesskit_unix = { version = "0.13.1", path = "../unix", optional = true, default-features = false } +[target.'cfg(target_os = "android")'.dependencies] +accesskit_android = { version = "0.1.0", path = "../android", features = ["embedded-dex"] } + [dev-dependencies.winit] -version = "0.30" +version = "0.30.5" default-features = false features = ["x11", "wayland", "wayland-dlopen", "wayland-csd-adwaita"] diff --git a/platforms/winit/README.md b/platforms/winit/README.md index 6926d59f..bbcf3f29 100644 --- a/platforms/winit/README.md +++ b/platforms/winit/README.md @@ -10,3 +10,7 @@ While this crate's API is purely blocking, it internally spawns asynchronous tas - If you use tokio, make sure to enable the `tokio` feature of this crate. - If you use another async runtime or if you don't use one at all, the default feature will suit your needs. + +## Android activity compatibility + +The Android implementation of this adapter currently only works with [GameActivity](https://developer.android.com/games/agdk/game-activity), which is one of the two activity implementations that winit currently supports. diff --git a/platforms/winit/src/platform_impl/android.rs b/platforms/winit/src/platform_impl/android.rs new file mode 100644 index 00000000..6de4310f --- /dev/null +++ b/platforms/winit/src/platform_impl/android.rs @@ -0,0 +1,50 @@ +// Copyright 2025 The AccessKit Authors. All rights reserved. +// Licensed under the Apache License, Version 2.0 (found in +// the LICENSE-APACHE file). + +use accesskit::{ActionHandler, ActivationHandler, DeactivationHandler, TreeUpdate}; +use accesskit_android::{ + jni::{objects::JObject, JavaVM}, + InjectingAdapter, +}; +use winit::{ + event::WindowEvent, event_loop::ActiveEventLoop, platform::android::ActiveEventLoopExtAndroid, + window::Window, +}; + +pub struct Adapter { + adapter: InjectingAdapter, +} + +impl Adapter { + pub fn new( + event_loop: &ActiveEventLoop, + _window: &Window, + activation_handler: impl 'static + ActivationHandler + Send, + action_handler: impl 'static + ActionHandler + Send, + _deactivation_handler: impl 'static + DeactivationHandler, + ) -> Self { + let app = event_loop.android_app(); + let vm = unsafe { JavaVM::from_raw(app.vm_as_ptr() as *mut _) }.unwrap(); + let mut env = vm.get_env().unwrap(); + let activity = unsafe { JObject::from_raw(app.activity_as_ptr() as *mut _) }; + let view = env + .get_field( + &activity, + "mSurfaceView", + "Lcom/google/androidgamesdk/GameActivity$InputEnabledSurfaceView;", + ) + .unwrap() + .l() + .unwrap(); + let adapter = + InjectingAdapter::new(&mut env, &view, activation_handler, action_handler).unwrap(); + Self { adapter } + } + + pub fn update_if_active(&mut self, updater: impl FnOnce() -> TreeUpdate) { + self.adapter.update_if_active(updater); + } + + pub fn process_event(&mut self, _window: &Window, _event: &WindowEvent) {} +} diff --git a/platforms/winit/src/platform_impl/mod.rs b/platforms/winit/src/platform_impl/mod.rs index 76197cbd..38bb1a8d 100644 --- a/platforms/winit/src/platform_impl/mod.rs +++ b/platforms/winit/src/platform_impl/mod.rs @@ -27,6 +27,10 @@ mod platform; #[path = "unix.rs"] mod platform; +#[cfg(target_os = "android")] +#[path = "android.rs"] +mod platform; + #[cfg(not(any( target_os = "windows", target_os = "macos", @@ -39,7 +43,8 @@ mod platform; target_os = "netbsd", target_os = "openbsd" ) - ) + ), + target_os = "android" )))] #[path = "null.rs"] mod platform; diff --git a/release-please-config.json b/release-please-config.json index 2b5f5167..addf935f 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -6,6 +6,7 @@ "packages": { "common": {}, "consumer": {}, + "platforms/android": {}, "platforms/atspi-common": {}, "platforms/macos": {}, "platforms/unix": {},