Skip to content

Precise scrolling #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Android-PowerUser merged 9 commits into from
Apr 14, 2025
Merged

Precise scrolling #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
cab4d4b
Add files via upload
Android-PowerUser Apr 14, 2025
828d489
Add files via upload
Android-PowerUser Apr 14, 2025
da9a442
Add files via upload
Android-PowerUser Apr 14, 2025
e007e1a
Add files via upload
Android-PowerUser Apr 14, 2025
647820f
Add files via upload
Android-PowerUser Apr 14, 2025
0f9cf5c
Add files via upload
Android-PowerUser Apr 14, 2025
fb565e8
Add files via upload
Android-PowerUser Apr 14, 2025
ffeec8d
Add files via upload
Android-PowerUser Apr 14, 2025
a0f8740
Add files via upload
Android-PowerUser Apr 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
354 changes: 354 additions & 0 deletions app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,36 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
showToast("Versuche nach oben zu scrollen", false)
serviceInstance?.scrollUp()
}
is Command.ScrollLeft -> {
Log.d(TAG, "Scrolling left")
showToast("Versuche nach links zu scrollen", false)
serviceInstance?.scrollLeft()
}
is Command.ScrollRight -> {
Log.d(TAG, "Scrolling right")
showToast("Versuche nach rechts zu scrollen", false)
serviceInstance?.scrollRight()
}
is Command.ScrollDownFromCoordinates -> {
Log.d(TAG, "Scrolling down from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
showToast("Versuche von Position (${command.x}, ${command.y}) nach unten zu scrollen", false)
serviceInstance?.scrollDown(command.x, command.y, command.distance, command.duration)
}
is Command.ScrollUpFromCoordinates -> {
Log.d(TAG, "Scrolling up from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
showToast("Versuche von Position (${command.x}, ${command.y}) nach oben zu scrollen", false)
serviceInstance?.scrollUp(command.x, command.y, command.distance, command.duration)
}
is Command.ScrollLeftFromCoordinates -> {
Log.d(TAG, "Scrolling left from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
showToast("Versuche von Position (${command.x}, ${command.y}) nach links zu scrollen", false)
serviceInstance?.scrollLeft(command.x, command.y, command.distance, command.duration)
}
is Command.ScrollRightFromCoordinates -> {
Log.d(TAG, "Scrolling right from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
showToast("Versuche von Position (${command.x}, ${command.y}) nach rechts zu scrollen", false)
serviceInstance?.scrollRight(command.x, command.y, command.distance, command.duration)
}
}
}

Expand Down Expand Up @@ -1096,6 +1126,60 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
}
}

/**
* Scroll down from specific coordinates with custom distance and duration
*
* @param x Starting X coordinate
* @param y Starting Y coordinate
* @param distance Distance in pixels to scroll
* @param duration Duration of the scroll gesture in milliseconds
*/
fun scrollDown(x: Float, y: Float, distance: Float, duration: Long) {
Log.d(TAG, "Scrolling down from ($x, $y) with distance $distance and duration $duration ms")
showToast("Scrolle nach unten von bestimmter Position...", false)

try {
// Create a path for the gesture (swipe from specified position upward by the specified distance)
val swipePath = Path()
swipePath.moveTo(x, y) // Start from specified position
swipePath.lineTo(x, y - distance) // Move upward by the specified distance

// Create a gesture builder and add the swipe
val gestureBuilder = GestureDescription.Builder()
val gesture = GestureDescription.StrokeDescription(
swipePath,
0, // start time
duration // custom duration in milliseconds
)
gestureBuilder.addStroke(gesture)

// Dispatch the gesture
val result = dispatchGesture(
gestureBuilder.build(),
object : GestureResultCallback() {
override fun onCompleted(gestureDescription: GestureDescription) {
Log.d(TAG, "Coordinate-based scroll down gesture completed")
showToast("Erfolgreich nach unten gescrollt von Position ($x, $y)", false)
}

override fun onCancelled(gestureDescription: GestureDescription) {
Log.e(TAG, "Coordinate-based scroll down gesture cancelled")
showToast("Scrollen nach unten von Position ($x, $y) abgebrochen", true)
}
},
null // handler
)

if (!result) {
Log.e(TAG, "Failed to dispatch coordinate-based scroll down gesture")
showToast("Fehler beim Scrollen nach unten von Position ($x, $y)", true)
}
} catch (e: Exception) {
Log.e(TAG, "Error scrolling down from coordinates: ${e.message}")
showToast("Fehler beim Scrollen nach unten von Position ($x, $y): ${e.message}", true)
}
}

/**
* Scroll up on the screen using gesture
*/
Expand Down Expand Up @@ -1150,6 +1234,276 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
}
}

/**
* Scroll up from specific coordinates with custom distance and duration
*
* @param x Starting X coordinate
* @param y Starting Y coordinate
* @param distance Distance in pixels to scroll
* @param duration Duration of the scroll gesture in milliseconds
*/
fun scrollUp(x: Float, y: Float, distance: Float, duration: Long) {
Log.d(TAG, "Scrolling up from ($x, $y) with distance $distance and duration $duration ms")
showToast("Scrolle nach oben von bestimmter Position...", false)

try {
// Create a path for the gesture (swipe from specified position downward by the specified distance)
val swipePath = Path()
swipePath.moveTo(x, y) // Start from specified position
swipePath.lineTo(x, y + distance) // Move downward by the specified distance

// Create a gesture builder and add the swipe
val gestureBuilder = GestureDescription.Builder()
val gesture = GestureDescription.StrokeDescription(
swipePath,
0, // start time
duration // custom duration in milliseconds
)
gestureBuilder.addStroke(gesture)

// Dispatch the gesture
val result = dispatchGesture(
gestureBuilder.build(),
object : GestureResultCallback() {
override fun onCompleted(gestureDescription: GestureDescription) {
Log.d(TAG, "Coordinate-based scroll up gesture completed")
showToast("Erfolgreich nach oben gescrollt von Position ($x, $y)", false)
}

override fun onCancelled(gestureDescription: GestureDescription) {
Log.e(TAG, "Coordinate-based scroll up gesture cancelled")
showToast("Scrollen nach oben von Position ($x, $y) abgebrochen", true)
}
},
null // handler
)

if (!result) {
Log.e(TAG, "Failed to dispatch coordinate-based scroll up gesture")
showToast("Fehler beim Scrollen nach oben von Position ($x, $y)", true)
}
} catch (e: Exception) {
Log.e(TAG, "Error scrolling up from coordinates: ${e.message}")
showToast("Fehler beim Scrollen nach oben von Position ($x, $y): ${e.message}", true)
}
}

/**
* Scroll left on the screen using gesture
*/
fun scrollLeft() {
Log.d(TAG, "Scrolling left")
showToast("Scrolle nach links...", false)

try {
// Get display metrics to calculate swipe coordinates
val displayMetrics = resources.displayMetrics
val screenHeight = displayMetrics.heightPixels
val screenWidth = displayMetrics.widthPixels

// Create a path for the gesture (swipe from middle-right to middle-left)
val swipePath = Path()
swipePath.moveTo(screenWidth * 0.7f, screenHeight / 2f) // Start from 70% across the screen
swipePath.lineTo(screenWidth * 0.3f, screenHeight / 2f) // Move to 30% across the screen

// Create a gesture builder and add the swipe
val gestureBuilder = GestureDescription.Builder()
val gesture = GestureDescription.StrokeDescription(
swipePath,
0, // start time
300 // duration in milliseconds
)
gestureBuilder.addStroke(gesture)

// Dispatch the gesture
val result = dispatchGesture(
gestureBuilder.build(),
object : GestureResultCallback() {
override fun onCompleted(gestureDescription: GestureDescription) {
Log.d(TAG, "Scroll left gesture completed")
showToast("Erfolgreich nach links gescrollt", false)
}

override fun onCancelled(gestureDescription: GestureDescription) {
Log.e(TAG, "Scroll left gesture cancelled")
showToast("Scrollen nach links abgebrochen", true)
}
},
null // handler
)

if (!result) {
Log.e(TAG, "Failed to dispatch scroll left gesture")
showToast("Fehler beim Scrollen nach links", true)
}
} catch (e: Exception) {
Log.e(TAG, "Error scrolling left: ${e.message}")
showToast("Fehler beim Scrollen nach links: ${e.message}", true)
}
}

/**
* Scroll left from specific coordinates with custom distance and duration
*
* @param x Starting X coordinate
* @param y Starting Y coordinate
* @param distance Distance in pixels to scroll
* @param duration Duration of the scroll gesture in milliseconds
*/
fun scrollLeft(x: Float, y: Float, distance: Float, duration: Long) {
Log.d(TAG, "Scrolling left from ($x, $y) with distance $distance and duration $duration ms")
showToast("Scrolle nach links von bestimmter Position...", false)

try {
// Create a path for the gesture (swipe from specified position leftward by the specified distance)
val swipePath = Path()
swipePath.moveTo(x, y) // Start from specified position
swipePath.lineTo(x - distance, y) // Move leftward by the specified distance

// Create a gesture builder and add the swipe
val gestureBuilder = GestureDescription.Builder()
val gesture = GestureDescription.StrokeDescription(
swipePath,
0, // start time
duration // custom duration in milliseconds
)
gestureBuilder.addStroke(gesture)

// Dispatch the gesture
val result = dispatchGesture(
gestureBuilder.build(),
object : GestureResultCallback() {
override fun onCompleted(gestureDescription: GestureDescription) {
Log.d(TAG, "Coordinate-based scroll left gesture completed")
showToast("Erfolgreich nach links gescrollt von Position ($x, $y)", false)
}

override fun onCancelled(gestureDescription: GestureDescription) {
Log.e(TAG, "Coordinate-based scroll left gesture cancelled")
showToast("Scrollen nach links von Position ($x, $y) abgebrochen", true)
}
},
null // handler
)

if (!result) {
Log.e(TAG, "Failed to dispatch coordinate-based scroll left gesture")
showToast("Fehler beim Scrollen nach links von Position ($x, $y)", true)
}
} catch (e: Exception) {
Log.e(TAG, "Error scrolling left from coordinates: ${e.message}")
showToast("Fehler beim Scrollen nach links von Position ($x, $y): ${e.message}", true)
}
}

/**
* Scroll right on the screen using gesture
*/
fun scrollRight() {
Log.d(TAG, "Scrolling right")
showToast("Scrolle nach rechts...", false)

try {
// Get display metrics to calculate swipe coordinates
val displayMetrics = resources.displayMetrics
val screenHeight = displayMetrics.heightPixels
val screenWidth = displayMetrics.widthPixels

// Create a path for the gesture (swipe from middle-left to middle-right)
val swipePath = Path()
swipePath.moveTo(screenWidth * 0.3f, screenHeight / 2f) // Start from 30% across the screen
swipePath.lineTo(screenWidth * 0.7f, screenHeight / 2f) // Move to 70% across the screen

// Create a gesture builder and add the swipe
val gestureBuilder = GestureDescription.Builder()
val gesture = GestureDescription.StrokeDescription(
swipePath,
0, // start time
300 // duration in milliseconds
)
gestureBuilder.addStroke(gesture)

// Dispatch the gesture
val result = dispatchGesture(
gestureBuilder.build(),
object : GestureResultCallback() {
override fun onCompleted(gestureDescription: GestureDescription) {
Log.d(TAG, "Scroll right gesture completed")
showToast("Erfolgreich nach rechts gescrollt", false)
}

override fun onCancelled(gestureDescription: GestureDescription) {
Log.e(TAG, "Scroll right gesture cancelled")
showToast("Scrollen nach rechts abgebrochen", true)
}
},
null // handler
)

if (!result) {
Log.e(TAG, "Failed to dispatch scroll right gesture")
showToast("Fehler beim Scrollen nach rechts", true)
}
} catch (e: Exception) {
Log.e(TAG, "Error scrolling right: ${e.message}")
showToast("Fehler beim Scrollen nach rechts: ${e.message}", true)
}
}

/**
* Scroll right from specific coordinates with custom distance and duration
*
* @param x Starting X coordinate
* @param y Starting Y coordinate
* @param distance Distance in pixels to scroll
* @param duration Duration of the scroll gesture in milliseconds
*/
fun scrollRight(x: Float, y: Float, distance: Float, duration: Long) {
Log.d(TAG, "Scrolling right from ($x, $y) with distance $distance and duration $duration ms")
showToast("Scrolle nach rechts von bestimmter Position...", false)

try {
// Create a path for the gesture (swipe from specified position rightward by the specified distance)
val swipePath = Path()
swipePath.moveTo(x, y) // Start from specified position
swipePath.lineTo(x + distance, y) // Move rightward by the specified distance

// Create a gesture builder and add the swipe
val gestureBuilder = GestureDescription.Builder()
val gesture = GestureDescription.StrokeDescription(
swipePath,
0, // start time
duration // custom duration in milliseconds
)
gestureBuilder.addStroke(gesture)

// Dispatch the gesture
val result = dispatchGesture(
gestureBuilder.build(),
object : GestureResultCallback() {
override fun onCompleted(gestureDescription: GestureDescription) {
Log.d(TAG, "Coordinate-based scroll right gesture completed")
showToast("Erfolgreich nach rechts gescrollt von Position ($x, $y)", false)
}

override fun onCancelled(gestureDescription: GestureDescription) {
Log.e(TAG, "Coordinate-based scroll right gesture cancelled")
showToast("Scrollen nach rechts von Position ($x, $y) abgebrochen", true)
}
},
null // handler
)

if (!result) {
Log.e(TAG, "Failed to dispatch coordinate-based scroll right gesture")
showToast("Fehler beim Scrollen nach rechts von Position ($x, $y)", true)
}
} catch (e: Exception) {
Log.e(TAG, "Error scrolling right from coordinates: ${e.message}")
showToast("Fehler beim Scrollen nach rechts von Position ($x, $y): ${e.message}", true)
}
}

/**
* Retrieve the latest screenshot from the standard screenshot folder
*/
Expand Down
6 changes: 6 additions & 0 deletions app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatScreen.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,12 @@ fun ChatScreen(
is Command.ShowRecentApps -> "Übersicht der letzten Apps öffnen"
is Command.ScrollDown -> "Nach unten scrollen"
is Command.ScrollUp -> "Nach oben scrollen"
is Command.ScrollLeft -> "Nach links scrollen"
is Command.ScrollRight -> "Nach rechts scrollen"
is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms"
is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms"
is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms"
is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms"
}

Text(
Expand Down
Loading