From 1258e40bf1892e1f1d025e49a4aff8ee71c6e024 Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Tue, 22 Apr 2025 16:26:56 +0200 Subject: [PATCH 1/6] Add files via upload --- .../ScreenOperatorAccessibilityService.kt | 166 ++++++++---------- 1 file changed, 76 insertions(+), 90 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index dbf9885e..0de17688 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -111,9 +111,12 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { serviceInstance?.tapAtCoordinates(command.x, command.y) } is Command.TakeScreenshot -> { - Log.d(TAG, "Taking screenshot") - showToast("Versuche Screenshot aufzunehmen", false) - serviceInstance?.takeScreenshot() + Log.d(TAG, "Taking screenshot with 300ms delay") + showToast("Versuche Screenshot aufzunehmen (mit 300ms Verzögerung)", false) + // Add a 300ms delay before taking the screenshot + mainHandler.postDelayed({ + serviceInstance?.takeScreenshot() + }, 300) // 300ms delay } is Command.PressHomeButton -> { Log.d(TAG, "Pressing home button") @@ -363,39 +366,33 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { // Recycle the node focusedNode.recycle() } else { - Log.e(TAG, "No focused editable node found") - showToast("Kein fokussiertes Textfeld gefunden", true) + Log.d(TAG, "No focused editable node found, trying to find any editable node") // Try to find any editable node val editableNode = findAnyEditableNode(rootNode!!) if (editableNode != null) { - Log.d(TAG, "Found editable node, trying to focus and write text") - showToast("Textfeld gefunden, versuche zu fokussieren und Text zu schreiben", false) + Log.d(TAG, "Found editable node") + showToast("Textfeld gefunden, schreibe Text: \"$text\"", false) // Focus the node first - val focusResult = editableNode.performAction(AccessibilityNodeInfo.ACTION_FOCUS) + editableNode.performAction(AccessibilityNodeInfo.ACTION_FOCUS) - if (focusResult) { - // Set the text in the editable field - val bundle = android.os.Bundle() - bundle.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) - - val result = editableNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, bundle) - - if (result) { - Log.d(TAG, "Successfully wrote text: $text") - showToast("Text erfolgreich geschrieben: \"$text\"", false) - } else { - Log.e(TAG, "Failed to write text") - showToast("Fehler beim Schreiben des Textes, versuche alternative Methode", true) - - // Try alternative method: paste text - tryPasteText(editableNode, text) - } + // Set the text in the editable field + val bundle = android.os.Bundle() + bundle.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) + + val result = editableNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, bundle) + + if (result) { + Log.d(TAG, "Successfully wrote text: $text") + showToast("Text erfolgreich geschrieben: \"$text\"", false) } else { - Log.e(TAG, "Failed to focus editable node") - showToast("Fehler beim Fokussieren des Textfeldes", true) + Log.e(TAG, "Failed to write text") + showToast("Fehler beim Schreiben des Textes, versuche alternative Methode", true) + + // Try alternative method: paste text + tryPasteText(editableNode, text) } // Recycle the node @@ -1139,7 +1136,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { // Wait a moment for the screenshot to be saved, then retrieve it handler.postDelayed({ retrieveLatestScreenshot(screenInfo) - }, 1000) // Wait 1 second for the screenshot to be saved + }, 800) // Wait 800ms for the screenshot to be saved (reduced from 1000ms) return } else { @@ -1176,70 +1173,59 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { val screenInfo = StringBuilder() screenInfo.append("Bildschirmelemente:\n") - // Capture information about all interactive elements - val interactiveElements = findAllInteractiveElements(rootNode!!) + // Find all interactive elements + val elements = findAllInteractiveElements(rootNode!!) - if (interactiveElements.isEmpty()) { - screenInfo.append("Keine interaktiven Elemente gefunden.") - } else { - screenInfo.append("Gefundene interaktive Elemente (${interactiveElements.size}):\n\n") + // Add information about each element + elements.forEachIndexed { index, element -> + screenInfo.append("${index + 1}. ") - interactiveElements.forEachIndexed { index, element -> - screenInfo.append("${index + 1}. ") - - // Get element ID if available - val elementId = getNodeId(element) - if (elementId.isNotEmpty()) { - screenInfo.append("ID: \"$elementId\" ") - } - - // Add element text if available - if (!element.text.isNullOrEmpty()) { - screenInfo.append("Text: \"${element.text}\" ") - } - - // Add content description if available - if (!element.contentDescription.isNullOrEmpty()) { - screenInfo.append("Beschreibung: \"${element.contentDescription}\" ") - } - - // Add element class name if available - if (element.className != null) { - screenInfo.append("Klasse: ${element.className} ") - } - - // Add element properties - val properties = mutableListOf() - if (element.isClickable) properties.add("klickbar") - if (element.isLongClickable) properties.add("lang-klickbar") - if (element.isCheckable) properties.add("auswählbar") - if (element.isChecked) properties.add("ausgewählt") - if (element.isEditable) properties.add("editierbar") - if (element.isFocusable) properties.add("fokussierbar") - if (element.isFocused) properties.add("fokussiert") - if (element.isPassword) properties.add("passwort") - if (element.isScrollable) properties.add("scrollbar") - - if (properties.isNotEmpty()) { - screenInfo.append("Eigenschaften: ${properties.joinToString(", ")} ") - } - - // Add element bounds - val rect = Rect() - element.getBoundsInScreen(rect) - screenInfo.append("Position: (${rect.left}, ${rect.top}, ${rect.right}, ${rect.bottom})") - - // Add a button name if we can infer one - val buttonName = getButtonName(element) - if (buttonName.isNotEmpty()) { - screenInfo.append(" Vermuteter Name: \"$buttonName\"") - } - - screenInfo.append("\n") - - // Recycle the element - element.recycle() + // Add text if available + if (!element.text.isNullOrEmpty()) { + screenInfo.append("Text: \"${element.text}\" ") + } + + // Add content description if available + if (!element.contentDescription.isNullOrEmpty()) { + screenInfo.append("Beschreibung: \"${element.contentDescription}\" ") } + + // Add element class name if available + if (element.className != null) { + screenInfo.append("Klasse: ${element.className} ") + } + + // Add element properties + val properties = mutableListOf() + if (element.isClickable) properties.add("klickbar") + if (element.isLongClickable) properties.add("lang-klickbar") + if (element.isCheckable) properties.add("auswählbar") + if (element.isChecked) properties.add("ausgewählt") + if (element.isEditable) properties.add("editierbar") + if (element.isFocusable) properties.add("fokussierbar") + if (element.isFocused) properties.add("fokussiert") + if (element.isPassword) properties.add("passwort") + if (element.isScrollable) properties.add("scrollbar") + + if (properties.isNotEmpty()) { + screenInfo.append("Eigenschaften: ${properties.joinToString(", ")} ") + } + + // Add element bounds + val rect = Rect() + element.getBoundsInScreen(rect) + screenInfo.append("Position: (${rect.left}, ${rect.top}, ${rect.right}, ${rect.bottom})") + + // Add a button name if we can infer one + val buttonName = getButtonName(element) + if (buttonName.isNotEmpty()) { + screenInfo.append(" Vermuteter Name: \"$buttonName\"") + } + + screenInfo.append("\n") + + // Recycle the element + element.recycle() } return screenInfo.toString() @@ -1594,7 +1580,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { } /** - * Show recent apps overview + * Show the recent apps screen */ fun showRecentApps() { Log.d(TAG, "Showing recent apps") From d48ffa471f1ade3c02cd944d8a19bc7228043fa8 Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Tue, 22 Apr 2025 16:51:30 +0200 Subject: [PATCH 2/6] Update ScreenOperatorAccessibilityService.kt --- .../ai/sample/ScreenOperatorAccessibilityService.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index 0de17688..65661f4a 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -111,12 +111,12 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { serviceInstance?.tapAtCoordinates(command.x, command.y) } is Command.TakeScreenshot -> { - Log.d(TAG, "Taking screenshot with 300ms delay") - showToast("Versuche Screenshot aufzunehmen (mit 300ms Verzögerung)", false) - // Add a 300ms delay before taking the screenshot + Log.d(TAG, "Taking screenshot with 1000ms delay") + showToast("Versuche Screenshot aufzunehmen (mit 1000ms Verzögerung)", false) + // Add a 1000ms delay before taking the screenshot mainHandler.postDelayed({ serviceInstance?.takeScreenshot() - }, 300) // 300ms delay + }, 1000) // 1000ms delay } is Command.PressHomeButton -> { Log.d(TAG, "Pressing home button") From 6d2ff05babda882d7fa1ef9224a513463b0b94cf Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Tue, 22 Apr 2025 17:04:33 +0200 Subject: [PATCH 3/6] Update ScreenOperatorAccessibilityService.kt --- .../ai/sample/ScreenOperatorAccessibilityService.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index 65661f4a..135cf356 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -111,12 +111,12 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { serviceInstance?.tapAtCoordinates(command.x, command.y) } is Command.TakeScreenshot -> { - Log.d(TAG, "Taking screenshot with 1000ms delay") - showToast("Versuche Screenshot aufzunehmen (mit 1000ms Verzögerung)", false) - // Add a 1000ms delay before taking the screenshot + Log.d(TAG, "Taking screenshot with 500ms delay") + showToast("Versuche Screenshot aufzunehmen (mit 500ms Verzögerung)", false) + // Add a 500ms delay before taking the screenshot, sure all commands executed before mainHandler.postDelayed({ serviceInstance?.takeScreenshot() - }, 1000) // 1000ms delay + }, 500) // 500ms delay } is Command.PressHomeButton -> { Log.d(TAG, "Pressing home button") @@ -1136,7 +1136,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { // Wait a moment for the screenshot to be saved, then retrieve it handler.postDelayed({ retrieveLatestScreenshot(screenInfo) - }, 800) // Wait 800ms for the screenshot to be saved (reduced from 1000ms) + }, 600) // Wait 800ms for the screenshot to be saved (reduced from 1000ms) return } else { From e3ed21deea05f522e13db72b19d3021cdcfcb521 Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Tue, 22 Apr 2025 17:18:26 +0200 Subject: [PATCH 4/6] Update ScreenOperatorAccessibilityService.kt --- .../ai/sample/ScreenOperatorAccessibilityService.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index 135cf356..2cdcee67 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -111,12 +111,12 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { serviceInstance?.tapAtCoordinates(command.x, command.y) } is Command.TakeScreenshot -> { - Log.d(TAG, "Taking screenshot with 500ms delay") - showToast("Versuche Screenshot aufzunehmen (mit 500ms Verzögerung)", false) - // Add a 500ms delay before taking the screenshot, sure all commands executed before + Log.d(TAG, "Taking screenshot with 700ms delay") + showToast("Versuche Screenshot aufzunehmen (mit 700ms Verzögerung)", false) + // Add a 700ms delay before taking the screenshot, sure all commands executed before mainHandler.postDelayed({ serviceInstance?.takeScreenshot() - }, 500) // 500ms delay + }, 700) // 700ms delay } is Command.PressHomeButton -> { Log.d(TAG, "Pressing home button") From 25400b0b0d21595323845795349bd350adacdbeb Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Tue, 22 Apr 2025 17:34:18 +0200 Subject: [PATCH 5/6] Update ScreenOperatorAccessibilityService.kt --- .../ai/sample/ScreenOperatorAccessibilityService.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index 2cdcee67..912d15b5 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -111,12 +111,12 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { serviceInstance?.tapAtCoordinates(command.x, command.y) } is Command.TakeScreenshot -> { - Log.d(TAG, "Taking screenshot with 700ms delay") - showToast("Versuche Screenshot aufzunehmen (mit 700ms Verzögerung)", false) - // Add a 700ms delay before taking the screenshot, sure all commands executed before + Log.d(TAG, "Taking screenshot with 850ms delay") + showToast("Versuche Screenshot aufzunehmen (mit 850ms Verzögerung)", false) + // Add a 850ms delay before taking the screenshot, sure all commands executed before mainHandler.postDelayed({ serviceInstance?.takeScreenshot() - }, 700) // 700ms delay + }, 850) // 850ms delay } is Command.PressHomeButton -> { Log.d(TAG, "Pressing home button") From e5549a3f043aa532f66db69821a606283dfb7edd Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Wed, 23 Apr 2025 06:51:35 +0200 Subject: [PATCH 6/6] Update ScreenOperatorAccessibilityService.kt --- .../com/google/ai/sample/ScreenOperatorAccessibilityService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index 912d15b5..debbb51b 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -1136,7 +1136,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { // Wait a moment for the screenshot to be saved, then retrieve it handler.postDelayed({ retrieveLatestScreenshot(screenInfo) - }, 600) // Wait 800ms for the screenshot to be saved (reduced from 1000ms) + }, 800) // Wait 800ms for the screenshot to be saved (reduced from 1000ms) return } else {