diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index dd31420c..8d49fe6a 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -135,6 +135,36 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { showToast("Versuche nach oben zu scrollen", false) serviceInstance?.scrollUp() } + is Command.ScrollLeft -> { + Log.d(TAG, "Scrolling left") + showToast("Versuche nach links zu scrollen", false) + serviceInstance?.scrollLeft() + } + is Command.ScrollRight -> { + Log.d(TAG, "Scrolling right") + showToast("Versuche nach rechts zu scrollen", false) + serviceInstance?.scrollRight() + } + is Command.ScrollDownFromCoordinates -> { + Log.d(TAG, "Scrolling down from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms") + showToast("Versuche von Position (${command.x}, ${command.y}) nach unten zu scrollen", false) + serviceInstance?.scrollDown(command.x, command.y, command.distance, command.duration) + } + is Command.ScrollUpFromCoordinates -> { + Log.d(TAG, "Scrolling up from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms") + showToast("Versuche von Position (${command.x}, ${command.y}) nach oben zu scrollen", false) + serviceInstance?.scrollUp(command.x, command.y, command.distance, command.duration) + } + is Command.ScrollLeftFromCoordinates -> { + Log.d(TAG, "Scrolling left from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms") + showToast("Versuche von Position (${command.x}, ${command.y}) nach links zu scrollen", false) + serviceInstance?.scrollLeft(command.x, command.y, command.distance, command.duration) + } + is Command.ScrollRightFromCoordinates -> { + Log.d(TAG, "Scrolling right from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms") + showToast("Versuche von Position (${command.x}, ${command.y}) nach rechts zu scrollen", false) + serviceInstance?.scrollRight(command.x, command.y, command.distance, command.duration) + } } } @@ -1096,6 +1126,60 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { } } + /** + * Scroll down from specific coordinates with custom distance and duration + * + * @param x Starting X coordinate + * @param y Starting Y coordinate + * @param distance Distance in pixels to scroll + * @param duration Duration of the scroll gesture in milliseconds + */ + fun scrollDown(x: Float, y: Float, distance: Float, duration: Long) { + Log.d(TAG, "Scrolling down from ($x, $y) with distance $distance and duration $duration ms") + showToast("Scrolle nach unten von bestimmter Position...", false) + + try { + // Create a path for the gesture (swipe from specified position upward by the specified distance) + val swipePath = Path() + swipePath.moveTo(x, y) // Start from specified position + swipePath.lineTo(x, y - distance) // Move upward by the specified distance + + // Create a gesture builder and add the swipe + val gestureBuilder = GestureDescription.Builder() + val gesture = GestureDescription.StrokeDescription( + swipePath, + 0, // start time + duration // custom duration in milliseconds + ) + gestureBuilder.addStroke(gesture) + + // Dispatch the gesture + val result = dispatchGesture( + gestureBuilder.build(), + object : GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription) { + Log.d(TAG, "Coordinate-based scroll down gesture completed") + showToast("Erfolgreich nach unten gescrollt von Position ($x, $y)", false) + } + + override fun onCancelled(gestureDescription: GestureDescription) { + Log.e(TAG, "Coordinate-based scroll down gesture cancelled") + showToast("Scrollen nach unten von Position ($x, $y) abgebrochen", true) + } + }, + null // handler + ) + + if (!result) { + Log.e(TAG, "Failed to dispatch coordinate-based scroll down gesture") + showToast("Fehler beim Scrollen nach unten von Position ($x, $y)", true) + } + } catch (e: Exception) { + Log.e(TAG, "Error scrolling down from coordinates: ${e.message}") + showToast("Fehler beim Scrollen nach unten von Position ($x, $y): ${e.message}", true) + } + } + /** * Scroll up on the screen using gesture */ @@ -1150,6 +1234,276 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { } } + /** + * Scroll up from specific coordinates with custom distance and duration + * + * @param x Starting X coordinate + * @param y Starting Y coordinate + * @param distance Distance in pixels to scroll + * @param duration Duration of the scroll gesture in milliseconds + */ + fun scrollUp(x: Float, y: Float, distance: Float, duration: Long) { + Log.d(TAG, "Scrolling up from ($x, $y) with distance $distance and duration $duration ms") + showToast("Scrolle nach oben von bestimmter Position...", false) + + try { + // Create a path for the gesture (swipe from specified position downward by the specified distance) + val swipePath = Path() + swipePath.moveTo(x, y) // Start from specified position + swipePath.lineTo(x, y + distance) // Move downward by the specified distance + + // Create a gesture builder and add the swipe + val gestureBuilder = GestureDescription.Builder() + val gesture = GestureDescription.StrokeDescription( + swipePath, + 0, // start time + duration // custom duration in milliseconds + ) + gestureBuilder.addStroke(gesture) + + // Dispatch the gesture + val result = dispatchGesture( + gestureBuilder.build(), + object : GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription) { + Log.d(TAG, "Coordinate-based scroll up gesture completed") + showToast("Erfolgreich nach oben gescrollt von Position ($x, $y)", false) + } + + override fun onCancelled(gestureDescription: GestureDescription) { + Log.e(TAG, "Coordinate-based scroll up gesture cancelled") + showToast("Scrollen nach oben von Position ($x, $y) abgebrochen", true) + } + }, + null // handler + ) + + if (!result) { + Log.e(TAG, "Failed to dispatch coordinate-based scroll up gesture") + showToast("Fehler beim Scrollen nach oben von Position ($x, $y)", true) + } + } catch (e: Exception) { + Log.e(TAG, "Error scrolling up from coordinates: ${e.message}") + showToast("Fehler beim Scrollen nach oben von Position ($x, $y): ${e.message}", true) + } + } + + /** + * Scroll left on the screen using gesture + */ + fun scrollLeft() { + Log.d(TAG, "Scrolling left") + showToast("Scrolle nach links...", false) + + try { + // Get display metrics to calculate swipe coordinates + val displayMetrics = resources.displayMetrics + val screenHeight = displayMetrics.heightPixels + val screenWidth = displayMetrics.widthPixels + + // Create a path for the gesture (swipe from middle-right to middle-left) + val swipePath = Path() + swipePath.moveTo(screenWidth * 0.7f, screenHeight / 2f) // Start from 70% across the screen + swipePath.lineTo(screenWidth * 0.3f, screenHeight / 2f) // Move to 30% across the screen + + // Create a gesture builder and add the swipe + val gestureBuilder = GestureDescription.Builder() + val gesture = GestureDescription.StrokeDescription( + swipePath, + 0, // start time + 300 // duration in milliseconds + ) + gestureBuilder.addStroke(gesture) + + // Dispatch the gesture + val result = dispatchGesture( + gestureBuilder.build(), + object : GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription) { + Log.d(TAG, "Scroll left gesture completed") + showToast("Erfolgreich nach links gescrollt", false) + } + + override fun onCancelled(gestureDescription: GestureDescription) { + Log.e(TAG, "Scroll left gesture cancelled") + showToast("Scrollen nach links abgebrochen", true) + } + }, + null // handler + ) + + if (!result) { + Log.e(TAG, "Failed to dispatch scroll left gesture") + showToast("Fehler beim Scrollen nach links", true) + } + } catch (e: Exception) { + Log.e(TAG, "Error scrolling left: ${e.message}") + showToast("Fehler beim Scrollen nach links: ${e.message}", true) + } + } + + /** + * Scroll left from specific coordinates with custom distance and duration + * + * @param x Starting X coordinate + * @param y Starting Y coordinate + * @param distance Distance in pixels to scroll + * @param duration Duration of the scroll gesture in milliseconds + */ + fun scrollLeft(x: Float, y: Float, distance: Float, duration: Long) { + Log.d(TAG, "Scrolling left from ($x, $y) with distance $distance and duration $duration ms") + showToast("Scrolle nach links von bestimmter Position...", false) + + try { + // Create a path for the gesture (swipe from specified position leftward by the specified distance) + val swipePath = Path() + swipePath.moveTo(x, y) // Start from specified position + swipePath.lineTo(x - distance, y) // Move leftward by the specified distance + + // Create a gesture builder and add the swipe + val gestureBuilder = GestureDescription.Builder() + val gesture = GestureDescription.StrokeDescription( + swipePath, + 0, // start time + duration // custom duration in milliseconds + ) + gestureBuilder.addStroke(gesture) + + // Dispatch the gesture + val result = dispatchGesture( + gestureBuilder.build(), + object : GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription) { + Log.d(TAG, "Coordinate-based scroll left gesture completed") + showToast("Erfolgreich nach links gescrollt von Position ($x, $y)", false) + } + + override fun onCancelled(gestureDescription: GestureDescription) { + Log.e(TAG, "Coordinate-based scroll left gesture cancelled") + showToast("Scrollen nach links von Position ($x, $y) abgebrochen", true) + } + }, + null // handler + ) + + if (!result) { + Log.e(TAG, "Failed to dispatch coordinate-based scroll left gesture") + showToast("Fehler beim Scrollen nach links von Position ($x, $y)", true) + } + } catch (e: Exception) { + Log.e(TAG, "Error scrolling left from coordinates: ${e.message}") + showToast("Fehler beim Scrollen nach links von Position ($x, $y): ${e.message}", true) + } + } + + /** + * Scroll right on the screen using gesture + */ + fun scrollRight() { + Log.d(TAG, "Scrolling right") + showToast("Scrolle nach rechts...", false) + + try { + // Get display metrics to calculate swipe coordinates + val displayMetrics = resources.displayMetrics + val screenHeight = displayMetrics.heightPixels + val screenWidth = displayMetrics.widthPixels + + // Create a path for the gesture (swipe from middle-left to middle-right) + val swipePath = Path() + swipePath.moveTo(screenWidth * 0.3f, screenHeight / 2f) // Start from 30% across the screen + swipePath.lineTo(screenWidth * 0.7f, screenHeight / 2f) // Move to 70% across the screen + + // Create a gesture builder and add the swipe + val gestureBuilder = GestureDescription.Builder() + val gesture = GestureDescription.StrokeDescription( + swipePath, + 0, // start time + 300 // duration in milliseconds + ) + gestureBuilder.addStroke(gesture) + + // Dispatch the gesture + val result = dispatchGesture( + gestureBuilder.build(), + object : GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription) { + Log.d(TAG, "Scroll right gesture completed") + showToast("Erfolgreich nach rechts gescrollt", false) + } + + override fun onCancelled(gestureDescription: GestureDescription) { + Log.e(TAG, "Scroll right gesture cancelled") + showToast("Scrollen nach rechts abgebrochen", true) + } + }, + null // handler + ) + + if (!result) { + Log.e(TAG, "Failed to dispatch scroll right gesture") + showToast("Fehler beim Scrollen nach rechts", true) + } + } catch (e: Exception) { + Log.e(TAG, "Error scrolling right: ${e.message}") + showToast("Fehler beim Scrollen nach rechts: ${e.message}", true) + } + } + + /** + * Scroll right from specific coordinates with custom distance and duration + * + * @param x Starting X coordinate + * @param y Starting Y coordinate + * @param distance Distance in pixels to scroll + * @param duration Duration of the scroll gesture in milliseconds + */ + fun scrollRight(x: Float, y: Float, distance: Float, duration: Long) { + Log.d(TAG, "Scrolling right from ($x, $y) with distance $distance and duration $duration ms") + showToast("Scrolle nach rechts von bestimmter Position...", false) + + try { + // Create a path for the gesture (swipe from specified position rightward by the specified distance) + val swipePath = Path() + swipePath.moveTo(x, y) // Start from specified position + swipePath.lineTo(x + distance, y) // Move rightward by the specified distance + + // Create a gesture builder and add the swipe + val gestureBuilder = GestureDescription.Builder() + val gesture = GestureDescription.StrokeDescription( + swipePath, + 0, // start time + duration // custom duration in milliseconds + ) + gestureBuilder.addStroke(gesture) + + // Dispatch the gesture + val result = dispatchGesture( + gestureBuilder.build(), + object : GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription) { + Log.d(TAG, "Coordinate-based scroll right gesture completed") + showToast("Erfolgreich nach rechts gescrollt von Position ($x, $y)", false) + } + + override fun onCancelled(gestureDescription: GestureDescription) { + Log.e(TAG, "Coordinate-based scroll right gesture cancelled") + showToast("Scrollen nach rechts von Position ($x, $y) abgebrochen", true) + } + }, + null // handler + ) + + if (!result) { + Log.e(TAG, "Failed to dispatch coordinate-based scroll right gesture") + showToast("Fehler beim Scrollen nach rechts von Position ($x, $y)", true) + } + } catch (e: Exception) { + Log.e(TAG, "Error scrolling right from coordinates: ${e.message}") + showToast("Fehler beim Scrollen nach rechts von Position ($x, $y): ${e.message}", true) + } + } + /** * Retrieve the latest screenshot from the standard screenshot folder */ diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatScreen.kt b/app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatScreen.kt index b4a9ced2..f39926d0 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatScreen.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatScreen.kt @@ -200,6 +200,12 @@ fun ChatScreen( is Command.ShowRecentApps -> "Übersicht der letzten Apps öffnen" is Command.ScrollDown -> "Nach unten scrollen" is Command.ScrollUp -> "Nach oben scrollen" + is Command.ScrollLeft -> "Nach links scrollen" + is Command.ScrollRight -> "Nach rechts scrollen" + is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" } Text( diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatViewModel.kt index 2843c157..867c7356 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/chat/ChatViewModel.kt @@ -123,6 +123,12 @@ class ChatViewModel( is Command.ShowRecentApps -> "Übersicht der letzten Apps öffnen" is Command.ScrollDown -> "Nach unten scrollen" is Command.ScrollUp -> "Nach oben scrollen" + is Command.ScrollLeft -> "Nach links scrollen" + is Command.ScrollRight -> "Nach rechts scrollen" + is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" + is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" + is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" + is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" } } @@ -177,6 +183,12 @@ class ChatViewModel( is Command.ShowRecentApps -> "Übersicht der letzten Apps öffnen" is Command.ScrollDown -> "Nach unten scrollen" is Command.ScrollUp -> "Nach oben scrollen" + is Command.ScrollLeft -> "Nach links scrollen" + is Command.ScrollRight -> "Nach rechts scrollen" + is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" } _commandExecutionStatus.value = "Führe aus: $commandDescription (${index + 1}/${commands.size})" diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt index e69e7cdb..5cb05e6c 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt @@ -446,6 +446,12 @@ fun PhotoReasoningScreen( is Command.ShowRecentApps -> "Übersicht der letzten Apps öffnen" is Command.ScrollDown -> "Nach unten scrollen" is Command.ScrollUp -> "Nach oben scrollen" + is Command.ScrollLeft -> "Nach links scrollen" + is Command.ScrollRight -> "Nach rechts scrollen" + is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" } Text( diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index 845f1551..7d8b974e 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -251,6 +251,12 @@ class PhotoReasoningViewModel( is Command.ShowRecentApps -> "Übersicht der letzten Apps öffnen" is Command.ScrollDown -> "Nach unten scrollen" is Command.ScrollUp -> "Nach oben scrollen" + is Command.ScrollLeft -> "Nach links scrollen" + is Command.ScrollRight -> "Nach rechts scrollen" + is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" + is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" + is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" + is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" } } @@ -305,6 +311,12 @@ class PhotoReasoningViewModel( is Command.ShowRecentApps -> "Übersicht der letzten Apps öffnen" is Command.ScrollDown -> "Nach unten scrollen" is Command.ScrollUp -> "Nach oben scrollen" + is Command.ScrollLeft -> "Nach links scrollen" + is Command.ScrollRight -> "Nach rechts scrollen" + is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" } _commandExecutionStatus.value = "Führe aus: $commandDescription (${index + 1}/${commands.size})" diff --git a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt index 39de909f..a5d65b4f 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt @@ -108,6 +108,30 @@ object CommandParser { Regex("(?i)\\b(?:nach oben scrollen|hoch scrollen|nach oben wischen|hoch wischen)\\b") ) + // Scroll left patterns - for scrolling left + private val SCROLL_LEFT_PATTERNS = listOf( + // Function-like patterns + Regex("(?i)\\bscrollLeft\\(\\)"), + Regex("(?i)\\bscrollLeftPage\\(\\)"), + Regex("(?i)\\bpageLeft\\(\\)"), + + // Natural language patterns + Regex("(?i)\\b(?:scroll|swipe|move|nach links) (?:left|nach links)\\b"), + Regex("(?i)\\b(?:nach links scrollen|links scrollen|nach links wischen|links wischen)\\b") + ) + + // Scroll right patterns - for scrolling right + private val SCROLL_RIGHT_PATTERNS = listOf( + // Function-like patterns + Regex("(?i)\\bscrollRight\\(\\)"), + Regex("(?i)\\bscrollRightPage\\(\\)"), + Regex("(?i)\\bpageRight\\(\\)"), + + // Natural language patterns + Regex("(?i)\\b(?:scroll|swipe|move|nach rechts) (?:right|nach rechts)\\b"), + Regex("(?i)\\b(?:nach rechts scrollen|rechts scrollen|nach rechts wischen|rechts wischen)\\b") + ) + // Buffer for storing partial text between calls private var textBuffer = "" @@ -166,6 +190,12 @@ object CommandParser { is Command.ShowRecentApps -> Log.d(TAG, "Command details: ShowRecentApps") is Command.ScrollDown -> Log.d(TAG, "Command details: ScrollDown") is Command.ScrollUp -> Log.d(TAG, "Command details: ScrollUp") + is Command.ScrollLeft -> Log.d(TAG, "Command details: ScrollLeft") + is Command.ScrollRight -> Log.d(TAG, "Command details: ScrollRight") + is Command.ScrollDownFromCoordinates -> Log.d(TAG, "Command details: ScrollDownFromCoordinates(${command.x}, ${command.y}, ${command.distance}, ${command.duration})") + is Command.ScrollUpFromCoordinates -> Log.d(TAG, "Command details: ScrollUpFromCoordinates(${command.x}, ${command.y}, ${command.distance}, ${command.duration})") + is Command.ScrollLeftFromCoordinates -> Log.d(TAG, "Command details: ScrollLeftFromCoordinates(${command.x}, ${command.y}, ${command.distance}, ${command.duration})") + is Command.ScrollRightFromCoordinates -> Log.d(TAG, "Command details: ScrollRightFromCoordinates(${command.x}, ${command.y}, ${command.distance}, ${command.duration})") } } } catch (e: Exception) { @@ -202,6 +232,12 @@ object CommandParser { // Look for scroll up commands findScrollUpCommands(text, commands) + + // Look for scroll left commands + findScrollLeftCommands(text, commands) + + // Look for scroll right commands + findScrollRightCommands(text, commands) } /** @@ -345,15 +381,38 @@ object CommandParser { * Find scroll down commands in the text */ private fun findScrollDownCommands(text: String, commands: MutableList) { - // Try each pattern - for (pattern in SCROLL_DOWN_PATTERNS) { - if (pattern.containsMatchIn(text)) { - // Check if this command is already in the list (avoid duplicates) - if (!commands.any { it is Command.ScrollDown }) { - Log.d(TAG, "Found scroll down command with pattern ${pattern.pattern}") - commands.add(Command.ScrollDown) - // Only add one scroll down command even if multiple matches are found - break + // First check for coordinate-based scroll down commands + val coordPattern = Regex("(?i)\\bscrollDown\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)") + val matches = coordPattern.findAll(text) + + for (match in matches) { + if (match.groupValues.size >= 5) { + try { + val x = match.groupValues[1].toFloat() + val y = match.groupValues[2].toFloat() + val distance = match.groupValues[3].toFloat() + val duration = match.groupValues[4].toLong() + + Log.d(TAG, "Found coordinate-based scroll down command: scrollDown($x, $y, $distance, $duration)") + commands.add(Command.ScrollDownFromCoordinates(x, y, distance, duration)) + } catch (e: Exception) { + Log.e(TAG, "Error parsing coordinate-based scroll down command: ${e.message}") + } + } + } + + // If no coordinate-based commands were found, look for simple scroll down commands + if (!commands.any { it is Command.ScrollDownFromCoordinates }) { + // Try each pattern + for (pattern in SCROLL_DOWN_PATTERNS) { + if (pattern.containsMatchIn(text)) { + // Check if this command is already in the list (avoid duplicates) + if (!commands.any { it is Command.ScrollDown }) { + Log.d(TAG, "Found scroll down command with pattern ${pattern.pattern}") + commands.add(Command.ScrollDown) + // Only add one scroll down command even if multiple matches are found + break + } } } } @@ -363,15 +422,120 @@ object CommandParser { * Find scroll up commands in the text */ private fun findScrollUpCommands(text: String, commands: MutableList) { - // Try each pattern - for (pattern in SCROLL_UP_PATTERNS) { - if (pattern.containsMatchIn(text)) { - // Check if this command is already in the list (avoid duplicates) - if (!commands.any { it is Command.ScrollUp }) { - Log.d(TAG, "Found scroll up command with pattern ${pattern.pattern}") - commands.add(Command.ScrollUp) - // Only add one scroll up command even if multiple matches are found - break + // First check for coordinate-based scroll up commands + val coordPattern = Regex("(?i)\\bscrollUp\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)") + val matches = coordPattern.findAll(text) + + for (match in matches) { + if (match.groupValues.size >= 5) { + try { + val x = match.groupValues[1].toFloat() + val y = match.groupValues[2].toFloat() + val distance = match.groupValues[3].toFloat() + val duration = match.groupValues[4].toLong() + + Log.d(TAG, "Found coordinate-based scroll up command: scrollUp($x, $y, $distance, $duration)") + commands.add(Command.ScrollUpFromCoordinates(x, y, distance, duration)) + } catch (e: Exception) { + Log.e(TAG, "Error parsing coordinate-based scroll up command: ${e.message}") + } + } + } + + // If no coordinate-based commands were found, look for simple scroll up commands + if (!commands.any { it is Command.ScrollUpFromCoordinates }) { + // Try each pattern + for (pattern in SCROLL_UP_PATTERNS) { + if (pattern.containsMatchIn(text)) { + // Check if this command is already in the list (avoid duplicates) + if (!commands.any { it is Command.ScrollUp }) { + Log.d(TAG, "Found scroll up command with pattern ${pattern.pattern}") + commands.add(Command.ScrollUp) + // Only add one scroll up command even if multiple matches are found + break + } + } + } + } + } + + /** + * Find scroll left commands in the text + */ + private fun findScrollLeftCommands(text: String, commands: MutableList) { + // First check for coordinate-based scroll left commands + val coordPattern = Regex("(?i)\\bscrollLeft\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)") + val matches = coordPattern.findAll(text) + + for (match in matches) { + if (match.groupValues.size >= 5) { + try { + val x = match.groupValues[1].toFloat() + val y = match.groupValues[2].toFloat() + val distance = match.groupValues[3].toFloat() + val duration = match.groupValues[4].toLong() + + Log.d(TAG, "Found coordinate-based scroll left command: scrollLeft($x, $y, $distance, $duration)") + commands.add(Command.ScrollLeftFromCoordinates(x, y, distance, duration)) + } catch (e: Exception) { + Log.e(TAG, "Error parsing coordinate-based scroll left command: ${e.message}") + } + } + } + + // If no coordinate-based commands were found, look for simple scroll left commands + if (!commands.any { it is Command.ScrollLeftFromCoordinates }) { + // Try each pattern + for (pattern in SCROLL_LEFT_PATTERNS) { + if (pattern.containsMatchIn(text)) { + // Check if this command is already in the list (avoid duplicates) + if (!commands.any { it is Command.ScrollLeft }) { + Log.d(TAG, "Found scroll left command with pattern ${pattern.pattern}") + commands.add(Command.ScrollLeft) + // Only add one scroll left command even if multiple matches are found + break + } + } + } + } + } + + /** + * Find scroll right commands in the text + */ + private fun findScrollRightCommands(text: String, commands: MutableList) { + // First check for coordinate-based scroll right commands + val coordPattern = Regex("(?i)\\bscrollRight\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)") + val matches = coordPattern.findAll(text) + + for (match in matches) { + if (match.groupValues.size >= 5) { + try { + val x = match.groupValues[1].toFloat() + val y = match.groupValues[2].toFloat() + val distance = match.groupValues[3].toFloat() + val duration = match.groupValues[4].toLong() + + Log.d(TAG, "Found coordinate-based scroll right command: scrollRight($x, $y, $distance, $duration)") + commands.add(Command.ScrollRightFromCoordinates(x, y, distance, duration)) + } catch (e: Exception) { + Log.e(TAG, "Error parsing coordinate-based scroll right command: ${e.message}") + } + } + } + + // If no coordinate-based commands were found, look for simple scroll right commands + if (!commands.any { it is Command.ScrollRightFromCoordinates }) { + // Try each pattern + for (pattern in SCROLL_RIGHT_PATTERNS) { + if (pattern.containsMatchIn(text)) { + // Check if this command is already in the list (avoid duplicates) + if (!commands.any { it is Command.ScrollRight }) { + Log.d(TAG, "Found scroll right command with pattern ${pattern.pattern}") + commands.add(Command.ScrollRight) + // Only add one scroll right command even if multiple matches are found + break + } } } } @@ -451,4 +615,34 @@ sealed class Command { * Command to scroll up */ object ScrollUp : Command() + + /** + * Command to scroll left + */ + object ScrollLeft : Command() + + /** + * Command to scroll right + */ + object ScrollRight : Command() + + /** + * Command to scroll down from specific coordinates with custom distance and duration + */ + data class ScrollDownFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command() + + /** + * Command to scroll up from specific coordinates with custom distance and duration + */ + data class ScrollUpFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command() + + /** + * Command to scroll left from specific coordinates with custom distance and duration + */ + data class ScrollLeftFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command() + + /** + * Command to scroll right from specific coordinates with custom distance and duration + */ + data class ScrollRightFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command() }