diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 70ff642..d71eed4 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -3,8 +3,9 @@ plugins { id("com.android.application") id("org.jetbrains.kotlin.android") - id("org.jetbrains.kotlin.plugin.serialization") version "1.9.0" + id("org.jetbrains.kotlin.plugin.serialization") version "1.9.20" id("com.google.android.libraries.mapsplatform.secrets-gradle-plugin") + id("kotlin-parcelize") } android { diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index 19a4470..74ebfbe 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -32,6 +32,7 @@ import com.google.ai.sample.GenerativeViewModelFactory import java.util.Date import java.util.Locale import java.util.concurrent.atomic.AtomicBoolean +import java.lang.NumberFormatException class ScreenOperatorAccessibilityService : AccessibilityService() { companion object { @@ -97,6 +98,10 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { showToast("Accessibility Service is not available. Please enable the service in settings.", true) return } + + val displayMetrics = serviceInstance!!.resources.displayMetrics + val screenWidth = displayMetrics.widthPixels + val screenHeight = displayMetrics.heightPixels // Execute the command when (command) { @@ -106,9 +111,11 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { serviceInstance?.findAndClickButtonByText(command.buttonText) } is Command.TapCoordinates -> { - Log.d(TAG, "Tapping at coordinates: (${command.x}, ${command.y})") - showToast("Trying to tap coordinates: (${command.x}, ${command.y})", false) - serviceInstance?.tapAtCoordinates(command.x, command.y) + val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth) + val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight) + Log.d(TAG, "Tapping at coordinates: (${command.x} -> $xPx, ${command.y} -> $yPx)") + showToast("Trying to tap coordinates: ($xPx, $yPx)", false) + serviceInstance?.tapAtCoordinates(xPx, yPx) } is Command.TakeScreenshot -> { Log.d(TAG, "Taking screenshot with 850ms delay") @@ -154,24 +161,44 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { serviceInstance?.scrollRight() } is Command.ScrollDownFromCoordinates -> { - Log.d(TAG, "Scrolling down from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms") - showToast("Trying to scroll down from position (${command.x}, ${command.y})", false) - serviceInstance?.scrollDown(command.x, command.y, command.distance, command.duration) + Log.d(TAG, "ScrollDownFromCoordinates: Original inputs x='${command.x}', y='${command.y}', distance='${command.distance}', duration='${command.duration}'") + Log.d(TAG, "ScrollDownFromCoordinates: Using screenWidth=$screenWidth, screenHeight=$screenHeight for conversions (distance uses screenHeight).") + val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth) + val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight) + val distancePx = serviceInstance!!.convertCoordinate(command.distance, screenHeight) + Log.d(TAG, "ScrollDownFromCoordinates: Converted to xPx=$xPx, yPx=$yPx, distancePx=$distancePx") + showToast("Trying to scroll down from position ($xPx, $yPx)", false) + serviceInstance?.scrollDown(xPx, yPx, distancePx, command.duration) } is Command.ScrollUpFromCoordinates -> { - Log.d(TAG, "Scrolling up from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms") - showToast("Trying to scroll up from position (${command.x}, ${command.y})", false) - serviceInstance?.scrollUp(command.x, command.y, command.distance, command.duration) + Log.d(TAG, "ScrollUpFromCoordinates: Original inputs x='${command.x}', y='${command.y}', distance='${command.distance}', duration='${command.duration}'") + Log.d(TAG, "ScrollUpFromCoordinates: Using screenWidth=$screenWidth, screenHeight=$screenHeight for conversions (distance uses screenHeight).") + val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth) + val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight) + val distancePx = serviceInstance!!.convertCoordinate(command.distance, screenHeight) + Log.d(TAG, "ScrollUpFromCoordinates: Converted to xPx=$xPx, yPx=$yPx, distancePx=$distancePx") + showToast("Trying to scroll up from position ($xPx, $yPx)", false) + serviceInstance?.scrollUp(xPx, yPx, distancePx, command.duration) } is Command.ScrollLeftFromCoordinates -> { - Log.d(TAG, "Scrolling left from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms") - showToast("Trying to scroll left from position (${command.x}, ${command.y})", false) - serviceInstance?.scrollLeft(command.x, command.y, command.distance, command.duration) + Log.d(TAG, "ScrollLeftFromCoordinates: Original inputs x='${command.x}', y='${command.y}', distance='${command.distance}', duration='${command.duration}'") + Log.d(TAG, "ScrollLeftFromCoordinates: Using screenWidth=$screenWidth, screenHeight=$screenHeight for conversions (distance uses screenWidth).") + val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth) + val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight) + val distancePx = serviceInstance!!.convertCoordinate(command.distance, screenWidth) + Log.d(TAG, "ScrollLeftFromCoordinates: Converted to xPx=$xPx, yPx=$yPx, distancePx=$distancePx") + showToast("Trying to scroll left from position ($xPx, $yPx)", false) + serviceInstance?.scrollLeft(xPx, yPx, distancePx, command.duration) } is Command.ScrollRightFromCoordinates -> { - Log.d(TAG, "Scrolling right from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms") - showToast("Trying to scroll right from position (${command.x}, ${command.y})", false) - serviceInstance?.scrollRight(command.x, command.y, command.distance, command.duration) + Log.d(TAG, "ScrollRightFromCoordinates: Original inputs x='${command.x}', y='${command.y}', distance='${command.distance}', duration='${command.duration}'") + Log.d(TAG, "ScrollRightFromCoordinates: Using screenWidth=$screenWidth, screenHeight=$screenHeight for conversions (distance uses screenWidth).") + val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth) + val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight) + val distancePx = serviceInstance!!.convertCoordinate(command.distance, screenWidth) + Log.d(TAG, "ScrollRightFromCoordinates: Converted to xPx=$xPx, yPx=$yPx, distancePx=$distancePx") + showToast("Trying to scroll right from position ($xPx, $yPx)", false) + serviceInstance?.scrollRight(xPx, yPx, distancePx, command.duration) } is Command.OpenApp -> { Log.d(TAG, "Opening app: ${command.packageName}") @@ -256,6 +283,25 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { // Show a toast to indicate the service is connected showToast("Accessibility Service is enabled and connected", false) } + + private fun convertCoordinate(coordinateString: String, screenSize: Int): Float { + return try { + if (coordinateString.endsWith("%")) { + val numericValue = coordinateString.removeSuffix("%").toFloat() + (numericValue / 100.0f) * screenSize + } else { + coordinateString.toFloat() + } + } catch (e: NumberFormatException) { + Log.e(TAG, "Error converting coordinate string: '$coordinateString'", e) + showToast("Error parsing coordinate: '$coordinateString'. Using 0f.", true) + 0f // Default to 0f or handle error as appropriate + } catch (e: Exception) { + Log.e(TAG, "Unexpected error converting coordinate string: '$coordinateString'", e) + showToast("Unexpected error parsing coordinate: '$coordinateString'. Using 0f.", true) + 0f // Default to 0f or handle error as appropriate + } + } override fun onInterrupt() { Log.d(TAG, "Accessibility service interrupted") @@ -1972,14 +2018,19 @@ fun pressEnterKey() { * @param duration Duration of the scroll gesture in milliseconds */ fun scrollDown(x: Float, y: Float, distance: Float, duration: Long) { - Log.d(TAG, "Scrolling down from ($x, $y) with distance $distance and duration $duration ms") + Log.d(TAG, "scrollDown method: Received x=$x, y=$y, distance=$distance, duration=$duration") showToast("Scrolling down from specific position...", false) try { // Create a path for the gesture (swipe from specified position upward by the specified distance) val swipePath = Path() - swipePath.moveTo(x, y) // Start from specified position - swipePath.lineTo(x, y - distance) // Move upward by the specified distance + val startX = x + val startY = y + val endX = x + val endY = y - distance + swipePath.moveTo(startX, startY) + swipePath.lineTo(endX, endY) + Log.d(TAG, "scrollDown method: Creating swipePath from ($startX, $startY) to ($endX, $endY) over $duration ms") // Create a gesture builder and add the swipe val gestureBuilder = GestureDescription.Builder() @@ -1995,21 +2046,23 @@ fun pressEnterKey() { gestureBuilder.build(), object : GestureResultCallback() { override fun onCompleted(gestureDescription: GestureDescription) { - Log.d(TAG, "Coordinate-based scroll down gesture completed") - showToast("Successfully scrolled down from position ($x, $y)", false) + super.onCompleted(gestureDescription) + Log.d(TAG, "scrollDown method: Gesture completed for path from ($startX, $startY) to ($endX, $endY)") + showToast("Successfully scrolled down from position ($startX, $startY)", false) } override fun onCancelled(gestureDescription: GestureDescription) { - Log.e(TAG, "Coordinate-based scroll down gesture cancelled") - showToast("Scroll down from position ($x, $y) cancelled", true) + super.onCancelled(gestureDescription) + Log.e(TAG, "scrollDown method: Gesture CANCELLED for path from ($startX, $startY) to ($endX, $endY). GestureDescription: $gestureDescription") + showToast("Scroll down from position ($startX, $startY) cancelled", true) } }, null // handler ) if (!result) { - Log.e(TAG, "Failed to dispatch coordinate-based scroll down gesture") - showToast("Error scrolling down from position ($x, $y)", true) + Log.e(TAG, "Failed to dispatch coordinate-based scroll down gesture for path from ($startX, $startY) to ($endX, $endY)") + showToast("Error scrolling down from position ($startX, $startY)", true) } } catch (e: Exception) { Log.e(TAG, "Error scrolling down from coordinates: ${e.message}") @@ -2082,14 +2135,19 @@ fun pressEnterKey() { * @param duration Duration of the scroll gesture in milliseconds */ fun scrollUp(x: Float, y: Float, distance: Float, duration: Long) { - Log.d(TAG, "Scrolling up from ($x, $y) with distance $distance and duration $duration ms") + Log.d(TAG, "scrollUp method: Received x=$x, y=$y, distance=$distance, duration=$duration") showToast("Scrolling up from specific position...", false) try { // Create a path for the gesture (swipe from specified position downward by the specified distance) val swipePath = Path() - swipePath.moveTo(x, y) // Start from specified position - swipePath.lineTo(x, y + distance) // Move downward by the specified distance + val startX = x + val startY = y + val endX = x + val endY = y + distance + swipePath.moveTo(startX, startY) + swipePath.lineTo(endX, endY) + Log.d(TAG, "scrollUp method: Creating swipePath from ($startX, $startY) to ($endX, $endY) over $duration ms") // Create a gesture builder and add the swipe val gestureBuilder = GestureDescription.Builder() @@ -2105,21 +2163,23 @@ fun pressEnterKey() { gestureBuilder.build(), object : GestureResultCallback() { override fun onCompleted(gestureDescription: GestureDescription) { - Log.d(TAG, "Coordinate-based scroll up gesture completed") - showToast("Successfully scrolled up from position ($x, $y)", false) + super.onCompleted(gestureDescription) + Log.d(TAG, "scrollUp method: Gesture completed for path from ($startX, $startY) to ($endX, $endY)") + showToast("Successfully scrolled up from position ($startX, $startY)", false) } override fun onCancelled(gestureDescription: GestureDescription) { - Log.e(TAG, "Coordinate-based scroll up gesture cancelled") - showToast("Scroll up from position ($x, $y) cancelled", true) + super.onCancelled(gestureDescription) + Log.e(TAG, "scrollUp method: Gesture CANCELLED for path from ($startX, $startY) to ($endX, $endY). GestureDescription: $gestureDescription") + showToast("Scroll up from position ($startX, $startY) cancelled", true) } }, null // handler ) if (!result) { - Log.e(TAG, "Failed to dispatch coordinate-based scroll up gesture") - showToast("Error scrolling up from position ($x, $y)", true) + Log.e(TAG, "Failed to dispatch coordinate-based scroll up gesture for path from ($startX, $startY) to ($endX, $endY)") + showToast("Error scrolling up from position ($startX, $startY)", true) } } catch (e: Exception) { Log.e(TAG, "Error scrolling up from coordinates: ${e.message}") @@ -2140,10 +2200,10 @@ fun pressEnterKey() { val screenHeight = displayMetrics.heightPixels val screenWidth = displayMetrics.widthPixels - // Create a path for the gesture (swipe from middle-right to middle-left) + // Create a path for the gesture (swipe from left to right, content moves left) val swipePath = Path() - swipePath.moveTo(screenWidth * 0.7f, screenHeight / 2f) // Start from 70% across the screen - swipePath.lineTo(screenWidth * 0.3f, screenHeight / 2f) // Move to 30% across the screen + swipePath.moveTo(screenWidth * 0.3f, screenHeight / 2f) // Start from 30% across the screen + swipePath.lineTo(screenWidth * 0.7f, screenHeight / 2f) // Move to 70% across the screen // Create a gesture builder and add the swipe val gestureBuilder = GestureDescription.Builder() @@ -2190,14 +2250,19 @@ fun pressEnterKey() { * @param duration Duration of the scroll gesture in milliseconds */ fun scrollLeft(x: Float, y: Float, distance: Float, duration: Long) { - Log.d(TAG, "Scrolling left from ($x, $y) with distance $distance and duration $duration ms") + Log.d(TAG, "scrollLeft method: Received x=$x, y=$y, distance=$distance, duration=$duration") showToast("Scrolling left from specific position...", false) try { - // Create a path for the gesture (swipe from specified position to the left by the specified distance) + // Create a path for the gesture (swipe L-R, content moves Left) val swipePath = Path() - swipePath.moveTo(x, y) // Start from specified position - swipePath.lineTo(x - distance, y) // Move to the left by the specified distance + val startX = x + val startY = y + val endX = x + distance // Finger swipes L-R, content moves Left + val endY = y + swipePath.moveTo(startX, startY) + swipePath.lineTo(endX, endY) + Log.d(TAG, "scrollLeft method: Creating swipePath from ($startX, $startY) to ($endX, $endY) over $duration ms") // Create a gesture builder and add the swipe val gestureBuilder = GestureDescription.Builder() @@ -2213,21 +2278,23 @@ fun pressEnterKey() { gestureBuilder.build(), object : GestureResultCallback() { override fun onCompleted(gestureDescription: GestureDescription) { - Log.d(TAG, "Coordinate-based scroll left gesture completed") - showToast("Successfully scrolled left from position ($x, $y)", false) + super.onCompleted(gestureDescription) + Log.d(TAG, "scrollLeft method: Gesture completed for path from ($startX, $startY) to ($endX, $endY)") + showToast("Successfully scrolled left from position ($startX, $startY)", false) } override fun onCancelled(gestureDescription: GestureDescription) { - Log.e(TAG, "Coordinate-based scroll left gesture cancelled") - showToast("Scroll left from position ($x, $y) cancelled", true) + super.onCancelled(gestureDescription) + Log.e(TAG, "scrollLeft method: Gesture CANCELLED for path from ($startX, $startY) to ($endX, $endY). GestureDescription: $gestureDescription") + showToast("Scroll left from position ($startX, $startY) cancelled", true) } }, null // handler ) if (!result) { - Log.e(TAG, "Failed to dispatch coordinate-based scroll left gesture") - showToast("Error scrolling left from position ($x, $y)", true) + Log.e(TAG, "Failed to dispatch coordinate-based scroll left gesture for path from ($startX, $startY) to ($endX, $endY)") + showToast("Error scrolling left from position ($startX, $startY)", true) } } catch (e: Exception) { Log.e(TAG, "Error scrolling left from coordinates: ${e.message}") @@ -2248,10 +2315,10 @@ fun pressEnterKey() { val screenHeight = displayMetrics.heightPixels val screenWidth = displayMetrics.widthPixels - // Create a path for the gesture (swipe from middle-left to middle-right) + // Create a path for the gesture (swipe from right to left, content moves right) val swipePath = Path() - swipePath.moveTo(screenWidth * 0.3f, screenHeight / 2f) // Start from 30% across the screen - swipePath.lineTo(screenWidth * 0.7f, screenHeight / 2f) // Move to 70% across the screen + swipePath.moveTo(screenWidth * 0.7f, screenHeight / 2f) // Start from 70% across the screen + swipePath.lineTo(screenWidth * 0.3f, screenHeight / 2f) // Move to 30% across the screen // Create a gesture builder and add the swipe val gestureBuilder = GestureDescription.Builder() @@ -2298,14 +2365,19 @@ fun pressEnterKey() { * @param duration Duration of the scroll gesture in milliseconds */ fun scrollRight(x: Float, y: Float, distance: Float, duration: Long) { - Log.d(TAG, "Scrolling right from ($x, $y) with distance $distance and duration $duration ms") + Log.d(TAG, "scrollRight method: Received x=$x, y=$y, distance=$distance, duration=$duration") showToast("Scrolling right from specific position...", false) try { - // Create a path for the gesture (swipe from specified position to the right by the specified distance) + // Create a path for the gesture (swipe R-L, content moves Right) val swipePath = Path() - swipePath.moveTo(x, y) // Start from specified position - swipePath.lineTo(x + distance, y) // Move to the right by the specified distance + val startX = x + val startY = y + val endX = x - distance // Finger swipes R-L, content moves Right + val endY = y + swipePath.moveTo(startX, startY) + swipePath.lineTo(endX, endY) + Log.d(TAG, "scrollRight method: Creating swipePath from ($startX, $startY) to ($endX, $endY) over $duration ms") // Create a gesture builder and add the swipe val gestureBuilder = GestureDescription.Builder() @@ -2321,21 +2393,23 @@ fun pressEnterKey() { gestureBuilder.build(), object : GestureResultCallback() { override fun onCompleted(gestureDescription: GestureDescription) { - Log.d(TAG, "Coordinate-based scroll right gesture completed") - showToast("Successfully scrolled right from position ($x, $y)", false) + super.onCompleted(gestureDescription) + Log.d(TAG, "scrollRight method: Gesture completed for path from ($startX, $startY) to ($endX, $endY)") + showToast("Successfully scrolled right from position ($startX, $startY)", false) } override fun onCancelled(gestureDescription: GestureDescription) { - Log.e(TAG, "Coordinate-based scroll right gesture cancelled") - showToast("Scroll right from position ($x, $y) cancelled", true) + super.onCancelled(gestureDescription) + Log.e(TAG, "scrollRight method: Gesture CANCELLED for path from ($startX, $startY) to ($endX, $endY). GestureDescription: $gestureDescription") + showToast("Scroll right from position ($startX, $startY) cancelled", true) } }, null // handler ) if (!result) { - Log.e(TAG, "Failed to dispatch coordinate-based scroll right gesture") - showToast("Error scrolling right from position ($x, $y)", true) + Log.e(TAG, "Failed to dispatch coordinate-based scroll right gesture for path from ($startX, $startY) to ($endX, $endY)") + showToast("Error scrolling right from position ($startX, $startY)", true) } } catch (e: Exception) { Log.e(TAG, "Error scrolling right from coordinates: ${e.message}") diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt index 9a00017..9e48218 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt @@ -95,10 +95,12 @@ import com.google.ai.sample.util.Command import com.google.ai.sample.util.SystemMessageEntry import com.google.ai.sample.util.SystemMessageEntryPreferences import com.google.ai.sample.util.UriSaver -import com.google.ai.sample.util.shareTextFile // Added for sharing +import com.google.ai.sample.util.shareTextFile +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch -import kotlinx.serialization.builtins.ListSerializer // Added for JSON -import kotlinx.serialization.json.Json // Added for JSON +import kotlinx.coroutines.withContext +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.json.Json import android.util.Log import kotlinx.serialization.SerializationException @@ -148,7 +150,7 @@ internal fun PhotoReasoningRoute( viewModel.updateSystemMessage(message, context) }, onReasonClicked = { inputText, selectedItems -> - coroutineScope.launch { + coroutineScope.launch { val bitmaps = selectedItems.mapNotNull { val imageRequest = imageRequestBuilder.data(it).precision(Precision.EXACT).build() try { @@ -205,7 +207,7 @@ fun PhotoReasoningScreen( LaunchedEffect(Unit) { systemMessageEntries = SystemMessageEntryPreferences.loadEntries(context) } - LaunchedEffect(showDatabaseListPopup) { // Reload entries when database popup becomes visible + LaunchedEffect(showDatabaseListPopup) { if (showDatabaseListPopup) { systemMessageEntries = SystemMessageEntryPreferences.loadEntries(context) } @@ -283,7 +285,7 @@ fun PhotoReasoningScreen( Spacer(modifier = Modifier.height(8.dp)) TextButton(onClick = { onEnableAccessibilityService() - Toast.makeText(context, "Open Accessibility Settings...", Toast.LENGTH_SHORT).show() + Toast.makeText(context, "Open Accessibility Settings..." as CharSequence, Toast.LENGTH_SHORT).show() }) { Text("Activate Accessibility Service") } } } @@ -324,7 +326,7 @@ fun PhotoReasoningScreen( } } else { onEnableAccessibilityService() - Toast.makeText(context, "Enable the Accessibility service for Screen Operator", Toast.LENGTH_LONG).show() + Toast.makeText(context, "Enable the Accessibility service for Screen Operator" as CharSequence, Toast.LENGTH_LONG).show() } }, modifier = Modifier.padding(all = 4.dp).align(Alignment.CenterVertically)) { Icon(Icons.Default.Send, stringResource(R.string.action_go), tint = MaterialTheme.colorScheme.primary) @@ -354,7 +356,6 @@ fun PhotoReasoningScreen( is Command.ClickButton -> "Click on button: \"${command.buttonText}\"" is Command.TapCoordinates -> "Tap coordinates: (${command.x}, ${command.y})" is Command.TakeScreenshot -> "Take screenshot" - // ... (other command types) ... else -> command::class.simpleName ?: "Unknown Command" } Text("${index + 1}. $commandText", color = MaterialTheme.colorScheme.onTertiaryContainer) @@ -380,7 +381,7 @@ fun PhotoReasoningScreen( SystemMessageEntryPreferences.deleteEntry(context, entry) systemMessageEntries = SystemMessageEntryPreferences.loadEntries(context) }, - onImportCompleted = { // Add this new lambda + onImportCompleted = { systemMessageEntries = SystemMessageEntryPreferences.loadEntries(context) } ) @@ -393,7 +394,7 @@ fun PhotoReasoningScreen( onSaveClicked = { title, guide, originalEntry -> val currentEntry = SystemMessageEntry(title.trim(), guide.trim()) if (title.isBlank() || guide.isBlank()) { - Toast.makeText(context, "Title and Guide cannot be empty.", Toast.LENGTH_SHORT).show() + Toast.makeText(context, "Title and Guide cannot be empty." as CharSequence, Toast.LENGTH_SHORT).show() return@EditEntryPopup } if (originalEntry == null) { @@ -403,13 +404,13 @@ fun PhotoReasoningScreen( showEditEntryPopup = false systemMessageEntries = SystemMessageEntryPreferences.loadEntries(context) } else { - Toast.makeText(context, "An entry with this title already exists.", Toast.LENGTH_SHORT).show() + Toast.makeText(context, "An entry with this title already exists." as CharSequence, Toast.LENGTH_SHORT).show() return@EditEntryPopup } } else { val existingEntryWithNewTitle = systemMessageEntries.find { it.title.equals(currentEntry.title, ignoreCase = true) && it.guide != originalEntry.guide } if (existingEntryWithNewTitle != null && originalEntry.title != currentEntry.title) { - Toast.makeText(context, "Another entry with this new title already exists.", Toast.LENGTH_SHORT).show() + Toast.makeText(context, "Another entry with this new title already exists." as CharSequence, Toast.LENGTH_SHORT).show() return@EditEntryPopup } SystemMessageEntryPreferences.updateEntry(context, originalEntry, currentEntry) @@ -429,8 +430,10 @@ fun DatabaseListPopup( onNewClicked: () -> Unit, onEntryClicked: (SystemMessageEntry) -> Unit, onDeleteClicked: (SystemMessageEntry) -> Unit, - onImportCompleted: () -> Unit // New lambda parameter + onImportCompleted: () -> Unit ) { + val TAG_IMPORT_PROCESS = "ImportProcess" + val scope = rememberCoroutineScope() var entryMenuToShow: SystemMessageEntry? by remember { mutableStateOf(null) } var selectionModeActive by rememberSaveable { mutableStateOf(false) } var selectedEntryTitles by rememberSaveable { mutableStateOf(emptySet()) } @@ -441,58 +444,136 @@ fun DatabaseListPopup( var remainingEntriesToImport by remember { mutableStateOf>(emptyList()) } var skipAllDuplicates by remember { mutableStateOf(false) } - fun processImportedEntries( + // processImportedEntries is defined within DatabaseListPopup, so it has access to context, onImportCompleted, etc. + fun processImportedEntries( imported: List, - currentSystemEntries: List // Pass current entries for comparison + currentSystemEntries: List ) { + val TAG_IMPORT_PROCESS_FUNCTION = "ImportProcessFunction" + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Starting processImportedEntries. Imported: ${imported.size}, Current: ${currentSystemEntries.size}, SkipAll: $skipAllDuplicates") + var newCount = 0 - var updatedCount = 0 + var updatedCount = 0 var skippedCount = 0 val entriesToProcess = imported.toMutableList() - fun continueProcessing(remainingToProcess: MutableList) { - while (remainingToProcess.isNotEmpty()) { - val newEntry = remainingToProcess.removeAt(0) - val existingEntry = currentSystemEntries.find { it.title.equals(newEntry.title, ignoreCase = true) } + while (entriesToProcess.isNotEmpty()) { + val newEntry = entriesToProcess.removeAt(0) + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Processing entry: Title='${newEntry.title}'. Remaining in batch: ${entriesToProcess.size}") + val existingEntry = currentSystemEntries.find { it.title.equals(newEntry.title, ignoreCase = true) } - if (existingEntry != null) { - if (skipAllDuplicates) { - skippedCount++ - continue - } - entryToConfirmOverwrite = Pair(existingEntry, newEntry) - remainingEntriesToImport = remainingToProcess.toList() // Save remaining for after dialog - return // Stop processing, let dialog handle this one - } else { - SystemMessageEntryPreferences.addEntry(context, newEntry) - newCount++ + if (existingEntry != null) { + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Duplicate found for title: '${newEntry.title}'. Existing guide: '${existingEntry.guide.take(50)}', New guide: '${newEntry.guide.take(50)}'") + if (skipAllDuplicates) { + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Skipping duplicate '${newEntry.title}' due to skipAllDuplicates flag.") + skippedCount++ + continue } + Log.d(TAG_IMPORT_PROCESS_FUNCTION, "Calling askForOverwrite for '${newEntry.title}'.") + entryToConfirmOverwrite = Pair(existingEntry, newEntry) + remainingEntriesToImport = entriesToProcess.toList() + return + } else { + Log.i(TAG_IMPORT_PROCESS_FUNCTION, "Adding new entry: Title='${newEntry.title}'") + SystemMessageEntryPreferences.addEntry(context, newEntry) + newCount++ } - // All processed or skipped - val summary = "Import finished: $newCount added, $updatedCount updated, $skippedCount skipped." - Toast.makeText(context, summary, Toast.LENGTH_LONG).show() - onImportCompleted() // Refresh the main list - skipAllDuplicates = false // Reset for next import operation } - continueProcessing(entriesToProcess) + Log.i(TAG_IMPORT_PROCESS_FUNCTION, "Finished processing batch. newCount=$newCount, updatedCount=$updatedCount, skippedCount=$skippedCount") + val summary = "Import finished: $newCount added, $updatedCount updated, $skippedCount skipped." + Toast.makeText(context, summary as CharSequence, Toast.LENGTH_LONG).show() + onImportCompleted() + skipAllDuplicates = false } val filePickerLauncher = rememberLauncherForActivityResult( contract = ActivityResultContracts.GetContent(), onResult = { uri: Uri? -> - uri?.let { + Log.d(TAG_IMPORT_PROCESS, "FilePickerLauncher onResult triggered.") + if (uri == null) { + Log.w(TAG_IMPORT_PROCESS, "URI is null, no file selected or operation cancelled.") + scope.launch(Dispatchers.Main) { + Toast.makeText(context, "No file selected." as CharSequence, Toast.LENGTH_SHORT).show() + } + return@rememberLauncherForActivityResult + } + + Log.i(TAG_IMPORT_PROCESS, "Selected file URI: $uri") + scope.launch(Dispatchers.Main) { + Toast.makeText(context, "File selected: $uri. Starting import..." as CharSequence, Toast.LENGTH_SHORT).show() + } + + scope.launch(Dispatchers.IO) { try { - context.contentResolver.openInputStream(it)?.use { inputStream -> - val jsonString = inputStream.bufferedReader().readText() - val parsedEntries = Json.decodeFromString(ListSerializer(SystemMessageEntry.serializer()), jsonString) - val currentSystemEntries = SystemMessageEntryPreferences.loadEntries(context) // Load fresh list - skipAllDuplicates = false // Reset for new import - processImportedEntries(parsedEntries, currentSystemEntries) + Log.d(TAG_IMPORT_PROCESS, "Attempting to open InputStream for URI: $uri on thread: ${Thread.currentThread().name}") + + var fileSize = -1L + try { + context.contentResolver.openFileDescriptor(uri, "r")?.use { pfd -> + fileSize = pfd.statSize + } + Log.i(TAG_IMPORT_PROCESS, "Estimated file size: $fileSize bytes.") + } catch (e: Exception) { + Log.w(TAG_IMPORT_PROCESS, "Could not determine file size for URI: $uri. Will proceed without size check.", e) } + + val MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024 + if (fileSize != -1L && fileSize > MAX_FILE_SIZE_BYTES) { + Log.e(TAG_IMPORT_PROCESS, "File size ($fileSize bytes) exceeds limit of $MAX_FILE_SIZE_BYTES bytes.") + withContext(Dispatchers.Main) { + Toast.makeText(context, "File is too large (max 10MB)." as CharSequence, Toast.LENGTH_LONG).show() + } + return@launch + } + if (fileSize == 0L) { + Log.w(TAG_IMPORT_PROCESS, "Imported file is empty (0 bytes).") + withContext(Dispatchers.Main) { + Toast.makeText(context, "Imported file is empty." as CharSequence, Toast.LENGTH_LONG).show() + } + return@launch + } + + context.contentResolver.openInputStream(uri)?.use { inputStream -> + Log.i(TAG_IMPORT_PROCESS, "InputStream opened. Reading text on thread: ${Thread.currentThread().name}") + val jsonString = inputStream.bufferedReader().readText() + Log.i(TAG_IMPORT_PROCESS, "File content read. Size: ${jsonString.length} chars.") + Log.v(TAG_IMPORT_PROCESS, "File content snippet: ${jsonString.take(500)}") + + if (jsonString.isBlank()) { + Log.w(TAG_IMPORT_PROCESS, "Imported file content is blank.") + withContext(Dispatchers.Main) { + Toast.makeText(context, "Imported file content is blank." as CharSequence, Toast.LENGTH_LONG).show() + } + return@use + } + + Log.d(TAG_IMPORT_PROCESS, "Attempting to parse JSON string on thread: ${Thread.currentThread().name}") + val parsedEntries = Json.decodeFromString(ListSerializer(SystemMessageEntry.serializer()), jsonString) + Log.i(TAG_IMPORT_PROCESS, "JSON parsed. Found ${parsedEntries.size} entries.") + + val currentSystemEntries = SystemMessageEntryPreferences.loadEntries(context) + Log.d(TAG_IMPORT_PROCESS, "Current system entries loaded: ${currentSystemEntries.size} entries.") + + withContext(Dispatchers.Main) { + Log.d(TAG_IMPORT_PROCESS, "Switching to Main thread for processImportedEntries: ${Thread.currentThread().name}") + skipAllDuplicates = false + processImportedEntries( + imported = parsedEntries, + currentSystemEntries = currentSystemEntries + ) + } + } ?: Log.w(TAG_IMPORT_PROCESS, "ContentResolver.openInputStream returned null for URI: $uri (second check).") } catch (e: Exception) { - Log.e("DatabaseListPopup", "Error reading or parsing imported file", e) - Toast.makeText(context, "Error importing file: ${e.message}", Toast.LENGTH_LONG).show() + Log.e(TAG_IMPORT_PROCESS, "Error during file import for URI: $uri on thread: ${Thread.currentThread().name}", e) + withContext(Dispatchers.Main) { + val errorMessage = if (e is OutOfMemoryError) { + "Out of memory. File may be too large or contain too many entries." + } else { + e.message ?: "Unknown error during import." + } + Toast.makeText(context, "Error importing file: $errorMessage" as CharSequence, Toast.LENGTH_LONG).show() + } } } } @@ -503,33 +584,45 @@ fun DatabaseListPopup( OverwriteConfirmationDialog( entryTitle = newEntry.title, onConfirm = { + Log.d(TAG_IMPORT_PROCESS, "Overwrite confirmed for title: '${newEntry.title}'") SystemMessageEntryPreferences.updateEntry(context, existingEntry, newEntry) - // updatedCount++ // This logic needs to be inside processImportedEntries or passed back - Toast.makeText(context, "Entry '${newEntry.title}' overwritten.", Toast.LENGTH_SHORT).show() + Toast.makeText(context, "Entry '${newEntry.title}' overwritten." as CharSequence, Toast.LENGTH_SHORT).show() entryToConfirmOverwrite = null - val currentSystemEntries = SystemMessageEntryPreferences.loadEntries(context) // Reload for next check - processImportedEntries(remainingEntriesToImport, currentSystemEntries) // Continue with remaining + val currentSystemEntriesAfterUpdate = SystemMessageEntryPreferences.loadEntries(context) + Log.d(TAG_IMPORT_PROCESS, "Continuing with remaining ${remainingEntriesToImport.size} entries after dialog (Confirm).") + processImportedEntries( + imported = remainingEntriesToImport, + currentSystemEntries = currentSystemEntriesAfterUpdate + ) }, - onDeny = { // User chose "No" for this specific entry - // skippedCount++ // This logic needs to be inside processImportedEntries or passed back + onDeny = { + Log.d(TAG_IMPORT_PROCESS, "Overwrite denied for title: '${newEntry.title}'") entryToConfirmOverwrite = null - val currentSystemEntries = SystemMessageEntryPreferences.loadEntries(context) // Reload for next check - // Here you could ask about "Skip All remaining?" - // For now, just continue processing without skipping all by default - processImportedEntries(remainingEntriesToImport, currentSystemEntries) // Continue with remaining + val currentSystemEntriesAfterDeny = SystemMessageEntryPreferences.loadEntries(context) + Log.d(TAG_IMPORT_PROCESS, "Continuing with remaining ${remainingEntriesToImport.size} entries after dialog (Deny).") + processImportedEntries( + imported = remainingEntriesToImport, + currentSystemEntries = currentSystemEntriesAfterDeny + ) }, - onSkipAll = { // This is now a conceptual third option in the dialog, or handled differently + onSkipAll = { + Log.d(TAG_IMPORT_PROCESS, "Skip All selected for title: '${newEntry.title}'") skipAllDuplicates = true entryToConfirmOverwrite = null - val currentSystemEntries = SystemMessageEntryPreferences.loadEntries(context) // Reload for next check - processImportedEntries(remainingEntriesToImport, currentSystemEntries) // Continue with remaining, now skipping + val currentSystemEntriesAfterSkipAll = SystemMessageEntryPreferences.loadEntries(context) + Log.d(TAG_IMPORT_PROCESS, "Continuing with remaining ${remainingEntriesToImport.size} entries after dialog (SkipAll).") + processImportedEntries( + imported = remainingEntriesToImport, + currentSystemEntries = currentSystemEntriesAfterSkipAll + ) }, - onDismiss = { // Dialog dismissed externally - entryToConfirmOverwrite = null - remainingEntriesToImport = emptyList() // Stop this import batch + onDismiss = { + Log.d(TAG_IMPORT_PROCESS, "Overwrite dialog dismissed for title: '${entryToConfirmOverwrite?.second?.title}'. Import process for this batch might halt.") + entryToConfirmOverwrite = null + remainingEntriesToImport = emptyList() skipAllDuplicates = false - Toast.makeText(context, "Import cancelled.", Toast.LENGTH_SHORT).show() - onImportCompleted() // Refresh to show any partial imports before dismissal + Toast.makeText(context, "Import cancelled for remaining items." as CharSequence, Toast.LENGTH_SHORT).show() + onImportCompleted() } ) } @@ -652,9 +745,9 @@ fun DatabaseListPopup( } } Spacer(modifier = Modifier.height(8.dp)) - Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.End, verticalAlignment = Alignment.CenterVertically) { + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically) { if (selectionModeActive) { - Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.padding(end = 8.dp)) { + Row(verticalAlignment = Alignment.CenterVertically) { Checkbox( checked = selectAllChecked, onCheckedChange = { isChecked -> @@ -665,27 +758,32 @@ fun DatabaseListPopup( ) Text("All", color = Color.Black, style = MaterialTheme.typography.bodyMedium) } + } else { + Spacer(modifier = Modifier.width(80.dp)) // Placeholder for alignment } - Button(onClick = { filePickerLauncher.launch("*/*") }, colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary), modifier = Modifier.padding(end = 8.dp)) { Text("Import") } - Button( - onClick = { - if (selectionModeActive) { - if (selectedEntryTitles.isEmpty()) { - Toast.makeText(context, "No entries selected for export.", Toast.LENGTH_SHORT).show() - } else { - val entriesToExport = entries.filter { selectedEntryTitles.contains(it.title) } - val jsonString = Json.encodeToString(ListSerializer(SystemMessageEntry.serializer()), entriesToExport) - shareTextFile(context, "system_messages_export.txt", jsonString) + + Row(verticalAlignment = Alignment.CenterVertically) { + Button(onClick = { filePickerLauncher.launch("*/*") }, colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary), modifier = Modifier.padding(end = 8.dp)) { Text("Import") } + Button( + onClick = { + if (selectionModeActive) { + if (selectedEntryTitles.isEmpty()) { + Toast.makeText(context, "No entries selected for export." as CharSequence, Toast.LENGTH_SHORT).show() + } else { + val entriesToExport = entries.filter { selectedEntryTitles.contains(it.title) } + val jsonString = Json.encodeToString(ListSerializer(SystemMessageEntry.serializer()), entriesToExport) + shareTextFile(context, "system_messages_export.txt", jsonString) + } + selectionModeActive = false + selectedEntryTitles = emptySet() + selectAllChecked = false + } else { + selectionModeActive = true } - selectionModeActive = false - selectedEntryTitles = emptySet() - selectAllChecked = false - } else { - selectionModeActive = true - } - }, - colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary) - ) { Text(if (selectionModeActive) "Share" else "Export") } + }, + colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary) + ) { Text("Export") } // Text is now always "Export" + } } } } @@ -698,7 +796,7 @@ fun OverwriteConfirmationDialog( entryTitle: String, onConfirm: () -> Unit, onDeny: () -> Unit, - onSkipAll: () -> Unit, // Added for conceptual completeness, might need UI adjustment + onSkipAll: () -> Unit, onDismiss: () -> Unit ) { AlertDialog( @@ -711,13 +809,6 @@ fun OverwriteConfirmationDialog( dismissButton = { TextButton(onClick = onDeny) { Text("No") } } - // To add "Skip All", AlertDialog might need to be replaced with a custom Dialog, - // or we could add a checkbox "Apply to all future duplicates in this session?" to this dialog. - // For this iteration, Skip All is handled by the calling logic if "No" is chosen. - // A more integrated "Skip All" button could be: - // neutralButton = { TextButton(onClick = onSkipAll) { Text("Skip All") } } - // However, standard AlertDialog doesn't have a direct neutralButton like this. - // Consider adding a TextButton next to No/Yes if UI permits, or a checkbox. ) } diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index d566512..ed39b87 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -1,5 +1,6 @@ package com.google.ai.sample.feature.multimodal +import android.content.Context import android.graphics.Bitmap import android.graphics.drawable.BitmapDrawable import android.net.Uri @@ -23,6 +24,8 @@ import com.google.ai.sample.util.ChatHistoryPreferences import com.google.ai.sample.util.Command import com.google.ai.sample.util.CommandParser import com.google.ai.sample.util.SystemMessagePreferences +import com.google.ai.sample.util.SystemMessageEntryPreferences // Added import +import com.google.ai.sample.util.SystemMessageEntry // Added import import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -90,15 +93,7 @@ class PhotoReasoningViewModel( ) { _uiState.value = PhotoReasoningUiState.Loading - // Get the system message - val systemMessageText = _systemMessage.value - - // Create the prompt with system message if available - val prompt = if (systemMessageText.isNotBlank()) { - "System Message: $systemMessageText\n\nFOLLOW THE INSTRUCTIONS STRICTLY: $userInput" - } else { - "FOLLOW THE INSTRUCTIONS STRICTLY: $userInput" - } + val prompt = "FOLLOW THE INSTRUCTIONS STRICTLY: $userInput" // Store the current user input and selected images currentUserInput = userInput @@ -452,7 +447,7 @@ class PhotoReasoningViewModel( /** * Update the system message */ - fun updateSystemMessage(message: String, context: android.content.Context) { + fun updateSystemMessage(message: String, context: Context) { _systemMessage.value = message // Save to SharedPreferences for persistence @@ -462,13 +457,31 @@ class PhotoReasoningViewModel( /** * Load the system message from SharedPreferences */ - fun loadSystemMessage(context: android.content.Context) { + fun loadSystemMessage(context: Context) { val message = SystemMessagePreferences.loadSystemMessage(context) _systemMessage.value = message // Also load chat history loadChatHistory(context) } + + /** + * Helper function to format database entries as text. + */ + private fun formatDatabaseEntriesAsText(context: Context): String { + val entries = SystemMessageEntryPreferences.loadEntries(context) + if (entries.isEmpty()) { + return "" + } + val builder = StringBuilder() + builder.append("Available System Guides:\n---\n") + for (entry in entries) { + builder.append("Title: ${entry.title}\n") + builder.append("Guide: ${entry.guide}\n") + builder.append("---\n") + } + return builder.toString() + } /** * Process commands found in the AI response @@ -513,7 +526,7 @@ class PhotoReasoningViewModel( /** * Save chat history to SharedPreferences */ - private fun saveChatHistory(context: android.content.Context?) { + private fun saveChatHistory(context: Context?) { context?.let { ChatHistoryPreferences.saveChatMessages(it, chatMessages) } @@ -522,7 +535,7 @@ class PhotoReasoningViewModel( /** * Load chat history from SharedPreferences */ - fun loadChatHistory(context: android.content.Context) { + fun loadChatHistory(context: Context) { val savedMessages = ChatHistoryPreferences.loadChatMessages(context) if (savedMessages.isNotEmpty()) { _chatState.clearMessages() @@ -532,18 +545,29 @@ class PhotoReasoningViewModel( _chatMessagesFlow.value = chatMessages // Rebuild the chat history for the AI - rebuildChatHistory() + rebuildChatHistory(context) // Pass context here } } /** * Rebuild the chat history for the AI based on the current messages */ - private fun rebuildChatHistory() { + private fun rebuildChatHistory(context: Context) { // Added context parameter // Convert the current chat messages to Content objects for the chat history val history = mutableListOf() + + // 1. Active System Message + if (_systemMessage.value.isNotBlank()) { + history.add(content(role = "user") { text(_systemMessage.value) }) + } + + // 2. Formatted Database Entries + val formattedDbEntries = formatDatabaseEntriesAsText(context) + if (formattedDbEntries.isNotBlank()) { + history.add(content(role = "user") { text(formattedDbEntries) }) + } - // Group messages by participant to create proper conversation turns + // 3. Group messages by participant to create proper conversation turns var currentUserContent = "" var currentModelContent = "" @@ -597,20 +621,30 @@ class PhotoReasoningViewModel( chat = generativeModel.startChat( history = history ) + } else { + // Ensure chat is reset even if history is empty (e.g. only system message was there and it's now blank) + chat = generativeModel.startChat(history = emptyList()) } } /** * Clear the chat history */ - fun clearChatHistory(context: android.content.Context? = null) { + fun clearChatHistory(context: Context? = null) { _chatState.clearMessages() _chatMessagesFlow.value = emptyList() - // Reset the chat with empty history - chat = generativeModel.startChat( - history = emptyList() - ) + val initialHistory = mutableListOf() + if (_systemMessage.value.isNotBlank()) { + initialHistory.add(content(role = "user") { text(_systemMessage.value) }) + } + context?.let { ctx -> + val formattedDbEntries = formatDatabaseEntriesAsText(ctx) + if (formattedDbEntries.isNotBlank()) { + initialHistory.add(content(role = "user") { text(formattedDbEntries) }) + } + } + chat = generativeModel.startChat(history = initialHistory.toList()) // Also clear from SharedPreferences if context is provided context?.let { @@ -627,7 +661,7 @@ class PhotoReasoningViewModel( */ fun addScreenshotToConversation( screenshotUri: Uri, - context: android.content.Context, + context: Context, screenInfo: String? = null ) { PhotoReasoningApplication.applicationScope.launch(Dispatchers.Main) { diff --git a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt index f2d65a8..a7bf47d 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt @@ -77,13 +77,13 @@ object CommandParser { // Tap coordinates patterns - expanded to catch more variations private val TAP_COORDINATES_PATTERNS = listOf( // Standard patterns - Regex("(?i)\\b(?:tap|click|press|tippe|klicke|tippe auf|klicke auf) (?:at|on|auf) (?:coordinates?|koordinaten|position|stelle|punkt)[:\\s]\\s*\\(?\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)?"), - Regex("(?i)\\b(?:tap|click|press|tippe|klicke|tippe auf|klicke auf) (?:at|on|auf) \\(?\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)?"), + Regex("(?i)\\b(?:tap|click|press|tippe|klicke|tippe auf|klicke auf) (?:at|on|auf) (?:coordinates?|koordinaten|position|stelle|punkt)[:\\s]\\s*\\(?\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)?"), + Regex("(?i)\\b(?:tap|click|press|tippe|klicke|tippe auf|klicke auf) (?:at|on|auf) \\(?\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)?"), // Function-like patterns - Regex("(?i)\\btapAtCoordinates\\(\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)"), - Regex("(?i)\\bclickAtPosition\\(\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)"), - Regex("(?i)\\btapAt\\(\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)") + Regex("(?i)\\btapAtCoordinates\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)"), + Regex("(?i)\\bclickAtPosition\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)"), + Regex("(?i)\\btapAt\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)") ) // Screenshot patterns - expanded for consistency @@ -475,13 +475,14 @@ object CommandParser { for (match in matches) { try { if (match.groupValues.size > 2) { - val x = match.groupValues[1].trim().toFloat() - val y = match.groupValues[2].trim().toFloat() + val xString = match.groupValues[1].trim() + val yString = match.groupValues[2].trim() // Check if this command is already in the list (avoid duplicates) - if (!commands.any { it is Command.TapCoordinates && it.x == x && it.y == y }) { - Log.d(TAG, "Found tap coordinates command with pattern ${pattern.pattern}: ($x, $y)") - commands.add(Command.TapCoordinates(x, y)) + // Note: Comparison now happens with strings directly. + if (!commands.any { it is Command.TapCoordinates && it.x == xString && it.y == yString }) { + Log.d(TAG, "Found tap coordinates command with pattern ${pattern.pattern}: ($xString, $yString)") + commands.add(Command.TapCoordinates(xString, yString)) } } } catch (e: Exception) { @@ -568,19 +569,19 @@ object CommandParser { */ private fun findScrollDownCommands(text: String, commands: MutableList) { // First check for coordinate-based scroll down commands - val coordPattern = Regex("(?i)\\bscrollDown\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)") + val coordPattern = Regex("(?i)\\bscrollDown\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+)\\s*\\)") val matches = coordPattern.findAll(text) for (match in matches) { if (match.groupValues.size >= 5) { try { - val x = match.groupValues[1].toFloat() - val y = match.groupValues[2].toFloat() - val distance = match.groupValues[3].toFloat() + val xString = match.groupValues[1].trim() + val yString = match.groupValues[2].trim() + val distanceString = match.groupValues[3].trim() val duration = match.groupValues[4].toLong() - Log.d(TAG, "Found coordinate-based scroll down command: scrollDown($x, $y, $distance, $duration)") - commands.add(Command.ScrollDownFromCoordinates(x, y, distance, duration)) + Log.d(TAG, "Found coordinate-based scroll down command: scrollDown($xString, $yString, $distanceString, $duration)") + commands.add(Command.ScrollDownFromCoordinates(xString, yString, distanceString, duration)) } catch (e: Exception) { Log.e(TAG, "Error parsing coordinate-based scroll down command: ${e.message}") } @@ -609,19 +610,19 @@ object CommandParser { */ private fun findScrollUpCommands(text: String, commands: MutableList) { // First check for coordinate-based scroll up commands - val coordPattern = Regex("(?i)\\bscrollUp\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)") + val coordPattern = Regex("(?i)\\bscrollUp\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+)\\s*\\)") val matches = coordPattern.findAll(text) for (match in matches) { if (match.groupValues.size >= 5) { try { - val x = match.groupValues[1].toFloat() - val y = match.groupValues[2].toFloat() - val distance = match.groupValues[3].toFloat() + val xString = match.groupValues[1].trim() + val yString = match.groupValues[2].trim() + val distanceString = match.groupValues[3].trim() val duration = match.groupValues[4].toLong() - Log.d(TAG, "Found coordinate-based scroll up command: scrollUp($x, $y, $distance, $duration)") - commands.add(Command.ScrollUpFromCoordinates(x, y, distance, duration)) + Log.d(TAG, "Found coordinate-based scroll up command: scrollUp($xString, $yString, $distanceString, $duration)") + commands.add(Command.ScrollUpFromCoordinates(xString, yString, distanceString, duration)) } catch (e: Exception) { Log.e(TAG, "Error parsing coordinate-based scroll up command: ${e.message}") } @@ -650,19 +651,19 @@ object CommandParser { */ private fun findScrollLeftCommands(text: String, commands: MutableList) { // First check for coordinate-based scroll left commands - val coordPattern = Regex("(?i)\\bscrollLeft\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)") + val coordPattern = Regex("(?i)\\bscrollLeft\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+)\\s*\\)") val matches = coordPattern.findAll(text) for (match in matches) { if (match.groupValues.size >= 5) { try { - val x = match.groupValues[1].toFloat() - val y = match.groupValues[2].toFloat() - val distance = match.groupValues[3].toFloat() + val xString = match.groupValues[1].trim() + val yString = match.groupValues[2].trim() + val distanceString = match.groupValues[3].trim() val duration = match.groupValues[4].toLong() - Log.d(TAG, "Found coordinate-based scroll left command: scrollLeft($x, $y, $distance, $duration)") - commands.add(Command.ScrollLeftFromCoordinates(x, y, distance, duration)) + Log.d(TAG, "Found coordinate-based scroll left command: scrollLeft($xString, $yString, $distanceString, $duration)") + commands.add(Command.ScrollLeftFromCoordinates(xString, yString, distanceString, duration)) } catch (e: Exception) { Log.e(TAG, "Error parsing coordinate-based scroll left command: ${e.message}") } @@ -691,19 +692,19 @@ object CommandParser { */ private fun findScrollRightCommands(text: String, commands: MutableList) { // First check for coordinate-based scroll right commands - val coordPattern = Regex("(?i)\\bscrollRight\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)") + val coordPattern = Regex("(?i)\\bscrollRight\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+)\\s*\\)") val matches = coordPattern.findAll(text) for (match in matches) { if (match.groupValues.size >= 5) { try { - val x = match.groupValues[1].toFloat() - val y = match.groupValues[2].toFloat() - val distance = match.groupValues[3].toFloat() + val xString = match.groupValues[1].trim() + val yString = match.groupValues[2].trim() + val distanceString = match.groupValues[3].trim() val duration = match.groupValues[4].toLong() - Log.d(TAG, "Found coordinate-based scroll right command: scrollRight($x, $y, $distance, $duration)") - commands.add(Command.ScrollRightFromCoordinates(x, y, distance, duration)) + Log.d(TAG, "Found coordinate-based scroll right command: scrollRight($xString, $yString, $distanceString, $duration)") + commands.add(Command.ScrollRightFromCoordinates(xString, yString, distanceString, duration)) } catch (e: Exception) { Log.e(TAG, "Error parsing coordinate-based scroll right command: ${e.message}") } @@ -796,7 +797,7 @@ sealed class Command { /** * Command to tap at the specified coordinates */ - data class TapCoordinates(val x: Float, val y: Float) : Command() + data class TapCoordinates(val x: String, val y: String) : Command() /** * Command to take a screenshot @@ -846,22 +847,22 @@ sealed class Command { /** * Command to scroll down from specific coordinates with custom distance and duration */ - data class ScrollDownFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command() + data class ScrollDownFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() /** * Command to scroll up from specific coordinates with custom distance and duration */ - data class ScrollUpFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command() + data class ScrollUpFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() /** * Command to scroll left from specific coordinates with custom distance and duration */ - data class ScrollLeftFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command() + data class ScrollLeftFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() /** * Command to scroll right from specific coordinates with custom distance and duration */ - data class ScrollRightFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command() + data class ScrollRightFromCoordinates(val x: String, val y: String, val distance: String, val duration: Long) : Command() /** * Command to open an app by package name diff --git a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntry.kt b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntry.kt index 20f9961..d816624 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntry.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntry.kt @@ -1,9 +1,12 @@ package com.google.ai.sample.util +import android.os.Parcelable +import kotlinx.parcelize.Parcelize import kotlinx.serialization.Serializable +@Parcelize @Serializable data class SystemMessageEntry( val title: String, val guide: String -) +) : Parcelable diff --git a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt index 165ec11..b73c40d 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt @@ -11,6 +11,7 @@ object SystemMessageEntryPreferences { private const val TAG = "SystemMessageEntryPrefs" private const val PREFS_NAME = "system_message_entry_prefs" private const val KEY_SYSTEM_MESSAGE_ENTRIES = "system_message_entries" + private const val KEY_DEFAULT_DB_ENTRIES_POPULATED = "default_db_entries_populated" // Added constant private fun getSharedPreferences(context: Context): SharedPreferences { return context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) @@ -19,7 +20,8 @@ object SystemMessageEntryPreferences { fun saveEntries(context: Context, entries: List) { try { val jsonString = Json.encodeToString(ListSerializer(SystemMessageEntry.serializer()), entries) - Log.d(TAG, "Saving entries: $jsonString") + Log.d(TAG, "Saving ${entries.size} entries. First entry title if exists: ${entries.firstOrNull()?.title}.") + // Log.v(TAG, "Saving JSON: $jsonString") // Verbose, uncomment if needed for deep debugging val editor = getSharedPreferences(context).edit() editor.putString(KEY_SYSTEM_MESSAGE_ENTRIES, jsonString) editor.apply() @@ -30,10 +32,38 @@ object SystemMessageEntryPreferences { fun loadEntries(context: Context): List { try { - val jsonString = getSharedPreferences(context).getString(KEY_SYSTEM_MESSAGE_ENTRIES, null) + val prefs = getSharedPreferences(context) + val defaultsPopulated = prefs.getBoolean(KEY_DEFAULT_DB_ENTRIES_POPULATED, false) + + if (!defaultsPopulated) { + Log.d(TAG, "Default entries not populated. Populating now.") + val defaultEntries = listOf( + SystemMessageEntry( + title = "Example Task: Web Browsing", + guide = "// TODO: Define a detailed guide for the AI on how to perform web browsing tasks. \n// Example: \"To search the web, first click on the search bar (element_id: 'search_bar'), then type your query using writeText('your query'), then click the search button (element_id: 'search_button').\"" + ), + SystemMessageEntry( + title = "Example Task: Sending an Email", + guide = "// TODO: Provide step-by-step instructions for composing and sending an email. \n// Specify UI elements to interact with (e.g., compose button, recipient field, subject field, body field, send button).\"" + ), + SystemMessageEntry( + title = "General App Navigation Guide", + guide = "// TODO: Describe common navigation patterns within this app or general Android OS that the AI should know. \n// Example: \"To go to settings, click on the 'Settings' icon. To return to the previous screen, use the back button.\"" + ) + ) + saveEntries(context, defaultEntries) // This saves them to KEY_SYSTEM_MESSAGE_ENTRIES + prefs.edit().putBoolean(KEY_DEFAULT_DB_ENTRIES_POPULATED, true).apply() + Log.d(TAG, "Populated and saved default database entries.") + // The logic will now fall through to load these just-saved entries. + } + + // Existing logic to load entries from KEY_SYSTEM_MESSAGE_ENTRIES: + val jsonString = prefs.getString(KEY_SYSTEM_MESSAGE_ENTRIES, null) if (jsonString != null) { - Log.d(TAG, "Loaded entries: $jsonString") - return Json.decodeFromString(ListSerializer(SystemMessageEntry.serializer()), jsonString) + // Log.v(TAG, "Loaded JSON: $jsonString") // Verbose + val loadedEntries = Json.decodeFromString(ListSerializer(SystemMessageEntry.serializer()), jsonString) + Log.d(TAG, "Loaded ${loadedEntries.size} entries. First entry title if exists: ${loadedEntries.firstOrNull()?.title}.") + return loadedEntries } Log.d(TAG, "No entries found, returning empty list.") return emptyList() @@ -44,19 +74,22 @@ object SystemMessageEntryPreferences { } fun addEntry(context: Context, entry: SystemMessageEntry) { + Log.d(TAG, "Adding entry: Title='${entry.title}'") val entries = loadEntries(context).toMutableList() entries.add(entry) saveEntries(context, entries) } fun updateEntry(context: Context, oldEntry: SystemMessageEntry, newEntry: SystemMessageEntry) { + Log.d(TAG, "Updating entry: OldTitle='${oldEntry.title}', NewTitle='${newEntry.title}'") val entries = loadEntries(context).toMutableList() - val index = entries.indexOfFirst { it.title == oldEntry.title } // Assuming title is unique for now + val index = entries.indexOfFirst { it.title == oldEntry.title } if (index != -1) { entries[index] = newEntry saveEntries(context, entries) + Log.i(TAG, "Entry updated successfully: NewTitle='${newEntry.title}'") } else { - Log.w(TAG, "Entry with title '${oldEntry.title}' not found for update.") + Log.w(TAG, "Entry with old title '${oldEntry.title}' not found for update.") // Optionally, add the new entry if the old one is not found // addEntry(context, newEntry) } diff --git a/app/src/test/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityServiceTest.kt b/app/src/test/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityServiceTest.kt new file mode 100644 index 0000000..a021a28 --- /dev/null +++ b/app/src/test/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityServiceTest.kt @@ -0,0 +1,123 @@ +package com.google.ai.sample + +import android.content.Context +import android.content.res.Resources +import android.util.DisplayMetrics +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.Mock +import org.mockito.Mockito.`when` +import org.mockito.MockitoAnnotations +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import java.lang.reflect.Method + +// Since ScreenOperatorAccessibilityService is an Android Service, we might need Robolectric +// if we were testing more of its lifecycle or Android-specific features. +// For testing a private method like convertCoordinate, it might be simpler, +// but if it accesses resources (like DisplayMetrics indirectly), Robolectric can be helpful. +// For now, let's assume we can mock essential parts if direct invocation is too complex. +// However, convertCoordinate itself doesn't use Android APIs directly, only its parameters. +// The service's executeCommand DOES use Android APIs (resources.displayMetrics). + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [Config.OLDEST_SDK]) // Configure for a specific SDK if necessary +class ScreenOperatorAccessibilityServiceTest { + + // We are testing a private method. We'll need an instance of the service + // or use reflection with a null instance if the method is static-like (which it is not). + // Let's instantiate it simply. Robolectric can help with service instantiation. + private lateinit var service: ScreenOperatorAccessibilityService + private lateinit var convertCoordinateMethod: Method + + @Mock + private lateinit var mockContext: Context + + @Mock + private lateinit var mockResources: Resources + + @Mock + private lateinit var mockDisplayMetrics: DisplayMetrics + + @Before + fun setUp() { + MockitoAnnotations.openMocks(this) // Initialize mocks + + // Mock Android framework components if needed by the method under test + // For convertCoordinate, it does not directly use Android context/resources. + // However, if we were testing executeCommand, we would need more extensive mocking. + `when`(mockContext.resources).thenReturn(mockResources) + `when`(mockResources.displayMetrics).thenReturn(mockDisplayMetrics) + + service = ScreenOperatorAccessibilityService() + // If ScreenOperatorAccessibilityService had dependencies injected via constructor, + // we would need to provide them here. For now, it has a default constructor. + + // Use reflection to make the private method accessible + convertCoordinateMethod = ScreenOperatorAccessibilityService::class.java.getDeclaredMethod( + "convertCoordinate", // Method name + String::class.java, // First parameter type (String) + Int::class.java // Second parameter type (Int) + ).apply { + isAccessible = true // Make it accessible + } + } + + private fun invokeConvertCoordinate(coordinateString: String, screenSize: Int): Float { + // The method is not static, so it needs an instance of the class + return convertCoordinateMethod.invoke(service, coordinateString, screenSize) as Float + } + + @Test + fun `convertCoordinate - percentage values`() { + assertEquals(500.0f, invokeConvertCoordinate("50%", 1000)) + assertEquals(255.0f, invokeConvertCoordinate("25.5%", 1000)) + assertEquals(0.0f, invokeConvertCoordinate("0%", 1000)) + assertEquals(1000.0f, invokeConvertCoordinate("100%", 1000)) + assertEquals(100.0f, invokeConvertCoordinate("10%", 1000)) // Test with whole number percentage + assertEquals(333.0f, invokeConvertCoordinate("33.3%", 1000)) + } + + @Test + fun `convertCoordinate - pixel values`() { + assertEquals(123.0f, invokeConvertCoordinate("123", 1000)) + assertEquals(123.45f, invokeConvertCoordinate("123.45", 1000)) + assertEquals(0.0f, invokeConvertCoordinate("0", 1000)) + assertEquals(1000.0f, invokeConvertCoordinate("1000", 1000)) + } + + @Test + fun `convertCoordinate - edge cases and error handling`() { + // Invalid percentage (non-numeric) + assertEquals(0.0f, invokeConvertCoordinate("abc%", 1000)) + // Invalid pixel (non-numeric) + assertEquals(0.0f, invokeConvertCoordinate("abc", 1000)) + // Invalid format (mix of valid and invalid) + assertEquals(0.0f, invokeConvertCoordinate("50%abc", 1000)) + // Empty string + assertEquals(0.0f, invokeConvertCoordinate("", 1000)) + // Percentage without number + assertEquals(0.0f, invokeConvertCoordinate("%", 1000)) + // Just a number with percent somewhere else + assertEquals(0.0f, invokeConvertCoordinate("50%20", 1000)) + // Negative percentage + assertEquals(-100.0f, invokeConvertCoordinate("-10%", 1000)) + // Negative pixel + assertEquals(-100.0f, invokeConvertCoordinate("-100", 1000)) + } + + @Test + fun `convertCoordinate - zero screen size`() { + assertEquals(0.0f, invokeConvertCoordinate("50%", 0)) + assertEquals(123.0f, invokeConvertCoordinate("123", 0)) // Pixel value should be unaffected by screen size + assertEquals(0.0f, invokeConvertCoordinate("0%", 0)) + } + + @Test + fun `convertCoordinate - large values`() { + assertEquals(20000.0f, invokeConvertCoordinate("200%", 10000)) + assertEquals(5000.0f, invokeConvertCoordinate("5000", 10000)) + } +} diff --git a/app/src/test/kotlin/com/google/ai/sample/util/CommandParserTest.kt b/app/src/test/kotlin/com/google/ai/sample/util/CommandParserTest.kt new file mode 100644 index 0000000..87e528d --- /dev/null +++ b/app/src/test/kotlin/com/google/ai/sample/util/CommandParserTest.kt @@ -0,0 +1,212 @@ +package com.google.ai.sample.util + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class CommandParserTest { + + @Test + fun `test tapAtCoordinates with pixel values`() { + val commandText = "tapAtCoordinates(100, 200)" + val commands = CommandParser.parseCommands(commandText, clearBuffer = true) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.TapCoordinates) + val tapCommand = commands[0] as Command.TapCoordinates + assertEquals("100", tapCommand.x) + assertEquals("200", tapCommand.y) + } + + @Test + fun `test tapAtCoordinates with percentage values`() { + val commandText = "tapAtCoordinates(50%, 25%)" + val commands = CommandParser.parseCommands(commandText, clearBuffer = true) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.TapCoordinates) + val tapCommand = commands[0] as Command.TapCoordinates + assertEquals("50%", tapCommand.x) + assertEquals("25%", tapCommand.y) + } + + @Test + fun `test tapAtCoordinates with mixed percentage and pixel values`() { + val commandText = "tapAtCoordinates(50%, 200)" + CommandParser.clearBuffer() // Clear buffer before test + val commands = CommandParser.parseCommands(commandText) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.TapCoordinates) + val tapCommand = commands[0] as Command.TapCoordinates + assertEquals("50%", tapCommand.x) + assertEquals("200", tapCommand.y) + } + + @Test + fun `test tapAtCoordinates with decimal percentage values`() { + val commandText = "tapAtCoordinates(10.5%, 80.2%)" + CommandParser.clearBuffer() // Clear buffer before test + val commands = CommandParser.parseCommands(commandText) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.TapCoordinates) + val tapCommand = commands[0] as Command.TapCoordinates + assertEquals("10.5%", tapCommand.x) + assertEquals("80.2%", tapCommand.y) + } + + @Test + fun `test scrollDown with pixel values`() { + val commandText = "scrollDown(50, 100, 100, 200)" + val commands = CommandParser.parseCommands(commandText, clearBuffer = true) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.ScrollDownFromCoordinates) + val scrollCommand = commands[0] as Command.ScrollDownFromCoordinates + assertEquals("50", scrollCommand.x) + assertEquals("100", scrollCommand.y) + assertEquals("100", scrollCommand.distance) // Expect String + assertEquals(200L, scrollCommand.duration) + } + + @Test + fun `test scrollDown with percentage x y and pixel distance`() { + val commandText = "scrollDown(10%, 90%, 100, 200)" + val commands = CommandParser.parseCommands(commandText, clearBuffer = true) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.ScrollDownFromCoordinates) + val scrollCommand = commands[0] as Command.ScrollDownFromCoordinates + assertEquals("10%", scrollCommand.x) + assertEquals("90%", scrollCommand.y) + assertEquals("100", scrollCommand.distance) // Expect String + assertEquals(200L, scrollCommand.duration) + } + + @Test + fun `test scrollDown with percentage x y and percentage distance`() { + val commandText = "scrollDown(10%, 20%, 30%, 500)" + CommandParser.clearBuffer() + val commands = CommandParser.parseCommands(commandText) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.ScrollDownFromCoordinates) + val scrollCommand = commands[0] as Command.ScrollDownFromCoordinates + assertEquals("10%", scrollCommand.x) + assertEquals("20%", scrollCommand.y) + assertEquals("30%", scrollCommand.distance) // Expect String + assertEquals(500L, scrollCommand.duration) + } + + @Test + fun `test scrollUp with percentage x y and pixel distance`() { + val commandText = "scrollUp(10.5%, 80.2%, 150, 250)" + val commands = CommandParser.parseCommands(commandText, clearBuffer = true) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.ScrollUpFromCoordinates) + val scrollCommand = commands[0] as Command.ScrollUpFromCoordinates + assertEquals("10.5%", scrollCommand.x) + assertEquals("80.2%", scrollCommand.y) + assertEquals("150", scrollCommand.distance) // Expect String + assertEquals(250L, scrollCommand.duration) + } + + @Test + fun `test scrollUp with percentage x y and percentage distance`() { + val commandText = "scrollUp(10%, 20%, \"30.5%\", 500)" // Quotes around distance for clarity, regex handles it + CommandParser.clearBuffer() + val commands = CommandParser.parseCommands(commandText) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.ScrollUpFromCoordinates) + val scrollCommand = commands[0] as Command.ScrollUpFromCoordinates + assertEquals("10%", scrollCommand.x) + assertEquals("20%", scrollCommand.y) + assertEquals("30.5%", scrollCommand.distance) // Expect String + assertEquals(500L, scrollCommand.duration) + } + + @Test + fun `test scrollLeft with percentage x y and pixel distance`() { + val commandText = "scrollLeft(5%, 15%, 50, 100)" + val commands = CommandParser.parseCommands(commandText, clearBuffer = true) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.ScrollLeftFromCoordinates) + val scrollCommand = commands[0] as Command.ScrollLeftFromCoordinates + assertEquals("5%", scrollCommand.x) + assertEquals("15%", scrollCommand.y) + assertEquals("50", scrollCommand.distance) // Expect String + assertEquals(100L, scrollCommand.duration) + } + + @Test + fun `test scrollLeft with percentage x y and percentage distance`() { + val commandText = "scrollLeft(5%, 10%, \"15.5%\", 300)" + CommandParser.clearBuffer() + val commands = CommandParser.parseCommands(commandText) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.ScrollLeftFromCoordinates) + val scrollCommand = commands[0] as Command.ScrollLeftFromCoordinates + assertEquals("5%", scrollCommand.x) + assertEquals("10%", scrollCommand.y) + assertEquals("15.5%", scrollCommand.distance) // Expect String + assertEquals(300L, scrollCommand.duration) + } + + @Test + fun `test scrollRight with percentage x y and pixel distance`() { + val commandText = "scrollRight(95%, 85%, 75, 150)" + val commands = CommandParser.parseCommands(commandText, clearBuffer = true) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.ScrollRightFromCoordinates) + val scrollCommand = commands[0] as Command.ScrollRightFromCoordinates + assertEquals("95%", scrollCommand.x) + assertEquals("85%", scrollCommand.y) + assertEquals("75", scrollCommand.distance) // Expect String + assertEquals(150L, scrollCommand.duration) + } + + @Test + fun `test scrollRight with percentage x y and percentage distance`() { + val commandText = "scrollRight(90%, 80%, \"25%\", 400)" + CommandParser.clearBuffer() + val commands = CommandParser.parseCommands(commandText) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.ScrollRightFromCoordinates) + val scrollCommand = commands[0] as Command.ScrollRightFromCoordinates + assertEquals("90%", scrollCommand.x) + assertEquals("80%", scrollCommand.y) + assertEquals("25%", scrollCommand.distance) // Expect String + assertEquals(400L, scrollCommand.duration) + } + + // Test cases for natural language commands + @Test + fun `test tap at coordinates with percentage values natural language`() { + val commandText = "tap at coordinates (50.5%, 25.2%)" + CommandParser.clearBuffer() + val commands = CommandParser.parseCommands(commandText) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.TapCoordinates) + val tapCommand = commands[0] as Command.TapCoordinates + assertEquals("50.5%", tapCommand.x) + assertEquals("25.2%", tapCommand.y) + } + + @Test + fun `test tap on 20 percent and 30 percent`() { + val commandText = "tap on 20%, 30%" + CommandParser.clearBuffer() + val commands = CommandParser.parseCommands(commandText) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.TapCoordinates) + val tapCommand = commands[0] as Command.TapCoordinates + assertEquals("20%", tapCommand.x) + assertEquals("30%", tapCommand.y) + } + + @Test + fun `test tap at 20% and 30%`() { + val commandText = "tap at 20% and 30%" + CommandParser.clearBuffer() + val commands = CommandParser.parseCommands(commandText) + assertEquals(1, commands.size) + assertTrue(commands[0] is Command.TapCoordinates) + val tapCommand = commands[0] as Command.TapCoordinates + assertEquals("20%", tapCommand.x) + assertEquals("30%", tapCommand.y) + } +} diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 diff --git a/local.properties b/local.properties index 15d500e..8dbb40e 100644 --- a/local.properties +++ b/local.properties @@ -1,8 +1 @@ -## This file must *NOT* be checked into Version Control Systems, -# as it contains information specific to your local configuration. -# -# Location of the SDK. This is only used by Gradle. -# For customization when using a Version Control System, please read the -# header note. -#Tue Apr 01 14:08:59 CEST 2025 -sdk.dir=D\:\\AndroidStudioSDK +sdk.dir=/system/sdk