diff --git a/app/src/main/kotlin/com/google/ai/sample/GenerativeAiViewModelFactory.kt b/app/src/main/kotlin/com/google/ai/sample/GenerativeAiViewModelFactory.kt index 7b6b06c..f3e2d09 100644 --- a/app/src/main/kotlin/com/google/ai/sample/GenerativeAiViewModelFactory.kt +++ b/app/src/main/kotlin/com/google/ai/sample/GenerativeAiViewModelFactory.kt @@ -18,7 +18,7 @@ enum class ModelOption(val displayName: String, val modelName: String) { val GenerativeViewModelFactory = object : ViewModelProvider.Factory { // Current selected model name - private var currentModelName = ModelOption.GEMINI_FLASH_LITE.modelName + private var currentModelName = ModelOption.GEMINI_FLASH_PREVIEW.modelName /** * Set the model to high reasoning capability (gemini-2.5-pro-preview-03-25) @@ -95,7 +95,7 @@ val GenerativeViewModelFactory = object : ViewModelProvider.Factory { // Add companion object with static methods for easier access object GenerativeAiViewModelFactory { // Current selected model name - duplicated from GenerativeViewModelFactory - private var currentModelName = ModelOption.GEMINI_FLASH_LITE.modelName + private var currentModelName = ModelOption.GEMINI_FLASH_PREVIEW.modelName /** * Set the model to high reasoning capability (gemini-2.5-pro-preview-03-25) diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt index 9e48218..45332fe 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt @@ -100,7 +100,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import kotlinx.serialization.builtins.ListSerializer -import kotlinx.serialization.json.Json +import kotlinx.serialization.json.Json import android.util.Log import kotlinx.serialization.SerializationException @@ -108,6 +108,19 @@ import kotlinx.serialization.SerializationException val DarkYellow1 = Color(0xFFF0A500) // A darker yellow val DarkYellow2 = Color(0xFFF3C100) // A slightly lighter dark yellow +@Composable +fun StopButton(onClick: () -> Unit) { + Button( + onClick = onClick, + colors = ButtonDefaults.buttonColors(containerColor = Color.Red), + modifier = Modifier + .fillMaxWidth() + .padding(8.dp) + ) { + Text("Stop", color = Color.White) + } +} + @Composable internal fun PhotoReasoningRoute( viewModel: PhotoReasoningViewModel = viewModel(factory = GenerativeViewModelFactory) @@ -175,7 +188,8 @@ internal fun PhotoReasoningRoute( onClearChatHistory = { mainActivity?.getPhotoReasoningViewModel()?.clearChatHistory(context) }, - isKeyboardOpen = isKeyboardOpen + isKeyboardOpen = isKeyboardOpen, + onStopClicked = { viewModel.onStopClicked() } ) } @@ -191,7 +205,8 @@ fun PhotoReasoningScreen( isAccessibilityServiceEnabled: Boolean = false, onEnableAccessibilityService: () -> Unit = {}, onClearChatHistory: () -> Unit = {}, - isKeyboardOpen: Boolean + isKeyboardOpen: Boolean, + onStopClicked: () -> Unit = {} ) { var userQuestion by rememberSaveable { mutableStateOf("") } val imageUris = rememberSaveable(saver = UriSaver()) { mutableStateListOf() } @@ -301,39 +316,45 @@ fun PhotoReasoningScreen( } } - Card(modifier = Modifier.fillMaxWidth()) { - Row(modifier = Modifier.padding(top = 16.dp)) { - Column(modifier = Modifier.padding(all = 4.dp).align(Alignment.CenterVertically)) { - IconButton(onClick = { pickMedia.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly)) }, modifier = Modifier.padding(bottom = 4.dp)) { - Icon(Icons.Rounded.Add, stringResource(R.string.add_image)) + val showStopButton = uiState is PhotoReasoningUiState.Loading || commandExecutionStatus.isNotEmpty() + + if (showStopButton) { + StopButton(onClick = onStopClicked) + } else { + Card(modifier = Modifier.fillMaxWidth()) { + Row(modifier = Modifier.padding(top = 16.dp)) { + Column(modifier = Modifier.padding(all = 4.dp).align(Alignment.CenterVertically)) { + IconButton(onClick = { pickMedia.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly)) }, modifier = Modifier.padding(bottom = 4.dp)) { + Icon(Icons.Rounded.Add, stringResource(R.string.add_image)) + } + IconButton(onClick = onClearChatHistory, modifier = Modifier.padding(top = 4.dp).drawBehind { + drawCircle(color = Color.Black, radius = size.minDimension / 2, style = androidx.compose.ui.graphics.drawscope.Stroke(width = 1.dp.toPx())) + }) { Text("New", style = MaterialTheme.typography.labelMedium, color = MaterialTheme.colorScheme.primary) } } - IconButton(onClick = onClearChatHistory, modifier = Modifier.padding(top = 4.dp).drawBehind { - drawCircle(color = Color.Black, radius = size.minDimension / 2, style = androidx.compose.ui.graphics.drawscope.Stroke(width = 1.dp.toPx())) - }) { Text("New", style = MaterialTheme.typography.labelMedium, color = MaterialTheme.colorScheme.primary) } - } - OutlinedTextField( - value = userQuestion, - label = { Text(stringResource(R.string.reason_label)) }, - placeholder = { Text(stringResource(R.string.reason_hint)) }, - onValueChange = { userQuestion = it }, - modifier = Modifier.weight(1f).padding(end = 8.dp) - ) - IconButton(onClick = { - if (isAccessibilityServiceEnabled) { - if (userQuestion.isNotBlank()) { - onReasonClicked(userQuestion, imageUris.toList()) - userQuestion = "" + OutlinedTextField( + value = userQuestion, + label = { Text(stringResource(R.string.reason_label)) }, + placeholder = { Text(stringResource(R.string.reason_hint)) }, + onValueChange = { userQuestion = it }, + modifier = Modifier.weight(1f).padding(end = 8.dp) + ) + IconButton(onClick = { + if (isAccessibilityServiceEnabled) { + if (userQuestion.isNotBlank()) { + onReasonClicked(userQuestion, imageUris.toList()) + userQuestion = "" + } + } else { + onEnableAccessibilityService() + Toast.makeText(context, "Enable the Accessibility service for Screen Operator" as CharSequence, Toast.LENGTH_LONG).show() } - } else { - onEnableAccessibilityService() - Toast.makeText(context, "Enable the Accessibility service for Screen Operator" as CharSequence, Toast.LENGTH_LONG).show() + }, modifier = Modifier.padding(all = 4.dp).align(Alignment.CenterVertically)) { + Icon(Icons.Default.Send, stringResource(R.string.action_go), tint = MaterialTheme.colorScheme.primary) } - }, modifier = Modifier.padding(all = 4.dp).align(Alignment.CenterVertically)) { - Icon(Icons.Default.Send, stringResource(R.string.action_go), tint = MaterialTheme.colorScheme.primary) } - } - LazyRow(modifier = Modifier.padding(all = 8.dp)) { - items(imageUris) { uri -> AsyncImage(uri, null, Modifier.padding(4.dp).requiredSize(72.dp)) } + LazyRow(modifier = Modifier.padding(all = 8.dp)) { + items(imageUris) { uri -> AsyncImage(uri, null, Modifier.padding(4.dp).requiredSize(72.dp)) } + } } } @@ -732,7 +753,7 @@ fun DatabaseListPopup( verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.End ) { - Text("Add a new system message guide", color = Color.Black.copy(alpha = 0.6f), style = MaterialTheme.typography.bodyMedium, modifier = Modifier.weight(1f)) + Text("This is also sent to the AI", color = Color.Black.copy(alpha = 0.6f), style = MaterialTheme.typography.bodyMedium, modifier = Modifier.weight(1f)) Button(onClick = onNewClicked, colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.primary), modifier = Modifier.padding(start = 8.dp)) { Text("New") } @@ -948,7 +969,7 @@ fun ErrorChatBubble( @Preview @Composable fun PhotoReasoningScreenPreviewWithContent() { - MaterialTheme { + MaterialTheme { PhotoReasoningScreen( uiState = PhotoReasoningUiState.Success("This is a preview of the photo reasoning screen."), commandExecutionStatus = "Command executed: Take screenshot", @@ -961,7 +982,8 @@ fun PhotoReasoningScreenPreviewWithContent() { PhotoReasoningMessage(text = "Hello, how can I help you?", participant = PhotoParticipant.USER), PhotoReasoningMessage(text = "I am here to help you. What do you want to know?", participant = PhotoParticipant.MODEL) ), - isKeyboardOpen = false + isKeyboardOpen = false, + onStopClicked = {} ) } } @@ -1059,7 +1081,7 @@ val SystemMessageEntrySaver = Saver>( @Composable @Preview(showSystemUi = true) fun PhotoReasoningScreenPreviewEmpty() { - MaterialTheme { PhotoReasoningScreen(isKeyboardOpen = false) } + MaterialTheme { PhotoReasoningScreen(isKeyboardOpen = false, onStopClicked = {}) } } @Preview(showBackground = true) @@ -1088,3 +1110,11 @@ fun DatabaseListPopupEmptyPreview() { DatabaseListPopup(onDismissRequest = {}, entries = emptyList(), onNewClicked = {}, onEntryClicked = {}, onDeleteClicked = {}, onImportCompleted = {}) } } + +@Preview(showBackground = true, name = "Stop Button Preview") +@Composable +fun StopButtonPreview() { + MaterialTheme { + StopButton {} + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningUiState.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningUiState.kt index 8baf475..d6cdbf6 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningUiState.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningUiState.kt @@ -44,4 +44,9 @@ sealed interface PhotoReasoningUiState { data class Error( val errorMessage: String ): PhotoReasoningUiState + + /** + * Operation was stopped by the user + */ + data object Stopped: PhotoReasoningUiState } diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index ed39b87..8a8f3e9 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -28,11 +28,15 @@ import com.google.ai.sample.util.SystemMessageEntryPreferences // Added import import com.google.ai.sample.util.SystemMessageEntry // Added import import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.Job import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +// import kotlinx.coroutines.isActive // Removed as we will use job.isActive import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import java.io.IOException +// import kotlin.coroutines.coroutineContext // Removed if not used +import java.util.concurrent.atomic.AtomicBoolean class PhotoReasoningViewModel( private var generativeModel: GenerativeModel, @@ -86,23 +90,27 @@ class PhotoReasoningViewModel( // Maximum number of retry attempts for API calls private val MAX_RETRY_ATTEMPTS = 3 + private var currentReasoningJob: Job? = null + private var commandProcessingJob: Job? = null + private val stopExecutionFlag = AtomicBoolean(false) fun reason( userInput: String, selectedImages: List ) { _uiState.value = PhotoReasoningUiState.Loading - + stopExecutionFlag.set(false) // Reset flag at the beginning of a new reason call + val prompt = "FOLLOW THE INSTRUCTIONS STRICTLY: $userInput" - + // Store the current user input and selected images currentUserInput = userInput currentSelectedImages = selectedImages - + // Clear previous commands _detectedCommands.value = emptyList() _commandExecutionStatus.value = "" - + // Add user message to chat history val userMessage = PhotoReasoningMessage( text = userInput, @@ -111,7 +119,7 @@ class PhotoReasoningViewModel( ) _chatState.addMessage(userMessage) _chatMessagesFlow.value = chatMessages - + // Add AI message with pending status val pendingAiMessage = PhotoReasoningMessage( text = "", @@ -121,89 +129,211 @@ class PhotoReasoningViewModel( _chatState.addMessage(pendingAiMessage) _chatMessagesFlow.value = chatMessages - // Use application scope to prevent cancellation when app goes to background - PhotoReasoningApplication.applicationScope.launch(Dispatchers.IO) { + currentReasoningJob?.cancel() // Cancel any previous reasoning job + currentReasoningJob = PhotoReasoningApplication.applicationScope.launch(Dispatchers.IO) { + var shouldContinueProcessing = true // Create content with the current images and prompt val inputContent = content { - for (bitmap in selectedImages) { - image(bitmap) + // Ensure line for original request: 136 + if (currentReasoningJob?.isActive != true) { + shouldContinueProcessing = false + // No return here + } + if (shouldContinueProcessing) { // Check flag before proceeding + for (bitmap in selectedImages) { + // Ensure line for original request: 138 + if (currentReasoningJob?.isActive != true) { + shouldContinueProcessing = false + break // Break from the for loop + } + if (!shouldContinueProcessing) break // Check flag again in case it was set by the outer check + image(bitmap) + } + } + if (shouldContinueProcessing) { // Check flag before proceeding + // Ensure line for original request: 141 + if (currentReasoningJob?.isActive != true) { + shouldContinueProcessing = false + // No return here + } + } + if (shouldContinueProcessing) { // Check flag before proceeding + text(prompt) } - text(prompt) } - - // Try to send the message with retry logic for 503 errors + + if (!shouldContinueProcessing) { + // If processing should not continue, we might need to update UI state + // For now, the existing check below should handle it. + // If specific UI updates are needed here, they can be added. + return@launch + } + + if (currentReasoningJob?.isActive != true) return@launch // Check for cancellation outside content block sendMessageWithRetry(inputContent, 0) } } - + + fun onStopClicked() { + stopExecutionFlag.set(true) + currentReasoningJob?.cancel() + commandProcessingJob?.cancel() + + val lastMessage = chatMessages.lastOrNull() + val statusMessage = "Operation stopped by user." + + if (lastMessage != null && lastMessage.participant == PhotoParticipant.MODEL && lastMessage.isPending) { + _chatState.replaceLastPendingMessage() // Remove pending message + _chatState.addMessage( + PhotoReasoningMessage( + text = statusMessage, + participant = PhotoParticipant.MODEL, + isPending = false + ) + ) + } else if (lastMessage != null && lastMessage.participant == PhotoParticipant.MODEL && !lastMessage.isPending) { + // If the last message was a successful model response, update it. + _chatState.updateLastMessageText(lastMessage.text + "\n\n[Stopped by user]") + } else { + // If no relevant model message, or last message was user/error, add a new model message + _chatState.addMessage( + PhotoReasoningMessage( + text = statusMessage, + participant = PhotoParticipant.MODEL, + isPending = false + ) + ) + } + _chatMessagesFlow.value = chatMessages + + + _uiState.value = PhotoReasoningUiState.Stopped + _commandExecutionStatus.value = "Stopped." + _detectedCommands.value = emptyList() + Log.d(TAG, "Stop clicked, operations cancelled, UI updated to Stopped state.") + } + /** * Send a message to the AI with retry logic for 503 errors - * + * * @param inputContent The content to send * @param retryCount The current retry count */ private suspend fun sendMessageWithRetry(inputContent: Content, retryCount: Int) { + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) { // Check for cancellation + _uiState.value = PhotoReasoningUiState.Success("Operation cancelled before sending.") + updateAiMessage("Operation cancelled.") + return + } + var shouldProceed = true // Flag to control further processing + try { // Send the message to the chat to maintain context val response = chat.sendMessage(inputContent) - + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) { // Check for cancellation + _uiState.value = PhotoReasoningUiState.Success("Operation cancelled after sending.") + updateAiMessage("Operation cancelled.") + return + } + var outputContent = "" - + // Process the response response.text?.let { modelResponse -> outputContent = modelResponse - + + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) { // Check for cancellation + _uiState.value = PhotoReasoningUiState.Success("Operation cancelled during response processing.") + updateAiMessage("Operation cancelled.") + shouldProceed = false // Signal to skip further processing + } + } + + if (shouldProceed) { // Only proceed if not cancelled in the 'let' block withContext(Dispatchers.Main) { - _uiState.value = PhotoReasoningUiState.Success(outputContent) - - // Update the AI message in chat history - updateAiMessage(outputContent) - - // Parse and execute commands from the response - processCommands(modelResponse) + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) { // Re-check for cancellation + _uiState.value = PhotoReasoningUiState.Success("Operation cancelled.") + updateAiMessage("Operation cancelled.") + // No return@withContext, logic will naturally skip due to outer 'if (shouldProceed)' and this check + } else { + _uiState.value = PhotoReasoningUiState.Success(outputContent) + + // Update the AI message in chat history + updateAiMessage(outputContent) + + // Parse and execute commands from the response + // Ensure modelResponse is accessible or passed correctly if needed here + // Assuming outputContent is what's needed for processCommands if modelResponse was scoped to 'let' + response.text?.let { modelResponse -> // Re-access modelResponse safely + processCommands(modelResponse) + } + } } } - + + // Save chat history after successful response - withContext(Dispatchers.Main) { - saveChatHistory(MainActivity.getInstance()?.applicationContext) + // Ensure this runs only if processing was successful and not cancelled + if (shouldProceed && currentReasoningJob?.isActive == true && !stopExecutionFlag.get()) { + withContext(Dispatchers.Main) { + // Ensure we are still active before saving + if (currentReasoningJob?.isActive == true && !stopExecutionFlag.get()) { + saveChatHistory(MainActivity.getInstance()?.applicationContext) + } + } } } catch (e: Exception) { + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) { // Check for cancellation during exception handling + _uiState.value = PhotoReasoningUiState.Error("Operation cancelled during error handling.") + updateAiMessage("Operation cancelled during error handling.") + return + } Log.e(TAG, "Error generating content: ${e.message}", e) - + // Check specifically for quota exceeded errors first if (isQuotaExceededError(e) && apiKeyManager != null) { + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) return // Check for cancellation handleQuotaExceededError(e, inputContent, retryCount) return } - + // Check for other 503 errors if (is503Error(e) && apiKeyManager != null) { + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) return // Check for cancellation handle503Error(e, inputContent, retryCount) return } - + // If we get here, it's not a 503 error or quota exceeded error - withContext(Dispatchers.Main) { - _uiState.value = PhotoReasoningUiState.Error(e.localizedMessage ?: "Unknown error") - _commandExecutionStatus.value = "Error during generation: ${e.localizedMessage}" - - // Update chat with error message - _chatState.replaceLastPendingMessage() - _chatState.addMessage( - PhotoReasoningMessage( - text = e.localizedMessage ?: "Unknown error", - participant = PhotoParticipant.ERROR - ) - ) - _chatMessagesFlow.value = chatMessages - - // Save chat history even after error - saveChatHistory(MainActivity.getInstance()?.applicationContext) + if (currentReasoningJob?.isActive == true && !stopExecutionFlag.get()) { + withContext(Dispatchers.Main) { + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) { + // If cancelled, potentially update UI or log, but don't proceed with error state for this exception + } else { + _uiState.value = PhotoReasoningUiState.Error(e.localizedMessage ?: "Unknown error") + _commandExecutionStatus.value = "Error during generation: ${e.localizedMessage}" + + // Update chat with error message + _chatState.replaceLastPendingMessage() + _chatState.addMessage( + PhotoReasoningMessage( + text = e.localizedMessage ?: "Unknown error", + participant = PhotoParticipant.ERROR + ) + ) + _chatMessagesFlow.value = chatMessages + + // Save chat history even after error + saveChatHistory(MainActivity.getInstance()?.applicationContext) + } + } + } else { + _uiState.value = PhotoReasoningUiState.Error("Operation cancelled during error processing.") + updateAiMessage("Operation cancelled during error processing.") } } } - + /** * Check if an exception represents a quota exceeded error * @@ -251,6 +381,7 @@ class PhotoReasoningViewModel( * Handle quota exceeded errors specifically */ private suspend fun handleQuotaExceededError(e: Exception, inputContent: Content, retryCount: Int) { + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) return // Check for cancellation // Mark the current API key as failed val currentKey = MainActivity.getInstance()?.getCurrentApiKey() if (currentKey != null && apiKeyManager != null) { @@ -327,17 +458,19 @@ class PhotoReasoningViewModel( ) // Retry the request with the new API key + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) return // Check for cancellation sendMessageWithRetry(inputContent, retryCount + 1) return } } } } - + /** * Handle 503 errors (excluding quota exceeded errors) */ private suspend fun handle503Error(e: Exception, inputContent: Content, retryCount: Int) { + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) return // Check for cancellation // Mark the current API key as failed val currentKey = MainActivity.getInstance()?.getCurrentApiKey() if (currentKey != null && apiKeyManager != null) { @@ -411,39 +544,52 @@ class PhotoReasoningViewModel( ) // Retry the request with the new API key + if (currentReasoningJob?.isActive != true || stopExecutionFlag.get()) return // Check for cancellation sendMessageWithRetry(inputContent, retryCount + 1) return } } } } - + /** * Update the AI message in chat history */ - private fun updateAiMessage(text: String) { - // Find the last AI message and update it + private fun updateAiMessage(text: String, isPending: Boolean = false) { + // Find the last AI message and update it or add a new one if no suitable message exists val messages = _chatState.messages.toMutableList() val lastAiMessageIndex = messages.indexOfLast { it.participant == PhotoParticipant.MODEL } - - if (lastAiMessageIndex >= 0) { - val updatedMessage = messages[lastAiMessageIndex].copy(text = text, isPending = false) + + if (lastAiMessageIndex >= 0 && messages[lastAiMessageIndex].isPending) { + // If last AI message is pending, update it + val updatedMessage = messages[lastAiMessageIndex].copy(text = text, isPending = isPending) messages[lastAiMessageIndex] = updatedMessage - - // Clear and re-add all messages to maintain order - _chatState.clearMessages() - for (message in messages) { - _chatState.addMessage(message) - } - - // Update the flow - _chatMessagesFlow.value = chatMessages - - // Save chat history after updating message - saveChatHistory(MainActivity.getInstance()?.applicationContext) + } else if (lastAiMessageIndex >=0 && !messages[lastAiMessageIndex].isPending && text.startsWith(messages[lastAiMessageIndex].text)) { + // If last AI message is not pending, but the new text is an extension, update it (e.g. for stop message) + val updatedMessage = messages[lastAiMessageIndex].copy(text = text, isPending = isPending) + messages[lastAiMessageIndex] = updatedMessage + } + else { + // Otherwise, add a new AI message + messages.add(PhotoReasoningMessage(text = text, participant = PhotoParticipant.MODEL, isPending = isPending)) + } + + // Clear and re-add all messages to maintain order + _chatState.clearMessages() + for (message in messages) { + _chatState.addMessage(message) + } + + // Update the flow + _chatMessagesFlow.value = chatMessages + + // Save chat history after updating message + // Only save if the operation wasn't stopped, or if it's a deliberate update after stopping + if (!stopExecutionFlag.get() || text.contains("stopped by user", ignoreCase = true)) { + saveChatHistory(MainActivity.getInstance()?.applicationContext) } } - + /** * Update the system message */ @@ -487,42 +633,68 @@ class PhotoReasoningViewModel( * Process commands found in the AI response */ private fun processCommands(text: String) { - PhotoReasoningApplication.applicationScope.launch(Dispatchers.Main) { + commandProcessingJob?.cancel() // Cancel any previous command processing + commandProcessingJob = PhotoReasoningApplication.applicationScope.launch(Dispatchers.Main) { + if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch // Check for cancellation try { // Parse commands from the text val commands = CommandParser.parseCommands(text) - + if (commands.isNotEmpty()) { + if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch Log.d(TAG, "Found ${commands.size} commands in response") - + // Update the detected commands val currentCommands = _detectedCommands.value.toMutableList() currentCommands.addAll(commands) _detectedCommands.value = currentCommands - + // Update status to show commands were detected - val commandDescriptions = commands.joinToString("; ") { command -> + val commandDescriptions = commands.joinToString("; ") { command -> command.toString() } _commandExecutionStatus.value = "Commands detected: $commandDescriptions" - + // Execute the commands for (command in commands) { + if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) { // Check for cancellation before executing each command + Log.d(TAG, "Command execution stopped before executing: $command") + _commandExecutionStatus.value = "Command execution stopped." + break // Exit loop if cancelled + } try { + Log.d(TAG, "Executing command: $command") ScreenOperatorAccessibilityService.executeCommand(command) + // Check immediately after execution attempt if a stop was requested + if (stopExecutionFlag.get()) { + Log.d(TAG, "Command execution stopped after attempting: $command") + _commandExecutionStatus.value = "Command execution stopped." + break + } } catch (e: Exception) { + if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) break // Exit loop if cancelled during error handling Log.e(TAG, "Error executing command: ${e.message}", e) _commandExecutionStatus.value = "Error during command execution: ${e.message}" } } + if (stopExecutionFlag.get()){ + _commandExecutionStatus.value = "Command processing loop was stopped." + } } } catch (e: Exception) { + if (commandProcessingJob?.isActive != true || stopExecutionFlag.get()) return@launch Log.e(TAG, "Error processing commands: ${e.message}", e) _commandExecutionStatus.value = "Error during command processing: ${e.message}" + } finally { + if (stopExecutionFlag.get()){ + _commandExecutionStatus.value = "Command processing finished after stop request." + } + // Reset flag after processing is complete or stopped to allow future executions + // No, don't reset here. Reset at the beginning of 'reason' or when stop is explicitly cleared. } } } - + /** * Save chat history to SharedPreferences */ @@ -788,23 +960,30 @@ class PhotoReasoningViewModel( */ private class ChatState { private val _messages = mutableListOf() - + val messages: List get() = _messages.toList() - + fun addMessage(message: PhotoReasoningMessage) { _messages.add(message) } - + fun clearMessages() { _messages.clear() } - + fun replaceLastPendingMessage() { val lastPendingIndex = _messages.indexOfLast { it.isPending } if (lastPendingIndex >= 0) { _messages.removeAt(lastPendingIndex) } } + + fun updateLastMessageText(newText: String) { + if (_messages.isNotEmpty()) { + val lastMessage = _messages.last() + _messages[_messages.size -1] = lastMessage.copy(text = newText, isPending = false) + } + } } } diff --git a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt index 6f16b21..0bf1334 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt @@ -30,10 +30,6 @@ object CommandParser { private val ALL_PATTERNS: List = listOf( // Enter key patterns PatternInfo("enterKey1", Regex("(?i)\\benter\\(\\)"), { Command.PressEnterKey }, CommandTypeEnum.PRESS_ENTER_KEY), - PatternInfo("enterKey2", Regex("(?i)\\bpressEnter\\(\\)"), { Command.PressEnterKey }, CommandTypeEnum.PRESS_ENTER_KEY), - PatternInfo("enterKey3", Regex("(?i)\\benterKey\\(\\)"), { Command.PressEnterKey }, CommandTypeEnum.PRESS_ENTER_KEY), - PatternInfo("enterKey4", Regex("(?i)\\b(?:press|hit|tap|drücke|tippe auf) (?:the )?enter(?: key| button)?\\b"), { Command.PressEnterKey }, CommandTypeEnum.PRESS_ENTER_KEY), - PatternInfo("enterKey5", Regex("(?i)\\b(?:press|hit|tap|drücke|tippe auf) (?:the )?return(?: key| button)?\\b"), { Command.PressEnterKey }, CommandTypeEnum.PRESS_ENTER_KEY), // Model selection patterns PatternInfo("highReasoning1", Regex("(?i)\\bhighReasoningModel\\(\\)"), { Command.UseHighReasoningModel }, CommandTypeEnum.USE_HIGH_REASONING_MODEL), diff --git a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt index b73c40d..a1891e3 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessageEntryPreferences.kt @@ -39,16 +39,12 @@ object SystemMessageEntryPreferences { Log.d(TAG, "Default entries not populated. Populating now.") val defaultEntries = listOf( SystemMessageEntry( - title = "Example Task: Web Browsing", - guide = "// TODO: Define a detailed guide for the AI on how to perform web browsing tasks. \n// Example: \"To search the web, first click on the search bar (element_id: 'search_bar'), then type your query using writeText('your query'), then click the search button (element_id: 'search_button').\"" + title = "Termux", + guide = "To write something in Termux you must be sure the ESC HOME banner is away. If not: `back()` `scrollRight(75%, 99%, 50%, 50)` `tapAtCoordinates(50%, 99%)` this in one message. Check if the banner has disappeared also at the bottom. To show the keyboard in Termux if the banner is gone: `tapAtCoordinates(50%, 99%)` And you must always `Enter()` twice.\"" ), SystemMessageEntry( - title = "Example Task: Sending an Email", - guide = "// TODO: Provide step-by-step instructions for composing and sending an email. \n// Specify UI elements to interact with (e.g., compose button, recipient field, subject field, body field, send button).\"" - ), - SystemMessageEntry( - title = "General App Navigation Guide", - guide = "// TODO: Describe common navigation patterns within this app or general Android OS that the AI should know. \n// Example: \"To go to settings, click on the 'Settings' icon. To return to the previous screen, use the back button.\"" + title = "Chromium-based Browser", + guide = "To see more in a screenshot, you may want to consider zooming out. To do this, tap the three vertical dots, select the appropriate location in the menu, and then tap the 'minus' symbol (multiple times). You can only zoom out to 50%.\"" ) ) saveEntries(context, defaultEntries) // This saves them to KEY_SYSTEM_MESSAGE_ENTRIES diff --git a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessagePreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessagePreferences.kt index 6d944ed..e347c7e 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/SystemMessagePreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/SystemMessagePreferences.kt @@ -14,7 +14,7 @@ object SystemMessagePreferences { private const val KEY_FIRST_START_COMPLETED = "first_start_completed" // New flag // Content from pasted_content.txt - private const val DEFAULT_SYSTEM_MESSAGE_ON_FIRST_START = """You are on an App on a Smartphone. You're app is called Screen Operator. You start from this app. Proceed step by step! DON'T USE TOOL CODE! You must operate the screen with exactly following commands: "`home()`" "`back()`" "`recentApps()`" for buttons and words: "`clickOnButton("sample")`" "`tapAtCoordinates(x, y)`" "`scrollDown()`" "`scrollUp()`" "`scrollLeft()`" "`scrollRight()`" "`scrollDown(x, y, how much pixel to scroll, duration in milliseconds)`" "`scrollUp(x, y, how much pixel to scroll, duration in milliseconds)`" "`scrollLeft(x, y, how much pixel to scroll, duration in milliseconds)`" "`scrollRight(x, y, how much pixel to scroll, duration in milliseconds)`" scroll status bar down: "`scrollUp(540, 0, 1100, 50)`" Only the Play Store and Settings can be opened this way: "`openApp("sample")`" You must open other apps from the home screen. "`takeScreenshot()`" To write text, search and click the textfield thereafter: "`writeText("sample text")`" You need to write the already existing text, if it should continue exist. If the keyboard is displayed, you can press "`Enter()`". Otherwise, you have to open the keyboard by clicking on the text field. You can see the screen and get additional Informations about them with: "`takeScreenshot()`" You need this command at the end of every message until you are finish. When you're done don't say "`takeScreenshot()`" Your task is:""" + private const val DEFAULT_SYSTEM_MESSAGE_ON_FIRST_START = """You are on an App on a Smartphone. You're app is called Screen Operator. You start from this app. Proceed step by step! DON'T USE TOOL CODE! You must operate the screen with exactly following commands: "`home()`" "`back()`" "`recentApps()`" for buttons and words: "`clickOnButton("sample")`" "`tapAtCoordinates(x, y)`" "`tapAtCoordinates(x percent of screen%, y percent of screen%)`""`scrollDown()`" "`scrollUp()`" "`scrollLeft()`" "`scrollRight()`" "`scrollDown(x, y, how much pixel to scroll, duration in milliseconds)`" "`scrollUp(x, y, how much pixel to scroll, duration in milliseconds)`" "`scrollLeft(x, y, how much pixel to scroll, duration in milliseconds)`" "`scrollRight(x, y, how much pixel to scroll, duration in milliseconds)`" "`scrollDown(x percent of screen%, y percent of screen%, how much percent to scroll%, duration in milliseconds)`" "`scrollUp(x percent of screen%, y percent of screen%, how much percent to scroll, duration in milliseconds)`" "`scrollLeft(x percent of screen%, y percent of screen%, how much percent to scroll, duration in milliseconds)`" "`scrollRight(x percent of screen%, y percent of screen%, how much percent to scroll, duration in milliseconds)`" scroll status bar down: "`scrollUp(540, 0, 1100, 50)`" "`takeScreenshot()`" To write text, search and click the textfield thereafter: "`writeText("sample text")`" You need to write the already existing text, if it should continue exist. If the keyboard is displayed, you can press "`Enter()`". Otherwise, you have to open the keyboard by clicking on the text field. You can see the screen and get additional Informations about them with: "`takeScreenshot()`" You need this command at the end of every message until you are finish. When you're done don't say "`takeScreenshot()`" Your task is:""" /** * Save system message to SharedPreferences diff --git a/app/src/test/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityServiceTest.kt b/app/src/test/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityServiceTest.kt deleted file mode 100644 index a021a28..0000000 --- a/app/src/test/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityServiceTest.kt +++ /dev/null @@ -1,123 +0,0 @@ -package com.google.ai.sample - -import android.content.Context -import android.content.res.Resources -import android.util.DisplayMetrics -import org.junit.Assert.assertEquals -import org.junit.Before -import org.junit.Test -import org.junit.runner.RunWith -import org.mockito.Mock -import org.mockito.Mockito.`when` -import org.mockito.MockitoAnnotations -import org.robolectric.RobolectricTestRunner -import org.robolectric.annotation.Config -import java.lang.reflect.Method - -// Since ScreenOperatorAccessibilityService is an Android Service, we might need Robolectric -// if we were testing more of its lifecycle or Android-specific features. -// For testing a private method like convertCoordinate, it might be simpler, -// but if it accesses resources (like DisplayMetrics indirectly), Robolectric can be helpful. -// For now, let's assume we can mock essential parts if direct invocation is too complex. -// However, convertCoordinate itself doesn't use Android APIs directly, only its parameters. -// The service's executeCommand DOES use Android APIs (resources.displayMetrics). - -@RunWith(RobolectricTestRunner::class) -@Config(sdk = [Config.OLDEST_SDK]) // Configure for a specific SDK if necessary -class ScreenOperatorAccessibilityServiceTest { - - // We are testing a private method. We'll need an instance of the service - // or use reflection with a null instance if the method is static-like (which it is not). - // Let's instantiate it simply. Robolectric can help with service instantiation. - private lateinit var service: ScreenOperatorAccessibilityService - private lateinit var convertCoordinateMethod: Method - - @Mock - private lateinit var mockContext: Context - - @Mock - private lateinit var mockResources: Resources - - @Mock - private lateinit var mockDisplayMetrics: DisplayMetrics - - @Before - fun setUp() { - MockitoAnnotations.openMocks(this) // Initialize mocks - - // Mock Android framework components if needed by the method under test - // For convertCoordinate, it does not directly use Android context/resources. - // However, if we were testing executeCommand, we would need more extensive mocking. - `when`(mockContext.resources).thenReturn(mockResources) - `when`(mockResources.displayMetrics).thenReturn(mockDisplayMetrics) - - service = ScreenOperatorAccessibilityService() - // If ScreenOperatorAccessibilityService had dependencies injected via constructor, - // we would need to provide them here. For now, it has a default constructor. - - // Use reflection to make the private method accessible - convertCoordinateMethod = ScreenOperatorAccessibilityService::class.java.getDeclaredMethod( - "convertCoordinate", // Method name - String::class.java, // First parameter type (String) - Int::class.java // Second parameter type (Int) - ).apply { - isAccessible = true // Make it accessible - } - } - - private fun invokeConvertCoordinate(coordinateString: String, screenSize: Int): Float { - // The method is not static, so it needs an instance of the class - return convertCoordinateMethod.invoke(service, coordinateString, screenSize) as Float - } - - @Test - fun `convertCoordinate - percentage values`() { - assertEquals(500.0f, invokeConvertCoordinate("50%", 1000)) - assertEquals(255.0f, invokeConvertCoordinate("25.5%", 1000)) - assertEquals(0.0f, invokeConvertCoordinate("0%", 1000)) - assertEquals(1000.0f, invokeConvertCoordinate("100%", 1000)) - assertEquals(100.0f, invokeConvertCoordinate("10%", 1000)) // Test with whole number percentage - assertEquals(333.0f, invokeConvertCoordinate("33.3%", 1000)) - } - - @Test - fun `convertCoordinate - pixel values`() { - assertEquals(123.0f, invokeConvertCoordinate("123", 1000)) - assertEquals(123.45f, invokeConvertCoordinate("123.45", 1000)) - assertEquals(0.0f, invokeConvertCoordinate("0", 1000)) - assertEquals(1000.0f, invokeConvertCoordinate("1000", 1000)) - } - - @Test - fun `convertCoordinate - edge cases and error handling`() { - // Invalid percentage (non-numeric) - assertEquals(0.0f, invokeConvertCoordinate("abc%", 1000)) - // Invalid pixel (non-numeric) - assertEquals(0.0f, invokeConvertCoordinate("abc", 1000)) - // Invalid format (mix of valid and invalid) - assertEquals(0.0f, invokeConvertCoordinate("50%abc", 1000)) - // Empty string - assertEquals(0.0f, invokeConvertCoordinate("", 1000)) - // Percentage without number - assertEquals(0.0f, invokeConvertCoordinate("%", 1000)) - // Just a number with percent somewhere else - assertEquals(0.0f, invokeConvertCoordinate("50%20", 1000)) - // Negative percentage - assertEquals(-100.0f, invokeConvertCoordinate("-10%", 1000)) - // Negative pixel - assertEquals(-100.0f, invokeConvertCoordinate("-100", 1000)) - } - - @Test - fun `convertCoordinate - zero screen size`() { - assertEquals(0.0f, invokeConvertCoordinate("50%", 0)) - assertEquals(123.0f, invokeConvertCoordinate("123", 0)) // Pixel value should be unaffected by screen size - assertEquals(0.0f, invokeConvertCoordinate("0%", 0)) - } - - @Test - fun `convertCoordinate - large values`() { - assertEquals(20000.0f, invokeConvertCoordinate("200%", 10000)) - assertEquals(5000.0f, invokeConvertCoordinate("5000", 10000)) - } -} diff --git a/app/src/test/kotlin/com/google/ai/sample/util/CommandParserTest.kt b/app/src/test/kotlin/com/google/ai/sample/util/CommandParserTest.kt deleted file mode 100644 index 87e528d..0000000 --- a/app/src/test/kotlin/com/google/ai/sample/util/CommandParserTest.kt +++ /dev/null @@ -1,212 +0,0 @@ -package com.google.ai.sample.util - -import org.junit.Assert.assertEquals -import org.junit.Assert.assertTrue -import org.junit.Test - -class CommandParserTest { - - @Test - fun `test tapAtCoordinates with pixel values`() { - val commandText = "tapAtCoordinates(100, 200)" - val commands = CommandParser.parseCommands(commandText, clearBuffer = true) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.TapCoordinates) - val tapCommand = commands[0] as Command.TapCoordinates - assertEquals("100", tapCommand.x) - assertEquals("200", tapCommand.y) - } - - @Test - fun `test tapAtCoordinates with percentage values`() { - val commandText = "tapAtCoordinates(50%, 25%)" - val commands = CommandParser.parseCommands(commandText, clearBuffer = true) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.TapCoordinates) - val tapCommand = commands[0] as Command.TapCoordinates - assertEquals("50%", tapCommand.x) - assertEquals("25%", tapCommand.y) - } - - @Test - fun `test tapAtCoordinates with mixed percentage and pixel values`() { - val commandText = "tapAtCoordinates(50%, 200)" - CommandParser.clearBuffer() // Clear buffer before test - val commands = CommandParser.parseCommands(commandText) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.TapCoordinates) - val tapCommand = commands[0] as Command.TapCoordinates - assertEquals("50%", tapCommand.x) - assertEquals("200", tapCommand.y) - } - - @Test - fun `test tapAtCoordinates with decimal percentage values`() { - val commandText = "tapAtCoordinates(10.5%, 80.2%)" - CommandParser.clearBuffer() // Clear buffer before test - val commands = CommandParser.parseCommands(commandText) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.TapCoordinates) - val tapCommand = commands[0] as Command.TapCoordinates - assertEquals("10.5%", tapCommand.x) - assertEquals("80.2%", tapCommand.y) - } - - @Test - fun `test scrollDown with pixel values`() { - val commandText = "scrollDown(50, 100, 100, 200)" - val commands = CommandParser.parseCommands(commandText, clearBuffer = true) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.ScrollDownFromCoordinates) - val scrollCommand = commands[0] as Command.ScrollDownFromCoordinates - assertEquals("50", scrollCommand.x) - assertEquals("100", scrollCommand.y) - assertEquals("100", scrollCommand.distance) // Expect String - assertEquals(200L, scrollCommand.duration) - } - - @Test - fun `test scrollDown with percentage x y and pixel distance`() { - val commandText = "scrollDown(10%, 90%, 100, 200)" - val commands = CommandParser.parseCommands(commandText, clearBuffer = true) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.ScrollDownFromCoordinates) - val scrollCommand = commands[0] as Command.ScrollDownFromCoordinates - assertEquals("10%", scrollCommand.x) - assertEquals("90%", scrollCommand.y) - assertEquals("100", scrollCommand.distance) // Expect String - assertEquals(200L, scrollCommand.duration) - } - - @Test - fun `test scrollDown with percentage x y and percentage distance`() { - val commandText = "scrollDown(10%, 20%, 30%, 500)" - CommandParser.clearBuffer() - val commands = CommandParser.parseCommands(commandText) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.ScrollDownFromCoordinates) - val scrollCommand = commands[0] as Command.ScrollDownFromCoordinates - assertEquals("10%", scrollCommand.x) - assertEquals("20%", scrollCommand.y) - assertEquals("30%", scrollCommand.distance) // Expect String - assertEquals(500L, scrollCommand.duration) - } - - @Test - fun `test scrollUp with percentage x y and pixel distance`() { - val commandText = "scrollUp(10.5%, 80.2%, 150, 250)" - val commands = CommandParser.parseCommands(commandText, clearBuffer = true) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.ScrollUpFromCoordinates) - val scrollCommand = commands[0] as Command.ScrollUpFromCoordinates - assertEquals("10.5%", scrollCommand.x) - assertEquals("80.2%", scrollCommand.y) - assertEquals("150", scrollCommand.distance) // Expect String - assertEquals(250L, scrollCommand.duration) - } - - @Test - fun `test scrollUp with percentage x y and percentage distance`() { - val commandText = "scrollUp(10%, 20%, \"30.5%\", 500)" // Quotes around distance for clarity, regex handles it - CommandParser.clearBuffer() - val commands = CommandParser.parseCommands(commandText) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.ScrollUpFromCoordinates) - val scrollCommand = commands[0] as Command.ScrollUpFromCoordinates - assertEquals("10%", scrollCommand.x) - assertEquals("20%", scrollCommand.y) - assertEquals("30.5%", scrollCommand.distance) // Expect String - assertEquals(500L, scrollCommand.duration) - } - - @Test - fun `test scrollLeft with percentage x y and pixel distance`() { - val commandText = "scrollLeft(5%, 15%, 50, 100)" - val commands = CommandParser.parseCommands(commandText, clearBuffer = true) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.ScrollLeftFromCoordinates) - val scrollCommand = commands[0] as Command.ScrollLeftFromCoordinates - assertEquals("5%", scrollCommand.x) - assertEquals("15%", scrollCommand.y) - assertEquals("50", scrollCommand.distance) // Expect String - assertEquals(100L, scrollCommand.duration) - } - - @Test - fun `test scrollLeft with percentage x y and percentage distance`() { - val commandText = "scrollLeft(5%, 10%, \"15.5%\", 300)" - CommandParser.clearBuffer() - val commands = CommandParser.parseCommands(commandText) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.ScrollLeftFromCoordinates) - val scrollCommand = commands[0] as Command.ScrollLeftFromCoordinates - assertEquals("5%", scrollCommand.x) - assertEquals("10%", scrollCommand.y) - assertEquals("15.5%", scrollCommand.distance) // Expect String - assertEquals(300L, scrollCommand.duration) - } - - @Test - fun `test scrollRight with percentage x y and pixel distance`() { - val commandText = "scrollRight(95%, 85%, 75, 150)" - val commands = CommandParser.parseCommands(commandText, clearBuffer = true) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.ScrollRightFromCoordinates) - val scrollCommand = commands[0] as Command.ScrollRightFromCoordinates - assertEquals("95%", scrollCommand.x) - assertEquals("85%", scrollCommand.y) - assertEquals("75", scrollCommand.distance) // Expect String - assertEquals(150L, scrollCommand.duration) - } - - @Test - fun `test scrollRight with percentage x y and percentage distance`() { - val commandText = "scrollRight(90%, 80%, \"25%\", 400)" - CommandParser.clearBuffer() - val commands = CommandParser.parseCommands(commandText) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.ScrollRightFromCoordinates) - val scrollCommand = commands[0] as Command.ScrollRightFromCoordinates - assertEquals("90%", scrollCommand.x) - assertEquals("80%", scrollCommand.y) - assertEquals("25%", scrollCommand.distance) // Expect String - assertEquals(400L, scrollCommand.duration) - } - - // Test cases for natural language commands - @Test - fun `test tap at coordinates with percentage values natural language`() { - val commandText = "tap at coordinates (50.5%, 25.2%)" - CommandParser.clearBuffer() - val commands = CommandParser.parseCommands(commandText) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.TapCoordinates) - val tapCommand = commands[0] as Command.TapCoordinates - assertEquals("50.5%", tapCommand.x) - assertEquals("25.2%", tapCommand.y) - } - - @Test - fun `test tap on 20 percent and 30 percent`() { - val commandText = "tap on 20%, 30%" - CommandParser.clearBuffer() - val commands = CommandParser.parseCommands(commandText) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.TapCoordinates) - val tapCommand = commands[0] as Command.TapCoordinates - assertEquals("20%", tapCommand.x) - assertEquals("30%", tapCommand.y) - } - - @Test - fun `test tap at 20% and 30%`() { - val commandText = "tap at 20% and 30%" - CommandParser.clearBuffer() - val commands = CommandParser.parseCommands(commandText) - assertEquals(1, commands.size) - assertTrue(commands[0] is Command.TapCoordinates) - val tapCommand = commands[0] as Command.TapCoordinates - assertEquals("20%", tapCommand.x) - assertEquals("30%", tapCommand.y) - } -} diff --git a/local.properties b/local.properties index 8dbb40e..04ef92a 100644 --- a/local.properties +++ b/local.properties @@ -1 +1 @@ -sdk.dir=/system/sdk +sdk.dir=/opt/android/sdk