From a55b26dfab8a2c763bc5da8f7f213bf3cc904073 Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Wed, 23 Apr 2025 11:39:22 +0200 Subject: [PATCH 1/4] Add files via upload --- .../multimodal/PhotoReasoningViewModel.kt | 382 +++++++++--------- 1 file changed, 192 insertions(+), 190 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index ad051c00..753ccc5a 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -14,6 +14,7 @@ import coil.size.Precision import com.google.ai.client.generativeai.GenerativeModel import com.google.ai.client.generativeai.type.content import com.google.ai.client.generativeai.type.Content +import com.google.ai.sample.ApiKeyManager import com.google.ai.sample.MainActivity import com.google.ai.sample.PhotoReasoningApplication import com.google.ai.sample.ScreenOperatorAccessibilityService @@ -27,9 +28,12 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import java.io.IOException +import java.net.HttpURLConnection class PhotoReasoningViewModel( - private val generativeModel: GenerativeModel + private var generativeModel: GenerativeModel, + private val apiKeyManager: ApiKeyManager? = null ) : ViewModel() { private val TAG = "PhotoReasoningViewModel" @@ -76,6 +80,9 @@ class PhotoReasoningViewModel( private var chat = generativeModel.startChat( history = emptyList() ) + + // Maximum number of retry attempts for API calls + private val MAX_RETRY_ATTEMPTS = 3 fun reason( userInput: String, @@ -121,63 +128,173 @@ class PhotoReasoningViewModel( // Use application scope to prevent cancellation when app goes to background PhotoReasoningApplication.applicationScope.launch(Dispatchers.IO) { - try { - // Create content with the current images and prompt - val inputContent = content { - for (bitmap in selectedImages) { - image(bitmap) - } - text(prompt) - } - - // Send the message to the chat to maintain context - val response = chat.sendMessage(inputContent) - - var outputContent = "" - - // Process the response - response.text?.let { modelResponse -> - outputContent = modelResponse - - withContext(Dispatchers.Main) { - _uiState.value = PhotoReasoningUiState.Success(outputContent) - - // Update the AI message in chat history - updateAiMessage(outputContent) - - // Parse and execute commands from the response - processCommands(modelResponse) - } + // Create content with the current images and prompt + val inputContent = content { + for (bitmap in selectedImages) { + image(bitmap) } + text(prompt) + } + + // Try to send the message with retry logic for 503 errors + sendMessageWithRetry(inputContent, 0) + } + } + + /** + * Send a message to the AI with retry logic for 503 errors + * + * @param inputContent The content to send + * @param retryCount The current retry count + */ + private suspend fun sendMessageWithRetry(inputContent: Content, retryCount: Int) { + try { + // Send the message to the chat to maintain context + val response = chat.sendMessage(inputContent) + + var outputContent = "" + + // Process the response + response.text?.let { modelResponse -> + outputContent = modelResponse - // Save chat history after successful response withContext(Dispatchers.Main) { - saveChatHistory(MainActivity.getInstance()?.applicationContext) + _uiState.value = PhotoReasoningUiState.Success(outputContent) + + // Update the AI message in chat history + updateAiMessage(outputContent) + + // Parse and execute commands from the response + processCommands(modelResponse) } - } catch (e: Exception) { - Log.e(TAG, "Error generating content: ${e.message}", e) - - withContext(Dispatchers.Main) { - _uiState.value = PhotoReasoningUiState.Error(e.localizedMessage ?: "Unknown error") - _commandExecutionStatus.value = "Fehler bei der Generierung: ${e.localizedMessage}" + } + + // Save chat history after successful response + withContext(Dispatchers.Main) { + saveChatHistory(MainActivity.getInstance()?.applicationContext) + } + } catch (e: Exception) { + Log.e(TAG, "Error generating content: ${e.message}", e) + + // Check if this is a 503 error + if (is503Error(e) && apiKeyManager != null) { + // Mark the current API key as failed + val currentKey = MainActivity.getInstance()?.getCurrentApiKey() + if (currentKey != null) { + apiKeyManager.markKeyAsFailed(currentKey) - // Update chat with error message - _chatState.replaceLastPendingMessage() - _chatState.addMessage( - PhotoReasoningMessage( - text = e.localizedMessage ?: "Unknown error", - participant = PhotoParticipant.ERROR - ) - ) - _chatMessagesFlow.value = chatMessages + // Check if we have only one key or if all keys are failed + val keyCount = apiKeyManager.getKeyCount() + val allKeysFailed = apiKeyManager.areAllKeysFailed() - // Save chat history even after error - saveChatHistory(MainActivity.getInstance()?.applicationContext) + if (keyCount <= 1 || (allKeysFailed && retryCount >= MAX_RETRY_ATTEMPTS)) { + // Only one key available or all keys have failed after multiple attempts + // Show the special message about waiting or adding more keys + withContext(Dispatchers.Main) { + _uiState.value = PhotoReasoningUiState.Error( + "Server überlastet (503). Bitte warten Sie 45 Sekunden oder fügen Sie weitere API-Schlüssel hinzu." + ) + _commandExecutionStatus.value = "Alle API-Schlüssel erschöpft. Bitte warten Sie 45 Sekunden oder fügen Sie weitere API-Schlüssel hinzu." + + // Update chat with error message + _chatState.replaceLastPendingMessage() + _chatState.addMessage( + PhotoReasoningMessage( + text = "Server überlastet (503). Bitte warten Sie 45 Sekunden oder fügen Sie weitere API-Schlüssel hinzu.", + participant = PhotoParticipant.ERROR + ) + ) + _chatMessagesFlow.value = chatMessages + + // Reset failed keys to allow retry after waiting + apiKeyManager.resetFailedKeys() + + // Show toast + MainActivity.getInstance()?.updateStatusMessage( + "Alle API-Schlüssel erschöpft. Bitte warten Sie 45 Sekunden oder fügen Sie weitere API-Schlüssel hinzu.", + true + ) + + // Save chat history even after error + saveChatHistory(MainActivity.getInstance()?.applicationContext) + } + } else if (retryCount < MAX_RETRY_ATTEMPTS) { + // Try to switch to the next available key + val nextKey = apiKeyManager.switchToNextAvailableKey() + if (nextKey != null) { + withContext(Dispatchers.Main) { + _commandExecutionStatus.value = "API-Schlüssel erschöpft. Wechsle zu nächstem Schlüssel..." + + // Show toast + MainActivity.getInstance()?.updateStatusMessage( + "API-Schlüssel erschöpft (503). Wechsle zu nächstem Schlüssel...", + false + ) + } + + // Create a new GenerativeModel with the new API key + val config = generativeModel.config + generativeModel = GenerativeModel( + modelName = generativeModel.modelName, + apiKey = nextKey, + generationConfig = config.generationConfig + ) + + // Create a new chat instance with the new model + chat = generativeModel.startChat( + history = emptyList() + ) + + // Retry the request with the new API key + sendMessageWithRetry(inputContent, retryCount + 1) + return + } + } } } + + // If we get here, either it's not a 503 error, or we've exhausted all API keys, or reached max retries + withContext(Dispatchers.Main) { + _uiState.value = PhotoReasoningUiState.Error(e.localizedMessage ?: "Unknown error") + _commandExecutionStatus.value = "Fehler bei der Generierung: ${e.localizedMessage}" + + // Update chat with error message + _chatState.replaceLastPendingMessage() + _chatState.addMessage( + PhotoReasoningMessage( + text = if (is503Error(e)) + "Server überlastet (503). Bitte warten Sie 45 Sekunden oder fügen Sie weitere API-Schlüssel hinzu." + else + e.localizedMessage ?: "Unknown error", + participant = PhotoParticipant.ERROR + ) + ) + _chatMessagesFlow.value = chatMessages + + // Save chat history even after error + saveChatHistory(MainActivity.getInstance()?.applicationContext) + } } } + /** + * Check if an exception represents a 503 Service Unavailable error + * + * @param e The exception to check + * @return True if the exception represents a 503 error, false otherwise + */ + private fun is503Error(e: Exception): Boolean { + // Check for HTTP 503 error in the exception message + val message = e.message?.lowercase() ?: "" + + // Check for common 503 error patterns + return message.contains("503") || + message.contains("service unavailable") || + message.contains("server unavailable") || + message.contains("server error") || + (e is IOException && message.contains("server")) + } + /** * Update the AI message in chat history */ @@ -365,6 +482,34 @@ class PhotoReasoningViewModel( } } + /** + * Save chat history to SharedPreferences + */ + fun saveChatHistory(context: android.content.Context?) { + if (context != null) { + ChatHistoryPreferences.saveChatHistory(context, chatMessages) + } + } + + /** + * Load chat history from SharedPreferences + */ + fun loadChatHistory(context: android.content.Context) { + val history = ChatHistoryPreferences.loadChatHistory(context) + _chatState.clearMessages() + history.forEach { _chatState.addMessage(it) } + _chatMessagesFlow.value = chatMessages + } + + /** + * Clear chat history + */ + fun clearChatHistory(context: android.content.Context) { + _chatState.clearMessages() + _chatMessagesFlow.value = chatMessages + ChatHistoryPreferences.clearChatHistory(context) + } + /** * Add a screenshot to the conversation * @@ -492,149 +637,6 @@ class PhotoReasoningViewModel( Log.e(TAG, "Error adding screenshot to conversation: ${e.message}", e) _commandExecutionStatus.value = "Fehler beim Hinzufügen des Screenshots: ${e.message}" Toast.makeText(context, "Fehler beim Hinzufügen des Screenshots: ${e.message}", Toast.LENGTH_SHORT).show() - - // Add error message to chat - _chatState.addMessage( - PhotoReasoningMessage( - text = "Fehler beim Hinzufügen des Screenshots: ${e.message}", - participant = PhotoParticipant.ERROR - ) - ) - _chatMessagesFlow.value = chatMessages - - // Save chat history after adding error message - saveChatHistory(context) - } - } - } - - /** - * Load saved chat history from SharedPreferences and initialize chat with history - */ - fun loadChatHistory(context: android.content.Context) { - val savedMessages = ChatHistoryPreferences.loadChatMessages(context) - if (savedMessages.isNotEmpty()) { - _chatState.clearMessages() - savedMessages.forEach { _chatState.addMessage(it) } - _chatMessagesFlow.value = chatMessages - - // Rebuild the chat history for the AI - rebuildChatHistory() - } - } - - /** - * Rebuild the chat history for the AI based on the current messages - */ - private fun rebuildChatHistory() { - // Convert the current chat messages to Content objects for the chat history - val history = mutableListOf() - - // Group messages by participant to create proper conversation turns - var currentUserContent = "" - var currentModelContent = "" - - for (message in chatMessages) { - when (message.participant) { - PhotoParticipant.USER -> { - // If we have model content and are now seeing a user message, - // add the model content to history and reset - if (currentModelContent.isNotEmpty()) { - history.add(content(role = "model") { text(currentModelContent) }) - currentModelContent = "" - } - - // Append to current user content - if (currentUserContent.isNotEmpty()) { - currentUserContent += "\n\n" - } - currentUserContent += message.text - } - PhotoParticipant.MODEL -> { - // If we have user content and are now seeing a model message, - // add the user content to history and reset - if (currentUserContent.isNotEmpty()) { - history.add(content(role = "user") { text(currentUserContent) }) - currentUserContent = "" - } - - // Append to current model content - if (currentModelContent.isNotEmpty()) { - currentModelContent += "\n\n" - } - currentModelContent += message.text - } - PhotoParticipant.ERROR -> { - // Errors are not included in the AI history - continue - } - } - } - - // Add any remaining content - if (currentUserContent.isNotEmpty()) { - history.add(content(role = "user") { text(currentUserContent) }) - } - if (currentModelContent.isNotEmpty()) { - history.add(content(role = "model") { text(currentModelContent) }) - } - - // Create a new chat with the rebuilt history - if (history.isNotEmpty()) { - chat = generativeModel.startChat( - history = history - ) - } - } - - /** - * Save current chat history to SharedPreferences - */ - fun saveChatHistory(context: android.content.Context?) { - context?.let { - ChatHistoryPreferences.saveChatMessages(it, chatMessages) - } - } - - /** - * Clear the chat history - */ - fun clearChatHistory(context: android.content.Context? = null) { - _chatState.clearMessages() - _chatMessagesFlow.value = emptyList() - - // Reset the chat with empty history - chat = generativeModel.startChat( - history = emptyList() - ) - - // Also clear from SharedPreferences if context is provided - context?.let { - ChatHistoryPreferences.clearChatMessages(it) - } - } - - /** - * Chat state management class - */ - private class ChatState { - private val _messages = mutableListOf() - - val messages: List - get() = _messages.toList() - - fun addMessage(message: PhotoReasoningMessage) { - _messages.add(message) - } - - fun clearMessages() { - _messages.clear() - } - - fun replaceLastPendingMessage() { - val lastPendingIndex = _messages.indexOfLast { it.isPending } - if (lastPendingIndex >= 0) { - _messages.removeAt(lastPendingIndex) } } } From 6c12cf07e2752bbdfe40d5d1939019aae952ae4f Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Wed, 23 Apr 2025 11:39:55 +0200 Subject: [PATCH 2/4] Add files via upload --- .../com/google/ai/sample/ApiKeyManager.kt | 134 +++++++++++++++++- .../ai/sample/GenerativeAiViewModelFactory.kt | 5 +- 2 files changed, 137 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/ApiKeyManager.kt b/app/src/main/kotlin/com/google/ai/sample/ApiKeyManager.kt index 61020e4c..d55e3e77 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ApiKeyManager.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ApiKeyManager.kt @@ -12,6 +12,7 @@ class ApiKeyManager(context: Context) { private val PREFS_NAME = "api_key_prefs" private val API_KEYS = "api_keys" private val CURRENT_KEY_INDEX = "current_key_index" + private val FAILED_KEYS = "failed_keys" private val prefs: SharedPreferences = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) @@ -73,6 +74,9 @@ class ApiKeyManager(context: Context) { setCurrentKeyIndex(0) } + // Clear this key from failed keys if it was previously marked as failed + removeFailedKey(apiKey) + Log.d(TAG, "Added new API key, total keys: ${keys.size}") return true } @@ -95,6 +99,9 @@ class ApiKeyManager(context: Context) { setCurrentKeyIndex(0) } + // Also remove from failed keys if present + removeFailedKey(apiKey) + Log.d(TAG, "Removed API key, remaining keys: ${keys.size}") } else { Log.d(TAG, "API key not found for removal") @@ -128,6 +135,122 @@ class ApiKeyManager(context: Context) { return prefs.getInt(CURRENT_KEY_INDEX, 0) } + /** + * Mark an API key as failed (e.g., due to 503 error) + * @param apiKey The API key to mark as failed + */ + fun markKeyAsFailed(apiKey: String) { + val failedKeys = getFailedKeys().toMutableList() + if (!failedKeys.contains(apiKey)) { + failedKeys.add(apiKey) + saveFailedKeys(failedKeys) + Log.d(TAG, "Marked API key as failed: ${apiKey.take(5)}...") + } + } + + /** + * Remove an API key from the failed keys list + * @param apiKey The API key to remove from failed keys + */ + fun removeFailedKey(apiKey: String) { + val failedKeys = getFailedKeys().toMutableList() + if (failedKeys.remove(apiKey)) { + saveFailedKeys(failedKeys) + Log.d(TAG, "Removed API key from failed keys: ${apiKey.take(5)}...") + } + } + + /** + * Get all failed API keys + * @return List of failed API keys + */ + fun getFailedKeys(): List { + val keysString = prefs.getString(FAILED_KEYS, "") ?: "" + return if (keysString.isEmpty()) { + emptyList() + } else { + keysString.split(",") + } + } + + /** + * Check if an API key is marked as failed + * @param apiKey The API key to check + * @return True if the key is marked as failed, false otherwise + */ + fun isKeyFailed(apiKey: String): Boolean { + return getFailedKeys().contains(apiKey) + } + + /** + * Reset all failed keys + */ + fun resetFailedKeys() { + prefs.edit().remove(FAILED_KEYS).apply() + Log.d(TAG, "Reset all failed keys") + } + + /** + * Check if all API keys are marked as failed + * @return True if all keys are failed, false otherwise + */ + fun areAllKeysFailed(): Boolean { + val keys = getApiKeys() + val failedKeys = getFailedKeys() + return keys.isNotEmpty() && failedKeys.size >= keys.size + } + + /** + * Get the count of available API keys + * @return The number of API keys + */ + fun getKeyCount(): Int { + return getApiKeys().size + } + + /** + * Switch to the next available API key that is not marked as failed + * @return The new API key or null if no valid keys are available + */ + fun switchToNextAvailableKey(): String? { + val keys = getApiKeys() + if (keys.isEmpty()) { + Log.d(TAG, "No API keys available to switch to") + return null + } + + val failedKeys = getFailedKeys() + val currentIndex = getCurrentKeyIndex() + + // If all keys are failed, reset failed keys and start from the beginning + if (failedKeys.size >= keys.size) { + Log.d(TAG, "All keys are marked as failed, resetting failed keys") + resetFailedKeys() + setCurrentKeyIndex(0) + return keys[0] + } + + // Find the next key that is not failed + var nextIndex = (currentIndex + 1) % keys.size + var attempts = 0 + + while (attempts < keys.size) { + if (!failedKeys.contains(keys[nextIndex])) { + setCurrentKeyIndex(nextIndex) + Log.d(TAG, "Switched to next available key at index $nextIndex") + return keys[nextIndex] + } + nextIndex = (nextIndex + 1) % keys.size + attempts++ + } + + // If we get here, all keys are failed (shouldn't happen due to earlier check) + Log.d(TAG, "Could not find a non-failed key, resetting failed keys") + resetFailedKeys() + setCurrentKeyIndex(0) + return keys[0] + } + /** * Save the list of API keys to SharedPreferences * @param keys The list of API keys to save @@ -137,11 +260,20 @@ class ApiKeyManager(context: Context) { prefs.edit().putString(API_KEYS, keysString).apply() } + /** + * Save the list of failed API keys to SharedPreferences + * @param keys The list of failed API keys to save + */ + private fun saveFailedKeys(keys: List) { + val keysString = keys.joinToString(",") + prefs.edit().putString(FAILED_KEYS, keysString).apply() + } + /** * Clear all stored API keys */ fun clearAllKeys() { - prefs.edit().remove(API_KEYS).remove(CURRENT_KEY_INDEX).apply() + prefs.edit().remove(API_KEYS).remove(CURRENT_KEY_INDEX).remove(FAILED_KEYS).apply() Log.d(TAG, "Cleared all API keys") } diff --git a/app/src/main/kotlin/com/google/ai/sample/GenerativeAiViewModelFactory.kt b/app/src/main/kotlin/com/google/ai/sample/GenerativeAiViewModelFactory.kt index b560e723..200f4fcd 100644 --- a/app/src/main/kotlin/com/google/ai/sample/GenerativeAiViewModelFactory.kt +++ b/app/src/main/kotlin/com/google/ai/sample/GenerativeAiViewModelFactory.kt @@ -1,5 +1,6 @@ package com.google.ai.sample +import android.content.Context import androidx.lifecycle.ViewModel import androidx.lifecycle.ViewModelProvider import androidx.lifecycle.viewmodel.CreationExtras @@ -67,7 +68,9 @@ val GenerativeViewModelFactory = object : ViewModelProvider.Factory { apiKey = apiKey, generationConfig = config ) - PhotoReasoningViewModel(generativeModel) + // Pass the ApiKeyManager to the ViewModel for key rotation + val apiKeyManager = ApiKeyManager.getInstance(application) + PhotoReasoningViewModel(generativeModel, apiKeyManager) } isAssignableFrom(ChatViewModel::class.java) -> { From 06c98679f3275ffd8a6caa3b58c813d88c0c5b98 Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Wed, 23 Apr 2025 11:50:53 +0200 Subject: [PATCH 3/4] Add files via upload --- .../multimodal/PhotoReasoningViewModel.kt | 156 +++++++++++++++--- 1 file changed, 131 insertions(+), 25 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index 753ccc5a..81ea4d58 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -29,7 +29,6 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import java.io.IOException -import java.net.HttpURLConnection class PhotoReasoningViewModel( private var generativeModel: GenerativeModel, @@ -233,11 +232,11 @@ class PhotoReasoningViewModel( } // Create a new GenerativeModel with the new API key - val config = generativeModel.config + val generationConfig = generativeModel.config.generationConfig generativeModel = GenerativeModel( modelName = generativeModel.modelName, apiKey = nextKey, - generationConfig = config.generationConfig + generationConfig = generationConfig ) // Create a new chat instance with the new model @@ -309,7 +308,9 @@ class PhotoReasoningViewModel( // Clear and re-add all messages to maintain order _chatState.clearMessages() - messages.forEach { _chatState.addMessage(it) } + for (message in messages) { + _chatState.addMessage(message) + } // Update the flow _chatMessagesFlow.value = chatMessages @@ -358,10 +359,10 @@ class PhotoReasoningViewModel( _detectedCommands.value = currentCommands // Update status to show commands were detected - val commandDescriptions = commands.map { - when (it) { - is Command.ClickButton -> "Klick auf Button: \"${it.buttonText}\"" - is Command.TapCoordinates -> "Tippen auf Koordinaten: (${it.x}, ${it.y})" + val commandDescriptions = commands.map { command -> + when (command) { + is Command.ClickButton -> "Klick auf Button: \"${command.buttonText}\"" + is Command.TapCoordinates -> "Tippen auf Koordinaten: (${command.x}, ${command.y})" is Command.TakeScreenshot -> "Screenshot aufnehmen" is Command.PressHomeButton -> "Home-Button drücken" is Command.PressBackButton -> "Zurück-Button drücken" @@ -370,12 +371,12 @@ class PhotoReasoningViewModel( is Command.ScrollUp -> "Nach oben scrollen" is Command.ScrollLeft -> "Nach links scrollen" is Command.ScrollRight -> "Nach rechts scrollen" - is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" - is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" - is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" - is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${it.x}, ${it.y}) mit Distanz ${it.distance}px und Dauer ${it.duration}ms" - is Command.OpenApp -> "App öffnen: \"${it.packageName}\"" - is Command.WriteText -> "Text schreiben: \"${it.text}\"" + is Command.ScrollDownFromCoordinates -> "Nach unten scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollUpFromCoordinates -> "Nach oben scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollLeftFromCoordinates -> "Nach links scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.ScrollRightFromCoordinates -> "Nach rechts scrollen von Position (${command.x}, ${command.y}) mit Distanz ${command.distance}px und Dauer ${command.duration}ms" + is Command.OpenApp -> "App öffnen: \"${command.packageName}\"" + is Command.WriteText -> "Text schreiben: \"${command.text}\"" is Command.UseHighReasoningModel -> "Wechsle zu leistungsfähigerem Modell (gemini-2.5-pro-preview-03-25)" is Command.UseLowReasoningModel -> "Wechsle zu schnellerem Modell (gemini-2.0-flash-lite)" } @@ -392,8 +393,8 @@ class PhotoReasoningViewModel( _commandExecutionStatus.value = "Befehle erkannt: ${commandDescriptions.joinToString(", ")}" // Check if accessibility service is enabled - val isServiceEnabled = mainActivity?.let { - ScreenOperatorAccessibilityService.isAccessibilityServiceEnabled(it) + val isServiceEnabled = mainActivity?.let { activity -> + ScreenOperatorAccessibilityService.isAccessibilityServiceEnabled(activity) } ?: false if (!isServiceEnabled) { @@ -487,7 +488,7 @@ class PhotoReasoningViewModel( */ fun saveChatHistory(context: android.content.Context?) { if (context != null) { - ChatHistoryPreferences.saveChatHistory(context, chatMessages) + ChatHistoryPreferences.saveChatMessages(context, chatMessages) } } @@ -495,19 +496,99 @@ class PhotoReasoningViewModel( * Load chat history from SharedPreferences */ fun loadChatHistory(context: android.content.Context) { - val history = ChatHistoryPreferences.loadChatHistory(context) - _chatState.clearMessages() - history.forEach { _chatState.addMessage(it) } - _chatMessagesFlow.value = chatMessages + val savedMessages = ChatHistoryPreferences.loadChatMessages(context) + if (savedMessages.isNotEmpty()) { + _chatState.clearMessages() + for (message in savedMessages) { + _chatState.addMessage(message) + } + _chatMessagesFlow.value = chatMessages + + // Rebuild the chat history for the AI + rebuildChatHistory() + } } /** - * Clear chat history + * Rebuild the chat history for the AI based on the current messages */ - fun clearChatHistory(context: android.content.Context) { + private fun rebuildChatHistory() { + // Convert the current chat messages to Content objects for the chat history + val history = mutableListOf() + + // Group messages by participant to create proper conversation turns + var currentUserContent = "" + var currentModelContent = "" + + for (message in chatMessages) { + when (message.participant) { + PhotoParticipant.USER -> { + // If we have model content and are now seeing a user message, + // add the model content to history and reset + if (currentModelContent.isNotEmpty()) { + history.add(content(role = "model") { text(currentModelContent) }) + currentModelContent = "" + } + + // Append to current user content + if (currentUserContent.isNotEmpty()) { + currentUserContent += "\n\n" + } + currentUserContent += message.text + } + PhotoParticipant.MODEL -> { + // If we have user content and are now seeing a model message, + // add the user content to history and reset + if (currentUserContent.isNotEmpty()) { + history.add(content(role = "user") { text(currentUserContent) }) + currentUserContent = "" + } + + // Append to current model content + if (currentModelContent.isNotEmpty()) { + currentModelContent += "\n\n" + } + currentModelContent += message.text + } + PhotoParticipant.ERROR -> { + // Errors are not included in the AI history + continue + } + } + } + + // Add any remaining content + if (currentUserContent.isNotEmpty()) { + history.add(content(role = "user") { text(currentUserContent) }) + } + if (currentModelContent.isNotEmpty()) { + history.add(content(role = "model") { text(currentModelContent) }) + } + + // Create a new chat with the rebuilt history + if (history.isNotEmpty()) { + chat = generativeModel.startChat( + history = history + ) + } + } + + /** + * Clear the chat history + */ + fun clearChatHistory(context: android.content.Context? = null) { _chatState.clearMessages() - _chatMessagesFlow.value = chatMessages - ChatHistoryPreferences.clearChatHistory(context) + _chatMessagesFlow.value = emptyList() + + // Reset the chat with empty history + chat = generativeModel.startChat( + history = emptyList() + ) + + // Also clear from SharedPreferences if context is provided + context?.let { + ChatHistoryPreferences.clearChatMessages(it) + } } /** @@ -640,4 +721,29 @@ class PhotoReasoningViewModel( } } } + + /** + * Chat state management class + */ + private class ChatState { + private val _messages = mutableListOf() + + val messages: List + get() = _messages.toList() + + fun addMessage(message: PhotoReasoningMessage) { + _messages.add(message) + } + + fun clearMessages() { + _messages.clear() + } + + fun replaceLastPendingMessage() { + val lastPendingIndex = _messages.indexOfLast { it.isPending } + if (lastPendingIndex >= 0) { + _messages.removeAt(lastPendingIndex) + } + } + } } From d856c30ceafd1c676443e88994f74c505bc9386c Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Wed, 23 Apr 2025 11:59:13 +0200 Subject: [PATCH 4/4] Add files via upload --- .../feature/multimodal/PhotoReasoningViewModel.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index 81ea4d58..900488dd 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -14,6 +14,7 @@ import coil.size.Precision import com.google.ai.client.generativeai.GenerativeModel import com.google.ai.client.generativeai.type.content import com.google.ai.client.generativeai.type.Content +import com.google.ai.client.generativeai.type.GenerationConfig import com.google.ai.sample.ApiKeyManager import com.google.ai.sample.MainActivity import com.google.ai.sample.PhotoReasoningApplication @@ -232,11 +233,12 @@ class PhotoReasoningViewModel( } // Create a new GenerativeModel with the new API key - val generationConfig = generativeModel.config.generationConfig + // Get the current model name and generation config if available + val modelName = generativeModel.modelName + // Create a new model with the same settings but new API key generativeModel = GenerativeModel( - modelName = generativeModel.modelName, - apiKey = nextKey, - generationConfig = generationConfig + modelName = modelName, + apiKey = nextKey ) // Create a new chat instance with the new model