Fix: Implement JSON serialization for SDK Content objects

Resolves "Serializer for class 'Content' is not found" error by introducing Data Transfer Objects (DTOs) and mappers for handling the serialization of `com.google.ai.client.generativeai.type.Content` and its `Part` types (specifically `TextPart` and `ImagePart`) using `kotlinx.serialization`.

Key changes:

1.  **DTOs Created (`PhotoReasoningDtos.kt`):**
    *   Defined `@Serializable` data classes: `ContentDto`, `PartDto` (sealed interface), `TextPartDto`, and `ImagePartDto`.
    *   `ImagePartDto` stores image data as a Base64 encoded string.

2.  **Image Utilities (`ImageUtils.kt`):**
    *   Added `bitmapToBase64` and `base64ToBitmap` functions to handle Bitmap to Base64 (PNG format) string conversion and vice-versa.

3.  **Mappers Implemented (`PhotoReasoningMappers.kt`):**
    *   Created extension functions to map between SDK `Content`/`Part` types and their corresponding DTOs.
    *   These mappers utilize `ImageUtils` for handling `ImagePart` bitmap data.

4.  **ViewModel Update (`PhotoReasoningViewModel.kt`):**
    *   In `sendMessageWithRetry`, SDK `Content` objects (for input and chat history) are now first mapped to their DTOs before being serialized to JSON using `Json.encodeToString()`.

5.  **Service Update (`ScreenCaptureService.kt`):**
    *   In the `ACTION_EXECUTE_AI_CALL` handler, incoming JSON strings are deserialized to `ContentDto` and `List<ContentDto>`.
    *   These DTOs are then mapped back to SDK `Content` types before being used for the AI call.

This ensures that `Content` objects, which may not be directly serializable by `kotlinx.serialization` if they are from an external SDK without `@Serializable` annotations, can be safely serialized for Intent extras and deserialized for use by the service. This fixes the runtime crash you previously encountered.

Author: google-labs-jules[bot]

app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt

@@ -27,6 +27,8 @@ import android.view.WindowManager
 import android.widget.Toast
 import com.google.ai.client.generativeai.GenerativeModel
 import com.google.ai.client.generativeai.type.Content
+import com.google.ai.sample.feature.multimodal.dtos.ContentDto
+import com.google.ai.sample.feature.multimodal.dtos.toSdk
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.SupervisorJob
@@ -183,10 +185,13 @@ class ScreenCaptureService : Service() {
                     var responseText: String? = null
                     var errorMessage: String? = null
                     try {
-                        // Deserialize chat history and input content.
-                        // Assumes Content/List<Content> are @Serializable or custom serializers are configured for Json object.
-                        val chatHistory = Json.decodeFromString<List<Content>>(chatHistoryJson)
-                        val inputContent = Json.decodeFromString<Content>(inputContentJson)
+                        // Deserialize JSON to DTOs.
+                        val chatHistoryDtos = Json.decodeFromString<List<ContentDto>>(chatHistoryJson)
+                        val inputContentDto = Json.decodeFromString<ContentDto>(inputContentJson)
+
+                        // Convert DTOs back to SDK types.
+                        val chatHistory = chatHistoryDtos.map { it.toSdk() } // Uses ContentDto.toSdk()
+                        val inputContent = inputContentDto.toSdk()           // Uses ContentDto.toSdk()
 
                         // Create a GenerativeModel instance for this specific call.
                         // This ensures the call uses the API key and model name provided by the ViewModel.
@@ -198,9 +203,9 @@ class ScreenCaptureService : Service() {
                         )
 
                         // Start a new chat session with the provided history for this call.
-                        val tempChat = generativeModel.startChat(history = chatHistory)
+                        val tempChat = generativeModel.startChat(history = chatHistory) // Use the mapped SDK history
                         Log.d(TAG, "Executing AI sendMessage with history size: ${chatHistory.size}")
-                        val aiResponse = tempChat.sendMessage(inputContent)
+                        val aiResponse = tempChat.sendMessage(inputContent) // Use the mapped SDK inputContent
                         responseText = aiResponse.text
                         Log.d(TAG, "AI call successful. Response text available: ${responseText != null}")
 

app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt

@@ -32,6 +32,7 @@ import com.google.ai.sample.util.CommandParser
 import com.google.ai.sample.util.SystemMessagePreferences
 import com.google.ai.sample.util.SystemMessageEntryPreferences // Added import
 import com.google.ai.sample.util.SystemMessageEntry // Added import
+import com.google.ai.sample.feature.multimodal.dtos.toDto // Added for DTO mapping
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.serialization.encodeToString
@@ -374,8 +375,11 @@ class PhotoReasoningViewModel(
             // This assumes Content and List<Content> are @Serializable or have custom serializers.
             // Add @file:UseSerializers(ContentSerializer::class, PartSerializer::class etc.) if needed at top of file
             // Or create DTOs. For this subtask, we'll assume direct serialization is possible.
-            val inputContentJson = Json.encodeToString(inputContent)
-            val chatHistoryJson = Json.encodeToString(chat.history) // chat.history is List<Content>
+            val inputContentDto = inputContent.toDto() // Use the mapper
+            val chatHistoryDtos = chat.history.map { it.toDto() } // Use the mapper for each item
+
+            val inputContentJson = Json.encodeToString(inputContentDto)
+            val chatHistoryJson = Json.encodeToString(chatHistoryDtos)
 
             val serviceIntent = Intent(context, ScreenCaptureService::class.java).apply {
                 action = ScreenCaptureService.ACTION_EXECUTE_AI_CALL

app/src/main/kotlin/com/google/ai/sample/feature/multimodal/dtos/PhotoReasoningDtos.kt

@@ -0,0 +1,41 @@
+package com.google.ai.sample.feature.multimodal.dtos
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+// Note: The Gemini SDK's Content object can have a nullable 'role'.
+// Parts are non-empty.
+
+@Serializable
+data class ContentDto(
+    val role: String? = null, // e.g., "user", "model", "function"
+    val parts: List<PartDto>
+)
+
+@Serializable
+sealed interface PartDto
+
+@Serializable
+@SerialName("text") // This helps ensure the type is clearly identified in JSON if needed, good practice for sealed types
+data class TextPartDto(val text: String) : PartDto
+
+@Serializable
+@SerialName("image")
+data class ImagePartDto(
+    val base64Image: String,
+    // While the SDK's ImagePart takes a Bitmap, mimeType isn't directly part of its constructor.
+    // We'll assume PNG for now or make it configurable if needed by the SDK upon reconstruction.
+    // For simplicity, we'll just store the image data. The SDK might infer mimetype on load,
+    // or the model might not need it explicitly if the image format is standard.
+    // Let's keep it simple: just the base64 string. The reconstruction to Bitmap won't need a mimeType.
+    // The SDK's `image(bitmap)` builder doesn't take a mimeType.
+) : PartDto
+
+// Placeholder for other Part types if they become necessary later.
+// Example:
+// @Serializable
+// @SerialName("blob")
+// data class BlobPartDto(
+//     val mimeType: String,
+//     val base64Data: String // Or ByteArray
+// ) : PartDto

app/src/main/kotlin/com/google/ai/sample/feature/multimodal/dtos/PhotoReasoningMappers.kt

@@ -0,0 +1,66 @@
+package com.google.ai.sample.feature.multimodal.dtos
+
+import android.graphics.Bitmap // Required for SDK ImagePart
+import com.google.ai.sample.util.ImageUtils // Our Base64<->Bitmap utils
+import com.google.ai.client.generativeai.type.Content // SDK Content
+import com.google.ai.client.generativeai.type.Part // SDK Part
+import com.google.ai.client.generativeai.type.TextPart // SDK TextPart
+import com.google.ai.client.generativeai.type.ImagePart // SDK ImagePart
+import com.google.ai.client.generativeai.type.content // SDK content builder
+
+// --- SDK to DTO Mappers ---
+
+fun Part.toDto(): PartDto {
+    return when (this) {
+        is TextPart -> TextPartDto(text = this.text)
+        is ImagePart -> {
+            // Assuming this.image is a Bitmap. The SDK's ImagePart constructor takes a Bitmap.
+            val base64Image = ImageUtils.bitmapToBase64(this.image)
+            ImagePartDto(base64Image = base64Image)
+        }
+        // Add other SDK Part types here if they become relevant
+        // e.g., is BlobPart -> BlobPartDto(...)
+        else -> throw IllegalArgumentException("Unsupported SDK Part type for DTO conversion: ${this.javaClass.name}")
+    }
+}
+
+fun Content.toDto(): ContentDto {
+    return ContentDto(
+        role = this.role, // SDK Content has a nullable 'role' (String)
+        parts = this.parts.map { it.toDto() } // parts is List<Part>
+    )
+}
+
+// --- DTO to SDK Mappers ---
+
+fun PartDto.toSdk(): Part {
+    return when (this) {
+        is TextPartDto -> TextPart(text = this.text)
+        is ImagePartDto -> {
+            val bitmap: Bitmap? = ImageUtils.base64ToBitmap(this.base64Image)
+            // The SDK's image builder part of `content { image(bitmap) }` expects a non-null Bitmap.
+            // If bitmap is null (due to bad Base64), we might throw an error or return a placeholder/skip.
+            // For now, let's throw an error, as a null bitmap in an ImagePart usually indicates a problem.
+            bitmap?.let { ImagePart(it) }
+                ?: throw IllegalArgumentException("Failed to convert Base64 to Bitmap for ImagePartDto, or Base64 was invalid.")
+        }
+        // Add other PartDto types here
+        // else -> throw IllegalArgumentException("Unsupported PartDto type for SDK conversion: ${this.javaClass.name}")
+    }
+}
+
+fun ContentDto.toSdk(): Content {
+    // The SDK's `content` builder is convenient here.
+    // It takes a role (optional) and a block to define parts.
+    val sdkParts = this.parts.map { it.toSdk() }
+    return content(this.role) {
+        sdkParts.forEach { sdkPart ->
+            when (sdkPart) {
+                is TextPart -> text(sdkPart.text)
+                is ImagePart -> image(sdkPart.image) // sdkPart.image is the Bitmap
+                // Add other SDK Part types here if they become relevant
+                else -> throw IllegalArgumentException("Unsupported SDK Part type during ContentDto.toSdk() parts mapping: ${sdkPart.javaClass.name}")
+            }
+        }
+    }
+}

app/src/main/kotlin/com/google/ai/sample/util/ImageUtils.kt

@@ -0,0 +1,37 @@
+package com.google.ai.sample.util // Or your chosen utility package
+
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.util.Base64
+import java.io.ByteArrayOutputStream
+
+object ImageUtils {
+
+    fun bitmapToBase64(bitmap: Bitmap, quality: Int = 80): String {
+        // Ensure quality is within a reasonable range (0-100)
+        val safeQuality = quality.coerceIn(0, 100)
+        val outputStream = ByteArrayOutputStream()
+        // Compress format can be PNG for lossless (but larger) or JPEG for lossy (smaller)
+        // PNG is generally safer for AI models if exact pixel data matters.
+        // Let's use PNG as it's lossless and doesn't require quality for compression itself in the same way JPEG does.
+        // For PNG, the 'quality' parameter is ignored by compress().
+        // If JPEG was used: bitmap.compress(Bitmap.CompressFormat.JPEG, safeQuality, outputStream)
+        bitmap.compress(Bitmap.CompressFormat.PNG, 100, outputStream) // Quality is ignored for PNG
+        val byteArray = outputStream.toByteArray()
+        return Base64.encodeToString(byteArray, Base64.NO_WRAP) // NO_WRAP is important for compact string
+    }
+
+    fun base64ToBitmap(base64String: String): Bitmap? {
+        return try {
+            val decodedBytes = Base64.decode(base64String, Base64.DEFAULT)
+            BitmapFactory.decodeByteArray(decodedBytes, 0, decodedBytes.size)
+        } catch (e: IllegalArgumentException) {
+            // Log error or handle - Base64 string might be invalid
+            android.util.Log.e("ImageUtils", "Error decoding Base64 string to Bitmap: ${e.message}")
+            null
+        } catch (e: Exception) {
+            android.util.Log.e("ImageUtils", "Unexpected error decoding Base64 string to Bitmap: ${e.message}")
+            null
+        }
+    }
+}