fix(images): preserve cross-client attachments and image understanding

src/apps/cli/README.md

@@ -129,6 +129,12 @@ terminal bracketed paste to attach a clipboard image or a local PNG, JPEG, GIF,
 or WebP path. Quoted paths, `file://` URLs, and POSIX shell-escaped paths are
 accepted. A message may contain up to five images, each no larger than 20 MiB.
 
+A primary model with image input support receives the image pixels directly.
+With a text-only primary model, configure an enabled image-understanding model
+and keep the `analyze_image` tool enabled for the agent. The host saves uploaded
+images as durable runtime attachments so analysis and restored turns can read
+them; image handling also applies when this CLI hosts mobile or peer sessions.
+
 Images are read when pasted, so later file changes do not alter the submitted
 turn and local absolute paths are not sent to the Runtime. Slash commands and
 Shell mode do not accept images. Shared TUI currently reports image paste as

src/apps/cli/src/peer_host/commands/dialog.rs

@@ -9,8 +9,8 @@ use tokio::sync::Mutex as AsyncMutex;
 
 use openbitfun_agent_runtime::sdk::AgentUserAnswersRequest;
 use openbitfun_runtime_ports::{
-    AgentDialogTurnRequest, AgentSubmissionSource, AgentTurnCancellationRequest,
-    DialogSubmissionPolicy, DialogTriggerSource,
+    AgentDialogTurnRequest, AgentInputAttachment, AgentSubmissionSource,
+    AgentTurnCancellationRequest, DialogSubmissionPolicy, DialogTriggerSource,
 };
 
 use crate::peer_host::args::{get_string, optional_string, request_value};
@@ -92,6 +92,42 @@ fn peer_dialog_metadata(request: &Value) -> Result<serde_json::Map<String, Value
     Ok(metadata)
 }
 
+fn peer_image_attachments(request: &Value) -> Result<Vec<AgentInputAttachment>, String> {
+    let images = match request.get("imageContexts") {
+        None | Some(Value::Null) => return Ok(Vec::new()),
+        Some(value) => serde_json::from_value::<
+            Vec<openbitfun_core::agentic::image_analysis::ImageContextData>,
+        >(value.clone())
+        .map_err(|error| format!("Invalid imageContexts: {error}"))?,
+    };
+    images
+        .into_iter()
+        .map(|image| {
+            if image.data_url.as_deref().is_none_or(str::is_empty)
+                && image.image_path.as_deref().is_none_or(str::is_empty)
+            {
+                return Err("An image attachment requires data_url or image_path".to_string());
+            }
+            let mut metadata = serde_json::Map::new();
+            if let Some(path) = image.image_path {
+                metadata.insert("imagePath".into(), json!(path));
+            }
+            if let Some(data) = image.data_url {
+                metadata.insert("dataUrl".into(), json!(data));
+            }
+            metadata.insert("mimeType".into(), json!(image.mime_type));
+            if let Some(details) = image.metadata {
+                metadata.insert("metadata".into(), details);
+            }
+            Ok(AgentInputAttachment {
+                kind: "remote_image".into(),
+                id: image.id,
+                metadata,
+            })
+        })
+        .collect()
+}
+
 pub(crate) async fn start_dialog_turn(
     state: &PeerHostState,
     args: &Value,
@@ -127,6 +163,7 @@ async fn submit_dialog_turn(state: &PeerHostState, args: &Value) -> Result<Value
     let turn_id =
         optional_string(request, "turnId").unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
     let metadata = peer_dialog_metadata(request)?;
+    let attachments = peer_image_attachments(request)?;
     let turn = PeerTurnKey::new(session_id.clone(), turn_id.clone());
     let stream_generation = state.turns.register_root(turn.clone())?;
     // Controller presence is not an admission requirement. This host owns and
@@ -160,7 +197,7 @@ async fn submit_dialog_turn(state: &PeerHostState, args: &Value) -> Result<Value
             policy,
             reply_route: None,
             prepended_reminders: Vec::new(),
-            attachments: Vec::new(),
+            attachments,
             metadata,
         })
         .await;
@@ -412,3 +449,36 @@ mod tests {
         assert!(other_session.lock().await.is_none());
     }
 }
+
+#[cfg(test)]
+mod image_attachment_tests {
+    use super::*;
+
+    #[test]
+    fn peer_image_attachments_preserve_inline_pixels_paths_and_legacy_text_requests() {
+        for request in [
+            json!({}),
+            json!({"imageContexts": null}),
+            json!({"imageContexts": []}),
+        ] {
+            assert!(peer_image_attachments(&request).unwrap().is_empty());
+        }
+        let request = json!({"imageContexts": [{
+            "id": "phone-image", "data_url": "data:image/png;base64,cGl4ZWxz",
+            "image_path": "/controller/image.png", "mime_type": "image/png",
+            "metadata": {"name": "screenshot.png"}
+        }]});
+        let attachments = peer_image_attachments(&request).unwrap();
+        let restored: Vec<AgentInputAttachment> =
+            serde_json::from_value(serde_json::to_value(&attachments).unwrap()).unwrap();
+        assert_eq!(restored[0].kind, "remote_image");
+        assert_eq!(
+            restored[0].metadata["dataUrl"],
+            request["imageContexts"][0]["data_url"]
+        );
+        assert_eq!(restored[0].metadata["imagePath"], "/controller/image.png");
+        assert_eq!(restored[0].metadata["metadata"]["name"], "screenshot.png");
+        assert!(peer_image_attachments(&json!({"imageContexts": [{}]})).is_err());
+        assert!(peer_image_attachments(&json!({"imageContexts": "bad"})).is_err());
+    }
+}

src/apps/mobile/README.md

@@ -16,6 +16,19 @@ and platform adapters. Product logic and stable contracts should remain in the
 platform-agnostic Rust layers and be exposed to these apps through explicit
 interfaces.
 
+## Image messages
+
+All three apps can send images with or without text. Camera photos are decoded
+on the phone and converted to a supported format before upload. Failed sends
+retain the draft and images; acknowledgement removes only the submitted content.
+
+Model selection belongs to the connected host. A primary model that supports
+images receives their pixels directly. For a text-only primary model, select an
+enabled image-understanding model in the host settings and keep `analyze_image`
+enabled for the agent. The receiving runtime saves inline attachments so the
+same images remain available after restoring a conversation, including sessions
+in SSH workspaces. An unavailable model or unreadable image produces an error.
+
 ## Shared visual contract
 
 HarmonyOS is the current visual baseline. The source contract in

src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/platform/ComposerImageEncoder.kt

@@ -0,0 +1,81 @@
+package com.openbitfun.mobile.app.platform
+
+import android.content.ContentResolver
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.graphics.ImageDecoder
+import android.graphics.Matrix
+import android.media.ExifInterface
+import android.net.Uri
+import android.os.Build
+import android.util.Base64
+import com.openbitfun.mobile.core.feature.session.ComposerImage
+import java.io.ByteArrayOutputStream
+import java.util.UUID
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+
+/** Decode camera formats and orientation on the device before crossing the relay. */
+internal suspend fun prepareComposerImage(resolver: ContentResolver, uri: Uri): ComposerImage =
+    withContext(Dispatchers.IO) {
+        val longestSide = 1920
+        var bitmap = if (Build.VERSION.SDK_INT >= 28) {
+            ImageDecoder.decodeBitmap(ImageDecoder.createSource(resolver, uri)) { decoder, info, _ ->
+                val scale = minOf(1.0, longestSide.toDouble() / maxOf(info.size.width, info.size.height))
+                decoder.setTargetSize(maxOf(1, (info.size.width * scale).toInt()), maxOf(1, (info.size.height * scale).toInt()))
+                decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE
+            }
+        } else {
+            val options = BitmapFactory.Options().apply { inJustDecodeBounds = true }
+            resolver.openInputStream(uri)?.use { BitmapFactory.decodeStream(it, null, options) }
+            require(options.outWidth > 0 && options.outHeight > 0) { "Invalid image" }
+            options.inSampleSize = 1
+            while (maxOf(options.outWidth, options.outHeight) / options.inSampleSize > longestSide) options.inSampleSize *= 2
+            options.inJustDecodeBounds = false
+            val decoded = resolver.openInputStream(uri)?.use { BitmapFactory.decodeStream(it, null, options) }
+                ?: error("Unable to decode image")
+            val orientation = resolver.openInputStream(uri)?.use {
+                ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
+            } ?: ExifInterface.ORIENTATION_NORMAL
+            val matrix = Matrix().apply {
+                when (orientation) {
+                    ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> setScale(-1f, 1f)
+                    ExifInterface.ORIENTATION_ROTATE_180 -> setRotate(180f)
+                    ExifInterface.ORIENTATION_FLIP_VERTICAL -> setScale(1f, -1f)
+                    ExifInterface.ORIENTATION_TRANSPOSE -> { setRotate(90f); postScale(-1f, 1f) }
+                    ExifInterface.ORIENTATION_ROTATE_90 -> setRotate(90f)
+                    ExifInterface.ORIENTATION_TRANSVERSE -> { setRotate(270f); postScale(-1f, 1f) }
+                    ExifInterface.ORIENTATION_ROTATE_270 -> setRotate(270f)
+                }
+            }
+            Bitmap.createBitmap(decoded, 0, 0, decoded.width, decoded.height, matrix, true).also {
+                if (it !== decoded) decoded.recycle()
+            }
+        }
+        try {
+            var quality = 85
+            while (true) {
+                val encoded = ByteArrayOutputStream().use {
+                    check(bitmap.compress(Bitmap.CompressFormat.JPEG, quality, it)) { "Image compression failed" }
+                    it.toByteArray()
+                }
+                if (encoded.size <= 1024 * 1024) {
+                    return@withContext ComposerImage(
+                        id = "android-${UUID.randomUUID()}",
+                        dataUrl = "data:image/jpeg;base64," + Base64.encodeToString(encoded, Base64.NO_WRAP),
+                        mimeType = "image/jpeg",
+                    )
+                }
+                if (quality > 45) { quality -= 10; continue }
+                check(minOf(bitmap.width, bitmap.height) > 64) { "Image exceeds upload limit" }
+                val smaller = Bitmap.createScaledBitmap(bitmap, maxOf(1, bitmap.width * 3 / 4), maxOf(1, bitmap.height * 3 / 4), true)
+                bitmap.recycle()
+                bitmap = smaller
+                quality = 75
+            }
+            @Suppress("UNREACHABLE_CODE")
+            error("Image encoding did not complete")
+        } finally {
+            bitmap.recycle()
+        }
+    }

src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ChatMessageBubble.kt

@@ -45,7 +45,7 @@ internal fun ChatMessageBubble(
     onCancelTool: (String, String) -> Unit,
     onAnswerTool: (String, String) -> Unit,
     onAnswerToolStructured: (String, List<QuestionAnswer>) -> Unit,
-    onRetry: (String) -> Unit,
+    onRetry: (ConversationRow) -> Unit,
     onOpenLink: (String, String) -> Unit,
     /** The file the preview surface is showing, so its card can say so. */
     previewingRemotePath: String,
@@ -96,7 +96,7 @@ internal fun ChatMessageBubble(
             row.showRetry -> ChatMessageRetryAction(
                 fromUser = fromUser,
                 enabled = enabled,
-                onRetry = { onRetry(row.text) },
+                onRetry = { onRetry(row) },
                 modifier = Modifier,
             )
 

src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationTimelineView.kt

@@ -72,7 +72,7 @@ internal fun ConversationTimelineView(
     onCancelTool: (String, String) -> Unit,
     onAnswerTool: (String, String) -> Unit,
     onAnswerToolStructured: (String, List<QuestionAnswer>) -> Unit,
-    onRetry: (String) -> Unit,
+    onRetry: (ConversationRow) -> Unit,
     onOpenFile: (String, String) -> Unit,
     previewingRemotePath: String,
     previewLoading: Boolean,

src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationView.kt

@@ -3,7 +3,6 @@ package com.openbitfun.mobile.app.ui.chat
 import android.app.Activity
 import android.content.Intent
 import android.speech.RecognizerIntent
-import android.util.Base64
 import android.widget.Toast
 import androidx.activity.compose.rememberLauncherForActivityResult
 import androidx.activity.result.PickVisualMediaRequest
@@ -17,9 +16,14 @@ import androidx.compose.material3.CircularProgressIndicator
 import androidx.compose.material3.ExperimentalMaterial3Api
 import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.LaunchedEffect
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.remember
+import androidx.compose.runtime.rememberCoroutineScope
+import androidx.compose.runtime.rememberUpdatedState
+import kotlinx.coroutines.launch
+import com.openbitfun.mobile.app.platform.prepareComposerImage
 import androidx.compose.runtime.saveable.rememberSaveable
 import androidx.compose.runtime.setValue
 import androidx.compose.ui.Modifier
@@ -43,7 +47,6 @@ import com.openbitfun.mobile.core.feature.session.conversationRows
 import com.openbitfun.mobile.core.feature.session.modelOptions
 import com.openbitfun.mobile.core.feature.session.selectedModelOption
 import com.openbitfun.mobile.core.feature.workspace.RemoteFileDownloadUiState
-import java.util.UUID
 
 internal const val CONVERSATION_TEST_TAG: String = "conversation"
 internal const val CONVERSATION_BACK_TEST_TAG: String = "conversation-back"
@@ -57,9 +60,6 @@ internal const val CONVERSATION_LOADING_TEST_TAG: String = "conversation-loading
  */
 internal const val CONVERSATION_LIST_TEST_TAG: String = "conversation-list"
 
-/** The relay refuses anything larger, and refusing here is a better error. */
-private const val MAX_IMAGE_BYTES = 8 * 1024 * 1024
-
 /**
  * Joins a dictated fragment onto whatever the composer already holds.
  *
@@ -128,22 +128,38 @@ internal fun ConversationView(
     // message survives session switches and process restarts via DraftStore.
     val draft = state.draft
     var images by remember(sessionId) { mutableStateOf<List<ComposerImage>>(emptyList()) }
+    LaunchedEffect(sessionId, state.lastSentMessage) {
+        state.lastSentMessage?.takeIf { it.sessionId == sessionId }?.let { sent ->
+            images = images.filterNot { it.id in sent.imageIds }
+        }
+    }
     var showSettings by rememberSaveable(sessionId) { mutableStateOf(false) }
     val context = LocalContext.current
 
+    val pickerScope = rememberCoroutineScope()
+    val currentSessionId by rememberUpdatedState(sessionId)
+    var preparingImage by remember(sessionId) { mutableStateOf(false) }
     val photoPicker = rememberLauncherForActivityResult(ActivityResultContracts.PickVisualMedia()) { uri ->
-        if (uri != null) {
-            val bytes = context.contentResolver.openInputStream(uri)?.use { it.readBytes() }
-            val mime = context.contentResolver.getType(uri) ?: "image/jpeg"
-            if (bytes != null && bytes.size <= MAX_IMAGE_BYTES && images.size < MAX_COMPOSER_IMAGES) {
-                images = images + ComposerImage(
-                    id = "android-" + UUID.randomUUID(),
-                    dataUrl = "data:" + mime + ";base64," + Base64.encodeToString(bytes, Base64.NO_WRAP),
-                    mimeType = mime,
-                )
+        if (uri != null && images.size < MAX_COMPOSER_IMAGES) {
+            val targetSession = sessionId
+            preparingImage = true
+            pickerScope.launch {
+                try {
+                    val prepared = prepareComposerImage(context.contentResolver, uri)
+                    if (currentSessionId == targetSession && images.size < MAX_COMPOSER_IMAGES) {
+                        images = images + prepared
+                    }
+                } catch (_: Exception) {
+                    if (currentSessionId == targetSession) {
+                        Toast.makeText(context, R.string.chat_image_prepare_failed, Toast.LENGTH_LONG).show()
+                    }
+                } finally {
+                    if (currentSessionId == targetSession) preparingImage = false
+                }
             }
         }
     }
+
     val voiceInput = rememberLauncherForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
         if (result.resultCode == Activity.RESULT_OK) {
             val text = result.data?.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS)?.firstOrNull().orEmpty()
@@ -208,8 +224,15 @@ internal fun ConversationView(
                 onAnswerToolStructured = { toolId, answers ->
                     onIntent(AnswerStructuredQuestion(sessionId, toolId, answers))
                 },
-                onRetry = { text ->
-                    onIntent(RemoteSessionIntent.SendMessage(sessionId, text, null))
+                onRetry = { row ->
+                    val retryImages = row.images.map { image ->
+                        images.firstOrNull { it.dataUrl == image.dataUrl } ?: ComposerImage(
+                            id = image.name,
+                            dataUrl = image.dataUrl,
+                            mimeType = image.dataUrl.substringAfter("data:").substringBefore(';'),
+                        )
+                    }
+                    onIntent(RemoteSessionIntent.SendMessage(sessionId, row.text, retryImages))
                 },
                 onOpenFile = onOpenFile,
                 previewingRemotePath = previewingRemotePath,
@@ -225,7 +248,7 @@ internal fun ConversationView(
             draft = draft,
             images = images,
             // An empty session id would send nowhere, so it reads as busy.
-            busy = state.busy || sessionId.isEmpty(),
+            busy = state.busy || preparingImage || sessionId.isEmpty(),
             streaming = activeTurn != null,
             phase = phase,
             model = timeline?.selectedModelOption(stringResource(R.string.models_unnamed)),
@@ -263,7 +286,6 @@ internal fun ConversationView(
                         images.takeIf { it.isNotEmpty() },
                     ),
                 )
-                images = emptyList()
             },
             onStop = {
                 onIntent(RemoteSessionIntent.CancelTurn(sessionId, activeTurn?.turnId))

src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/GeneralChatScreen.kt

@@ -345,8 +345,8 @@ internal fun GeneralChatScreen(
                         onCancelTool = { _, _ -> },
                         onAnswerTool = { _, _ -> },
                         onAnswerToolStructured = { _, _ -> },
-                        onRetry = { text ->
-                            viewModel.dispatch(GeneralChatIntent.UpdateDraft(text))
+                        onRetry = { row ->
+                            viewModel.dispatch(GeneralChatIntent.UpdateDraft(row.text))
                             viewModel.dispatch(GeneralChatIntent.Send)
                         },
                         // No desktop behind this surface, so a link to a file has

src/apps/mobile/android/app/src/main/res/values-zh/strings.xml

@@ -434,4 +434,5 @@
     <string name="account_open_github">打开 GitHub 授权</string>
     <string name="account_device_link_invalid">请使用当前版本的 OpenBitFun 设备二维码。</string>
     <string name="account_device_link_unavailable">该设备已离线，或不属于当前 GitHub 账户。</string>
+    <string name="chat_image_prepare_failed">无法处理这张图片，请选择其他图片。</string>
 </resources>

src/apps/mobile/android/app/src/main/res/values/strings.xml

@@ -449,4 +449,5 @@
     <string name="account_open_github">Open GitHub authorization</string>
     <string name="account_device_link_invalid">Use a current OpenBitFun device QR code.</string>
     <string name="account_device_link_unavailable">This device is offline or does not belong to your GitHub account.</string>
+    <string name="chat_image_prepare_failed">This image could not be prepared. Choose another image.</string>
 </resources>