From a25fd86874e1de9bdec03be8b4e2e0210ac959ed Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Thu, 10 Sep 2026 15:46:23 +0800 Subject: [PATCH 1/2] fix(images): preserve attachments across clients and runtime model paths --- src/apps/cli/README.md | 6 + src/apps/cli/src/peer_host/commands/dialog.rs | 76 ++++++- src/apps/mobile/README.md | 13 ++ .../app/platform/ComposerImageEncoder.kt | 81 +++++++ .../mobile/app/ui/chat/ChatMessageBubble.kt | 4 +- .../app/ui/chat/ConversationTimelineView.kt | 2 +- .../mobile/app/ui/chat/ConversationView.kt | 58 +++-- .../mobile/app/ui/chat/GeneralChatScreen.kt | 4 +- .../app/src/main/res/values-zh/strings.xml | 1 + .../app/src/main/res/values/strings.xml | 1 + .../runtime/AppRootRuntimeComposition.ets | 13 +- .../viewmodel/RemoteTranscriptController.ets | 1 - .../main/ets/services/ImagePickerService.ets | 15 +- .../services/RemoteChatCommandController.ets | 3 + .../src/test/RemoteControllersUnit.test.ets | 6 + .../Features/Chat/ChatTimelineView.swift | 6 +- .../Features/Chat/ComposerBar.swift | 13 +- .../MobileAppModel+GeneralChat.swift | 23 ++ .../MobileAppModel+RemoteSession.swift | 16 +- .../Infrastructure/MobileAppModel.swift | 27 ++- .../feature/session/RemoteSessionStore.kt | 21 +- .../feature/session/RemoteSessionUiState.kt | 30 +++ .../feature/session/RemoteSessionStoreTest.kt | 56 +++++ .../src/agentic/coordination/coordinator.rs | 40 +++- .../src/agentic/coordination/scheduler.rs | 30 ++- .../src/agentic/execution/execution_engine.rs | 206 +++++++++++++---- .../src/agentic/image_analysis/attachments.rs | 183 +++++++++++++++ .../image_analysis/image_processing.rs | 213 +++++++++++++++++- .../core/src/agentic/image_analysis/mod.rs | 1 + .../core/src/agentic/persistence/manager.rs | 68 ++++++ .../implementations/analyze_image_tool.rs | 88 +++++++- .../generated/remote-surface-registry.json | 5 +- .../src/remote_surface/capabilities.rs | 5 + .../services/services-core/src/json_store.rs | 11 + src/mobile-web/AGENTS.md | 1 + src/mobile-web/package.json | 1 + src/mobile-web/src/i18n/messages.ts | 13 +- src/mobile-web/src/pages/ChatPage.tsx | 30 +-- .../src/services/imageCompressor.ts | 22 +- .../tests/image-compressor.test.mjs | 58 +++++ .../src/flow_chat/hooks/useMessageSender.ts | 4 +- .../src/flow_chat/utils/imagePayload.test.ts | 67 ++++++ .../src/flow_chat/utils/imagePayload.ts | 12 +- .../api/generated/remoteSurface.ts | 5 +- .../peer-device/PeerConnectionManager.test.ts | 3 + .../peer-device/PeerConnectionManager.ts | 5 + 46 files changed, 1385 insertions(+), 162 deletions(-) create mode 100644 src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/platform/ComposerImageEncoder.kt create mode 100644 src/crates/assembly/core/src/agentic/image_analysis/attachments.rs create mode 100644 src/mobile-web/tests/image-compressor.test.mjs create mode 100644 src/web-ui/src/flow_chat/utils/imagePayload.test.ts diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index d9397ad744..e96a483b2b 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -129,6 +129,12 @@ terminal bracketed paste to attach a clipboard image or a local PNG, JPEG, GIF, or WebP path. Quoted paths, `file://` URLs, and POSIX shell-escaped paths are accepted. A message may contain up to five images, each no larger than 20 MiB. +A primary model with image input support receives the image pixels directly. +With a text-only primary model, configure an enabled image-understanding model +and keep the `analyze_image` tool enabled for the agent. The host saves uploaded +images as durable runtime attachments so analysis and restored turns can read +them; image handling also applies when this CLI hosts mobile or peer sessions. + Images are read when pasted, so later file changes do not alter the submitted turn and local absolute paths are not sent to the Runtime. Slash commands and Shell mode do not accept images. Shared TUI currently reports image paste as diff --git a/src/apps/cli/src/peer_host/commands/dialog.rs b/src/apps/cli/src/peer_host/commands/dialog.rs index 6923b2bfbd..b8c18c6552 100644 --- a/src/apps/cli/src/peer_host/commands/dialog.rs +++ b/src/apps/cli/src/peer_host/commands/dialog.rs @@ -9,8 +9,8 @@ use tokio::sync::Mutex as AsyncMutex; use openbitfun_agent_runtime::sdk::AgentUserAnswersRequest; use openbitfun_runtime_ports::{ - AgentDialogTurnRequest, AgentSubmissionSource, AgentTurnCancellationRequest, - DialogSubmissionPolicy, DialogTriggerSource, + AgentDialogTurnRequest, AgentInputAttachment, AgentSubmissionSource, + AgentTurnCancellationRequest, DialogSubmissionPolicy, DialogTriggerSource, }; use crate::peer_host::args::{get_string, optional_string, request_value}; @@ -92,6 +92,42 @@ fn peer_dialog_metadata(request: &Value) -> Result Result, String> { + let images = match request.get("imageContexts") { + None | Some(Value::Null) => return Ok(Vec::new()), + Some(value) => serde_json::from_value::< + Vec, + >(value.clone()) + .map_err(|error| format!("Invalid imageContexts: {error}"))?, + }; + images + .into_iter() + .map(|image| { + if image.data_url.as_deref().is_none_or(str::is_empty) + && image.image_path.as_deref().is_none_or(str::is_empty) + { + return Err("An image attachment requires data_url or image_path".to_string()); + } + let mut metadata = serde_json::Map::new(); + if let Some(path) = image.image_path { + metadata.insert("imagePath".into(), json!(path)); + } + if let Some(data) = image.data_url { + metadata.insert("dataUrl".into(), json!(data)); + } + metadata.insert("mimeType".into(), json!(image.mime_type)); + if let Some(details) = image.metadata { + metadata.insert("metadata".into(), details); + } + Ok(AgentInputAttachment { + kind: "remote_image".into(), + id: image.id, + metadata, + }) + }) + .collect() +} + pub(crate) async fn start_dialog_turn( state: &PeerHostState, args: &Value, @@ -127,6 +163,7 @@ async fn submit_dialog_turn(state: &PeerHostState, args: &Value) -> Result Result = + serde_json::from_value(serde_json::to_value(&attachments).unwrap()).unwrap(); + assert_eq!(restored[0].kind, "remote_image"); + assert_eq!( + restored[0].metadata["dataUrl"], + request["imageContexts"][0]["data_url"] + ); + assert_eq!(restored[0].metadata["imagePath"], "/controller/image.png"); + assert_eq!(restored[0].metadata["metadata"]["name"], "screenshot.png"); + assert!(peer_image_attachments(&json!({"imageContexts": [{}]})).is_err()); + assert!(peer_image_attachments(&json!({"imageContexts": "bad"})).is_err()); + } +} diff --git a/src/apps/mobile/README.md b/src/apps/mobile/README.md index 02b2c1f380..79ecc1ce0f 100644 --- a/src/apps/mobile/README.md +++ b/src/apps/mobile/README.md @@ -16,6 +16,19 @@ and platform adapters. Product logic and stable contracts should remain in the platform-agnostic Rust layers and be exposed to these apps through explicit interfaces. +## Image messages + +All three apps can send images with or without text. Camera photos are decoded +on the phone and converted to a supported format before upload. Failed sends +retain the draft and images; acknowledgement removes only the submitted content. + +Model selection belongs to the connected host. A primary model that supports +images receives their pixels directly. For a text-only primary model, select an +enabled image-understanding model in the host settings and keep `analyze_image` +enabled for the agent. The receiving runtime saves inline attachments so the +same images remain available after restoring a conversation, including sessions +in SSH workspaces. An unavailable model or unreadable image produces an error. + ## Shared visual contract HarmonyOS is the current visual baseline. The source contract in diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/platform/ComposerImageEncoder.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/platform/ComposerImageEncoder.kt new file mode 100644 index 0000000000..130c00e196 --- /dev/null +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/platform/ComposerImageEncoder.kt @@ -0,0 +1,81 @@ +package com.openbitfun.mobile.app.platform + +import android.content.ContentResolver +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.ImageDecoder +import android.graphics.Matrix +import android.media.ExifInterface +import android.net.Uri +import android.os.Build +import android.util.Base64 +import com.openbitfun.mobile.core.feature.session.ComposerImage +import java.io.ByteArrayOutputStream +import java.util.UUID +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** Decode camera formats and orientation on the device before crossing the relay. */ +internal suspend fun prepareComposerImage(resolver: ContentResolver, uri: Uri): ComposerImage = + withContext(Dispatchers.IO) { + val longestSide = 1920 + var bitmap = if (Build.VERSION.SDK_INT >= 28) { + ImageDecoder.decodeBitmap(ImageDecoder.createSource(resolver, uri)) { decoder, info, _ -> + val scale = minOf(1.0, longestSide.toDouble() / maxOf(info.size.width, info.size.height)) + decoder.setTargetSize(maxOf(1, (info.size.width * scale).toInt()), maxOf(1, (info.size.height * scale).toInt())) + decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE + } + } else { + val options = BitmapFactory.Options().apply { inJustDecodeBounds = true } + resolver.openInputStream(uri)?.use { BitmapFactory.decodeStream(it, null, options) } + require(options.outWidth > 0 && options.outHeight > 0) { "Invalid image" } + options.inSampleSize = 1 + while (maxOf(options.outWidth, options.outHeight) / options.inSampleSize > longestSide) options.inSampleSize *= 2 + options.inJustDecodeBounds = false + val decoded = resolver.openInputStream(uri)?.use { BitmapFactory.decodeStream(it, null, options) } + ?: error("Unable to decode image") + val orientation = resolver.openInputStream(uri)?.use { + ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) + } ?: ExifInterface.ORIENTATION_NORMAL + val matrix = Matrix().apply { + when (orientation) { + ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> setScale(-1f, 1f) + ExifInterface.ORIENTATION_ROTATE_180 -> setRotate(180f) + ExifInterface.ORIENTATION_FLIP_VERTICAL -> setScale(1f, -1f) + ExifInterface.ORIENTATION_TRANSPOSE -> { setRotate(90f); postScale(-1f, 1f) } + ExifInterface.ORIENTATION_ROTATE_90 -> setRotate(90f) + ExifInterface.ORIENTATION_TRANSVERSE -> { setRotate(270f); postScale(-1f, 1f) } + ExifInterface.ORIENTATION_ROTATE_270 -> setRotate(270f) + } + } + Bitmap.createBitmap(decoded, 0, 0, decoded.width, decoded.height, matrix, true).also { + if (it !== decoded) decoded.recycle() + } + } + try { + var quality = 85 + while (true) { + val encoded = ByteArrayOutputStream().use { + check(bitmap.compress(Bitmap.CompressFormat.JPEG, quality, it)) { "Image compression failed" } + it.toByteArray() + } + if (encoded.size <= 1024 * 1024) { + return@withContext ComposerImage( + id = "android-${UUID.randomUUID()}", + dataUrl = "data:image/jpeg;base64," + Base64.encodeToString(encoded, Base64.NO_WRAP), + mimeType = "image/jpeg", + ) + } + if (quality > 45) { quality -= 10; continue } + check(minOf(bitmap.width, bitmap.height) > 64) { "Image exceeds upload limit" } + val smaller = Bitmap.createScaledBitmap(bitmap, maxOf(1, bitmap.width * 3 / 4), maxOf(1, bitmap.height * 3 / 4), true) + bitmap.recycle() + bitmap = smaller + quality = 75 + } + @Suppress("UNREACHABLE_CODE") + error("Image encoding did not complete") + } finally { + bitmap.recycle() + } + } diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ChatMessageBubble.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ChatMessageBubble.kt index be72e08f8d..cd6d6eb71a 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ChatMessageBubble.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ChatMessageBubble.kt @@ -45,7 +45,7 @@ internal fun ChatMessageBubble( onCancelTool: (String, String) -> Unit, onAnswerTool: (String, String) -> Unit, onAnswerToolStructured: (String, List) -> Unit, - onRetry: (String) -> Unit, + onRetry: (ConversationRow) -> Unit, onOpenLink: (String, String) -> Unit, /** The file the preview surface is showing, so its card can say so. */ previewingRemotePath: String, @@ -96,7 +96,7 @@ internal fun ChatMessageBubble( row.showRetry -> ChatMessageRetryAction( fromUser = fromUser, enabled = enabled, - onRetry = { onRetry(row.text) }, + onRetry = { onRetry(row) }, modifier = Modifier, ) diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationTimelineView.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationTimelineView.kt index dc3c5a038c..92e19bf40f 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationTimelineView.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationTimelineView.kt @@ -72,7 +72,7 @@ internal fun ConversationTimelineView( onCancelTool: (String, String) -> Unit, onAnswerTool: (String, String) -> Unit, onAnswerToolStructured: (String, List) -> Unit, - onRetry: (String) -> Unit, + onRetry: (ConversationRow) -> Unit, onOpenFile: (String, String) -> Unit, previewingRemotePath: String, previewLoading: Boolean, diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationView.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationView.kt index c78fd7bc14..e25a293351 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationView.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/ConversationView.kt @@ -3,7 +3,6 @@ package com.openbitfun.mobile.app.ui.chat import android.app.Activity import android.content.Intent import android.speech.RecognizerIntent -import android.util.Base64 import android.widget.Toast import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.PickVisualMediaRequest @@ -17,9 +16,14 @@ import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.rememberUpdatedState +import kotlinx.coroutines.launch +import com.openbitfun.mobile.app.platform.prepareComposerImage import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier @@ -43,7 +47,6 @@ import com.openbitfun.mobile.core.feature.session.conversationRows import com.openbitfun.mobile.core.feature.session.modelOptions import com.openbitfun.mobile.core.feature.session.selectedModelOption import com.openbitfun.mobile.core.feature.workspace.RemoteFileDownloadUiState -import java.util.UUID internal const val CONVERSATION_TEST_TAG: String = "conversation" internal const val CONVERSATION_BACK_TEST_TAG: String = "conversation-back" @@ -57,9 +60,6 @@ internal const val CONVERSATION_LOADING_TEST_TAG: String = "conversation-loading */ internal const val CONVERSATION_LIST_TEST_TAG: String = "conversation-list" -/** The relay refuses anything larger, and refusing here is a better error. */ -private const val MAX_IMAGE_BYTES = 8 * 1024 * 1024 - /** * Joins a dictated fragment onto whatever the composer already holds. * @@ -128,22 +128,38 @@ internal fun ConversationView( // message survives session switches and process restarts via DraftStore. val draft = state.draft var images by remember(sessionId) { mutableStateOf>(emptyList()) } + LaunchedEffect(sessionId, state.lastSentMessage) { + state.lastSentMessage?.takeIf { it.sessionId == sessionId }?.let { sent -> + images = images.filterNot { it.id in sent.imageIds } + } + } var showSettings by rememberSaveable(sessionId) { mutableStateOf(false) } val context = LocalContext.current + val pickerScope = rememberCoroutineScope() + val currentSessionId by rememberUpdatedState(sessionId) + var preparingImage by remember(sessionId) { mutableStateOf(false) } val photoPicker = rememberLauncherForActivityResult(ActivityResultContracts.PickVisualMedia()) { uri -> - if (uri != null) { - val bytes = context.contentResolver.openInputStream(uri)?.use { it.readBytes() } - val mime = context.contentResolver.getType(uri) ?: "image/jpeg" - if (bytes != null && bytes.size <= MAX_IMAGE_BYTES && images.size < MAX_COMPOSER_IMAGES) { - images = images + ComposerImage( - id = "android-" + UUID.randomUUID(), - dataUrl = "data:" + mime + ";base64," + Base64.encodeToString(bytes, Base64.NO_WRAP), - mimeType = mime, - ) + if (uri != null && images.size < MAX_COMPOSER_IMAGES) { + val targetSession = sessionId + preparingImage = true + pickerScope.launch { + try { + val prepared = prepareComposerImage(context.contentResolver, uri) + if (currentSessionId == targetSession && images.size < MAX_COMPOSER_IMAGES) { + images = images + prepared + } + } catch (_: Exception) { + if (currentSessionId == targetSession) { + Toast.makeText(context, R.string.chat_image_prepare_failed, Toast.LENGTH_LONG).show() + } + } finally { + if (currentSessionId == targetSession) preparingImage = false + } } } } + val voiceInput = rememberLauncherForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> if (result.resultCode == Activity.RESULT_OK) { val text = result.data?.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS)?.firstOrNull().orEmpty() @@ -208,8 +224,15 @@ internal fun ConversationView( onAnswerToolStructured = { toolId, answers -> onIntent(AnswerStructuredQuestion(sessionId, toolId, answers)) }, - onRetry = { text -> - onIntent(RemoteSessionIntent.SendMessage(sessionId, text, null)) + onRetry = { row -> + val retryImages = row.images.map { image -> + images.firstOrNull { it.dataUrl == image.dataUrl } ?: ComposerImage( + id = image.name, + dataUrl = image.dataUrl, + mimeType = image.dataUrl.substringAfter("data:").substringBefore(';'), + ) + } + onIntent(RemoteSessionIntent.SendMessage(sessionId, row.text, retryImages)) }, onOpenFile = onOpenFile, previewingRemotePath = previewingRemotePath, @@ -225,7 +248,7 @@ internal fun ConversationView( draft = draft, images = images, // An empty session id would send nowhere, so it reads as busy. - busy = state.busy || sessionId.isEmpty(), + busy = state.busy || preparingImage || sessionId.isEmpty(), streaming = activeTurn != null, phase = phase, model = timeline?.selectedModelOption(stringResource(R.string.models_unnamed)), @@ -263,7 +286,6 @@ internal fun ConversationView( images.takeIf { it.isNotEmpty() }, ), ) - images = emptyList() }, onStop = { onIntent(RemoteSessionIntent.CancelTurn(sessionId, activeTurn?.turnId)) diff --git a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/GeneralChatScreen.kt b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/GeneralChatScreen.kt index 9f81037425..cf0a908cff 100644 --- a/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/GeneralChatScreen.kt +++ b/src/apps/mobile/android/app/src/main/kotlin/com/openbitfun/mobile/app/ui/chat/GeneralChatScreen.kt @@ -345,8 +345,8 @@ internal fun GeneralChatScreen( onCancelTool = { _, _ -> }, onAnswerTool = { _, _ -> }, onAnswerToolStructured = { _, _ -> }, - onRetry = { text -> - viewModel.dispatch(GeneralChatIntent.UpdateDraft(text)) + onRetry = { row -> + viewModel.dispatch(GeneralChatIntent.UpdateDraft(row.text)) viewModel.dispatch(GeneralChatIntent.Send) }, // No desktop behind this surface, so a link to a file has diff --git a/src/apps/mobile/android/app/src/main/res/values-zh/strings.xml b/src/apps/mobile/android/app/src/main/res/values-zh/strings.xml index f4907a0c0b..fbbe76c17d 100644 --- a/src/apps/mobile/android/app/src/main/res/values-zh/strings.xml +++ b/src/apps/mobile/android/app/src/main/res/values-zh/strings.xml @@ -434,4 +434,5 @@ 打开 GitHub 授权 请使用当前版本的 OpenBitFun 设备二维码。 该设备已离线,或不属于当前 GitHub 账户。 + 无法处理这张图片,请选择其他图片。 diff --git a/src/apps/mobile/android/app/src/main/res/values/strings.xml b/src/apps/mobile/android/app/src/main/res/values/strings.xml index 3e2a3fea3d..05bbe0fc29 100644 --- a/src/apps/mobile/android/app/src/main/res/values/strings.xml +++ b/src/apps/mobile/android/app/src/main/res/values/strings.xml @@ -449,4 +449,5 @@ Open GitHub authorization Use a current OpenBitFun device QR code. This device is offline or does not belong to your GitHub account. + This image could not be prepared. Choose another image. diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/runtime/AppRootRuntimeComposition.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/runtime/AppRootRuntimeComposition.ets index 53dfe0f83c..2f5396ad63 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/runtime/AppRootRuntimeComposition.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/runtime/AppRootRuntimeComposition.ets @@ -375,6 +375,17 @@ export abstract class AppRootRuntimeComposition { onTimelineReady: () => { this.remotePageState.setConversationLoading(false); }, + onComposerSubmitted: (sessionId: string, text: string, images: SelectedImageAttachment[]) => { + if (this.remotePageState.activeSession.sessionId !== sessionId) { + return; + } + if (this.remotePageState.chatInput.trim() === text.trim()) { + this.remotePageState.setChatInput(''); + } + this.remotePageState.setSelectedImages(this.remotePageState.selectedImages.filter( + (image: SelectedImageAttachment) => !images.some((sent: SelectedImageAttachment) => sent.id === image.id) + )); + }, onSendSucceeded: (turnId: string, pendingActiveId: string, localMessageId: string) => { this.chatTimelineStore.acknowledgeOptimisticTurn(localMessageId, turnId); if (turnId.length > 0) { @@ -407,8 +418,6 @@ export abstract class AppRootRuntimeComposition { localMessageId: string, pendingActiveId: string ) => { - this.remotePageState.setChatInput(rawText); - this.remotePageState.setSelectedImages(images); this.chatTimelineStore.markOptimisticMessageFailed(localMessageId); if (pendingActiveId.length > 0) { this.chatTimelineStore.clearPendingActiveTurn(pendingActiveId); diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteTranscriptController.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteTranscriptController.ets index c3e8a32c70..bf4e714516 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteTranscriptController.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/pages/viewmodel/RemoteTranscriptController.ets @@ -105,7 +105,6 @@ export class RemoteTranscriptController { const activeTurnId = this.remoteActiveTurnId(); const canSteerRunningTurn = queuedBehindRunningTurn && activeTurnId.length > 0 && this.remote.supportsHostCapability(REMOTE_CAPABILITY_DIALOG_STEER_V1); - this.remote.clearComposer(); const imageContexts: RemoteImageContext[] = images.length > 0 ? runtime.imagePicker.toRemoteContexts(images) : []; if (canSteerRunningTurn) { diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/services/ImagePickerService.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/services/ImagePickerService.ets index b2af92116f..137b1f7be7 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/services/ImagePickerService.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/services/ImagePickerService.ets @@ -112,16 +112,15 @@ export class ImagePickerService { } private async encodeImage(uri: string, originalSize: number): Promise { - if (originalSize <= this.transportImageBudgetBytes) { + // Photo picker URIs do not reliably carry a file extension or MIME type. + // Decode on the device so HEIC and other camera formats become actual JPEG. + if (originalSize <= this.transportImageBudgetBytes && /\.(png|jpe?g|gif|webp)(\?|$)/i.test(uri)) { return this.readOriginalImage(uri, originalSize); } try { return await this.compressImage(uri, originalSize); } catch (_err) { - if (originalSize <= this.maxImageBytes) { - return this.readOriginalImage(uri, originalSize); - } throw new Error(RemoteI18n.t('errors.imageCompressionFailed')); } } @@ -166,7 +165,6 @@ export class ImagePickerService { pixelMap = await source.createPixelMap({ desiredSize: targetSize }); packer = image.createImagePacker(); - let bestBytes: Uint8Array | undefined = undefined; for (let index = 0; index < this.compressionQualities.length; index += 1) { const packed = await packer.packToData(pixelMap, { format: 'image/jpeg', @@ -174,7 +172,6 @@ export class ImagePickerService { bufferSize: this.maxImageBytes + 1024 * 1024 }); const bytes = new Uint8Array(packed); - bestBytes = bytes; if (bytes.length <= this.transportImageBudgetBytes) { return { bytes, @@ -184,11 +181,7 @@ export class ImagePickerService { } } - return { - bytes: bestBytes || new Uint8Array(0), - mimeType: 'image/jpeg', - compressed: true - }; + throw new Error(RemoteI18n.t('errors.imageCompressionFailed')); } catch (err) { throw OhosError.wrap('image compress', err); } finally { diff --git a/src/apps/mobile/harmonyos/entry/src/main/ets/services/RemoteChatCommandController.ets b/src/apps/mobile/harmonyos/entry/src/main/ets/services/RemoteChatCommandController.ets index 046482b511..7087fe336c 100644 --- a/src/apps/mobile/harmonyos/entry/src/main/ets/services/RemoteChatCommandController.ets +++ b/src/apps/mobile/harmonyos/entry/src/main/ets/services/RemoteChatCommandController.ets @@ -41,6 +41,7 @@ export interface RemoteChatCommandCallbacks { // before any request has been sent. Without it the loading skeleton would // outlive the messages it is standing in for. onTimelineReady: () => void; + onComposerSubmitted?: (sessionId: string, text: string, images: SelectedImageAttachment[]) => void; onSendSucceeded: (turnId: string, pendingActiveId: string, localMessageId: string) => void; onSteerSucceeded?: ( steeringId: string, @@ -334,6 +335,7 @@ export class RemoteChatCommandController { const turnId = images.length > 0 ? await this.client.sendMessageWithImages(sessionId, text, agentType, imageContexts, requestedTurnId) : await this.client.sendMessage(sessionId, text, agentType, requestedTurnId); + this.callbacks.onComposerSubmitted?.(sessionId, rawText, images); this.callbacks.onSendSucceeded(turnId, pendingActiveId, localMessageId); this.callbacks.onStatusText(RemoteI18n.t('status.sentWaiting')); } catch (err) { @@ -367,6 +369,7 @@ export class RemoteChatCommandController { displayText, imageContexts ); + this.callbacks.onComposerSubmitted?.(sessionId, displayText, images); if (this.callbacks.onSteerSucceeded) { this.callbacks.onSteerSucceeded(result.steeringId, result.turnId, displayText, images); } diff --git a/src/apps/mobile/harmonyos/entry/src/test/RemoteControllersUnit.test.ets b/src/apps/mobile/harmonyos/entry/src/test/RemoteControllersUnit.test.ets index f577fc5b4e..f57c5f0db4 100644 --- a/src/apps/mobile/harmonyos/entry/src/test/RemoteControllersUnit.test.ets +++ b/src/apps/mobile/harmonyos/entry/src/test/RemoteControllersUnit.test.ets @@ -2131,12 +2131,16 @@ export default function remoteControllersUnitTest() { it('sends prepared remote messages and reports failed payloads', 0, async () => { const client = new FakeRemoteChatCommandClient(); const succeeded: string[] = []; + const submittedImages: string[] = []; const failed: string[] = []; const busyEvents: boolean[] = []; const controller = new RemoteChatCommandController(client, { onMessagesLoaded: (_messages: ChatMessage[], _hasMoreMessages: boolean) => {}, onMessageCountKnown: (_pollVersion: number, _knownMessageCount: number) => {}, onTimelineReady: () => {}, + onComposerSubmitted: (sessionId: string, _text: string, images: SelectedImageAttachment[]) => { + submittedImages.push(`${sessionId}:${images[0].id}`); + }, onSendSucceeded: (turnId: string, pendingActiveId: string, localMessageId: string) => { succeeded.push(`${turnId}:${pendingActiveId}:${localMessageId}`); }, @@ -2192,6 +2196,8 @@ export default function remoteControllersUnitTest() { expect(client.sendRequests[1]).assertEqual('text:session-1:Retry text:code'); expect(client.sendTurnIds[0]).assertEqual('local-1'); expect(client.sendTurnIds[1]).assertEqual('local-2'); + expect(submittedImages.length).assertEqual(1); + expect(submittedImages[0]).assertEqual('session-1:image-1'); expect(succeeded[0]).assertEqual('turn-1:pending-1:local-1'); expect(failed[0]).assertEqual('Retry text:0:local-2:pending-2'); expect(busyEvents[0]).assertTrue(); diff --git a/src/apps/mobile/ios/OpenBitFun/Features/Chat/ChatTimelineView.swift b/src/apps/mobile/ios/OpenBitFun/Features/Chat/ChatTimelineView.swift index 471a12c5b9..53fc8eec51 100644 --- a/src/apps/mobile/ios/OpenBitFun/Features/Chat/ChatTimelineView.swift +++ b/src/apps/mobile/ios/OpenBitFun/Features/Chat/ChatTimelineView.swift @@ -191,7 +191,7 @@ private struct ConversationRowView: View { .foregroundStyle(OpenBitFunTheme.muted) } if row.showRetry { - Button { model.retryMessage(row.text) } label: { + Button { model.retryMessage(row.text, images: row.images) } label: { Label(model.localized("重新发送"), systemImage: "arrow.clockwise") .font(MobileDesignTypography.labelSmall.font) .foregroundStyle(OpenBitFunTheme.statusDanger) @@ -220,7 +220,7 @@ private struct ConversationRowView: View { if let error = row.error, !error.isEmpty { assistantFailure(error) } else if row.showRetry { - Button { model.retryMessage(row.text) } label: { + Button { model.retryMessage(row.text, images: row.images) } label: { Label(model.localized("重试"), systemImage: "arrow.clockwise") .font(MobileDesignTypography.labelSmall.font) .foregroundStyle(OpenBitFunTheme.statusDanger) @@ -242,7 +242,7 @@ private struct ConversationRowView: View { .lineSpacing(MobileDesignTypography.bodySmall.lineSpacing) .textSelection(.enabled) if row.showRetry { - Button(model.localized("重试")) { model.retryMessage(row.text) } + Button(model.localized("重试")) { model.retryMessage(row.text, images: row.images) } .font(MobileDesignTypography.bodySmall.font.weight(.medium)) .foregroundStyle(MobileDesignColors.fileLink) .buttonStyle(.plain) diff --git a/src/apps/mobile/ios/OpenBitFun/Features/Chat/ComposerBar.swift b/src/apps/mobile/ios/OpenBitFun/Features/Chat/ComposerBar.swift index 85bad29810..2d4f35d1f1 100644 --- a/src/apps/mobile/ios/OpenBitFun/Features/Chat/ComposerBar.swift +++ b/src/apps/mobile/ios/OpenBitFun/Features/Chat/ComposerBar.swift @@ -513,15 +513,18 @@ struct ComposerBar: View { } private func importPickedImages(_ items: [PhotosPickerItem]) async { + let sessionID = model.selectedSessionID + let deviceKey = model.remoteExpectedDeviceKey for item in items { - guard let data = try? await item.loadTransferable(type: Data.self) else { + guard let data = try? await item.loadTransferable(type: Data.self), + let prepared = await Task.detached(priority: .userInitiated, operation: { + MobileAppModel.prepareComposerImage(data) + }).value else { model.showToast(model.localized("无法读取所选图片")) continue } - let mimeType = item.supportedContentTypes - .compactMap(\.preferredMIMEType) - .first ?? "image/jpeg" - model.addComposerImage(data: data, mimeType: mimeType) + guard model.selectedSessionID == sessionID, model.remoteExpectedDeviceKey == deviceKey else { break } + model.addComposerImage(data: prepared, mimeType: "image/jpeg") } pickerItems = [] } diff --git a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+GeneralChat.swift b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+GeneralChat.swift index 3879585f79..5a8e5c2832 100644 --- a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+GeneralChat.swift +++ b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+GeneralChat.swift @@ -1,4 +1,6 @@ import Foundation +import UIKit +import ImageIO import OpenBitFunMobileCore extension MobileAppModel { @@ -25,6 +27,27 @@ extension MobileAppModel { composerImages.append(ComposerAttachment(id: UUID().uuidString, data: data, mimeType: mimeType)) } + nonisolated static func prepareComposerImage(_ data: Data) -> Data? { + guard let source = CGImageSourceCreateWithData(data as CFData, nil) else { return nil } + var dimension = 1920 + while dimension >= 64 { + let options: [CFString: Any] = [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceCreateThumbnailWithTransform: true, + kCGImageSourceThumbnailMaxPixelSize: dimension + ] + guard let cgImage = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) else { return nil } + let image = UIImage(cgImage: cgImage) + for quality in [0.85, 0.75, 0.65, 0.5] { + if let encoded = image.jpegData(compressionQuality: quality), encoded.count <= 1024 * 1024 { + return encoded + } + } + dimension = dimension * 3 / 4 + } + return nil + } + func removeComposerImage(id: String) { composerImages.removeAll { $0.id == id } } diff --git a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+RemoteSession.swift b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+RemoteSession.swift index 71ca48c81e..9d7cb19ff6 100644 --- a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+RemoteSession.swift +++ b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel+RemoteSession.swift @@ -703,15 +703,14 @@ extension MobileAppModel { func sendRemote() { let value = draft.trimmingCharacters(in: .whitespacesAndNewlines) guard !value.isEmpty || !composerImages.isEmpty, - !isSending, + !isSending, !busy, + let coreAdapter, connectionPhase != .disconnected, let sessionID = visibleSessions.first(where: { $0.id == selectedSessionID })?.id else { return } let images = composerImages - draft = "" - composerImages = [] isSending = true busy = true - coreAdapter?.sendRemote(sessionID: sessionID, content: value, images: images) + coreAdapter.sendRemote(sessionID: sessionID, content: value, images: images) } func approveTool(_ toolID: String) { @@ -869,6 +868,15 @@ extension MobileAppModel { ) } setPublishedIfChanged(\.busy, to: ready.busy) + if let sent = ready.lastSentMessage, + sent.sessionId == selectedSessionID, + sent.id != lastAppliedRemoteSendID { + lastAppliedRemoteSendID = sent.id + if draft.trimmingCharacters(in: .whitespacesAndNewlines) == sent.content { + draft = "" + } + composerImages.removeAll { sent.imageIds.contains($0.id) } + } setPublishedIfChanged(\.remoteQuery, to: ready.query) setPublishedIfChanged(\.remoteAgentFilter, to: ready.agentFilter.name) setPublishedIfChanged(\.remoteHasMore, to: ready.hasMore) diff --git a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel.swift b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel.swift index 8cb9d1f8c9..8dae448fa6 100644 --- a/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel.swift +++ b/src/apps/mobile/ios/OpenBitFun/Infrastructure/MobileAppModel.swift @@ -32,6 +32,7 @@ final class MobileAppModel: ObservableObject { @Published var messages: [ChatMessage] @Published var timelineRows: [MobileConversationRow] = [] @Published var draft = "" + var lastAppliedRemoteSendID: String? @Published var drawerOpen = false @Published var settingsOpen = false @Published var remoteControlSettingsOpen = false @@ -374,18 +375,24 @@ final class MobileAppModel: ObservableObject { coreAdapter?.cancelRemoteTurn(sessionID: selectedSessionID, turnID: activeTurnID) } - func retryMessage(_ text: String) { + func retryMessage(_ text: String, images: [MobileTimelineImage] = []) { let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines) - guard !normalized.isEmpty, !busy, !isSending else { return } - if surface == .remote { - guard remoteSessionSelected, connectionPhase != .disconnected else { return } - isSending = true - busy = true - coreAdapter?.sendRemote(sessionID: selectedSessionID, content: normalized, images: []) - } else { - draft = normalized - send() + guard !normalized.isEmpty || !images.isEmpty, !busy, !isSending else { return } + let attachments = images.compactMap { image -> ComposerAttachment? in + if let retained = composerImages.first(where: { $0.dataURL == image.dataURL }) { return retained } + guard let comma = image.dataURL.firstIndex(of: ","), + let data = Data(base64Encoded: String(image.dataURL[image.dataURL.index(after: comma)...])) else { return nil } + let mime = image.dataURL.prefix(upTo: comma).dropFirst(5).split(separator: ";").first.map(String.init) ?? "image/jpeg" + return ComposerAttachment(id: UUID().uuidString, data: data, mimeType: mime) + } + guard attachments.count == images.count else { + showToast(localized("无法读取所选图片")) + return } + guard remoteSessionSelected, connectionPhase != .disconnected, let coreAdapter else { return } + isSending = true + busy = true + coreAdapter.sendRemote(sessionID: selectedSessionID, content: normalized, images: attachments) } func renameSelectedSession(_ title: String) { diff --git a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStore.kt b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStore.kt index 6dda11fb32..9be000d13f 100644 --- a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStore.kt +++ b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStore.kt @@ -1335,8 +1335,9 @@ public class RemoteSessionStore internal constructor( private fun sendMessage(intent: RemoteSessionIntent.SendMessage) { val sessionId = intent.sessionId.trim() val content = intent.content - if (sessionId.isEmpty() || content.trim().isEmpty()) return + if (sessionId.isEmpty() || (content.trim().isEmpty() && intent.images.isNullOrEmpty())) return val current = _state.value as? RemoteSessionUiState.Ready ?: return + if (current.busy || current.selectedSessionId != sessionId) return val wireImages = intent.images?.map { image -> com.openbitfun.mobile.core.protocol.ImageAttachment( name = image.id, @@ -1386,17 +1387,27 @@ public class RemoteSessionStore internal constructor( if (!isCurrentWork(operationToken)) return@launch response.turnId?.let(timelineStore::setLocalActiveTurn) controller.nudge() - deletePersistedDraft(sessionId) val ready = ((_state.value as? RemoteSessionUiState.Ready) ?: current) - if (ready.selectedSessionId == sessionId) _state.value = ready.copy(draft = "") + if (ready.selectedSessionId == sessionId) { + // An acknowledgement owns only the submitted draft. Keep + // newer typing and let each native picker remove only the + // acknowledged images; failed sends retain their pixels. + val draftUnchanged = ready.draft == current.draft && ready.draft.trim() == content.trim() + if (draftUnchanged) deletePersistedDraft(sessionId) + _state.value = ready.copy( + draft = if (draftUnchanged) "" else ready.draft, + lastSentMessage = SentChatMessage( + local.id, sessionId, content, intent.images.orEmpty().map { it.id }, + ), + ) + } setBusy((_state.value as? RemoteSessionUiState.Ready) ?: current, false) } catch (cancelled: CancellationException) { throw cancelled } catch (error: Throwable) { if (isCurrentWork(operationToken)) { timelineStore.markOptimisticMessageFailed(local.id) - setBusy((_state.value as? RemoteSessionUiState.Ready) ?: current, false) - handleFailure(error, current) + handleFailure(error, (_state.value as? RemoteSessionUiState.Ready) ?: current) } } } diff --git a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionUiState.kt b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionUiState.kt index e47b333c57..dcfc4b6fb4 100644 --- a/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionUiState.kt +++ b/src/apps/mobile/shared/core-feature/src/commonMain/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionUiState.kt @@ -184,6 +184,14 @@ public data class ComposerImage public constructor( public val mimeType: String, ) +/** A host acknowledgement, used to consume only the draft that was sent. */ +public data class SentChatMessage public constructor( + public val id: String, + public val sessionId: String, + public val content: String, + public val imageIds: List, +) + public sealed interface RemoteSessionUiState { public data object Idle : RemoteSessionUiState @@ -222,7 +230,29 @@ public sealed interface RemoteSessionUiState { public val draft: String, /** Monotonic authority revision for session-list projection on this store. */ public val revision: Long, + public val lastSentMessage: SentChatMessage?, ) : RemoteSessionUiState { + /** Preserve the existing Swift/Kotlin initializer when adding acknowledgement state. */ + public constructor( + sessions: List, + selectedSessionId: String?, + timeline: ChatTimelineState?, + busy: Boolean, + permissionMode: SessionPermissionMode?, + permissionModeFailure: PermissionModeFailure?, + query: String, + agentFilter: SessionAgentFilter, + hasMore: Boolean, + hasMoreMessages: Boolean, + modelCatalog: RemoteModelCatalog?, + modelCatalogFailure: ModelCatalogFailure?, + draft: String, + revision: Long, + ) : this( + sessions, selectedSessionId, timeline, busy, permissionMode, permissionModeFailure, + query, agentFilter, hasMore, hasMoreMessages, modelCatalog, modelCatalogFailure, draft, revision, null, + ) + public constructor( sessions: List, selectedSessionId: String?, diff --git a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStoreTest.kt b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStoreTest.kt index 2d60fe745e..f896abca00 100644 --- a/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStoreTest.kt +++ b/src/apps/mobile/shared/core-feature/src/commonTest/kotlin/com/openbitfun/mobile/core/feature/session/RemoteSessionStoreTest.kt @@ -1051,6 +1051,62 @@ class RemoteSessionStoreTest { store.dispatch(RemoteSessionIntent.Stop) } + @Test + fun imageOnlyMessagesAreSentAndAcknowledgedForNativePickers() = runTest { + val transport = FakeSessionTransport() + val store = RemoteSessionStore.create(this, transport) + store.dispatch(RemoteSessionIntent.Open("s-code")) + runCurrent() + val images = listOf(ComposerImage("photo-1", "data:image/png;base64,abc", "image/png")) + store.dispatch(RemoteSessionIntent.SendMessage("s-code", "", images)) + runCurrent() + val sent = transport.commands.last { it.cmd == "send_message" } + assertEquals("", sent.content) + assertEquals(images.single().dataUrl, sent.imageContexts?.single()?.dataUrl) + assertNull(sent.imageContexts?.single()?.imagePath) + val ready = assertIs(store.state.value) + assertFalse(ready.busy) + assertEquals(listOf("photo-1"), ready.lastSentMessage?.imageIds) + assertEquals("s-code", ready.lastSentMessage?.sessionId) + store.stop() + } + + @Test + fun imageSendFailureDoesNotConsumeAttachmentsAndAckKeepsNewTyping() = runTest { + val transport = FakeSessionTransport() + val store = RemoteSessionStore.create(this, transport) + store.dispatch(RemoteSessionIntent.Open("s-code")) + runCurrent() + val images = listOf(ComposerImage("photo-1", "data:image/png;base64,abc", "image/png")) + store.dispatch(RemoteSessionIntent.UpdateDraft("look")) + transport.sendMessageFailure = RelayFailure.NetworkUnreachable + val failureGate = CompletableDeferred() + transport.commandGates["send_message"] = failureGate + store.dispatch(RemoteSessionIntent.SendMessage("s-code", "look", images)) + runCurrent() + store.dispatch(RemoteSessionIntent.UpdateDraft("edited while waiting")) + failureGate.complete(Unit) + runCurrent() + val failed = assertIs(store.state.value) + assertEquals("edited while waiting", failed.draft) + assertNull(failed.lastSentMessage) + assertFalse(failed.busy) + + transport.sendMessageFailure = null + val gate = CompletableDeferred() + transport.commandGates["send_message"] = gate + store.dispatch(RemoteSessionIntent.SendMessage("s-code", "look", images)) + runCurrent() + store.dispatch(RemoteSessionIntent.UpdateDraft("next question")) + gate.complete(Unit) + runCurrent() + val ready = assertIs(store.state.value) + assertEquals("next question", ready.draft) + assertEquals(listOf("photo-1"), ready.lastSentMessage?.imageIds) + assertFalse(ready.busy) + store.stop() + } + @Test fun sendMessageFallsBackToTheLocallyCreatedRecordWhenTheFilterHidesTheSession() = runTest { val transport = FakeSessionTransport() diff --git a/src/crates/assembly/core/src/agentic/coordination/coordinator.rs b/src/crates/assembly/core/src/agentic/coordination/coordinator.rs index c491ca2151..cce3f9412f 100644 --- a/src/crates/assembly/core/src/agentic/coordination/coordinator.rs +++ b/src/crates/assembly/core/src/agentic/coordination/coordinator.rs @@ -5859,13 +5859,34 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet .map(|_task| ()) } + pub(crate) async fn prepare_input_images( + &self, + session_id: &str, + images: &mut [ImageContextData], + ) -> OpenBitFunResult<()> { + if images.is_empty() { + return Ok(()); + } + let session = self + .session_manager + .get_session(session_id) + .ok_or_else(|| OpenBitFunError::NotFound(format!("Session not found: {session_id}")))?; + let workspace = Self::build_workspace_binding(&session.config).await; + let context = + crate::agentic::tools::framework::ToolUseContext::for_tool_listing(workspace, None); + crate::agentic::image_analysis::attachments::prepare_inline_image_attachments( + images, &context, + ) + .await + } + #[allow(clippy::too_many_arguments)] async fn start_dialog_turn_internal( &self, session_id: String, user_input: String, original_user_input: Option, - image_contexts: Option>, + mut image_contexts: Option>, turn_id: Option, agent_type: String, workspace_path: Option, @@ -6192,6 +6213,10 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet let mut user_message_metadata = extra_user_message_metadata; + if let Some(images) = image_contexts.as_mut() { + self.prepare_input_images(&session_id, images).await?; + } + // Build image metadata for workspace turn persistence (before image_contexts is consumed) // Also stores original_text so the UI can display the user's actual input // instead of the vision-enhanced text. @@ -6342,6 +6367,19 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet None, ) .await?; + if image_contexts + .as_ref() + .is_some_and(|images| !images.is_empty()) + { + let config_service = crate::service::config::get_global_config_service().await?; + let ai_config: crate::service::config::types::AIConfig = + config_service.get_config(Some("ai")).await?; + crate::agentic::image_analysis::image_processing::validate_image_input_model( + &ai_config, + &resolved_model_id, + session.config.enable_tools, + )?; + } let reasoning_preset = self .session_manager .reconcile_session_reasoning_preset_for_turn(&session_id, "turn_admission") diff --git a/src/crates/assembly/core/src/agentic/coordination/scheduler.rs b/src/crates/assembly/core/src/agentic/coordination/scheduler.rs index d9dab519d2..f932f203a7 100644 --- a/src/crates/assembly/core/src/agentic/coordination/scheduler.rs +++ b/src/crates/assembly/core/src/agentic/coordination/scheduler.rs @@ -494,7 +494,7 @@ impl DialogScheduler { turn_id: String, content: String, display_content: Option, - attachments: Vec, + mut attachments: Vec, metadata: serde_json::Map, ) -> Result { if content.trim().is_empty() && attachments.is_empty() { @@ -503,7 +503,9 @@ impl DialogScheduler { // Reject a malformed attachment here rather than at the round boundary: // the caller is still holding the user's message and can surface the // failure, whereas the injection consumer would have to drop it. - agent_dialog_turn_image_contexts(&attachments).map_err(|error| error.to_string())?; + let mut images = agent_dialog_turn_image_contexts(&attachments) + .map_err(|error| error.to_string())? + .unwrap_or_default(); let _operation_guard = self.lock_session_operation(&session_id).await; let active_turn_id = match self .session_manager @@ -521,6 +523,23 @@ impl DialogScheduler { _ => None, }; + if active_turn_id.as_deref() == Some(turn_id.as_str()) && !images.is_empty() { + self.coordinator + .prepare_input_images(&session_id, &mut images) + .await + .map_err(|error| error.to_string())?; + for (attachment, image) in attachments.iter_mut().zip(&images) { + if let Some(path) = &image.image_path { + attachment + .metadata + .insert("imagePath".into(), serde_json::json!(path)); + } + attachment + .metadata + .insert("mimeType".into(), serde_json::json!(image.mime_type)); + } + } + let steering_id = Uuid::new_v4().to_string(); match resolve_dialog_steering_action( active_turn_id.as_deref(), @@ -1233,6 +1252,13 @@ impl DialogScheduler { )); } + if let Some(images) = queued_turn.image_contexts.as_mut() { + self.coordinator + .prepare_input_images(&session_id, images) + .await + .map_err(SchedulerSubmitError::Core)?; + } + // OpenCode-compatible semantics: accepting a new prompt while history // is staged permanently discards the hidden suffix before the Turn starts. self.coordinator diff --git a/src/crates/assembly/core/src/agentic/execution/execution_engine.rs b/src/crates/assembly/core/src/agentic/execution/execution_engine.rs index a444458065..f2ff99ee6d 100644 --- a/src/crates/assembly/core/src/agentic/execution/execution_engine.rs +++ b/src/crates/assembly/core/src/agentic/execution/execution_engine.rs @@ -25,9 +25,9 @@ use crate::agentic::execution::conditional_instructions::{ build_conditional_instruction_reminder, successful_workspace_read_paths, }; use crate::agentic::execution::types::FinishReason; +use crate::agentic::image_analysis::image_processing::process_image_contexts_in_workspace; use crate::agentic::image_analysis::{ - build_multimodal_message_with_images, process_image_contexts_for_provider, ImageContextData, - ImageLimits, + build_multimodal_message_with_images, ImageContextData, ImageLimits, }; use crate::agentic::round_preempt::RoundInjectionKind; use crate::agentic::session::{ @@ -40,6 +40,7 @@ use crate::agentic::session::{ INTERRUPTED_TURN_RESOLVED_MODEL_ID_METADATA_KEY, }; use crate::agentic::skill_agent_snapshot::build_skill_agent_tool_listing_sections_from_snapshot; +use crate::agentic::tools::framework::ToolUseContext; use crate::agentic::tools::implementations::{SkillTool, TaskTool}; use crate::agentic::tools::product_runtime::{ collect_product_loaded_deferred_tool_specs, GetToolSpecTool, @@ -75,7 +76,6 @@ use openbitfun_core_types::{ModelRequestContext, SessionModelBindingPolicy}; use openbitfun_runtime_ports::{resolve_permission_mode, PermissionMode, PermissionModeLayers}; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; -use std::path::Path; use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::Arc; use tokio_util::sync::CancellationToken; @@ -573,6 +573,7 @@ struct CompressionModelSummaryInput<'a> { prepended_prompt_reminders: &'a PrependedPromptReminders, tool_definitions: &'a Option>, workspace: Option<&'a WorkspaceBinding>, + workspace_services: Option<&'a crate::agentic::workspace::WorkspaceServices>, dialog_turn_id: &'a str, runtime_messages: &'a [Message], ai_client: Arc, @@ -1999,11 +2000,8 @@ impl ExecutionEngine { let mut final_ai_messages = Self::build_ai_messages_for_send( input.messages, &input.ai_client.config.format, - input - .context - .workspace - .as_ref() - .map(|workspace| workspace.root_path()), + input.context.workspace.as_ref(), + input.context.workspace_services.as_ref(), &input.context.dialog_turn_id, input.primary_model_facts.supports_image_inputs, input.prepended_reminders, @@ -2068,7 +2066,8 @@ impl ExecutionEngine { async fn build_ai_messages_for_send( messages: &[Message], provider: &str, - workspace_path: Option<&Path>, + workspace: Option<&WorkspaceBinding>, + workspace_services: Option<&crate::agentic::workspace::WorkspaceServices>, current_turn_id: &str, attach_images: bool, prepended_reminders: &[&str], @@ -2077,6 +2076,8 @@ impl ExecutionEngine { const MAX_IMAGE_BEARING_MESSAGE_ROUNDS: usize = 2; let limits = ImageLimits::for_provider(provider); + let image_context = + ToolUseContext::for_tool_listing(workspace.cloned(), workspace_services.cloned()); let trimmed_reminders = prepended_reminders .iter() @@ -2116,7 +2117,10 @@ impl ExecutionEngine { if !attach_images { // Primary model is text-only (or images are disabled). Convert to text-only // placeholder so providers that don't support image inputs won't error. - result.push(AIMessage::from(msg)); + let mut text_message = msg.clone(); + text_message.content = + MessageContent::Text(Self::render_multimodal_as_text(text, images)); + result.push(AIMessage::from(&text_message)); continue; } @@ -2145,10 +2149,10 @@ impl ExecutionEngine { prompt }; - match process_image_contexts_for_provider( + match process_image_contexts_in_workspace( &filtered_images, provider, - workspace_path, + &image_context, ) .await { @@ -2183,7 +2187,11 @@ impl ExecutionEngine { "Failed to rebuild multimodal payload, falling back to text-only message: message_id={}, provider={}, turn_id={:?}, current_turn_id={}, error={}", msg.id, provider, msg.metadata.turn_id, current_turn_id, err ); - result.push(AIMessage::from(msg)); + let mut unavailable = msg.clone(); + unavailable.content = MessageContent::Text(format!( + "{text}\n\n[Image pixels from this older message are unavailable. Ask for a new attachment if the current task requires inspecting them.]" + )); + result.push(AIMessage::from(&unavailable)); } else { return Err(err); } @@ -2270,7 +2278,11 @@ impl ExecutionEngine { } content.push_str("]\n"); - content.push_str("Note: the primary model cannot inspect image pixels directly. If an image path is available, use analyze_image to inspect it, or use a user-provided image skill with that path.\n"); + if images.iter().any(|image| image.image_path.is_some()) { + content.push_str("The primary model cannot inspect image pixels directly. Use analyze_image with the exact attached path and the user's question before answering about image content. The configured image understanding model reads the pixels; do not ask the user to describe an attached image instead.\n"); + } else { + content.push_str("Image pixels from this older message are unavailable. If the task requires them, ask the user to attach the image again.\n"); + } content } @@ -2280,6 +2292,7 @@ impl ExecutionEngine { runtime_messages: &[Message], dialog_turn_id: &str, workspace: Option<&WorkspaceBinding>, + workspace_services: Option<&crate::agentic::workspace::WorkspaceServices>, provider: &str, attach_images: bool, prepended_prompt_reminders: &PrependedPromptReminders, @@ -2288,7 +2301,8 @@ impl ExecutionEngine { let mut compression_messages = Self::build_ai_messages_for_send( runtime_messages, provider, - workspace.map(|workspace| workspace.root_path()), + workspace, + workspace_services, dialog_turn_id, attach_images, &prepended_reminders, @@ -2399,6 +2413,7 @@ impl ExecutionEngine { input.runtime_messages, input.dialog_turn_id, input.workspace, + input.workspace_services, &input.ai_client.config.format, input.primary_supports_image_understanding, input.prepended_prompt_reminders, @@ -2438,6 +2453,7 @@ impl ExecutionEngine { prepended_prompt_reminders: &PrependedPromptReminders, primary_supports_image_understanding: bool, workspace: Option<&WorkspaceBinding>, + workspace_services: Option<&crate::agentic::workspace::WorkspaceServices>, trace_config: Option, ) -> OpenBitFunResult> { let max_initial_recent = context_window.saturating_div(2).max(1); @@ -2480,6 +2496,7 @@ impl ExecutionEngine { runtime_messages: &plan.summary_request_messages, dialog_turn_id, workspace, + workspace_services, tool_definitions, prepended_prompt_reminders, primary_supports_image_understanding, @@ -2800,6 +2817,7 @@ impl ExecutionEngine { primary_supports_image_understanding: bool, compression_contract_limit: usize, workspace: Option<&WorkspaceBinding>, + workspace_services: Option<&crate::agentic::workspace::WorkspaceServices>, ) -> OpenBitFunResult)>> { let mut session = self .session_manager @@ -2879,6 +2897,7 @@ impl ExecutionEngine { prepended_prompt_reminders, primary_supports_image_understanding, workspace, + workspace_services, trace_config, ) .await; @@ -3162,6 +3181,7 @@ impl ExecutionEngine { &scaffold.prepended_prompt_reminders, scaffold.primary_supports_image_understanding, context.workspace.as_ref(), + context.workspace_services.as_ref(), trace_config, ) .await?; @@ -3845,22 +3865,41 @@ impl ExecutionEngine { let compression_trigger_budget = Self::compression_trigger_budget(context_window, ai_client.config.max_tokens); - // If the primary model is text-only, do not send image payloads to the provider. - // Instead, keep a text-only placeholder (including `image_id`). - if !primary_supports_image_understanding { - for msg in messages.iter_mut() { - let MessageContent::Multimodal { text, images } = &msg.content else { - continue; - }; - - let original_text = text.clone(); - let original_images = images.clone(); - - // Replace multimodal messages with text-only versions to avoid provider errors. - let next_text = Self::render_multimodal_as_text(&original_text, &original_images); + // Project images at the provider boundary on every round. Keep the + // canonical pixels and paths so steering, retries, and model switches + // retain the same attachments. - msg.content = MessageContent::Text(next_text); - msg.metadata.tokens = None; + let attachment_context = ToolUseContext::for_tool_listing( + context.workspace.clone(), + context.workspace_services.clone(), + ); + // Older turn metadata may still hold inline pixels even when its context + // snapshot predates durable attachments. Recover those pixels without + // changing the persisted record shape or requiring manual migration. + for message in &mut messages { + if let MessageContent::Multimodal { images, .. } = &mut message.content { + if !images + .iter() + .any(|image| image.data_url.is_some() && image.image_path.is_none()) + { + continue; + } + if let Err(error) = + crate::agentic::image_analysis::attachments::prepare_inline_image_attachments( + images, + &attachment_context, + ) + .await + { + if message.metadata.turn_id.as_deref() == Some(context.dialog_turn_id.as_str()) + { + return Err(error); + } + warn!( + "Unable to recover historical image attachment: message_id={}, error={}", + message.id, error + ); + } } } @@ -4039,6 +4078,7 @@ impl ExecutionEngine { primary_supports_image_understanding, context_profile_policy.compression_contract_limit, context.workspace.as_ref(), + context.workspace_services.as_ref(), ) .await { @@ -4218,13 +4258,26 @@ impl ExecutionEngine { messages.len() ); + if !primary_supports_image_understanding + && messages.iter().any(|message| { + message.metadata.turn_id.as_deref() == Some(context.dialog_turn_id.as_str()) + && matches!(&message.content, MessageContent::Multimodal { images, .. } if !images.is_empty()) + }) + { + if !available_tools.iter().any(|name| name == "analyze_image") { + return Err(OpenBitFunError::validation( + "This agent cannot analyze image attachments with the selected text-only model. Enable analyze_image for this agent or select a multimodal model.", + )); + } + // Includes images accepted as steering after the turn began. + crate::agentic::image_analysis::resolve_vision_model_from_global_config().await?; + } + let ai_messages = Self::build_ai_messages_for_send( &messages, &ai_client.config.format, - context - .workspace - .as_ref() - .map(|workspace| workspace.root_path()), + context.workspace.as_ref(), + context.workspace_services.as_ref(), &context.dialog_turn_id, primary_supports_image_understanding, &send_prepended_reminders, @@ -4278,6 +4331,7 @@ impl ExecutionEngine { primary_supports_image_understanding, context_profile_policy.compression_contract_limit, context.workspace.as_ref(), + context.workspace_services.as_ref(), ) .await { @@ -4668,21 +4722,14 @@ impl ExecutionEngine { InternalReminderKind::GoalObjectiveUpdated } }; - // Attachments rebuild into the same multimodal user - // message a turn-boundary submission would have - // produced; a bad payload degrades to text rather than - // dropping the user's steering message entirely. - let images = match agent_dialog_turn_image_contexts(&injection.attachments) - { - Ok(images) => images.unwrap_or_default(), - Err(error) => { - warn!( - "Dropping unusable steering attachments, injecting text only: session_id={}, steering_id={}, error={}", - context.session_id, injection_id, error - ); - Vec::new() - } - }; + // Use the same durable input path as turn-boundary messages. + // A malformed image must not silently become a text-only turn. + let mut images = agent_dialog_turn_image_contexts(&injection.attachments) + .map_err(|error| OpenBitFunError::validation(error.to_string()))? + .unwrap_or_default(); + crate::agentic::image_analysis::attachments::prepare_inline_image_attachments( + &mut images, &attachment_context, + ).await?; let user_msg = if images.is_empty() { Message::internal_reminder(reminder_kind, wrapped) } else { @@ -5291,6 +5338,67 @@ mod tests { use std::sync::Arc; use std::time::Duration; + #[tokio::test] + async fn image_inputs_keep_pixels_for_native_models_and_tool_paths_for_text_models() { + let mut image = crate::agentic::image_analysis::attachments::test_image(); + image.image_path = + Some("openbitfun://runtime/current/attachments/images/example.png".into()); + let messages = vec![ + Message::user_multimodal("Read this screenshot".into(), vec![image.clone()]) + .with_turn_id("turn".into()), + Message::internal_reminder_multimodal( + crate::agentic::core::InternalReminderKind::UserSteering, + "Also check the second image", + vec![image.clone()], + ) + .with_turn_id("turn".into()), + ]; + for provider in ["openai", "anthropic", "responses", "gemini"] { + let native = ExecutionEngine::build_ai_messages_for_send( + &messages, + provider, + None, + None, + "turn", + true, + &[], + ) + .await + .unwrap(); + assert_eq!(native.len(), 2); + for message in native { + let parts: serde_json::Value = + serde_json::from_str(message.content.as_deref().unwrap()).unwrap(); + assert!( + parts.as_array().unwrap().iter().any(|part| { + matches!(part["type"].as_str(), Some("image") | Some("image_url")) + || part.get("inline_data").is_some() + }), + "{provider}: {parts}" + ); + } + } + let text_only = ExecutionEngine::build_ai_messages_for_send( + &messages, + "openai", + None, + None, + "turn", + false, + &[], + ) + .await + .unwrap(); + assert_eq!(text_only.len(), 2); + for message in text_only { + let text = message.content.unwrap(); + assert!(text.contains("analyze_image")); + assert!(text.contains(image.image_path.as_deref().unwrap())); + assert!(!text.contains("base64")); + } + assert!(messages.iter().all(|message| matches!(&message.content, crate::agentic::core::MessageContent::Multimodal { images, .. } if images[0].data_url.is_some()))); + } + #[tokio::test] async fn compression_cancellation_before_preparation_does_not_start_work() { let token = tokio_util::sync::CancellationToken::new(); diff --git a/src/crates/assembly/core/src/agentic/image_analysis/attachments.rs b/src/crates/assembly/core/src/agentic/image_analysis/attachments.rs new file mode 100644 index 0000000000..dadd0c6911 --- /dev/null +++ b/src/crates/assembly/core/src/agentic/image_analysis/attachments.rs @@ -0,0 +1,183 @@ +//! Durable attachment preparation shared by every Agent input surface. + +use super::{decode_data_url, ImageContextData}; +use crate::agentic::tools::framework::{build_tool_runtime_artifact_reference, ToolUseContext}; +use crate::util::errors::{OpenBitFunError, OpenBitFunResult}; +use openbitfun_services_core::json_store::JsonFileStore; +use sha2::{Digest, Sha256}; +use std::path::Path; + +/// Inline pixels belong to the receiving Runtime, even in an SSH workspace. +/// Publish a stable artifact before admitting the message so text-only models +/// can call `analyze_image` and restored native multimodal turns can reload it. +/// Keep the inline payload for the transcript and for older message consumers. +pub(crate) async fn prepare_inline_image_attachments( + images: &mut [ImageContextData], + context: &ToolUseContext, +) -> OpenBitFunResult<()> { + if !images.iter().any(|image| image.data_url.is_some()) { + return Ok(()); + } + let runtime_root = context.current_workspace_runtime_root()?; + prepare_in_runtime_root( + images, + &runtime_root, + context.current_workspace_scope().as_deref(), + context.should_emit_runtime_uri(), + ) + .await +} + +async fn prepare_in_runtime_root( + images: &mut [ImageContextData], + runtime_root: &Path, + workspace_scope: Option<&str>, + emit_runtime_uri: bool, +) -> OpenBitFunResult<()> { + for image in images { + let Some(data_url) = image.data_url.as_deref() else { + continue; + }; + let (bytes, _) = decode_data_url(data_url)?; + let format = image::guess_format(&bytes).map_err(|error| { + OpenBitFunError::validation(format!("Invalid image attachment {}: {error}", image.id)) + })?; + let (extension, mime_type) = match format { + image::ImageFormat::Png => ("png", "image/png"), + image::ImageFormat::Jpeg => ("jpg", "image/jpeg"), + image::ImageFormat::Gif => ("gif", "image/gif"), + image::ImageFormat::WebP => ("webp", "image/webp"), + image::ImageFormat::Bmp => ("bmp", "image/bmp"), + _ => { + return Err(OpenBitFunError::validation(format!( + "Unsupported image attachment format: {format:?}. Use PNG, JPEG, GIF, WebP or BMP." + ))) + } + }; + let relative_path = format!( + "attachments/images/{:x}.{extension}", + Sha256::digest(&bytes) + ); + let path = runtime_root.join(&relative_path); + // A replay or queued dispatch can prepare the same pixels twice. The + // content address lets it reuse a complete, unchanged artifact. + if tokio::fs::read(&path).await.ok().as_deref() != Some(bytes.as_slice()) { + persist_attachment(&path, bytes).await?; + } + // Never reuse a controller-side path when pixels were supplied. A + // runtime URI explicitly selects host artifact IO in remote workspaces. + image.image_path = Some( + build_tool_runtime_artifact_reference( + &relative_path, + Some(runtime_root), + workspace_scope, + emit_runtime_uri, + ) + .map_err(|error| OpenBitFunError::validation(error.to_string()))?, + ); + image.mime_type = mime_type.to_string(); + } + Ok(()) +} + +#[cfg(test)] +pub(crate) fn test_image() -> ImageContextData { + use base64::Engine; + let mut bytes = std::io::Cursor::new(Vec::new()); + image::DynamicImage::new_rgb8(8, 6) + .write_to(&mut bytes, image::ImageFormat::Png) + .unwrap(); + ImageContextData { + id: "image-1".into(), + image_path: None, + data_url: Some(format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(bytes.into_inner()) + )), + mime_type: "image/png".into(), + metadata: Some(serde_json::json!({"name": "screenshot.png"})), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn inline_pixels_survive_redaction_and_reuse_the_same_artifact_on_retry() { + let root = tempfile::tempdir().unwrap(); + let mut images = vec![test_image()]; + images[0].image_path = Some("/controller-only/wrong.png".into()); + images[0].id = "../../not-a-filename".into(); + let original = images[0].data_url.clone(); + prepare_in_runtime_root(&mut images, root.path(), None, false) + .await + .unwrap(); + let path = images[0].image_path.clone().unwrap(); + assert!(Path::new(&path).starts_with(root.path())); + assert_eq!(images[0].data_url, original); + assert_eq!( + images[0].metadata.as_ref().unwrap()["name"], + "screenshot.png" + ); + prepare_in_runtime_root(&mut images, root.path(), None, false) + .await + .unwrap(); + assert_eq!(images[0].image_path.as_deref(), Some(path.as_str())); + + images[0].data_url = None; + let restored: Vec = + serde_json::from_value(serde_json::to_value(images).unwrap()).unwrap(); + let processed = + super::super::process_image_contexts_for_provider(&restored, "openai", None) + .await + .unwrap(); + assert_eq!((processed[0].width, processed[0].height), (8, 6)); + } + + #[tokio::test] + async fn remote_upload_references_runtime_artifacts_without_exposing_host_paths() { + let root = tempfile::tempdir().unwrap(); + let mut images = vec![test_image(), test_image()]; + images[1].id = "image-2".into(); + prepare_in_runtime_root(&mut images, root.path(), Some("remote-workspace"), true) + .await + .unwrap(); + let reference = images[0].image_path.as_deref().unwrap(); + assert!(reference.starts_with("openbitfun://runtime/remote-workspace/attachments/images/")); + assert!(!reference.contains(root.path().to_str().unwrap())); + assert_eq!(images[0].image_path, images[1].image_path); + assert_ne!(images[0].id, images[1].id); + let parsed = + crate::agentic::tools::workspace_paths::parse_openbitfun_runtime_uri(reference) + .unwrap(); + assert!(root.path().join(parsed.relative_path).is_file()); + } + + #[tokio::test] + async fn malformed_inline_payload_is_rejected_without_using_a_supplied_path() { + let root = tempfile::tempdir().unwrap(); + for data_url in [ + "data:image/png,hello", + "data:image/png;base64,???", + "data:image/png;base64,aGVsbG8=", + ] { + let mut images = vec![test_image()]; + images[0].image_path = Some("/valid-looking/file.png".into()); + images[0].data_url = Some(data_url.into()); + assert!( + prepare_in_runtime_root(&mut images, root.path(), None, false) + .await + .is_err() + ); + } + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); + } +} + +async fn persist_attachment(path: &Path, bytes: Vec) -> OpenBitFunResult<()> { + JsonFileStore + .write_bytes_atomic_strict(path, bytes) + .await + .map_err(|error| OpenBitFunError::io(format!("Failed to save image attachment: {error}"))) +} diff --git a/src/crates/assembly/core/src/agentic/image_analysis/image_processing.rs b/src/crates/assembly/core/src/agentic/image_analysis/image_processing.rs index 1065115e64..6e1b3c44c8 100644 --- a/src/crates/assembly/core/src/agentic/image_analysis/image_processing.rs +++ b/src/crates/assembly/core/src/agentic/image_analysis/image_processing.rs @@ -1,6 +1,7 @@ //! Shared image processing utilities used by both API-side image analysis and tool-driven image analysis. use super::types::{ImageContextData, ImageLimits}; +use crate::agentic::tools::framework::ToolUseContext; use crate::service::config::get_global_config_service; use crate::service::config::types::{AIConfig as ServiceAIConfig, AIModelConfig}; use crate::util::errors::{OpenBitFunError, OpenBitFunResult}; @@ -109,6 +110,73 @@ pub async fn resolve_vision_model_from_global_config() -> OpenBitFunResult OpenBitFunResult<()> { + let model = ai_config + .models + .iter() + .find(|model| model.id == primary_model_id) + .ok_or_else(|| { + OpenBitFunError::validation(format!("Model not found: {primary_model_id}")) + })?; + if model.supports_image_understanding() { + return Ok(()); + } + if !tools_enabled { + return Err(OpenBitFunError::validation( + "This model cannot read images while tools are disabled. Select a multimodal model or enable tools and configure an image understanding model.", + )); + } + resolve_vision_model_from_ai_config(ai_config).map(|_| ()) +} + +#[cfg(test)] +mod input_model_tests { + use super::*; + use crate::service::config::types::{ModelCapability, ModelCategory}; + + #[test] + fn native_images_do_not_require_a_fallback_model_but_text_models_do() { + let native = AIModelConfig { + id: "native".into(), + category: ModelCategory::Multimodal, + capabilities: vec![ModelCapability::ImageUnderstanding], + enabled: true, + ..Default::default() + }; + let text = AIModelConfig { + id: "text".into(), + category: ModelCategory::GeneralChat, + capabilities: vec![ModelCapability::TextChat], + enabled: true, + ..Default::default() + }; + let mut config = ServiceAIConfig::default(); + config.models = vec![native, text]; + assert!(validate_image_input_model(&config, "native", false).is_ok()); + assert!(validate_image_input_model(&config, "text", true) + .unwrap_err() + .to_string() + .contains("not configured")); + config.default_models.image_understanding = Some("native".into()); + assert!(validate_image_input_model(&config, "text", true).is_ok()); + assert!(validate_image_input_model(&config, "text", false).is_err()); + config.models[0].enabled = false; + assert!(validate_image_input_model(&config, "text", true) + .unwrap_err() + .to_string() + .contains("disabled")); + config.default_models.image_understanding = Some("text".into()); + assert!(validate_image_input_model(&config, "text", true) + .unwrap_err() + .to_string() + .contains("does not support")); + } +} + pub fn resolve_image_path(path: &str, workspace_path: Option<&Path>) -> OpenBitFunResult { let path_buf = PathBuf::from(path); @@ -141,6 +209,11 @@ pub fn decode_data_url(data_url: &str) -> OpenBitFunResult<(Vec, Option limits.max_width || orig_height > limits.max_height; - if !needs_resize && image_data.len() <= effective_max { + let provider_accepts_format = matches!( + guessed_format, + Some(ImageFormat::Png | ImageFormat::Jpeg | ImageFormat::Gif | ImageFormat::WebP) + ); + if !needs_resize && image_data.len() <= effective_max && provider_accepts_format { let mime_type = detect_mime_type_from_bytes(&image_data, fallback_mime)?; return Ok(ProcessedImage { data: image_data, @@ -264,6 +341,11 @@ pub fn optimize_image_with_size_limit( } } + if encoded.0.len() > effective_max { + return Err(OpenBitFunError::validation( + "Image cannot fit the provider upload limit", + )); + } Ok(ProcessedImage { data: encoded.0, mime_type: encoded.1, @@ -366,6 +448,29 @@ pub async fn process_image_contexts_for_provider( image_contexts: &[ImageContextData], provider: &str, workspace_path: Option<&Path>, +) -> OpenBitFunResult> { + process_image_contexts(image_contexts, provider, workspace_path, None).await +} + +pub(crate) async fn process_image_contexts_in_workspace( + image_contexts: &[ImageContextData], + provider: &str, + context: &ToolUseContext, +) -> OpenBitFunResult> { + process_image_contexts( + image_contexts, + provider, + context.workspace_root(), + Some(context), + ) + .await +} + +async fn process_image_contexts( + image_contexts: &[ImageContextData], + provider: &str, + workspace_path: Option<&Path>, + context: Option<&ToolUseContext>, ) -> OpenBitFunResult> { let limits = ImageLimits::for_provider(provider); @@ -384,8 +489,19 @@ pub async fn process_image_contexts_for_provider( let (data, data_url_mime) = decode_data_url(data_url)?; (data, data_url_mime.or_else(|| Some(ctx.mime_type.clone()))) } else if let Some(path_str) = &ctx.image_path { - let path = resolve_image_path(path_str, workspace_path)?; - let data = load_image_from_path(&path, workspace_path).await?; + let data = if let Some(context) = context { + let resolved = context.resolve_tool_path(path_str)?; + context + .file_system_for_path(&resolved)? + .read_file(&resolved.resolved_path) + .await + .map_err(|error| { + OpenBitFunError::io(format!("Failed to read image: {error}")) + })? + } else { + let path = resolve_image_path(path_str, workspace_path)?; + load_image_from_path(&path, workspace_path).await? + }; let detected_mime = detect_mime_type_from_bytes(&data, Some(&ctx.mime_type)).ok(); (data, detected_mime.or_else(|| Some(ctx.mime_type.clone()))) } else { @@ -603,3 +719,94 @@ mod resize_reporting_tests { assert!((processed.scale() - 1.0).abs() < f64::EPSILON); } } + +#[cfg(test)] +mod workspace_image_tests { + use super::*; + use crate::agentic::workspace::{ + WorkspaceBinding, WorkspaceCommandOptions, WorkspaceCommandResult, WorkspaceDirEntry, + WorkspaceFileSystem, WorkspaceServices, WorkspaceShell, + }; + use async_trait::async_trait; + use std::sync::Arc; + + struct RemoteFiles(Vec); + #[async_trait] + impl WorkspaceFileSystem for RemoteFiles { + async fn read_file(&self, _path: &str) -> anyhow::Result> { + Ok(self.0.clone()) + } + async fn read_file_text(&self, _path: &str) -> anyhow::Result { + anyhow::bail!("not text") + } + async fn write_file(&self, _path: &str, _contents: &[u8]) -> anyhow::Result<()> { + anyhow::bail!("read only") + } + async fn exists(&self, _path: &str) -> anyhow::Result { + Ok(true) + } + async fn is_file(&self, _path: &str) -> anyhow::Result { + Ok(true) + } + async fn is_dir(&self, _path: &str) -> anyhow::Result { + Ok(false) + } + async fn read_dir(&self, _path: &str) -> anyhow::Result> { + Ok(vec![]) + } + } + struct NoShell; + #[async_trait] + impl WorkspaceShell for NoShell { + async fn exec_with_options( + &self, + _command: &str, + _options: WorkspaceCommandOptions, + ) -> anyhow::Result { + anyhow::bail!("must not execute") + } + } + + #[tokio::test] + async fn image_path_reads_the_remote_host_even_when_the_same_path_exists_locally() { + let local = tempfile::tempdir().unwrap(); + let path = local.path().join("photo.png"); + std::fs::write(&path, b"local private content must never be read").unwrap(); + let identity = crate::service::remote_ssh::workspace_state::workspace_session_identity( + local.path().to_str().unwrap(), + Some("remote-1"), + Some("host"), + ) + .unwrap(); + let workspace = WorkspaceBinding::new_remote( + Some("remote-workspace".into()), + local.path().to_path_buf(), + "remote-1".into(), + "remote-session".into(), + identity, + ); + let mut image = crate::agentic::image_analysis::attachments::test_image(); + let bytes = decode_data_url(image.data_url.as_deref().unwrap()) + .unwrap() + .0; + image.data_url = None; + image.image_path = Some(path.to_str().unwrap().to_string()); + let context = ToolUseContext::for_tool_listing( + Some(workspace.clone()), + Some(WorkspaceServices { + fs: Arc::new(RemoteFiles(bytes)), + shell: Arc::new(NoShell), + }), + ); + let processed = process_image_contexts_in_workspace(&[image.clone()], "openai", &context) + .await + .unwrap(); + assert_eq!((processed[0].width, processed[0].height), (8, 6)); + let disconnected = ToolUseContext::for_tool_listing(Some(workspace), None); + assert!( + process_image_contexts_in_workspace(&[image], "openai", &disconnected) + .await + .is_err() + ); + } +} diff --git a/src/crates/assembly/core/src/agentic/image_analysis/mod.rs b/src/crates/assembly/core/src/agentic/image_analysis/mod.rs index aef9a8e435..1ec37bde65 100644 --- a/src/crates/assembly/core/src/agentic/image_analysis/mod.rs +++ b/src/crates/assembly/core/src/agentic/image_analysis/mod.rs @@ -2,6 +2,7 @@ //! //! Implements image pre-understanding functionality, converting image content to text descriptions +pub(crate) mod attachments; pub mod enhancer; pub mod image_processing; pub mod processor; diff --git a/src/crates/assembly/core/src/agentic/persistence/manager.rs b/src/crates/assembly/core/src/agentic/persistence/manager.rs index 6637435fb5..938280ed02 100644 --- a/src/crates/assembly/core/src/agentic/persistence/manager.rs +++ b/src/crates/assembly/core/src/agentic/persistence/manager.rs @@ -7740,3 +7740,71 @@ mod tests { .is_none()); } } + +#[cfg(test)] +mod image_persistence_tests { + use super::*; + use crate::agentic::image_analysis::{attachments, process_image_contexts_for_provider}; + use crate::agentic::tools::framework::ToolUseContext; + use crate::agentic::workspace::WorkspaceBinding; + + #[tokio::test] + async fn persisted_image_messages_reopen_after_inline_pixels_are_redacted() { + let dir = tempfile::tempdir().unwrap(); + let mut context = ToolUseContext::for_tool_listing( + Some(WorkspaceBinding::new( + Some("workspace".into()), + dir.path().to_path_buf(), + )), + None, + ); + context.custom_data.insert( + "__openbitfun_test_runtime_root".into(), + serde_json::json!(dir.path()), + ); + let mut images = vec![attachments::test_image()]; + attachments::prepare_inline_image_attachments(&mut images, &context) + .await + .unwrap(); + let original = Message::user_multimodal("Read the screenshot".into(), images); + let persisted = PersistenceManager::sanitize_message_for_persistence(&original); + let json = serde_json::to_value(persisted.as_ref()).unwrap(); + assert!(!json.to_string().contains("data:image/")); + let restored: Message = serde_json::from_value(json).unwrap(); + let MessageContent::Multimodal { images, .. } = restored.content else { + panic!("image message") + }; + assert!(images[0].data_url.is_none()); + assert!(images[0].metadata.as_ref().unwrap()["has_data_url"] + .as_bool() + .unwrap()); + let processed = process_image_contexts_for_provider(&images, "openai", None) + .await + .unwrap(); + assert_eq!((processed[0].width, processed[0].height), (8, 6)); + let MessageContent::Multimodal { + images: live_images, + .. + } = &original.content + else { + panic!("live image") + }; + assert!(live_images[0].data_url.is_some()); + + // A pre-upgrade image record has no new required fields or migration marker. + let legacy: crate::agentic::image_analysis::ImageContextData = + serde_json::from_value(serde_json::json!({ + "id": "old-image", "image_path": images[0].image_path, + "data_url": null, "mime_type": "image/png", "metadata": {"has_data_url": true} + })) + .unwrap(); + let round_trip = serde_json::from_value(serde_json::to_value(legacy).unwrap()).unwrap(); + assert_eq!( + process_image_contexts_for_provider(&[round_trip], "anthropic", None) + .await + .unwrap() + .len(), + 1 + ); + } +} diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/analyze_image_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/analyze_image_tool.rs index 09f2849991..efb940a394 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/analyze_image_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/analyze_image_tool.rs @@ -65,7 +65,7 @@ impl AnalyzeImageTool { let local_path = Path::new(input_path); if local_path.is_absolute() && !crate::agentic::tools::workspace_paths::is_openbitfun_tool_uri(input_path) - && (!context.is_some_and(|ctx| ctx.is_remote()) || local_path.is_file()) + && !context.is_some_and(|ctx| ctx.is_remote()) { return Ok(ResolvedImagePath::Local(local_path.to_path_buf())); } @@ -480,3 +480,89 @@ Divide by that factor to map anything in `analysis` back to source pixels — bu )]) } } + +#[cfg(test)] +mod image_path_tests { + use super::*; + use crate::agentic::workspace::WorkspaceBinding; + + #[tokio::test] + async fn analyze_image_reads_uploaded_runtime_pixels_in_an_ssh_session() { + let root = tempfile::tempdir().unwrap(); + let identity = crate::service::remote_ssh::workspace_state::workspace_session_identity( + "/ssh/workspace", + Some("remote-1"), + Some("host"), + ) + .unwrap(); + let mut context = ToolUseContext::for_tool_listing( + Some(WorkspaceBinding::new_remote( + Some("remote-workspace".into()), + PathBuf::from("/ssh/workspace"), + "remote-1".into(), + "remote-session".into(), + identity, + )), + None, + ); + context + .custom_data + .insert("__openbitfun_test_runtime_root".into(), json!(root.path())); + let mut images = vec![crate::agentic::image_analysis::attachments::test_image()]; + let expected = + crate::agentic::image_analysis::decode_data_url(images[0].data_url.as_deref().unwrap()) + .unwrap() + .0; + crate::agentic::image_analysis::attachments::prepare_inline_image_attachments( + &mut images, + &context, + ) + .await + .unwrap(); + let reference = images[0].image_path.as_deref().unwrap(); + assert!(reference.starts_with("openbitfun://runtime/remote-workspace/")); + let resolved = AnalyzeImageTool::resolve_path(reference, Some(&context)).unwrap(); + assert!(matches!(resolved, ResolvedImagePath::Local(_))); + assert_eq!( + AnalyzeImageTool::read_image_bytes(&resolved, Some(&context)) + .await + .unwrap(), + expected + ); + } + + #[tokio::test] + async fn analyze_image_never_falls_back_to_an_existing_controller_file() { + let local = tempfile::tempdir().unwrap(); + let path = local.path().join("private.png"); + std::fs::write(&path, b"private controller content").unwrap(); + let identity = crate::service::remote_ssh::workspace_state::workspace_session_identity( + local.path().to_str().unwrap(), + Some("remote-1"), + Some("host"), + ) + .unwrap(); + let context = ToolUseContext::for_tool_listing( + Some(WorkspaceBinding::new_remote( + Some("remote-workspace".into()), + local.path().to_path_buf(), + "remote-1".into(), + "remote-session".into(), + identity, + )), + None, + ); + let resolved = + AnalyzeImageTool::resolve_path(path.to_str().unwrap(), Some(&context)).unwrap(); + assert!(matches!( + resolved, + ResolvedImagePath::RemoteWorkspace { .. } + )); + let error = AnalyzeImageTool::read_image_bytes(&resolved, Some(&context)) + .await + .unwrap_err(); + assert!(error + .to_string() + .contains("workspace filesystem services are unavailable")); + } +} diff --git a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json index 864000a584..c69940ecca 100644 --- a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json +++ b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "digest": "fnv1a64:4347c91266c12da9", + "digest": "fnv1a64:57f26e05119c5457", "retiredCommandPrefixes": [ { "prefix": "lsp_", @@ -10,6 +10,7 @@ "capabilities": { "ids": [ "idempotent_dialog_submit", + "inline_image_attachments_v1", "targeted_session_rollback", "token_usage_statistics", "miniapp_agent_context_files_v1", @@ -23,6 +24,7 @@ ], "desktop": [ "idempotent_dialog_submit", + "inline_image_attachments_v1", "targeted_session_rollback", "token_usage_statistics", "miniapp_agent_context_files_v1", @@ -36,6 +38,7 @@ ], "cli": [ "idempotent_dialog_submit", + "inline_image_attachments_v1", "targeted_session_rollback", "token_usage_statistics", "product_control_v1", diff --git a/src/crates/contracts/product-domains/src/remote_surface/capabilities.rs b/src/crates/contracts/product-domains/src/remote_surface/capabilities.rs index 74aa2f0d33..c48279cd7f 100644 --- a/src/crates/contracts/product-domains/src/remote_surface/capabilities.rs +++ b/src/crates/contracts/product-domains/src/remote_surface/capabilities.rs @@ -25,6 +25,8 @@ pub enum PeerHostCapability { /// `start_dialog_turn` / `start_acp_dialog_turn` may be retried with the /// same `(sessionId, turnId)`; the host coalesces duplicate attempts. IdempotentDialogSubmit, + /// Inline imageContexts are durably prepared by the receiving Runtime. + InlineImageAttachmentsV1, /// Identity-based `rollback_session_to_turn` is implemented. TargetedSessionRollback, /// `get_token_usage_statistics` is implemented. @@ -51,6 +53,7 @@ impl PeerHostCapability { /// Every capability id, in wire order. pub const ALL: &'static [PeerHostCapability] = &[ Self::IdempotentDialogSubmit, + Self::InlineImageAttachmentsV1, Self::TargetedSessionRollback, Self::TokenUsageStatistics, Self::MiniappAgentContextFilesV1, @@ -67,6 +70,7 @@ impl PeerHostCapability { pub const fn key(self) -> &'static str { match self { Self::IdempotentDialogSubmit => "idempotent_dialog_submit", + Self::InlineImageAttachmentsV1 => "inline_image_attachments_v1", Self::TargetedSessionRollback => "targeted_session_rollback", Self::TokenUsageStatistics => "token_usage_statistics", Self::MiniappAgentContextFilesV1 => "miniapp_agent_context_files_v1", @@ -92,6 +96,7 @@ const DESKTOP_CAPABILITIES: &[PeerHostCapability] = PeerHostCapability::ALL; /// ProductControl providers, or presentation surface. const CLI_CAPABILITIES: &[PeerHostCapability] = &[ PeerHostCapability::IdempotentDialogSubmit, + PeerHostCapability::InlineImageAttachmentsV1, PeerHostCapability::TargetedSessionRollback, PeerHostCapability::TokenUsageStatistics, PeerHostCapability::ProductControlV1, diff --git a/src/crates/services/services-core/src/json_store.rs b/src/crates/services/services-core/src/json_store.rs index 2cacc4b385..1c71536225 100644 --- a/src/crates/services/services-core/src/json_store.rs +++ b/src/crates/services/services-core/src/json_store.rs @@ -265,6 +265,17 @@ impl JsonFileStore { .await } + /// Publish binary runtime artifacts with the same strict atomic replacement + /// used by text artifacts. Readers must never observe a partial attachment. + pub async fn write_bytes_atomic_strict( + &self, + path: &Path, + bytes: Vec, + ) -> Result<(), JsonFileStoreError> { + self.write_bytes_atomic_with_policy(path, bytes, AtomicWritePolicy::StrictReplace) + .await + } + /// Atomically publish a new UTF-8 text file, failing without changing the /// target when another process creates it first. pub async fn write_text_atomic_create_new( diff --git a/src/mobile-web/AGENTS.md b/src/mobile-web/AGENTS.md index 6d856f1bb2..7fa9baf9a1 100644 --- a/src/mobile-web/AGENTS.md +++ b/src/mobile-web/AGENTS.md @@ -50,6 +50,7 @@ Run the focused mobile-web checks after changes: ```bash pnpm --dir src/mobile-web run test:ui-components pnpm --dir src/mobile-web run test:account-login # account login without an online desktop +pnpm --dir src/mobile-web run test:images # image preparation and upload limits pnpm --dir src/mobile-web run test:workspace-identity # SSH host scope and legacy cache records pnpm --dir src/mobile-web run type-check pnpm run build:mobile-web diff --git a/src/mobile-web/package.json b/src/mobile-web/package.json index f3410146d1..86b3fb12fb 100644 --- a/src/mobile-web/package.json +++ b/src/mobile-web/package.json @@ -11,6 +11,7 @@ "dev": "vite", "test:ui-components": "node --test tests/mobile-ui-components.test.mjs", "test:account-login": "node --test tests/account-login.test.mjs", + "test:images": "node --test tests/image-compressor.test.mjs", "test:workspace-identity": "node --test tests/workspace-identity.test.mjs", "type-check": "tsc --noEmit", "build": "vite build", diff --git a/src/mobile-web/src/i18n/messages.ts b/src/mobile-web/src/i18n/messages.ts index 53cdc1ff76..6f45d4da0d 100644 --- a/src/mobile-web/src/i18n/messages.ts +++ b/src/mobile-web/src/i18n/messages.ts @@ -193,13 +193,14 @@ export const messages: Record = { loadingOlderMessages: 'Loading older messages...', showResponse: 'Show response', hideResponse: 'Hide response', - analyzingImage: 'Analyzing image with image understanding model...', + analyzingImage: 'Sending image attachments...', inputPlaceholder: 'How can I help you...', collapsedInputPlaceholder: 'Ask OpenBitFun', collapsedStreamingPlaceholder: 'Replying...', messageQueued: 'Message queued; it will run after the current step', - imageAnalyzingPlaceholder: 'Analyzing image...', + imageAnalyzingPlaceholder: 'Sending images...', imageAttachmentFallback: '(see attached images)', + imagePreparationFailed: 'This image could not be prepared. Choose a PNG, JPEG, GIF or WebP image.', modelSelection: 'Select model', modelAuto: 'Auto', modelAutoDesc: 'Automatically route to the best model', @@ -465,13 +466,14 @@ export const messages: Record = { loadingOlderMessages: '正在加载更早的消息...', showResponse: 'Show response', hideResponse: 'Hide response', - analyzingImage: '正在使用图像理解模型分析图片...', + analyzingImage: '正在发送图片附件...', inputPlaceholder: '有什么可以帮您...', collapsedInputPlaceholder: '向 OpenBitFun 提问', collapsedStreamingPlaceholder: '正在回复...', messageQueued: '已加入队列,将在当前步骤结束后处理', - imageAnalyzingPlaceholder: '正在分析图片...', + imageAnalyzingPlaceholder: '正在发送图片...', imageAttachmentFallback: '(见附带图片)', + imagePreparationFailed: '无法处理这张图片,请选择 PNG、JPEG、GIF 或 WebP 图片。', modelSelection: '选择模型', modelAuto: 'Auto', modelAutoDesc: '自动路由到最合适的模型', @@ -742,8 +744,9 @@ export const messages: Record = { collapsedInputPlaceholder: '向 OpenBitFun 提問', collapsedStreamingPlaceholder: '正在回覆...', messageQueued: '已加入隊列,將在當前步驟結束後處理', - imageAnalyzingPlaceholder: '正在分析圖片...', + imageAnalyzingPlaceholder: '正在傳送圖片...', imageAttachmentFallback: '(見附帶圖片)', + imagePreparationFailed: '無法處理這張圖片,請選擇 PNG、JPEG、GIF 或 WebP 圖片。', modelSelection: '選擇模型', modelAuto: 'Auto', modelAutoDesc: '自動路由到最合適的模型', diff --git a/src/mobile-web/src/pages/ChatPage.tsx b/src/mobile-web/src/pages/ChatPage.tsx index 629c0ea053..9cd26cebd5 100644 --- a/src/mobile-web/src/pages/ChatPage.tsx +++ b/src/mobile-web/src/pages/ChatPage.tsx @@ -913,34 +913,24 @@ const ChatPage: React.FC = ({ }, []); const handleFileChange = useCallback(async (e: React.ChangeEvent) => { - const files = e.target.files; - if (!files) return; + const files = Array.from(e.target.files ?? []); + e.target.value = ''; + const targetEpoch = captureChatTargetEpoch(); + if (targetEpoch === null) return; const maxImages = 5; const remaining = maxImages - pendingImages.length; - const toProcess = Array.from(files).slice(0, remaining); - const { compressImageFile } = await import('../services/imageCompressor'); - for (const file of toProcess) { + for (const file of files.slice(0, remaining)) { try { const compressed = await compressImageFile(file); - setPendingImages((prev) => { - if (prev.length >= maxImages) return prev; - return [...prev, { name: compressed.name, dataUrl: compressed.dataUrl }]; - }); + if (!isChatTargetCurrent(targetEpoch)) return; + setPendingImages(prev => prev.length >= maxImages ? prev : [...prev, compressed]); } catch { - const reader = new FileReader(); - reader.onload = () => { - const dataUrl = reader.result as string; - setPendingImages((prev) => { - if (prev.length >= maxImages) return prev; - return [...prev, { name: file.name, dataUrl }]; - }); - }; - reader.readAsDataURL(file); + if (!isChatTargetCurrent(targetEpoch)) return; + setError(t('chat.imagePreparationFailed')); } } - e.target.value = ''; - }, [pendingImages.length]); + }, [captureChatTargetEpoch, isChatTargetCurrent, pendingImages.length, setError, t]); const removeImage = useCallback((idx: number) => { setPendingImages((prev) => prev.filter((_, i) => i !== idx)); diff --git a/src/mobile-web/src/services/imageCompressor.ts b/src/mobile-web/src/services/imageCompressor.ts index ce07eb506d..9a9231d7b3 100644 --- a/src/mobile-web/src/services/imageCompressor.ts +++ b/src/mobile-web/src/services/imageCompressor.ts @@ -25,11 +25,14 @@ export async function compressImageDataUrl( dataUrl: string, fileName: string, ): Promise<{ dataUrl: string; name: string }> { - if (dataUrlByteSize(dataUrl) <= MAX_OUTPUT_BYTES) { + // Decode even small inputs: HEIC/SVG and corrupt bytes must not pass through + // as model-ready images just because their payload is short. + const img = await loadImage(dataUrl); + if (/^data:image\/(png|jpeg|gif|webp);base64,/.test(dataUrl) + && dataUrlByteSize(dataUrl) <= MAX_OUTPUT_BYTES + && Math.max(img.width, img.height) <= MAX_DIMENSION) { return { dataUrl, name: fileName }; } - - const img = await loadImage(dataUrl); let { width, height } = img; if (width > MAX_DIMENSION || height > MAX_DIMENSION) { @@ -41,7 +44,8 @@ export async function compressImageDataUrl( const canvas = document.createElement('canvas'); canvas.width = width; canvas.height = height; - const ctx = canvas.getContext('2d')!; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Image conversion is unavailable'); ctx.drawImage(img, 0, 0, width, height); let quality = JPEG_QUALITY; @@ -52,10 +56,12 @@ export async function compressImageDataUrl( result = canvas.toDataURL('image/jpeg', quality); } - if (dataUrlByteSize(result) > MAX_OUTPUT_BYTES) { - const scale = 0.75; - canvas.width = Math.round(width * scale); - canvas.height = Math.round(height * scale); + while (dataUrlByteSize(result) > MAX_OUTPUT_BYTES) { + if (canvas.width <= 64 || canvas.height <= 64) { + throw new Error('Image exceeds the upload size limit'); + } + canvas.width = Math.max(1, Math.floor(canvas.width * 0.75)); + canvas.height = Math.max(1, Math.floor(canvas.height * 0.75)); ctx.drawImage(img, 0, 0, canvas.width, canvas.height); result = canvas.toDataURL('image/jpeg', 0.6); } diff --git a/src/mobile-web/tests/image-compressor.test.mjs b/src/mobile-web/tests/image-compressor.test.mjs new file mode 100644 index 0000000000..4826a12349 --- /dev/null +++ b/src/mobile-web/tests/image-compressor.test.mjs @@ -0,0 +1,58 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import test from 'node:test'; +import ts from 'typescript'; + +const source = await readFile(new URL('../src/services/imageCompressor.ts', import.meta.url), 'utf8'); +const code = ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2020 }, +}).outputText; +const { compressImageDataUrl } = await import(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); + +function browserImages({ width = 100, height = 60, invalid = false, encode } = {}) { + globalThis.Image = class { + width = width; + height = height; + set src(_) { queueMicrotask(() => invalid ? this.onerror(new Error('decode failed')) : this.onload()); } + }; + let draws = 0; + globalThis.document = { + createElement: () => ({ width: 0, height: 0, + getContext: () => ({ drawImage: () => { draws += 1; } }), + toDataURL(type, quality) { return encode?.(this.width, this.height, quality) ?? `data:${type};base64,YQ==`; }, + }), + }; + return () => draws; +} + +test('small supported screenshots retain original pixels', async () => { + const draws = browserImages(); + const dataUrl = 'data:image/png;base64,YQ=='; + assert.deepEqual(await compressImageDataUrl(dataUrl, 'screen.png'), { dataUrl, name: 'screen.png' }); + assert.equal(draws(), 0); +}); + +test('small camera formats are decoded and converted instead of passed to providers', async () => { + const draws = browserImages(); + const result = await compressImageDataUrl('data:image/heic;base64,YQ==', 'camera.heic'); + assert.equal(result.name, 'camera.jpg'); + assert.match(result.dataUrl, /^data:image\/jpeg;base64,/); + assert.equal(draws(), 1); +}); + +test('large dimensions and incompressible payloads keep shrinking until the wire budget fits', async () => { + const sizes = []; + browserImages({ width: 8000, height: 6000, encode(width, height) { + sizes.push([width, height]); + return 'data:image/jpeg;base64,' + (width > 1000 ? 'a'.repeat(2 * 1024 * 1024) : 'YQ=='); + } }); + const result = await compressImageDataUrl('data:image/png;base64,YQ==', 'screen.png'); + assert.equal(sizes[0][0], 1920); + assert.ok(sizes.at(-1)[0] <= 1000); + assert.ok(result.dataUrl.length < 1024 * 1024); +}); + +test('unreadable photos fail before upload instead of using raw bytes as a fallback', async () => { + browserImages({ invalid: true }); + await assert.rejects(compressImageDataUrl('data:image/heic;base64,YQ==', 'camera.heic')); +}); diff --git a/src/web-ui/src/flow_chat/hooks/useMessageSender.ts b/src/web-ui/src/flow_chat/hooks/useMessageSender.ts index 60a4135d5d..21ecb71dac 100644 --- a/src/web-ui/src/flow_chat/hooks/useMessageSender.ts +++ b/src/web-ui/src/flow_chat/hooks/useMessageSender.ts @@ -4,7 +4,7 @@ * * Image handling is fully delegated to the backend coordinator which * decides whether to pre-analyse via a vision model or attach images - * directly. The frontend only uploads clipboard images and passes + * directly. The frontend prepares compatible attachment payloads and passes * ImageContextData[] through to the backend. */ @@ -222,7 +222,7 @@ export function useMessageSender(props: UseMessageSenderProps): UseMessageSender : aiTrimmedMessage; } // Always pass imageContexts to the backend; the coordinator decides - // whether to pre-analyse via a vision model or attach directly. + // whether to expose a path to analyze_image or attach pixels directly. await flowChatManager.sendMessage( fullMessage, sessionId || undefined, diff --git a/src/web-ui/src/flow_chat/utils/imagePayload.test.ts b/src/web-ui/src/flow_chat/utils/imagePayload.test.ts new file mode 100644 index 0000000000..634d265ca1 --- /dev/null +++ b/src/web-ui/src/flow_chat/utils/imagePayload.test.ts @@ -0,0 +1,67 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { ImageContext } from '@/shared/types/context'; +import { buildImagePayload } from './imagePayload'; + +const mocks = vi.hoisted(() => ({ + surfaceId: 'local', + advertised: false, + invoke: vi.fn(), + assertCurrent: vi.fn(), +})); +vi.mock('@/infrastructure/api/service-api/ApiClient', () => ({ api: { invoke: mocks.invoke } })); +vi.mock('@/infrastructure/peer-device/deviceSurface', () => ({ + getActiveSurfaceScope: () => ({ surfaceId: mocks.surfaceId, assertCurrent: mocks.assertCurrent }), + isLocalSurface: (id: string) => id === 'local', +})); +vi.mock('@/infrastructure/peer-device/PeerConnectionManager', () => ({ + peerConnectionManager: { get: () => ({ getState: () => ({ capabilities: { inlineImageAttachmentsV1: mocks.advertised } }) }) }, +})); +const clipboard: ImageContext = { + id: 'clipboard-1', type: 'image', timestamp: 1, imagePath: '/controller/stale.png', + imageName: 'screen.png', fileSize: 3, mimeType: 'image/png', source: 'clipboard', + isLocal: false, dataUrl: 'data:image/png;base64,YQ==', +}; + +beforeEach(() => { + mocks.surfaceId = 'local'; + mocks.advertised = false; + mocks.invoke.mockReset(); + mocks.assertCurrent.mockReset(); +}); + +describe('image payload host ownership', () => { + it.each(['local', 'peer-cli'])('retains pixels for the receiving %s Runtime without a controller path', async surface => { + mocks.surfaceId = surface; + mocks.advertised = surface !== 'local'; + const result = await buildImagePayload([clipboard]); + expect(mocks.invoke).not.toHaveBeenCalled(); + expect(result?.imageContexts[0].data_url).toBe(clipboard.dataUrl); + expect(result?.imageContexts[0].image_path).toBeUndefined(); + expect(result?.imageDisplayData[0].dataUrl).toBe(clipboard.dataUrl); + }); + + it('preserves the old host upload path when the new capability was not advertised', async () => { + mocks.surfaceId = 'old-peer'; + mocks.invoke.mockResolvedValue([{ id: clipboard.id, image_path: '/peer/upload.png' }]); + const result = await buildImagePayload([clipboard]); + expect(mocks.invoke).toHaveBeenCalledWith('upload_image_contexts', expect.any(Object)); + expect(result?.imageContexts[0].image_path).toBe('/peer/upload.png'); + expect(result?.imageContexts[0].data_url).toBe(clipboard.dataUrl); + expect(mocks.assertCurrent).toHaveBeenCalled(); + }); + + it('does not submit across a surface switch or hide a rejected legacy upload', async () => { + mocks.surfaceId = 'old-peer'; + mocks.invoke.mockRejectedValueOnce(new Error('unavailable')); + await expect(buildImagePayload([clipboard])).rejects.toThrow('unavailable'); + mocks.invoke.mockResolvedValue([{ id: clipboard.id, image_path: '/peer/upload.png' }]); + mocks.assertCurrent.mockImplementation(() => { throw new Error('surface changed'); }); + await expect(buildImagePayload([clipboard])).rejects.toThrow('surface changed'); + }); + + it('retains remote workspace paths when no inline pixels exist', async () => { + const result = await buildImagePayload([{ ...clipboard, isLocal: true, dataUrl: undefined, imagePath: '/remote/workspace/chart.png' }]); + expect(result?.imageContexts[0].image_path).toBe('/remote/workspace/chart.png'); + expect(result?.imageContexts[0].data_url).toBeUndefined(); + }); +}); diff --git a/src/web-ui/src/flow_chat/utils/imagePayload.ts b/src/web-ui/src/flow_chat/utils/imagePayload.ts index af272affa1..3b85da52f4 100644 --- a/src/web-ui/src/flow_chat/utils/imagePayload.ts +++ b/src/web-ui/src/flow_chat/utils/imagePayload.ts @@ -1,6 +1,8 @@ import type { ImageContext } from '@/shared/types/context'; import type { ImageContextData as ImageInputContextData } from '@/infrastructure/api/service-api/ImageContextTypes'; import { api } from '@/infrastructure/api/service-api/ApiClient'; +import { getActiveSurfaceScope, isLocalSurface } from '@/infrastructure/peer-device/deviceSurface'; +import { peerConnectionManager } from '@/infrastructure/peer-device/PeerConnectionManager'; export interface ImageDisplayData { id: string; @@ -20,10 +22,13 @@ export async function buildImagePayload(imageContexts: ImageContext[]): Promise< return undefined; } + const scope = getActiveSurfaceScope(); + const acceptsInline = isLocalSurface(scope.surfaceId) + || peerConnectionManager.get(scope.surfaceId)?.getState().capabilities.inlineImageAttachmentsV1 === true; const clipboardImages = imageContexts.filter(ctx => !ctx.isLocal && ctx.dataUrl); const uploadedImagePaths = new Map(); - if (clipboardImages.length > 0) { + if (!acceptsInline && clipboardImages.length > 0) { const uploadResults = await api.invoke>( 'upload_image_contexts', { @@ -43,6 +48,7 @@ export async function buildImagePayload(imageContexts: ImageContext[]): Promise< } ); + scope.assertCurrent('prepare image attachments'); for (const result of uploadResults) { if (result.image_path) { uploadedImagePaths.set(result.id, result.image_path); @@ -54,7 +60,9 @@ export async function buildImagePayload(imageContexts: ImageContext[]): Promise< imageContexts: imageContexts.map(ctx => ({ id: ctx.id, image_path: ctx.isLocal ? ctx.imagePath : uploadedImagePaths.get(ctx.id), - data_url: undefined, + // Retain pixels for durable host storage and Detached Dispatch. Older + // peer hosts still receive their upload path through the legacy branch. + data_url: ctx.dataUrl || undefined, mime_type: ctx.mimeType, metadata: { name: ctx.imageName, diff --git a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts index cc2017bb32..313b39daf4 100644 --- a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts +++ b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts @@ -1,6 +1,6 @@ // Generated by scripts/generate-interactive-capabilities.mjs; do not edit. // Source: openbitfun_product_domains::remote_surface (Product Operation Registry). -export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:4347c91266c12da9" as const; +export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:57f26e05119c5457" as const; /** * Registered Tauri commands the Peer Device controller keeps on the controller @@ -153,6 +153,7 @@ export const PEER_CONTROLLER_LOCAL_COMMANDS: ReadonlySet = new Set([ /** Every capability id a peer host may advertise in `peer_mode_ping`. */ export const PEER_HOST_CAPABILITY_IDS = [ "idempotent_dialog_submit", + "inline_image_attachments_v1", "targeted_session_rollback", "token_usage_statistics", "miniapp_agent_context_files_v1", @@ -173,6 +174,7 @@ export const PEER_HOST_ADVERTISED_CAPABILITIES: Readonly< > = { desktop: [ "idempotent_dialog_submit", + "inline_image_attachments_v1", "targeted_session_rollback", "token_usage_statistics", "miniapp_agent_context_files_v1", @@ -186,6 +188,7 @@ export const PEER_HOST_ADVERTISED_CAPABILITIES: Readonly< ], cli: [ "idempotent_dialog_submit", + "inline_image_attachments_v1", "targeted_session_rollback", "token_usage_statistics", "product_control_v1", diff --git a/src/web-ui/src/infrastructure/peer-device/PeerConnectionManager.test.ts b/src/web-ui/src/infrastructure/peer-device/PeerConnectionManager.test.ts index 792272e66c..f70dfcddb5 100644 --- a/src/web-ui/src/infrastructure/peer-device/PeerConnectionManager.test.ts +++ b/src/web-ui/src/infrastructure/peer-device/PeerConnectionManager.test.ts @@ -26,6 +26,7 @@ describe('PeerConnectionManager attach', () => { health: 'ready', capabilities: { idempotentDialogSubmit: true, + inlineImageAttachmentsV1: true, targetedSessionRollback: false, tokenUsageStatistics: true, miniAppAgentContextFilesV1: true, @@ -125,6 +126,7 @@ describe('PeerConnectionManager attach', () => { const desktop = await makeManager('desktop').connect('peer-1', 'Studio'); expect(desktop.getState().capabilities.hostKind).toBe('desktop'); + expect(desktop.getState().capabilities.inlineImageAttachmentsV1).toBe(false); const cli = await makeManager('cli').connect('peer-2', 'Studio'); expect(cli.getState().capabilities.hostKind).toBe('cli'); @@ -601,6 +603,7 @@ function createRpc(options: { failCommands?: Set } = {}) { host_type: 'desktop', capabilities: { idempotent_dialog_submit: true, + inline_image_attachments_v1: true, token_usage_statistics: true, miniapp_agent_context_files_v1: true, wsl_workspaces_v1: true, diff --git a/src/web-ui/src/infrastructure/peer-device/PeerConnectionManager.ts b/src/web-ui/src/infrastructure/peer-device/PeerConnectionManager.ts index a43da39712..045f7affbc 100644 --- a/src/web-ui/src/infrastructure/peer-device/PeerConnectionManager.ts +++ b/src/web-ui/src/infrastructure/peer-device/PeerConnectionManager.ts @@ -40,6 +40,8 @@ export type PeerHostKind = 'desktop' | 'cli'; export interface PeerHostCapabilities { readonly idempotentDialogSubmit: boolean; + /** Only true after explicit negotiation; older hosts need the upload API. */ + readonly inlineImageAttachmentsV1?: boolean; readonly targetedSessionRollback: boolean; readonly tokenUsageStatistics: boolean; /** MiniApp Agent runs accept immutable virtual context-file snapshots. */ @@ -156,6 +158,7 @@ interface PeerModePingResult { const NO_CAPABILITIES: PeerHostCapabilities = { idempotentDialogSubmit: false, + inlineImageAttachmentsV1: false, targetedSessionRollback: false, tokenUsageStatistics: false, miniAppAgentContextFilesV1: false, @@ -484,6 +487,7 @@ export class PeerConnectionManager { return { idempotentDialogSubmit: caps?.idempotent_dialog_submit === true, + inlineImageAttachmentsV1: caps?.inline_image_attachments_v1 === true, targetedSessionRollback: caps?.targeted_session_rollback === true, tokenUsageStatistics: caps?.token_usage_statistics === true, miniAppAgentContextFilesV1: caps?.miniapp_agent_context_files_v1 === true, @@ -713,6 +717,7 @@ function capabilitiesEqual( b: PeerHostCapabilities, ): boolean { return a.idempotentDialogSubmit === b.idempotentDialogSubmit && + a.inlineImageAttachmentsV1 === b.inlineImageAttachmentsV1 && a.targetedSessionRollback === b.targetedSessionRollback && a.tokenUsageStatistics === b.tokenUsageStatistics && a.miniAppAgentContextFilesV1 === b.miniAppAgentContextFilesV1 && From 044f81ab2f9f483167b3c3c578d2cb8b98b942c5 Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Thu, 10 Sep 2026 15:49:58 +0800 Subject: [PATCH 2/2] fix(images): recover inline pixels over stale historical paths --- .../core/src/agentic/execution/execution_engine.rs | 10 ++++------ src/web-ui/src/flow_chat/hooks/useMessageSender.ts | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/crates/assembly/core/src/agentic/execution/execution_engine.rs b/src/crates/assembly/core/src/agentic/execution/execution_engine.rs index f2ff99ee6d..36ec85425f 100644 --- a/src/crates/assembly/core/src/agentic/execution/execution_engine.rs +++ b/src/crates/assembly/core/src/agentic/execution/execution_engine.rs @@ -3874,14 +3874,12 @@ impl ExecutionEngine { context.workspace_services.clone(), ); // Older turn metadata may still hold inline pixels even when its context - // snapshot predates durable attachments. Recover those pixels without - // changing the persisted record shape or requiring manual migration. + // snapshot predates durable attachments. Inline pixels also supersede + // old temporary/controller paths, which may no longer exist. Recover + // them without changing the record shape or requiring manual migration. for message in &mut messages { if let MessageContent::Multimodal { images, .. } = &mut message.content { - if !images - .iter() - .any(|image| image.data_url.is_some() && image.image_path.is_none()) - { + if !images.iter().any(|image| image.data_url.is_some()) { continue; } if let Err(error) = diff --git a/src/web-ui/src/flow_chat/hooks/useMessageSender.ts b/src/web-ui/src/flow_chat/hooks/useMessageSender.ts index 21ecb71dac..540691fb7b 100644 --- a/src/web-ui/src/flow_chat/hooks/useMessageSender.ts +++ b/src/web-ui/src/flow_chat/hooks/useMessageSender.ts @@ -3,8 +3,8 @@ * Encapsulates session creation, image uploads, and message assembly. * * Image handling is fully delegated to the backend coordinator which - * decides whether to pre-analyse via a vision model or attach images - * directly. The frontend prepares compatible attachment payloads and passes + * exposes a path to the image analysis tool or attaches pixels + * directly for a multimodal model. The frontend prepares compatible attachment payloads and passes * ImageContextData[] through to the backend. */