From 09710bd9e2921dcfa3563cbe18847ef8faeb24dc Mon Sep 17 00:00:00 2001 From: EthanCheung Date: Fri, 18 Sep 2026 23:14:15 +0800 Subject: [PATCH 1/4] fix(web-ui): mount the image picker so WebKitGTK delivers change The composer's Add-image flow created a detached and clicked it. On WebKitGTK (Linux) the native chooser opens, the user picks a file, and the engine never fires change on the detached input, so every selection was silently dropped - no thumbnail, no log, no error. WebView2 and WKWebView deliver change either way, which is why only Linux users hit this. Mount the picker offscreen before clicking it (display:none is avoided on purpose: some WebKit builds refuse to open a chooser for it), and reclaim the node on both the change path and the cancel path via a one-shot window focus listener. Verified on Linux/WebKitGTK 2.52 with an isolated desktop instance: picker selection now lands in the composer. --- .../src/flow_chat/components/ChatInput.tsx | 31 +++++++++++-- .../components/ChatInputImageIntake.test.ts | 43 +++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts diff --git a/src/web-ui/src/flow_chat/components/ChatInput.tsx b/src/web-ui/src/flow_chat/components/ChatInput.tsx index 7bc052c411..c39eea69be 100644 --- a/src/web-ui/src/flow_chat/components/ChatInput.tsx +++ b/src/web-ui/src/flow_chat/components/ChatInput.tsx @@ -5719,16 +5719,38 @@ export const ChatInput: React.FC = ({ input.type = 'file'; input.accept = CHAT_INPUT_CONFIG.image.acceptedTypes.join(','); input.multiple = true; - + + // WebKitGTK never fires `change` on a detached file input after the native + // chooser closes, so a detached picker silently dropped every selection on + // Linux. Mounting the element offscreen keeps WebKitGTK on the same path as + // WebView2 and WKWebView. `display: none` is deliberately avoided because + // some WebKit builds refuse to open a chooser for a display:none input. + input.style.position = 'fixed'; + input.style.left = '-9999px'; + input.style.top = '0'; + input.style.width = '1px'; + input.style.height = '1px'; + input.style.opacity = '0'; + + const dismissPicker = () => { + window.removeEventListener('focus', dismissPicker); + input.onchange = null; + input.remove(); + }; + // Cancelling the chooser never fires `change`; reclaim the node the next + // time the window regains focus. + window.addEventListener('focus', dismissPicker); + input.onchange = async (e) => { + dismissPicker(); const files = (e.target as HTMLInputElement).files; if (!files || files.length === 0) return; - + const fileArray = Array.from(files).slice(0, remaining); if (files.length > remaining) { notificationService.warning(t('input.maxImagesWarning', { count: CHAT_INPUT_CONFIG.image.maxCount }), { duration: 3000 }); } - + for (const file of fileArray) { try { const imageContext = await createImageContextFromFile(file); @@ -5742,7 +5764,8 @@ export const ChatInput: React.FC = ({ } } }; - + + document.body.appendChild(input); input.click(); }, [addContext, currentImageCount, t]); diff --git a/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts b/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts new file mode 100644 index 0000000000..9f523a051a --- /dev/null +++ b/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts @@ -0,0 +1,43 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +function readChatInputComponent(): string { + return readFileSync(fileURLToPath(new URL('./ChatInput.tsx', import.meta.url)), 'utf8') + .replace(/\r\n/g, '\n'); +} + +describe('composer image file intake', () => { + const component = readChatInputComponent(); + + it('mounts the picker before clicking it so WebKitGTK delivers change', () => { + // WebKitGTK (the Linux webview) never fires `change` on a detached file + // input after the native chooser closes, which silently dropped every + // "Add image" selection on Linux. The picker must therefore be appended to + // the document before `input.click()`. + expect(component).toMatch( + /const handleImageInput = useCallback\(\(\) => \{[\s\S]*?document\.body\.appendChild\(input\);\s*\n\s*input\.click\(\);/, + ); + }); + + it('hides the mounted picker offscreen instead of display:none', () => { + expect(component).toContain("input.style.position = 'fixed'"); + expect(component).toContain("input.style.left = '-9999px'"); + // Some WebKit builds refuse to open a chooser for a display:none input, + // so the mounted picker must stay rendered, just offscreen. + expect(component).not.toContain("input.style.display = 'none'"); + }); + + it('reclaims the picker on both the change and the cancel path', () => { + // Selecting a file must release the node before the async intake loop. + expect(component).toMatch( + /input\.onchange = async \(e\) => \{\s*\n\s*dismissPicker\(\);/, + ); + // Cancelling the chooser never fires `change`; the one-shot focus listener + // is the only reclaim for that path. + expect(component).toContain("window.addEventListener('focus', dismissPicker);"); + expect(component).toMatch( + /const dismissPicker = \(\) => \{\s*\n\s*window\.removeEventListener\('focus', dismissPicker\);\s*\n\s*input\.onchange = null;\s*\n\s*input\.remove\(\);/, + ); + }); +}); From 39357b8a9dbff030653e0a56047e1db223a29c03 Mon Sep 17 00:00:00 2001 From: EthanCheung Date: Sat, 19 Sep 2026 00:26:54 +0800 Subject: [PATCH 2/4] fix(desktop): read pasted clipboard images through the host on WebKitGTK WebKitGTK delivers paste events with empty DataTransfer items and types, and its Async Clipboard API promise never settles, so a pasted image was unreachable from the page: the composer's file branch never ran and the paste landed as nothing. Add a get_clipboard_image desktop command that reads the clipboard through the same wl-paste/xclip tools as get_clipboard_files, sniffs magic bytes, and returns base64 pixels. The composer now listens for paste directly and, only when the webview reported zero types (the WebKitGTK shape; WebView2 and WKWebView deliver images in-band), asks the host for the image and reuses the existing clipboard-image intake with identical limits and errors. Registers the command in the remote-surface registry (LocalOnly) and in the attachments capability evidence; regenerates the capability artifacts. --- docs/interactive-capabilities/README.md | 4 +- .../technical/product-control-open-audit.json | 1 + .../technical/tauri-command-map.json | 82 +++++-------- .../desktop/src/api/clipboard_file_api.rs | 108 +++++++++++++++++- src/apps/desktop/src/lib.rs | 1 + .../generated/remote-surface-registry.json | 50 +++----- .../src/remote_surface/table.rs | 1 + .../interactive-capabilities/catalog.json | 1 + .../src/flow_chat/components/ChatInput.tsx | 50 +++++++- .../components/ChatInputImageIntake.test.ts | 14 +++ .../utils/externalFileIntake.test.ts | 11 ++ .../src/flow_chat/utils/externalFileIntake.ts | 15 +++ .../api/generated/remoteSurface.ts | 5 +- .../api/service-api/WorkspaceAPI.ts | 17 +++ 14 files changed, 258 insertions(+), 102 deletions(-) diff --git a/docs/interactive-capabilities/README.md b/docs/interactive-capabilities/README.md index be4c726782..a20c81ff1d 100644 --- a/docs/interactive-capabilities/README.md +++ b/docs/interactive-capabilities/README.md @@ -27,9 +27,9 @@ OpenBitFun Playbook currently contains **22 features**, **21 settings pages**, a - Generated per-item interaction audit: `docs/interactive-capabilities/technical/product-control-open-audit.json` - Generated low-level audit map: `docs/interactive-capabilities/technical/tauri-command-map.json` -说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **663** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 +说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **661** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 -Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **663** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. +Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **661** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. ## 控制边界 / Control boundary diff --git a/docs/interactive-capabilities/technical/product-control-open-audit.json b/docs/interactive-capabilities/technical/product-control-open-audit.json index 4bd8df1d79..a97fa23eb9 100644 --- a/docs/interactive-capabilities/technical/product-control-open-audit.json +++ b/docs/interactive-capabilities/technical/product-control-open-audit.json @@ -733,6 +733,7 @@ }, "evidence": [ "command:get_clipboard_files", + "command:get_clipboard_image", "command:resolve_browser_dropped_file_paths", "command:set_file_drop_preview_target", "command:upload_image_contexts", diff --git a/docs/interactive-capabilities/technical/tauri-command-map.json b/docs/interactive-capabilities/technical/tauri-command-map.json index 2957f8c3a7..3cd134500a 100644 --- a/docs/interactive-capabilities/technical/tauri-command-map.json +++ b/docs/interactive-capabilities/technical/tauri-command-map.json @@ -1,13 +1,13 @@ { "schemaVersion": 2, "generatedFrom": "src/shared/interactive-capabilities/catalog.json", - "catalogDigest": "6587344a6b5e75a80457ea4ba2670bf37cf3038bdb436089ae402eb7b5a5a025", - "commandCount": 663, + "catalogDigest": "5473331e06ecd4bfafbf9b547d48b4fe1561ccf69d86fdfde51ea10dc919531c", + "commandCount": 661, "coverage": { - "commandCount": 663, - "documentedCommandCount": 615, - "implementationCommandCount": 48, - "implementationDigest": "f6d38a24a70708988cb47ada81d07eccf0668684e8734c9ee5155b9ffa7e3db8" + "commandCount": 661, + "documentedCommandCount": 614, + "implementationCommandCount": 47, + "implementationDigest": "401fd0a5f64377e65573d1b0883fbfa844570096adfafd4c11a999d749ba5859" }, "commands": [ { @@ -268,22 +268,6 @@ "signature": "fn account_online_devices() -> Result, String>", "remoteWorkspacePolicy": "WorkspaceAgnostic" }, - { - "id": "account_relay_capabilities", - "moduleId": "remote_connect", - "capabilityId": "feature.remote-connect", - "capabilityIds": [ - "feature.remote-connect" - ], - "documentedItemIds": [ - "feature.remote-connect:devices" - ], - "visibility": "documented", - "rustPath": "api::remote_connect_api::account_relay_capabilities", - "sourceFile": "src/apps/desktop/src/api/remote_connect_api.rs", - "signature": "fn account_relay_capabilities() -> Result, String>", - "remoteWorkspacePolicy": "WorkspaceAgnostic" - }, { "id": "account_status", "moduleId": "remote_connect", @@ -342,22 +326,6 @@ "signature": "fn account_unsubscribe_session(request: UnsubscribeSessionRequest) -> Result<(), String>", "remoteWorkspacePolicy": "WorkspaceAgnostic" }, - { - "id": "account_update_device_alias", - "moduleId": "remote_connect", - "capabilityId": "feature.remote-connect", - "capabilityIds": [ - "feature.remote-connect" - ], - "documentedItemIds": [ - "feature.remote-connect:devices" - ], - "visibility": "documented", - "rustPath": "api::remote_connect_api::account_update_device_alias", - "sourceFile": "src/apps/desktop/src/api/remote_connect_api.rs", - "signature": "fn account_update_device_alias( request: AccountUpdateDeviceAliasRequest, ) -> Result<(), String>", - "remoteWorkspacePolicy": "WorkspaceAgnostic" - }, { "id": "acknowledge_external_ecosystems_command", "moduleId": "external_sources", @@ -2217,7 +2185,7 @@ "visibility": "documented", "rustPath": "download_skill_market", "sourceFile": "src/apps/desktop/src/api/skill_api.rs", - "signature": "fn download_skill_market( state: State<'_, AppState>, request: SkillMarketDownloadRequest, ) -> Result", + "signature": "fn download_skill_market( _state: State<'_, AppState>, request: SkillMarketDownloadRequest, ) -> Result", "remoteWorkspacePolicy": "LegacyUnaudited" }, { @@ -2233,7 +2201,7 @@ "visibility": "documented", "rustPath": "api::update_api::download_update", "sourceFile": "src/apps/desktop/src/api/update_api.rs", - "signature": "fn download_update( app: AppHandle, request: DownloadUpdateRequest, ) -> Result", + "signature": "fn download_update( app: AppHandle, request: PendingUpdateRequest, ) -> Result", "remoteWorkspacePolicy": "WorkspaceAgnostic" }, { @@ -2824,6 +2792,22 @@ "signature": "fn get_clipboard_files() -> Result", "remoteWorkspacePolicy": "LocalOnly" }, + { + "id": "get_clipboard_image", + "moduleId": "clipboard_file", + "capabilityId": "feature.files-editor", + "capabilityIds": [ + "feature.files-editor" + ], + "documentedItemIds": [ + "feature.files-editor:attachments" + ], + "visibility": "documented", + "rustPath": "get_clipboard_image", + "sourceFile": "src/apps/desktop/src/api/clipboard_file_api.rs", + "signature": "fn get_clipboard_image() -> Result", + "remoteWorkspacePolicy": "LocalOnly" + }, { "id": "get_config", "moduleId": "config", @@ -3222,20 +3206,6 @@ "signature": "fn get_latest_insights() -> Result, String>", "remoteWorkspacePolicy": "LocalOnly" }, - { - "id": "get_local_models_dev_catalogs", - "moduleId": "commands", - "capabilityId": "feature.projects", - "capabilityIds": [ - "feature.projects" - ], - "documentedItemIds": [], - "visibility": "implementation", - "rustPath": "get_local_models_dev_catalogs", - "sourceFile": "src/apps/desktop/src/api/commands.rs", - "signature": "fn get_local_models_dev_catalogs( ) -> Result", - "remoteWorkspacePolicy": "LocalOnly" - }, { "id": "get_mcp_prompt", "moduleId": "mcp", @@ -5167,7 +5137,7 @@ "visibility": "documented", "rustPath": "list_skill_market", "sourceFile": "src/apps/desktop/src/api/skill_api.rs", - "signature": "fn list_skill_market( state: State<'_, AppState>, request: SkillMarketListRequest, ) -> Result", + "signature": "fn list_skill_market( _state: State<'_, AppState>, request: SkillMarketListRequest, ) -> Result, String>", "remoteWorkspacePolicy": "LegacyUnaudited" }, { @@ -8400,7 +8370,7 @@ "visibility": "documented", "rustPath": "search_skill_market", "sourceFile": "src/apps/desktop/src/api/skill_api.rs", - "signature": "fn search_skill_market( state: State<'_, AppState>, request: SkillMarketSearchRequest, ) -> Result", + "signature": "fn search_skill_market( _state: State<'_, AppState>, request: SkillMarketSearchRequest, ) -> Result, String>", "remoteWorkspacePolicy": "LegacyUnaudited" }, { diff --git a/src/apps/desktop/src/api/clipboard_file_api.rs b/src/apps/desktop/src/api/clipboard_file_api.rs index 4ffeb52a05..95729f1f2e 100644 --- a/src/apps/desktop/src/api/clipboard_file_api.rs +++ b/src/apps/desktop/src/api/clipboard_file_api.rs @@ -324,6 +324,90 @@ pub async fn get_clipboard_files() -> Result { } } +/// Image bytes read from the system clipboard, base64-encoded for the webview. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ClipboardImageResponse { + pub base64: Option, + pub mime_type: Option, +} + +impl Default for ClipboardImageResponse { + fn default() -> Self { + Self { + base64: None, + mime_type: None, + } + } +} + +/// Sniffs the image format from magic bytes so a tool that misreports success +/// cannot inject arbitrary text as an attachment payload. +pub(crate) fn sniff_image_mime(bytes: &[u8]) -> Option<&'static str> { + if bytes.starts_with(&[0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A]) { + Some("image/png") + } else if bytes.len() >= 3 && bytes[0] == 0xFF && bytes[1] == 0xD8 && bytes[2] == 0xFF { + Some("image/jpeg") + } else { + None + } +} + +/// Reads a clipboard image on Linux. +/// +/// WebKitGTK delivers paste events with empty `DataTransfer` items, so the +/// webview itself can never see a pasted image; reading the Wayland/X11 +/// clipboard through the same tools as `get_clipboard_files` is the only +/// delivery path. Other platforms return `None`: their webviews deliver +/// clipboard images to the page directly and never need this fallback. +#[cfg(target_os = "linux")] +fn read_clipboard_image_internal() -> Option<(String, String)> { + use base64::Engine as _; + use std::process::Command; + + let read_target = |program: &str, args: &[&str]| -> Option> { + Command::new(program) + .args(args) + .output() + .ok() + .filter(|output| output.status.success() && !output.stdout.is_empty()) + .map(|output| output.stdout) + }; + + for mime in ["image/png", "image/jpeg"] { + let bytes = read_target("wl-paste", &["-t", mime]) + .or_else(|| read_target("xclip", &["-selection", "clipboard", "-t", mime, "-o"])); + if let Some(bytes) = bytes { + if let Some(sniffed) = sniff_image_mime(&bytes) { + let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes); + return Some((encoded, sniffed.to_string())); + } + } + } + None +} + +#[cfg(target_os = "linux")] +fn get_clipboard_image_internal() -> Option<(String, String)> { + read_clipboard_image_internal() +} + +#[cfg(not(target_os = "linux"))] +fn get_clipboard_image_internal() -> Option<(String, String)> { + None +} + +#[tauri::command] +pub async fn get_clipboard_image() -> Result { + Ok(match get_clipboard_image_internal() { + Some((base64, mime_type)) => ClipboardImageResponse { + base64: Some(base64), + mime_type: Some(mime_type), + }, + None => ClipboardImageResponse::default(), + }) +} + /// Pastes clipboard files between controller-local paths. /// /// The remote file provider exposes no copy primitive, so a remote workspace path is refused here @@ -485,10 +569,32 @@ pub(crate) fn copy_directory_recursive(source: &Path, target: &Path) -> Result<( mod tests { use super::{ copy_directory_recursive, decode_file_uri, generate_unique_path, - parse_clipboard_path_segments, parse_uri_list, + parse_clipboard_path_segments, parse_uri_list, sniff_image_mime, }; use std::path::Path; + #[test] + fn sniff_image_mime_detects_png_header() { + assert_eq!( + sniff_image_mime(&[0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A, 0, 0]), + Some("image/png") + ); + } + + #[test] + fn sniff_image_mime_detects_jpeg_header() { + assert_eq!( + sniff_image_mime(&[0xFF, 0xD8, 0xFF, 0xE0, 0, 0]), + Some("image/jpeg") + ); + } + + #[test] + fn sniff_image_mime_rejects_non_image_payloads() { + assert_eq!(sniff_image_mime(b"image/png but not really"), None); + assert_eq!(sniff_image_mime(&[]), None); + } + #[test] fn decode_unix_file_uri() { assert_eq!( diff --git a/src/apps/desktop/src/lib.rs b/src/apps/desktop/src/lib.rs index b101edb7a1..203b26dbe1 100644 --- a/src/apps/desktop/src/lib.rs +++ b/src/apps/desktop/src/lib.rs @@ -1431,6 +1431,7 @@ pub async fn run() { stop_file_watch, get_watched_paths, get_clipboard_files, + get_clipboard_image, api::browser_file_drop_api::resolve_browser_dropped_file_paths, api::file_drop_preview_api::set_file_drop_preview_target, paste_files, diff --git a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json index fa29cdcaeb..c3d6e899be 100644 --- a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json +++ b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "digest": "fnv1a64:d3892fea19448d1f", + "digest": "fnv1a64:221007508a2b1fd6", "retiredCommandPrefixes": [ { "prefix": "lsp_", @@ -257,18 +257,6 @@ "reason": "the controller keeps this command; peer hosts refuse it before dispatch" } }, - { - "id": "account_relay_capabilities", - "surface": "tauri_command", - "remoteWorkspace": "WorkspaceAgnostic", - "peer": { - "kind": "controller_local" - }, - "cliPeer": { - "kind": "unsupported", - "reason": "the controller keeps this command; peer hosts refuse it before dispatch" - } - }, { "id": "account_status", "surface": "tauri_command", @@ -317,18 +305,6 @@ "reason": "the controller keeps this command; peer hosts refuse it before dispatch" } }, - { - "id": "account_update_device_alias", - "surface": "tauri_command", - "remoteWorkspace": "WorkspaceAgnostic", - "peer": { - "kind": "controller_local" - }, - "cliPeer": { - "kind": "unsupported", - "reason": "the controller keeps this command; peer hosts refuse it before dispatch" - } - }, { "id": "acknowledge_external_ecosystems_command", "surface": "tauri_command", @@ -2306,6 +2282,18 @@ "reason": "the CLI peer host has no handler for this command" } }, + { + "id": "get_clipboard_image", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no handler for this command" + } + }, { "id": "get_config", "surface": "tauri_command", @@ -2598,18 +2586,6 @@ "reason": "the controller keeps this command; peer hosts refuse it before dispatch" } }, - { - "id": "get_local_models_dev_catalogs", - "surface": "tauri_command", - "remoteWorkspace": "LocalOnly", - "peer": { - "kind": "controller_local" - }, - "cliPeer": { - "kind": "unsupported", - "reason": "the controller keeps this command; peer hosts refuse it before dispatch" - } - }, { "id": "get_mcp_prompt", "surface": "tauri_command", diff --git a/src/crates/contracts/product-domains/src/remote_surface/table.rs b/src/crates/contracts/product-domains/src/remote_surface/table.rs index dbf37e8ae1..0040bdcdd4 100644 --- a/src/crates/contracts/product-domains/src/remote_surface/table.rs +++ b/src/crates/contracts/product-domains/src/remote_surface/table.rs @@ -274,6 +274,7 @@ pub(super) const OPERATIONS: &[OperationDefinition] = &[ op("get_baseline_snapshot_diff", Unaudited, Proxied, CLI_NOT_IMPLEMENTED), op("get_chat_mcp_catalog", Unsupported, Proxied, HANDLED), op("get_clipboard_files", LocalOnly, Proxied, CLI_NOT_IMPLEMENTED), + op("get_clipboard_image", LocalOnly, Proxied, CLI_NOT_IMPLEMENTED), op("get_config", Unaudited, Proxied, HANDLED), op("get_configs", Unaudited, Proxied, HANDLED), op("get_current_workspace", Agnostic, Proxied, HANDLED), diff --git a/src/shared/interactive-capabilities/catalog.json b/src/shared/interactive-capabilities/catalog.json index 9ddb30e8af..95aa0609b3 100644 --- a/src/shared/interactive-capabilities/catalog.json +++ b/src/shared/interactive-capabilities/catalog.json @@ -1542,6 +1542,7 @@ }, "evidence": [ "command:get_clipboard_files", + "command:get_clipboard_image", "command:resolve_browser_dropped_file_paths", "command:set_file_drop_preview_target", "command:upload_image_contexts", diff --git a/src/web-ui/src/flow_chat/components/ChatInput.tsx b/src/web-ui/src/flow_chat/components/ChatInput.tsx index c39eea69be..451115de88 100644 --- a/src/web-ui/src/flow_chat/components/ChatInput.tsx +++ b/src/web-ui/src/flow_chat/components/ChatInput.tsx @@ -237,6 +237,7 @@ import { buildExternalFileContexts, partitionExternalDropFiles, resolveExternalFileIntakeAvailability, + shouldAttemptNativeClipboardImageRead, type ExternalFileSource, } from '../utils/externalFileIntake'; import { selectInterruptedTurnRecovery } from '../utils/interruptedTurnRecovery'; @@ -4765,6 +4766,28 @@ export const ChatInput: React.FC = ({ } }, [addContext, contextStore, isExternalFileIntakeRequestCurrent, t]); + /** + * Host-side clipboard image read for engines that deliver paste events with + * empty DataTransfer (WebKitGTK on Linux). Reuses the clipboard-image + * intake, so limits and error reporting stay identical to the in-page path. + */ + const readPastedClipboardImage = useCallback(async (request: ExternalFileIntakeRequest) => { + try { + const image = await workspaceAPI.getClipboardImage(); + if (!image || !isExternalFileIntakeRequestCurrent(request)) return; + const binary = atob(image.base64); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index++) { + bytes[index] = binary.charCodeAt(index); + } + const extension = image.mimeType === 'image/png' ? 'png' : 'jpg'; + const file = new File([bytes], `clipboard-image.${extension}`, { type: image.mimeType }); + await addClipboardImageFiles(request, [file]); + } catch (error) { + log.warn('Native clipboard image read failed', { error }); + } + }, [addClipboardImageFiles, isExternalFileIntakeRequestCurrent]); + const addExternalPaths = useCallback(async ( request: ExternalFileIntakeRequest, source: ExternalFileSource, @@ -4946,9 +4969,32 @@ export const ChatInput: React.FC = ({ () => addClipboardImageFiles(request, [file]), ); }; + const handlePasteFallback = (event: Event) => { + // WebKitGTK fires paste with zero DataTransfer types; the in-page file + // branch can never run there, so ask the host to read the clipboard. + const clipboardData = (event as ClipboardEvent).clipboardData; + if (!clipboardData) return; + if (!shouldAttemptNativeClipboardImageRead(Array.from(clipboardData.types ?? []))) return; + if (!externalFileAvailability.supported) return; + const request = captureExternalFileIntakeRequest(); + void enqueueExternalFileIntake( + request, + () => readPastedClipboardImage(request), + ); + }; inputElement.addEventListener('imagePaste', handleImagePaste); - return () => inputElement.removeEventListener('imagePaste', handleImagePaste); - }, [addClipboardImageFiles, captureExternalFileIntakeRequest, enqueueExternalFileIntake]); + inputElement.addEventListener('paste', handlePasteFallback); + return () => { + inputElement.removeEventListener('imagePaste', handleImagePaste); + inputElement.removeEventListener('paste', handlePasteFallback); + }; + }, [ + addClipboardImageFiles, + captureExternalFileIntakeRequest, + enqueueExternalFileIntake, + externalFileAvailability, + readPastedClipboardImage, + ]); useWindowsFileDropPreview({ targetRef: fileDropTargetRef ?? externalFileDropTargetRef, diff --git a/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts b/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts index 9f523a051a..352b21f2fb 100644 --- a/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts +++ b/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts @@ -40,4 +40,18 @@ describe('composer image file intake', () => { /const dismissPicker = \(\) => \{\s*\n\s*window\.removeEventListener\('focus', dismissPicker\);\s*\n\s*input\.onchange = null;\s*\n\s*input\.remove\(\);/, ); }); + + it('falls back to a host clipboard read for empty-typed pastes', () => { + // WebKitGTK fires paste with zero DataTransfer types, so the composer must + // listen for paste directly and ask the host for the clipboard image. + expect(component).toContain( + 'shouldAttemptNativeClipboardImageRead(Array.from(clipboardData.types ?? []))', + ); + expect(component).toMatch( + /inputElement\.addEventListener\('paste', handlePasteFallback\);/, + ); + expect(component).toMatch( + /const image = await workspaceAPI\.getClipboardImage\(\);/, + ); + }); }); diff --git a/src/web-ui/src/flow_chat/utils/externalFileIntake.test.ts b/src/web-ui/src/flow_chat/utils/externalFileIntake.test.ts index aaa5fe9504..e3b9cf975a 100644 --- a/src/web-ui/src/flow_chat/utils/externalFileIntake.test.ts +++ b/src/web-ui/src/flow_chat/utils/externalFileIntake.test.ts @@ -6,6 +6,7 @@ import { normalizeExternalFilePath, partitionExternalDropFiles, resolveExternalFileIntakeAvailability, + shouldAttemptNativeClipboardImageRead, } from './externalFileIntake'; const metadata = (isDir: boolean, size = 12): FileMetadata => ({ @@ -176,4 +177,14 @@ describe('buildExternalFileContexts', () => { expect(result.contexts.map((context) => context.type)).toEqual(['image', 'image', 'file']); expect(result.failures).toEqual([{ path: '/tmp/c.gif', reason: 'image-limit' }]); }); + + it('attempts the native clipboard image read only for empty-typed pastes', () => { + // WebKitGTK delivers paste with no types at all; only that shape falls + // back to the host clipboard read. + expect(shouldAttemptNativeClipboardImageRead([])).toBe(true); + // Engines that report anything (files or text) deliver images in-band. + expect(shouldAttemptNativeClipboardImageRead(['Files'])).toBe(false); + expect(shouldAttemptNativeClipboardImageRead(['text/plain'])).toBe(false); + expect(shouldAttemptNativeClipboardImageRead(['Files', 'text/plain'])).toBe(false); + }); }); diff --git a/src/web-ui/src/flow_chat/utils/externalFileIntake.ts b/src/web-ui/src/flow_chat/utils/externalFileIntake.ts index 29bd7a603a..74c1818868 100644 --- a/src/web-ui/src/flow_chat/utils/externalFileIntake.ts +++ b/src/web-ui/src/flow_chat/utils/externalFileIntake.ts @@ -62,6 +62,21 @@ export function normalizeExternalFilePath(path: string): string { return repositoryPathKey(path); } +/** + * Whether a paste event should fall back to a native host clipboard read. + * + * WebKitGTK (the Linux webview) delivers paste events with NO DataTransfer + * types or items at all, so the in-page file branch can never see a pasted + * image. Engines that report any types (WebView2, WKWebView, Chromium) + * deliver clipboard images in-band, so only fully empty-typed pastes use the + * host fallback. + */ +export function shouldAttemptNativeClipboardImageRead( + types: ReadonlyArray, +): boolean { + return types.length === 0; +} + export function getContextLocalPath(context: ContextItem): string | undefined { if (context.type === 'file') return context.filePath; if (context.type === 'directory') return context.directoryPath; diff --git a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts index 775123b68e..f2b2811177 100644 --- a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts +++ b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts @@ -1,6 +1,6 @@ // Generated by scripts/generate-interactive-capabilities.mjs; do not edit. // Source: openbitfun_product_domains::remote_surface (Product Operation Registry). -export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:d3892fea19448d1f" as const; +export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:221007508a2b1fd6" as const; /** * Registered Tauri commands the Peer Device controller keeps on the controller @@ -23,12 +23,10 @@ export const PEER_CONTROLLER_LOCAL_COMMANDS: ReadonlySet = new Set([ "account_login", "account_logout", "account_online_devices", - "account_relay_capabilities", "account_status", "account_subscribe_session", "account_token_expired", "account_unsubscribe_session", - "account_update_device_alias", "appearance_market_browse", "appearance_market_download_release", "appearance_market_get_listing", @@ -78,7 +76,6 @@ export const PEER_CONTROLLER_LOCAL_COMMANDS: ReadonlySet = new Set([ "get_announcement_tips", "get_frontend_update_status", "get_latest_insights", - "get_local_models_dev_catalogs", "get_pending_announcements", "get_pending_update", "get_prevent_sleep_enabled", diff --git a/src/web-ui/src/infrastructure/api/service-api/WorkspaceAPI.ts b/src/web-ui/src/infrastructure/api/service-api/WorkspaceAPI.ts index 788f42ad26..054a8bbf03 100644 --- a/src/web-ui/src/infrastructure/api/service-api/WorkspaceAPI.ts +++ b/src/web-ui/src/infrastructure/api/service-api/WorkspaceAPI.ts @@ -1121,6 +1121,23 @@ export class WorkspaceAPI { } } + /** + * Reads an image from the system clipboard, or null when the clipboard holds + * no image. WebKitGTK delivers paste events with empty DataTransfer items, + * so pasted images are only reachable through this host read. + */ + async getClipboardImage(): Promise<{ base64: string; mimeType: string } | null> { + try { + const response = await api.invoke('get_clipboard_image'); + if (response?.base64 && response?.mimeType) { + return { base64: response.base64, mimeType: response.mimeType }; + } + return null; + } catch (error) { + throw createTauriCommandError('get_clipboard_image', error); + } + } + async resolveBrowserDroppedFilePaths(token: string, fileCount: number): Promise { try { return await api.invoke('resolve_browser_dropped_file_paths', { From c34d09e0cb724431335e9f134a802ed731370531 Mon Sep 17 00:00:00 2001 From: EthanCheung Date: Sat, 19 Sep 2026 00:51:13 +0800 Subject: [PATCH 3/4] fix(web-ui): stop claiming empty-typed drags in the composer drop zone ContextDropZone called preventDefault on every dragenter/dragover, which turned the composer into a DOM drop target for OS file drags. On WebKitGTK an OS file drag reports no dataTransfer types, so once the zone was claimed, the in-page handler saw nothing to handle while the claim itself interfered with the drag pass-through. Claim only drags the zone can positively identify and handle: internal context payloads, typed non-file drags, and Files drags when a DOM file handler is actually provided (the Windows desktop runtime). Empty-typed drags stay unclaimed and keep flowing to the pane-level native drop path. Known limitation: with this change the zone no longer swallows empty-typed drags, but dropping an OS file directly on the composer still does not attach on WebKitGTK 2.52 - a document-level dragover preventDefault keeps the subtree claimed and the native window drop handler still never sees the release over the composer. Drops elsewhere in the conversation pane work. Full on-composer native drops need follow-up at the WebKitGTK/wry interaction level. --- docs/interactive-capabilities/README.md | 4 +- .../technical/tauri-command-map.json | 66 ++++++++++++++++--- .../generated/remote-surface-registry.json | 38 ++++++++++- .../api/generated/remoteSurface.ts | 5 +- .../drag-drop/ContextDropZone.test.tsx | 19 ++++++ .../drag-drop/ContextDropZone.tsx | 34 ++++++++-- 6 files changed, 147 insertions(+), 19 deletions(-) diff --git a/docs/interactive-capabilities/README.md b/docs/interactive-capabilities/README.md index a20c81ff1d..c280bb8829 100644 --- a/docs/interactive-capabilities/README.md +++ b/docs/interactive-capabilities/README.md @@ -27,9 +27,9 @@ OpenBitFun Playbook currently contains **22 features**, **21 settings pages**, a - Generated per-item interaction audit: `docs/interactive-capabilities/technical/product-control-open-audit.json` - Generated low-level audit map: `docs/interactive-capabilities/technical/tauri-command-map.json` -说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **661** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 +说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **664** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 -Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **661** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. +Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **664** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. ## 控制边界 / Control boundary diff --git a/docs/interactive-capabilities/technical/tauri-command-map.json b/docs/interactive-capabilities/technical/tauri-command-map.json index 3cd134500a..08386ae0b9 100644 --- a/docs/interactive-capabilities/technical/tauri-command-map.json +++ b/docs/interactive-capabilities/technical/tauri-command-map.json @@ -1,13 +1,13 @@ { "schemaVersion": 2, "generatedFrom": "src/shared/interactive-capabilities/catalog.json", - "catalogDigest": "5473331e06ecd4bfafbf9b547d48b4fe1561ccf69d86fdfde51ea10dc919531c", - "commandCount": 661, + "catalogDigest": "6587344a6b5e75a80457ea4ba2670bf37cf3038bdb436089ae402eb7b5a5a025", + "commandCount": 664, "coverage": { - "commandCount": 661, - "documentedCommandCount": 614, - "implementationCommandCount": 47, - "implementationDigest": "401fd0a5f64377e65573d1b0883fbfa844570096adfafd4c11a999d749ba5859" + "commandCount": 664, + "documentedCommandCount": 616, + "implementationCommandCount": 48, + "implementationDigest": "f6d38a24a70708988cb47ada81d07eccf0668684e8734c9ee5155b9ffa7e3db8" }, "commands": [ { @@ -268,6 +268,22 @@ "signature": "fn account_online_devices() -> Result, String>", "remoteWorkspacePolicy": "WorkspaceAgnostic" }, + { + "id": "account_relay_capabilities", + "moduleId": "remote_connect", + "capabilityId": "feature.remote-connect", + "capabilityIds": [ + "feature.remote-connect" + ], + "documentedItemIds": [ + "feature.remote-connect:devices" + ], + "visibility": "documented", + "rustPath": "api::remote_connect_api::account_relay_capabilities", + "sourceFile": "src/apps/desktop/src/api/remote_connect_api.rs", + "signature": "fn account_relay_capabilities() -> Result, String>", + "remoteWorkspacePolicy": "WorkspaceAgnostic" + }, { "id": "account_status", "moduleId": "remote_connect", @@ -326,6 +342,22 @@ "signature": "fn account_unsubscribe_session(request: UnsubscribeSessionRequest) -> Result<(), String>", "remoteWorkspacePolicy": "WorkspaceAgnostic" }, + { + "id": "account_update_device_alias", + "moduleId": "remote_connect", + "capabilityId": "feature.remote-connect", + "capabilityIds": [ + "feature.remote-connect" + ], + "documentedItemIds": [ + "feature.remote-connect:devices" + ], + "visibility": "documented", + "rustPath": "api::remote_connect_api::account_update_device_alias", + "sourceFile": "src/apps/desktop/src/api/remote_connect_api.rs", + "signature": "fn account_update_device_alias( request: AccountUpdateDeviceAliasRequest, ) -> Result<(), String>", + "remoteWorkspacePolicy": "WorkspaceAgnostic" + }, { "id": "acknowledge_external_ecosystems_command", "moduleId": "external_sources", @@ -2185,7 +2217,7 @@ "visibility": "documented", "rustPath": "download_skill_market", "sourceFile": "src/apps/desktop/src/api/skill_api.rs", - "signature": "fn download_skill_market( _state: State<'_, AppState>, request: SkillMarketDownloadRequest, ) -> Result", + "signature": "fn download_skill_market( state: State<'_, AppState>, request: SkillMarketDownloadRequest, ) -> Result", "remoteWorkspacePolicy": "LegacyUnaudited" }, { @@ -2201,7 +2233,7 @@ "visibility": "documented", "rustPath": "api::update_api::download_update", "sourceFile": "src/apps/desktop/src/api/update_api.rs", - "signature": "fn download_update( app: AppHandle, request: PendingUpdateRequest, ) -> Result", + "signature": "fn download_update( app: AppHandle, request: DownloadUpdateRequest, ) -> Result", "remoteWorkspacePolicy": "WorkspaceAgnostic" }, { @@ -3206,6 +3238,20 @@ "signature": "fn get_latest_insights() -> Result, String>", "remoteWorkspacePolicy": "LocalOnly" }, + { + "id": "get_local_models_dev_catalogs", + "moduleId": "commands", + "capabilityId": "feature.projects", + "capabilityIds": [ + "feature.projects" + ], + "documentedItemIds": [], + "visibility": "implementation", + "rustPath": "get_local_models_dev_catalogs", + "sourceFile": "src/apps/desktop/src/api/commands.rs", + "signature": "fn get_local_models_dev_catalogs( ) -> Result", + "remoteWorkspacePolicy": "LocalOnly" + }, { "id": "get_mcp_prompt", "moduleId": "mcp", @@ -5137,7 +5183,7 @@ "visibility": "documented", "rustPath": "list_skill_market", "sourceFile": "src/apps/desktop/src/api/skill_api.rs", - "signature": "fn list_skill_market( _state: State<'_, AppState>, request: SkillMarketListRequest, ) -> Result, String>", + "signature": "fn list_skill_market( state: State<'_, AppState>, request: SkillMarketListRequest, ) -> Result", "remoteWorkspacePolicy": "LegacyUnaudited" }, { @@ -8370,7 +8416,7 @@ "visibility": "documented", "rustPath": "search_skill_market", "sourceFile": "src/apps/desktop/src/api/skill_api.rs", - "signature": "fn search_skill_market( _state: State<'_, AppState>, request: SkillMarketSearchRequest, ) -> Result, String>", + "signature": "fn search_skill_market( state: State<'_, AppState>, request: SkillMarketSearchRequest, ) -> Result", "remoteWorkspacePolicy": "LegacyUnaudited" }, { diff --git a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json index c3d6e899be..528c381d86 100644 --- a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json +++ b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "digest": "fnv1a64:221007508a2b1fd6", + "digest": "fnv1a64:01a7a1c7756afe23", "retiredCommandPrefixes": [ { "prefix": "lsp_", @@ -257,6 +257,18 @@ "reason": "the controller keeps this command; peer hosts refuse it before dispatch" } }, + { + "id": "account_relay_capabilities", + "surface": "tauri_command", + "remoteWorkspace": "WorkspaceAgnostic", + "peer": { + "kind": "controller_local" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the controller keeps this command; peer hosts refuse it before dispatch" + } + }, { "id": "account_status", "surface": "tauri_command", @@ -305,6 +317,18 @@ "reason": "the controller keeps this command; peer hosts refuse it before dispatch" } }, + { + "id": "account_update_device_alias", + "surface": "tauri_command", + "remoteWorkspace": "WorkspaceAgnostic", + "peer": { + "kind": "controller_local" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the controller keeps this command; peer hosts refuse it before dispatch" + } + }, { "id": "acknowledge_external_ecosystems_command", "surface": "tauri_command", @@ -2586,6 +2610,18 @@ "reason": "the controller keeps this command; peer hosts refuse it before dispatch" } }, + { + "id": "get_local_models_dev_catalogs", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "controller_local" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the controller keeps this command; peer hosts refuse it before dispatch" + } + }, { "id": "get_mcp_prompt", "surface": "tauri_command", diff --git a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts index f2b2811177..f0ea6434c2 100644 --- a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts +++ b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts @@ -1,6 +1,6 @@ // Generated by scripts/generate-interactive-capabilities.mjs; do not edit. // Source: openbitfun_product_domains::remote_surface (Product Operation Registry). -export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:221007508a2b1fd6" as const; +export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:01a7a1c7756afe23" as const; /** * Registered Tauri commands the Peer Device controller keeps on the controller @@ -23,10 +23,12 @@ export const PEER_CONTROLLER_LOCAL_COMMANDS: ReadonlySet = new Set([ "account_login", "account_logout", "account_online_devices", + "account_relay_capabilities", "account_status", "account_subscribe_session", "account_token_expired", "account_unsubscribe_session", + "account_update_device_alias", "appearance_market_browse", "appearance_market_download_release", "appearance_market_get_listing", @@ -76,6 +78,7 @@ export const PEER_CONTROLLER_LOCAL_COMMANDS: ReadonlySet = new Set([ "get_announcement_tips", "get_frontend_update_status", "get_latest_insights", + "get_local_models_dev_catalogs", "get_pending_announcements", "get_pending_update", "get_prevent_sleep_enabled", diff --git a/src/web-ui/src/shared/context-system/drag-drop/ContextDropZone.test.tsx b/src/web-ui/src/shared/context-system/drag-drop/ContextDropZone.test.tsx index b3f26200dc..e7b58348fc 100644 --- a/src/web-ui/src/shared/context-system/drag-drop/ContextDropZone.test.tsx +++ b/src/web-ui/src/shared/context-system/drag-drop/ContextDropZone.test.tsx @@ -82,4 +82,23 @@ describe('conversation area drops', () => { expect(drag('transcript', 'dragover', ['text/plain']).defaultPrevented).toBe(false); expect(drag('transcript', 'drop', ['text/plain']).defaultPrevented).toBe(false); }); + it('does not claim empty-typed OS file drags (WebKitGTK) so the native pane drop path owns them', () => { + // On WebKitGTK an OS file drag reports NO dataTransfer types. Claiming it + // (preventDefault) would make WebKit swallow the drop at the DOM level and + // stop forwarding it to the native window drag handler. The zone must stay + // unclaimed so the pane-level native drop path can receive the file. + function LinuxPane() { + const ref = useRef(null); + return
+ +
Input
+ +
; + } + root = createRoot(container); + act(() => root.render()); + expect(drag('composer', 'dragenter', []).defaultPrevented).toBe(false); + expect(drag('composer', 'dragover', []).defaultPrevented).toBe(false); + expect(drag('composer', 'drop', []).defaultPrevented).toBe(false); + }); }); diff --git a/src/web-ui/src/shared/context-system/drag-drop/ContextDropZone.tsx b/src/web-ui/src/shared/context-system/drag-drop/ContextDropZone.tsx index 050530c769..ac04225ae0 100644 --- a/src/web-ui/src/shared/context-system/drag-drop/ContextDropZone.tsx +++ b/src/web-ui/src/shared/context-system/drag-drop/ContextDropZone.tsx @@ -127,8 +127,30 @@ export const ContextDropZone: React.FC = ({ }, [dropTarget]); + /** + * Whether this zone should claim the drag as a DOM drop target. + * + * Claiming means calling preventDefault on dragenter/dragover, which turns + * the composer into a DOM drop target. On WebKitGTK (Linux) an OS file drag + * reports NO dataTransfer types, and a claimed DOM target makes WebKit stop + * forwarding the drop to the native window drag handler, silently swallowing + * the file. Empty-typed drags therefore stay unclaimed so the pane-level + * native drop path (wry drag events) owns them. + */ + const shouldClaimDrag = useCallback((e: DropEvent): boolean => { + if (!e.dataTransfer) return false; + const types = Array.from(e.dataTransfer.types); + if (types.includes('Files')) { + return !disabled && Boolean(onExternalFilesDrop); + } + // Internal context drags and typed non-file drags keep the historical + // claiming behavior; only unidentifiable empty-typed drags step aside. + return types.length > 0 || Boolean(dragManager.getCurrentPayload()); + }, [disabled, onExternalFilesDrop]); + const handleDragEnter = useCallback((e: DropEvent) => { if (!e.dataTransfer) return; + if (!shouldClaimDrag(e)) return; e.preventDefault(); e.stopPropagation(); @@ -149,10 +171,11 @@ export const ContextDropZone: React.FC = ({ dragManager.handleDragEnter(dropTargetRef.current, nativeDragEvent(e)); } } - }, [disabled, onExternalFilesDrop]); - + }, [disabled, onExternalFilesDrop, shouldClaimDrag]); + const handleDragOver = useCallback((e: DropEvent) => { if (!e.dataTransfer) return; + if (!shouldClaimDrag(e)) return; e.preventDefault(); e.stopPropagation(); @@ -160,7 +183,7 @@ export const ContextDropZone: React.FC = ({ e.dataTransfer.dropEffect = disabled || !onExternalFilesDrop ? 'none' : 'copy'; return; } - + const payload = dragManager.getCurrentPayload(); if (payload && dropTargetRef.current.canAccept(payload)) { e.dataTransfer.dropEffect = 'copy'; @@ -168,7 +191,7 @@ export const ContextDropZone: React.FC = ({ } else { e.dataTransfer.dropEffect = 'none'; } - }, [disabled, onExternalFilesDrop]); + }, [disabled, onExternalFilesDrop, shouldClaimDrag]); const handleDragLeave = useCallback((e: DropEvent) => { if (!e.dataTransfer) return; @@ -190,6 +213,7 @@ export const ContextDropZone: React.FC = ({ const handleDrop = useCallback((e: DropEvent) => { if (!e.dataTransfer) return; + if (!shouldClaimDrag(e)) return; e.preventDefault(); e.stopPropagation(); @@ -205,7 +229,7 @@ export const ContextDropZone: React.FC = ({ return; } dragManager.handleDrop(dropTargetRef.current, nativeDragEvent(e)); - }, [disabled, onExternalFilesDrop]); + }, [disabled, onExternalFilesDrop, shouldClaimDrag]); useEffect(() => { const target = extendedTargetRef?.current; From 610801ce40c1c459f8f1854b4ce9489ea1abee16 Mon Sep 17 00:00:00 2001 From: EthanCheung Date: Sun, 20 Sep 2026 18:33:57 +0800 Subject: [PATCH 4/4] fix(desktop): surface clipboard tool absence and use the process manager Review follow-up for the WebKitGTK clipboard-image fallback: - get_clipboard_image now distinguishes "clipboard holds no image" from "neither wl-paste nor xclip could be spawned". The latter returns an explicit clipboard_image_unsupported error that the composer surfaces as a localized warning instead of silently doing nothing. - Linux clipboard tool spawns go through the repository process-manager facade (openbitfun_core::util::process_manager::create_command) instead of bare std::process::Command, matching the repo rule for GUI-hosted child processes. --- .../desktop/src/api/clipboard_file_api.rs | 98 ++++++++++++++----- .../src/flow_chat/components/ChatInput.tsx | 7 +- src/web-ui/src/locales/en-US/flow-chat.json | 1 + src/web-ui/src/locales/zh-CN/flow-chat.json | 1 + src/web-ui/src/locales/zh-TW/flow-chat.json | 1 + 5 files changed, 80 insertions(+), 28 deletions(-) diff --git a/src/apps/desktop/src/api/clipboard_file_api.rs b/src/apps/desktop/src/api/clipboard_file_api.rs index 95729f1f2e..f5bfc66d44 100644 --- a/src/apps/desktop/src/api/clipboard_file_api.rs +++ b/src/apps/desktop/src/api/clipboard_file_api.rs @@ -251,10 +251,9 @@ mod macos_clipboard { #[cfg(target_os = "linux")] mod linux_clipboard { use super::parse_uri_list; - use std::process::Command; fn read_xclip_uri_list() -> Option { - let output = Command::new("xclip") + let output = openbitfun_core::util::process_manager::create_command("xclip") .args(["-selection", "clipboard", "-t", "text/uri-list", "-o"]) .output() .ok()?; @@ -267,7 +266,7 @@ mod linux_clipboard { } fn read_wl_paste_uri_list() -> Option { - let output = Command::new("wl-paste") + let output = openbitfun_core::util::process_manager::create_command("wl-paste") .args(["-t", "text/uri-list"]) .output() .ok()?; @@ -353,59 +352,104 @@ pub(crate) fn sniff_image_mime(bytes: &[u8]) -> Option<&'static str> { } } +/// Outcome of a Linux clipboard-image probe. +enum ClipboardImageRead { + /// An image payload, base64-encoded with its sniffed MIME type. + Image(String, String), + /// The reader tools ran; the clipboard simply holds no image payload. + Empty, + /// Neither `wl-paste` nor `xclip` could be spawned. Surfaced to the user + /// instead of silently reporting "no image": a plain-text clipboard and a + /// machine missing the reader tools must stay distinguishable. + ToolsUnavailable(String), +} + /// Reads a clipboard image on Linux. /// /// WebKitGTK delivers paste events with empty `DataTransfer` items, so the /// webview itself can never see a pasted image; reading the Wayland/X11 /// clipboard through the same tools as `get_clipboard_files` is the only -/// delivery path. Other platforms return `None`: their webviews deliver -/// clipboard images to the page directly and never need this fallback. +/// delivery path. #[cfg(target_os = "linux")] -fn read_clipboard_image_internal() -> Option<(String, String)> { +fn read_clipboard_image_internal() -> ClipboardImageRead { use base64::Engine as _; - use std::process::Command; - let read_target = |program: &str, args: &[&str]| -> Option> { - Command::new(program) + /// `Ok(None)` = the tool ran and reported no such payload; + /// `Err` = the tool could not be run at all. + let read_target = |program: &str, args: &[&str]| -> Result>, String> { + let output = openbitfun_core::util::process_manager::create_command(program) .args(args) .output() - .ok() - .filter(|output| output.status.success() && !output.stdout.is_empty()) - .map(|output| output.stdout) + .map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + format!("{program} is not installed") + } else { + format!("failed to spawn {program}: {error}") + } + })?; + Ok(output + .status + .success() + .then_some(output.stdout) + .filter(|stdout| !stdout.is_empty())) }; + let mut runnable_tools = 0usize; + let mut last_tool_error = String::new(); for mime in ["image/png", "image/jpeg"] { - let bytes = read_target("wl-paste", &["-t", mime]) - .or_else(|| read_target("xclip", &["-selection", "clipboard", "-t", mime, "-o"])); - if let Some(bytes) = bytes { - if let Some(sniffed) = sniff_image_mime(&bytes) { - let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes); - return Some((encoded, sniffed.to_string())); + for (program, args) in [ + ("wl-paste", vec!["-t", mime]), + ("xclip", vec!["-selection", "clipboard", "-t", mime, "-o"]), + ] { + match read_target(program, &args) { + Ok(Some(bytes)) => { + if let Some(sniffed) = sniff_image_mime(&bytes) { + let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes); + return ClipboardImageRead::Image(encoded, sniffed.to_string()); + } + runnable_tools += 1; + } + Ok(None) => runnable_tools += 1, + Err(error) => last_tool_error = error, } } } - None + + if runnable_tools > 0 { + return ClipboardImageRead::Empty; + } + ClipboardImageRead::ToolsUnavailable(format!( + "clipboard_image_unsupported: reading a clipboard image needs wl-paste (Wayland) or \ + xclip (X11), but neither is available ({last_tool_error}); install wl-clipboard or \ + xclip and retry" + )) } #[cfg(target_os = "linux")] -fn get_clipboard_image_internal() -> Option<(String, String)> { +fn get_clipboard_image_internal() -> ClipboardImageRead { read_clipboard_image_internal() } #[cfg(not(target_os = "linux"))] -fn get_clipboard_image_internal() -> Option<(String, String)> { - None +fn get_clipboard_image_internal() -> ClipboardImageRead { + // Other platforms deliver clipboard images to the page directly and never + // need the host fallback. + ClipboardImageRead::Empty } #[tauri::command] pub async fn get_clipboard_image() -> Result { - Ok(match get_clipboard_image_internal() { - Some((base64, mime_type)) => ClipboardImageResponse { + match get_clipboard_image_internal() { + ClipboardImageRead::Image(base64, mime_type) => Ok(ClipboardImageResponse { base64: Some(base64), mime_type: Some(mime_type), - }, - None => ClipboardImageResponse::default(), - }) + }), + ClipboardImageRead::Empty => Ok(ClipboardImageResponse::default()), + ClipboardImageRead::ToolsUnavailable(error) => { + log::warn!("Clipboard image read unsupported: {}", error); + Err(error) + } + } } /// Pastes clipboard files between controller-local paths. diff --git a/src/web-ui/src/flow_chat/components/ChatInput.tsx b/src/web-ui/src/flow_chat/components/ChatInput.tsx index 451115de88..6289832200 100644 --- a/src/web-ui/src/flow_chat/components/ChatInput.tsx +++ b/src/web-ui/src/flow_chat/components/ChatInput.tsx @@ -4785,8 +4785,13 @@ export const ChatInput: React.FC = ({ await addClipboardImageFiles(request, [file]); } catch (error) { log.warn('Native clipboard image read failed', { error }); + if (String(error).startsWith('clipboard_image_unsupported:')) { + notificationService.warning(t('input.clipboardImageToolsUnavailable'), { + duration: 4000, + }); + } } - }, [addClipboardImageFiles, isExternalFileIntakeRequestCurrent]); + }, [addClipboardImageFiles, isExternalFileIntakeRequestCurrent, t]); const addExternalPaths = useCallback(async ( request: ExternalFileIntakeRequest, diff --git a/src/web-ui/src/locales/en-US/flow-chat.json b/src/web-ui/src/locales/en-US/flow-chat.json index ed94122d7b..99f83a969a 100644 --- a/src/web-ui/src/locales/en-US/flow-chat.json +++ b/src/web-ui/src/locales/en-US/flow-chat.json @@ -579,6 +579,7 @@ "addImage": "Add image", "maxImagesWarning": "Maximum {{count}} images allowed", "imagePasteFailed": "Image paste failed", + "clipboardImageToolsUnavailable": "The clipboard image could not be read: install wl-clipboard (Wayland) or xclip (X11) and try again.", "externalFiles": { "clipboardPathsUnavailable": "The clipboard reported files, but their local paths could not be read.", "dropPathsUnavailable": "This file drop did not expose local paths. Copy the files and paste them into the composer instead.", diff --git a/src/web-ui/src/locales/zh-CN/flow-chat.json b/src/web-ui/src/locales/zh-CN/flow-chat.json index 3ce8e9f6d6..eb738033fc 100644 --- a/src/web-ui/src/locales/zh-CN/flow-chat.json +++ b/src/web-ui/src/locales/zh-CN/flow-chat.json @@ -579,6 +579,7 @@ "addImage": "添加图片", "maxImagesWarning": "最多只能选择{{count}}张图片", "imagePasteFailed": "图片粘贴失败", + "clipboardImageToolsUnavailable": "无法读取剪贴板图片：请安装 wl-clipboard（Wayland）或 xclip（X11）后重试。", "externalFiles": { "clipboardPathsUnavailable": "剪贴板中包含文件，但无法读取其本机路径。", "dropPathsUnavailable": "此次拖放未提供本机路径，请复制这些文件后粘贴到输入框。", diff --git a/src/web-ui/src/locales/zh-TW/flow-chat.json b/src/web-ui/src/locales/zh-TW/flow-chat.json index cbe3e5f1ce..b8b6ec2a4c 100644 --- a/src/web-ui/src/locales/zh-TW/flow-chat.json +++ b/src/web-ui/src/locales/zh-TW/flow-chat.json @@ -579,6 +579,7 @@ "addImage": "新增圖片", "maxImagesWarning": "最多隻能選擇{{count}}張圖片", "imagePasteFailed": "圖片粘貼失敗", + "clipboardImageToolsUnavailable": "無法讀取剪貼簿圖片：請安裝 wl-clipboard（Wayland）或 xclip（X11）後重試。", "externalFiles": { "clipboardPathsUnavailable": "剪貼簿中包含檔案，但無法讀取其本機路徑。", "dropPathsUnavailable": "此次拖放未提供本機路徑，請複製這些檔案後貼到輸入框。",