Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 2 additions & 21 deletions src/apps/desktop/capabilities/default.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,9 @@
"core:window:allow-unmaximize",
"core:window:allow-unminimize",
"core:window:allow-set-min-size",
"dialog:default",
"dialog:allow-open",
"dialog:allow-save",
"dialog:allow-ask",
"dialog:allow-confirm",
"dialog:allow-message",
"opener:default",
"opener:allow-open-url",
{
"identifier": "opener:allow-open-path",
"allow": [
{ "path": "$APPDATA/**" },
{ "path": "$HOME/**" }
]
},
"opener:allow-open-path",
"opener:allow-reveal-item-in-dir",
"fs:default",
"fs:allow-read-file",
Expand Down Expand Up @@ -96,13 +84,6 @@
"allow": [
{ "path": "$HOME/**" }
]
},
"notification:default",
"notification:allow-notify",
"notification:allow-show",
"notification:allow-request-permission",
"notification:allow-check-permissions",
"notification:allow-permission-state",
"notification:allow-is-permission-granted"
}
]
}
31 changes: 2 additions & 29 deletions src/apps/desktop/src/api/computer_use_api.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
//! Tauri commands for Computer use (permissions + settings deep links).

use crate::api::app_state::AppState;
use crate::computer_use::DesktopComputerUseHost;
use bitfun_core::agentic::tools::computer_use_host::ComputerUseHost;
use bitfun_core::service::config::types::AIConfig;
use serde::{Deserialize, Serialize};
use tauri::State;

Expand All @@ -27,36 +24,12 @@ pub struct ComputerUseOpenSettingsRequest {
pub async fn computer_use_get_status(
state: State<'_, AppState>,
) -> Result<ComputerUseStatusResponse, String> {
let ai: AIConfig = state
.config_service
.get_config(Some("ai"))
.await
.map_err(|e| e.to_string())?;

let host = DesktopComputerUseHost::new();
let snap = host
.permission_snapshot()
.await
.map_err(|e| e.to_string())?;

Ok(ComputerUseStatusResponse {
computer_use_enabled: ai.computer_use_enabled,
accessibility_granted: snap.accessibility_granted,
screen_capture_granted: snap.screen_capture_granted,
platform_note: snap.platform_note,
})
Err("computer_use_get_status error".to_string())
}

#[tauri::command]
pub async fn computer_use_request_permissions() -> Result<(), String> {
let host = DesktopComputerUseHost::new();
host.request_accessibility_permission()
.await
.map_err(|e| e.to_string())?;
host.request_screen_capture_permission()
.await
.map_err(|e| e.to_string())?;
Ok(())
Err("computer_use_request_permissions error".to_string())
}

#[tauri::command]
Expand Down
120 changes: 9 additions & 111 deletions src/apps/desktop/src/computer_use/desktop_host.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,16 @@
use anyhow::anyhow;
use async_trait::async_trait;
use bitfun_core::agentic::tools::computer_use_host::{
clamp_point_crop_half_extent, ActionRecord, AppClickParams, AppInfo, AppSelector,
AppStateSnapshot, AppWaitPredicate, ClickTarget, ComputerScreenshot, ComputerUseDisplayInfo,
clamp_point_crop_half_extent, ActionRecord, AppSelector,
AppStateSnapshot, ClickTarget, ComputerScreenshot, ComputerUseDisplayInfo,
ComputerUseHost, ComputerUseImageContentRect, ComputerUseImageGlobalBounds,
ComputerUseImplicitScreenshotCenter, ComputerUseInteractionScreenshotKind,
ComputerUseInteractionState, ComputerUseLastMutationKind, ComputerUseNavigateQuadrant,
ComputerUseNavigationRect, ComputerUsePermissionSnapshot, ComputerUseScreenshotParams,
ComputerUseScreenshotRefinement, ComputerUseSessionSnapshot, InteractiveActionResult,
InteractiveClickParams, InteractiveScrollParams, InteractiveTypeTextParams, InteractiveView,
InteractiveViewOpts, LoopDetectionResult, OcrRegionNative, ScreenshotCropCenter,
UiElementLocateQuery, UiElementLocateResult, VisualActionResult, VisualClickParams, VisualMark,
VisualMarkView, VisualMarkViewOpts, COMPUTER_USE_QUADRANT_CLICK_READY_MAX_LONG_EDGE,
ComputerUseScreenshotRefinement, ComputerUseSessionSnapshot,
LoopDetectionResult, OcrRegionNative, ScreenshotCropCenter,
UiElementLocateQuery, UiElementLocateResult, VisualMark,
COMPUTER_USE_QUADRANT_CLICK_READY_MAX_LONG_EDGE,
COMPUTER_USE_QUADRANT_EDGE_EXPAND_PX,
};
#[cfg(any(target_os = "macos", target_os = "windows"))]
Expand All @@ -24,11 +23,9 @@ use bitfun_core::agentic::tools::computer_use_optimizer::ComputerUseOptimizer;
use bitfun_core::util::errors::{BitFunError, BitFunResult};
use image::codecs::jpeg::JpegEncoder;
use image::{DynamicImage, Rgb, RgbImage};
use log::{debug, info, warn};
use log::{debug, warn};
use resvg::tiny_skia::{Pixmap, Transform};
use resvg::usvg;
use screenshots::display_info::DisplayInfo;
use screenshots::Screen;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
Expand Down Expand Up @@ -1069,8 +1066,8 @@ end tell"#])
Err(BitFunError::Tool(format!("Unknown mouse button: {}", s)))
}

fn map_key(name: &str) -> BitFunResult<Key> {
Err(BitFunError::Tool(format!("Unknown key name: {}", s)))
fn map_key(name: &str) -> BitFunResult<()> {
Err(BitFunError::Tool(format!("Unknown key name: {}", name)))
}

fn encode_jpeg(rgb: &RgbImage, quality: u8) -> BitFunResult<Vec<u8>> {
Expand Down Expand Up @@ -1629,104 +1626,6 @@ end tell"#])
(0.0, 0.0)
}

/// Resolve a screen capture from cache (if still valid and same screen) or capture fresh.
///
/// Phase 2 fix: when the model has called `desktop.focus_display`, we
/// commit to that screen instead of trusting the mouse pointer. This is
/// the explicit fix for the user's original complaint — on multi-monitor
/// setups the cursor often lives on a different screen than the one the
/// user is reasoning about (e.g. focus is on the laptop screen, mouse
/// is parked on the secondary monitor) and the legacy "screen at mouse
/// pointer" heuristic captured the wrong display.
fn resolve_screenshot_capture(
cached: Option<ScreenshotCacheEntry>,
mouse_x: f64,
mouse_y: f64,
preferred_display_id: Option<u32>,
) -> BitFunResult<(image::RgbaImage, Screen)> {
let mx = mouse_x.round() as i32;
let my = mouse_y.round() as i32;
let target_display_id = preferred_display_id
.or_else(|| Screen::from_point(mx, my).ok().map(|s| s.display_info.id));

if let Some(cache) = cached {
let screen_id_match = Some(cache.screen.display_info.id) == target_display_id;
if cache.capture_time.elapsed() < Duration::from_millis(SCREENSHOT_CACHE_TTL_MS)
&& screen_id_match
{
debug!(
"Using cached screenshot (age: {}ms)",
cache.capture_time.elapsed().as_millis()
);
return Ok((cache.rgba, cache.screen));
}
}

let screen = if let Some(id) = preferred_display_id {
Self::find_screen_by_id(id)
.or_else(|| Screen::from_point(mx, my).ok())
.or_else(|| Screen::from_point(0, 0).ok())
.ok_or_else(|| {
BitFunError::tool("Screen capture init: no display available".to_string())
})?
} else {
Screen::from_point(mx, my)
.or_else(|_| Screen::from_point(0, 0))
.map_err(|e| BitFunError::tool(format!("Screen capture init: {}", e)))?
};
let rgba = screen.capture().map_err(|e| {
BitFunError::tool(format!(
"Screenshot failed (on macOS grant Screen Recording for BitFun): {}",
e
))
})?;
Ok((rgba, screen))
}

/// Find a [`Screen`] by its display id from the host's enumeration.
fn find_screen_by_id(display_id: u32) -> Option<Screen> {
Screen::all()
.ok()
.and_then(|all| all.into_iter().find(|s| s.display_info.id == display_id))
}

/// Snapshot of all attached displays, with `is_active` / `has_pointer`
/// flags resolved relative to `preferred_display_id` and the current
/// mouse position.
fn enumerate_displays(
preferred_display_id: Option<u32>,
mouse_x: f64,
mouse_y: f64,
) -> Vec<ComputerUseDisplayInfo> {
let mx = mouse_x.round() as i32;
let my = mouse_y.round() as i32;
let pointer_display_id = Screen::from_point(mx, my).ok().map(|s| s.display_info.id);
let active_id = preferred_display_id.or(pointer_display_id);

let screens = match Screen::all() {
Ok(v) => v,
Err(_) => return vec![],
};
screens
.into_iter()
.map(|s| {
let d = s.display_info;
ComputerUseDisplayInfo {
display_id: d.id,
is_primary: d.is_primary,
is_active: Some(d.id) == active_id,
has_pointer: Some(d.id) == pointer_display_id,
origin_x: d.x,
origin_y: d.y,
width_logical: d.width,
height_logical: d.height,
scale_factor: d.scale_factor,
foreground_app: None,
}
})
.collect()
}

fn chord_includes_return_or_enter(keys: &[String]) -> bool {
keys.iter()
.any(|s| matches!(s.to_lowercase().as_str(), "return" | "enter" | "kp_enter"))
Expand Down Expand Up @@ -2239,7 +2138,6 @@ impl ComputerUseHost for DesktopComputerUseHost {
};

let (mouse_x, mouse_y) = Self::current_mouse_position();
let displays = Self::enumerate_displays(preferred_display_id, mouse_x, mouse_y);
let active_display_id = None;

let (click_ready, screenshot_kind, mut recommended_next_action) =
Expand Down
5 changes: 0 additions & 5 deletions src/apps/desktop/src/computer_use/screen_ocr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,6 @@ pub fn find_text_matches(
return windows_backend::find_text_matches(shot, &query);
}

#[cfg(target_os = "linux")]
{
return linux_backend::find_text_matches(shot, &query);
}

#[allow(unreachable_code)]
Err(BitFunError::tool(
"move_to_text OCR is not supported on this platform.".to_string(),
Expand Down
62 changes: 3 additions & 59 deletions src/apps/desktop/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
#![allow(non_snake_case)]
//! BitFun Desktop - Tauri-based desktop application with TransportAdapter architecture

// pub mod api;
pub mod computer_use;
pub mod api;
pub mod logging;
pub mod macos_menubar;
pub mod theme;

use bitfun_core::agentic::tools::computer_use_capability::set_computer_use_desktop_available;
use bitfun_core::agentic::tools::computer_use_host::ComputerUseHostRef;
use bitfun_core::infrastructure::ai::AIClientFactory;
use bitfun_core::infrastructure::{get_path_manager_arc, try_get_path_manager_arc};
use bitfun_core::service::workspace::get_global_workspace_service;
use bitfun_core::util::{elapsed_ms, TimingCollector};
use bitfun_transport::{TauriTransportAdapter, TransportAdapter};
use serde::Deserialize;
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
Expand Down Expand Up @@ -74,7 +71,7 @@ pub struct OhosPlatform {
pub feature: Vec<String>,
}

impl Defaut for OhosPlatform {
impl Default for OhosPlatform {
fn default() -> Self {
Self {
version: "6.0.0".to_string(),
Expand All @@ -93,7 +90,7 @@ impl Defaut for OhosPlatform {
/// Tauri application entry point
#[cfg_attr(mobile, tauri::mobile_entry_point)]
pub fn run() {
let runtime = tokio::runtime::Builder::new_multi_thread
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(16)
.enable_all()
.build()
Expand Down Expand Up @@ -186,19 +183,10 @@ pub async fn _run() {

let path_manager = get_path_manager_arc();

// setup_panic_hook();

let run_result = tauri::Builder::default()
.plugin(logging::build_log_plugin(log_targets))
.plugin(tauri_plugin_opener::init())
// .plugin(tauri_plugin_dialog::init())
.plugin(tauri_plugin_fs::init())
// .plugin(
// tauri_plugin_autostart::Builder::new()
// .app_name("BitFun")
// .build(),
// )
// .plugin(tauri_plugin_notification::init())
.manage(app_state)
.manage(coordinator_state)
.manage(scheduler_state)
Expand Down Expand Up @@ -381,7 +369,6 @@ pub async fn _run() {
api::agentic_api::set_subagent_timeout,
api::agentic_api::delete_session,
api::agentic_api::restore_session,
webdriver_bridge_result,
api::agentic_api::list_sessions,
api::agentic_api::confirm_tool_execution,
api::agentic_api::reject_tool_execution,
Expand Down Expand Up @@ -925,49 +912,6 @@ fn init_mcp_servers(app_handle: tauri::AppHandle) {
});
}

fn setup_panic_hook() {
std::panic::set_hook(Box::new(move |panic_info| {
let location = panic_info
.location()
.map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
.unwrap_or_else(|| "unknown location".to_string());

let message = panic_info
.payload()
.downcast_ref::<&str>()
.copied()
.or_else(|| {
panic_info
.payload()
.downcast_ref::<String>()
.map(String::as_str)
})
.unwrap_or("unknown panic message");

log::error!("Application panic at {}: {}", location, message);

// Known wry bug: WKWebView.URL() returns nil after navigating to an
// invalid address, causing url_from_webview to panic on unwrap().
// This is non-fatal — the webview is still alive — so we log and
// continue instead of killing the process.
// See: https://github.com/tauri-apps/wry/pull/1554
if location.contains("wry") && location.contains("wkwebview") {
log::warn!("Suppressed non-fatal wry/wkwebview panic, application continues");
return;
}

if message.contains("WSAStartup") || message.contains("10093") || message.contains("hyper")
{
log::error!("Network-related crash detected, possible solutions:");
log::error!(" 1) Restart the application");
log::error!(" 2) Check Windows network service status");
log::error!(" 3) Run as administrator");
}

std::process::exit(1);
}));
}

fn start_event_loop_with_transport(
event_queue: Arc<bitfun_core::agentic::events::EventQueue>,
event_router: Arc<bitfun_core::agentic::events::EventRouter>,
Expand Down
Loading
Loading