33use anyhow:: anyhow;
44use async_trait:: async_trait;
55use bitfun_core:: agentic:: tools:: computer_use_host:: {
6- clamp_point_crop_half_extent, ActionRecord , AppClickParams , AppInfo , AppSelector ,
7- AppStateSnapshot , AppWaitPredicate , ClickTarget , ComputerScreenshot , ComputerUseDisplayInfo ,
6+ clamp_point_crop_half_extent, ActionRecord , AppSelector ,
7+ AppStateSnapshot , ClickTarget , ComputerScreenshot , ComputerUseDisplayInfo ,
88 ComputerUseHost , ComputerUseImageContentRect , ComputerUseImageGlobalBounds ,
99 ComputerUseImplicitScreenshotCenter , ComputerUseInteractionScreenshotKind ,
1010 ComputerUseInteractionState , ComputerUseLastMutationKind , ComputerUseNavigateQuadrant ,
1111 ComputerUseNavigationRect , ComputerUsePermissionSnapshot , ComputerUseScreenshotParams ,
12- ComputerUseScreenshotRefinement , ComputerUseSessionSnapshot , InteractiveActionResult ,
13- InteractiveClickParams , InteractiveScrollParams , InteractiveTypeTextParams , InteractiveView ,
14- InteractiveViewOpts , LoopDetectionResult , OcrRegionNative , ScreenshotCropCenter ,
15- UiElementLocateQuery , UiElementLocateResult , VisualActionResult , VisualClickParams , VisualMark ,
16- VisualMarkView , VisualMarkViewOpts , COMPUTER_USE_QUADRANT_CLICK_READY_MAX_LONG_EDGE ,
12+ ComputerUseScreenshotRefinement , ComputerUseSessionSnapshot ,
13+ LoopDetectionResult , OcrRegionNative , ScreenshotCropCenter ,
14+ UiElementLocateQuery , UiElementLocateResult , VisualMark ,
15+ COMPUTER_USE_QUADRANT_CLICK_READY_MAX_LONG_EDGE ,
1716 COMPUTER_USE_QUADRANT_EDGE_EXPAND_PX ,
1817} ;
1918#[ cfg( any( target_os = "macos" , target_os = "windows" ) ) ]
@@ -24,11 +23,9 @@ use bitfun_core::agentic::tools::computer_use_optimizer::ComputerUseOptimizer;
2423use bitfun_core:: util:: errors:: { BitFunError , BitFunResult } ;
2524use image:: codecs:: jpeg:: JpegEncoder ;
2625use image:: { DynamicImage , Rgb , RgbImage } ;
27- use log:: { debug, info , warn} ;
26+ use log:: { debug, warn} ;
2827use resvg:: tiny_skia:: { Pixmap , Transform } ;
2928use resvg:: usvg;
30- use screenshots:: display_info:: DisplayInfo ;
31- use screenshots:: Screen ;
3229use std:: sync:: atomic:: { AtomicU64 , Ordering } ;
3330use std:: sync:: { Mutex , OnceLock } ;
3431use std:: time:: { Duration , Instant } ;
@@ -1069,8 +1066,8 @@ end tell"#])
10691066 Err ( BitFunError :: Tool ( format ! ( "Unknown mouse button: {}" , s) ) )
10701067 }
10711068
1072- fn map_key ( name : & str ) -> BitFunResult < Key > {
1073- Err ( BitFunError :: Tool ( format ! ( "Unknown key name: {}" , s ) ) )
1069+ fn map_key ( name : & str ) -> BitFunResult < ( ) > {
1070+ Err ( BitFunError :: Tool ( format ! ( "Unknown key name: {}" , name ) ) )
10741071 }
10751072
10761073 fn encode_jpeg ( rgb : & RgbImage , quality : u8 ) -> BitFunResult < Vec < u8 > > {
@@ -1629,104 +1626,6 @@ end tell"#])
16291626 ( 0.0 , 0.0 )
16301627 }
16311628
1632- /// Resolve a screen capture from cache (if still valid and same screen) or capture fresh.
1633- ///
1634- /// Phase 2 fix: when the model has called `desktop.focus_display`, we
1635- /// commit to that screen instead of trusting the mouse pointer. This is
1636- /// the explicit fix for the user's original complaint — on multi-monitor
1637- /// setups the cursor often lives on a different screen than the one the
1638- /// user is reasoning about (e.g. focus is on the laptop screen, mouse
1639- /// is parked on the secondary monitor) and the legacy "screen at mouse
1640- /// pointer" heuristic captured the wrong display.
1641- fn resolve_screenshot_capture (
1642- cached : Option < ScreenshotCacheEntry > ,
1643- mouse_x : f64 ,
1644- mouse_y : f64 ,
1645- preferred_display_id : Option < u32 > ,
1646- ) -> BitFunResult < ( image:: RgbaImage , Screen ) > {
1647- let mx = mouse_x. round ( ) as i32 ;
1648- let my = mouse_y. round ( ) as i32 ;
1649- let target_display_id = preferred_display_id
1650- . or_else ( || Screen :: from_point ( mx, my) . ok ( ) . map ( |s| s. display_info . id ) ) ;
1651-
1652- if let Some ( cache) = cached {
1653- let screen_id_match = Some ( cache. screen . display_info . id ) == target_display_id;
1654- if cache. capture_time . elapsed ( ) < Duration :: from_millis ( SCREENSHOT_CACHE_TTL_MS )
1655- && screen_id_match
1656- {
1657- debug ! (
1658- "Using cached screenshot (age: {}ms)" ,
1659- cache. capture_time. elapsed( ) . as_millis( )
1660- ) ;
1661- return Ok ( ( cache. rgba , cache. screen ) ) ;
1662- }
1663- }
1664-
1665- let screen = if let Some ( id) = preferred_display_id {
1666- Self :: find_screen_by_id ( id)
1667- . or_else ( || Screen :: from_point ( mx, my) . ok ( ) )
1668- . or_else ( || Screen :: from_point ( 0 , 0 ) . ok ( ) )
1669- . ok_or_else ( || {
1670- BitFunError :: tool ( "Screen capture init: no display available" . to_string ( ) )
1671- } ) ?
1672- } else {
1673- Screen :: from_point ( mx, my)
1674- . or_else ( |_| Screen :: from_point ( 0 , 0 ) )
1675- . map_err ( |e| BitFunError :: tool ( format ! ( "Screen capture init: {}" , e) ) ) ?
1676- } ;
1677- let rgba = screen. capture ( ) . map_err ( |e| {
1678- BitFunError :: tool ( format ! (
1679- "Screenshot failed (on macOS grant Screen Recording for BitFun): {}" ,
1680- e
1681- ) )
1682- } ) ?;
1683- Ok ( ( rgba, screen) )
1684- }
1685-
1686- /// Find a [`Screen`] by its display id from the host's enumeration.
1687- fn find_screen_by_id ( display_id : u32 ) -> Option < Screen > {
1688- Screen :: all ( )
1689- . ok ( )
1690- . and_then ( |all| all. into_iter ( ) . find ( |s| s. display_info . id == display_id) )
1691- }
1692-
1693- /// Snapshot of all attached displays, with `is_active` / `has_pointer`
1694- /// flags resolved relative to `preferred_display_id` and the current
1695- /// mouse position.
1696- fn enumerate_displays (
1697- preferred_display_id : Option < u32 > ,
1698- mouse_x : f64 ,
1699- mouse_y : f64 ,
1700- ) -> Vec < ComputerUseDisplayInfo > {
1701- let mx = mouse_x. round ( ) as i32 ;
1702- let my = mouse_y. round ( ) as i32 ;
1703- let pointer_display_id = Screen :: from_point ( mx, my) . ok ( ) . map ( |s| s. display_info . id ) ;
1704- let active_id = preferred_display_id. or ( pointer_display_id) ;
1705-
1706- let screens = match Screen :: all ( ) {
1707- Ok ( v) => v,
1708- Err ( _) => return vec ! [ ] ,
1709- } ;
1710- screens
1711- . into_iter ( )
1712- . map ( |s| {
1713- let d = s. display_info ;
1714- ComputerUseDisplayInfo {
1715- display_id : d. id ,
1716- is_primary : d. is_primary ,
1717- is_active : Some ( d. id ) == active_id,
1718- has_pointer : Some ( d. id ) == pointer_display_id,
1719- origin_x : d. x ,
1720- origin_y : d. y ,
1721- width_logical : d. width ,
1722- height_logical : d. height ,
1723- scale_factor : d. scale_factor ,
1724- foreground_app : None ,
1725- }
1726- } )
1727- . collect ( )
1728- }
1729-
17301629 fn chord_includes_return_or_enter ( keys : & [ String ] ) -> bool {
17311630 keys. iter ( )
17321631 . any ( |s| matches ! ( s. to_lowercase( ) . as_str( ) , "return" | "enter" | "kp_enter" ) )
@@ -2239,7 +2138,6 @@ impl ComputerUseHost for DesktopComputerUseHost {
22392138 } ;
22402139
22412140 let ( mouse_x, mouse_y) = Self :: current_mouse_position ( ) ;
2242- let displays = Self :: enumerate_displays ( preferred_display_id, mouse_x, mouse_y) ;
22432141 let active_display_id = None ;
22442142
22452143 let ( click_ready, screenshot_kind, mut recommended_next_action) =
0 commit comments