diff --git a/src/apps/desktop/AGENTS.md b/src/apps/desktop/AGENTS.md index d6514b4364..1e5e36c0ab 100644 --- a/src/apps/desktop/AGENTS.md +++ b/src/apps/desktop/AGENTS.md @@ -269,6 +269,12 @@ For native sidebar material and appearance bootstrap, run and `pnpm --dir src/web-ui run test:run src/infrastructure/appearance/adapters/ThemeTokenAppearanceAdapter.test.ts`. For embedded browser preview encoding and target correlation, run `cargo test -p openbitfun-desktop --lib api::browser_api::tests`. +For conversation-owned browser lifetime, build Desktop and Web UI, then run +`node tests/e2e/scripts/run-browser-session-isolation.mjs`. Its isolated native +fixture delays A's model response until B is selected, exercises background +opening and concurrent reads through real Agent tools, and verifies native +visibility and retained document state across session switches. It scripts only +model HTTP, not browser or session behavior; it does not establish remote coverage. After browser command registration changes, also run `cargo test -p openbitfun-desktop --lib remote_workspace_policy`. After changing updater command registration, also run diff --git a/src/apps/desktop/src/api/browser_api.rs b/src/apps/desktop/src/api/browser_api.rs index a0050a650e..c3765774da 100644 --- a/src/apps/desktop/src/api/browser_api.rs +++ b/src/apps/desktop/src/api/browser_api.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::{Mutex, OnceLock}; use tauri::Manager; +use tokio::sync::watch; const VIDEO_DECODER_MODE_ENV: &str = "OPENBITFUN_BROWSER_VIDEO_DECODER_MODE"; @@ -103,6 +104,8 @@ pub struct WebviewCreateRequest { pub height: f64, #[serde(default)] pub open_request_id: Option, + #[serde(default)] + pub conversation_id: Option, } #[derive(Debug, Deserialize)] @@ -121,6 +124,33 @@ struct BrowserTargetRecord { url: String, title: Option, open_request_id: Option, + conversation_id: Option, + navigation: Option, +} + +#[derive(Clone, Debug)] +struct BrowserNavigation { + started: bool, + completed: watch::Sender, +} + +impl BrowserNavigation { + fn new() -> Self { + Self { + started: false, + completed: watch::channel(false).0, + } + } + + fn page_load(&mut self, event: &str) { + if event == "started" { + self.started = true; + } else if event == "finished" && self.started { + // Redirects may finish at a different URL. A finish from the old + // document before this navigation starts must not release readers. + self.completed.send_replace(true); + } + } } #[derive(Default)] @@ -130,10 +160,9 @@ struct BrowserTargetRegistry { } impl BrowserTargetRegistry { - fn active_label_for_open_request(&self, request_id: &str) -> Option { + fn label_for_open_request(&self, request_id: &str) -> Option { self.records.iter().find_map(|(label, record)| { - (record.active && record.open_request_id.as_deref() == Some(request_id)) - .then(|| label.clone()) + (record.open_request_id.as_deref() == Some(request_id)).then(|| label.clone()) }) } } @@ -157,12 +186,18 @@ pub(crate) fn validate_browser_label(label: &str) -> Result<(), String> { } } -fn register_browser_target(label: &str, url: &str, open_request_id: Option) { +fn register_browser_target( + label: &str, + url: &str, + open_request_id: Option, + conversation_id: Option, +) { lock_browser_targets().records.insert( label.to_string(), BrowserTargetRecord { url: url.to_string(), open_request_id, + conversation_id, ..BrowserTargetRecord::default() }, ); @@ -180,16 +215,41 @@ fn associate_browser_open_request(label: &str, open_request_id: Option<&str>) { } } -pub(crate) fn update_browser_target_url(label: &str, url: &str) { +pub(crate) fn update_browser_target_page_load(label: &str, event: &str, url: &str) { if let Some(record) = lock_browser_targets().records.get_mut(label) { record.url = url.to_string(); // A navigation invalidates the cached document title. The async // automation host refreshes it from the page before target discovery // is returned to ControlHub. record.title = None; + if let Some(navigation) = &mut record.navigation { + navigation.page_load(event); + } } } +/// Native navigation can leave the previous completed document readable while +/// the replacement response is in flight. Do not expose that document to tools. +pub(crate) async fn wait_for_browser_navigation(label: &str) -> Result<(), String> { + let pending = lock_browser_targets() + .records + .get(label) + .and_then(|record| record.navigation.as_ref()) + .map(|navigation| navigation.completed.subscribe()); + if let Some(mut pending) = pending { + tokio::time::timeout( + std::time::Duration::from_secs(15), + pending.wait_for(|done| *done), + ) + .await + .map_err(|_| { + format!("Navigation on {label} is still loading; wait before reading this page") + })? + .map_err(|_| format!("Navigation on {label} was superseded or its page was closed"))?; + } + Ok(()) +} + pub(crate) fn update_browser_target_metadata(label: &str, url: &str, title: &str) { if let Some(record) = lock_browser_targets().records.get_mut(label) { record.url = url.to_string(); @@ -202,16 +262,24 @@ pub(crate) fn unregister_browser_target(label: &str) { } pub(crate) fn list_browser_targets(app: &tauri::AppHandle) -> Vec { - let mut registry = lock_browser_targets(); - registry - .records - .retain(|label, _| app.get_webview(label).is_some()); - let mut targets = registry - .records + let (records, views) = { + let mut registry = lock_browser_targets(); + // This only snapshots local handles; it performs no native UI calls. + // Take it while registration is excluded, so a newly registered page + // cannot be pruned against an older handle snapshot. + let views = app.webviews(); + registry + .records + .retain(|label, _| views.contains_key(label)); + (registry.records.clone(), views) + }; + // Native URL reads may dispatch to the UI thread. Page-load callbacks on + // that thread update this registry, so never hold its mutex across native IO. + let mut targets = records .iter() .map(|(label, record)| { - let url = app - .get_webview(label) + let url = views + .get(label) .and_then(|webview| { std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| webview.url())) .ok() @@ -227,6 +295,7 @@ pub(crate) fn list_browser_targets(app: &tauri::AppHandle) -> Vec Vec Option { let label = { let registry = lock_browser_targets(); - registry.active_label_for_open_request(request_id) + registry.label_for_open_request(request_id) }?; list_browser_targets(app) .into_iter() - .find(|target| target.id == label && target.active) + .find(|target| target.id == label) } fn validate_webview_bounds(x: f64, y: f64, width: f64, height: f64) -> Result<(), String> { @@ -288,9 +357,21 @@ pub async fn browser_webview_create( let mut builder = tauri::webview::WebviewBuilder::new(request.label, tauri::WebviewUrl::External(url)) .initialization_script(video_decoder_compatibility_script()) + // Navigation behavior must also exist before a background session + // has ever mounted a frontend panel, and after every document load. + .initialization_script(include_str!( + "../../../../shared/browser/blankTargetIntercept.js" + )) .transparent(false) .background_color(tauri::window::Color(0, 0, 0, 255)); + // Session automation must continue while its presentation is hidden. + // WebKit otherwise suspends hidden documents after a period of inactivity. + if request.conversation_id.is_some() { + builder = builder + .background_throttling(tauri::utils::config::BackgroundThrottlingPolicy::Disabled); + } + #[cfg(any(debug_assertions, feature = "devtools"))] { builder = builder.devtools(true); @@ -308,7 +389,12 @@ pub async fn browser_webview_create( .hide() .map_err(|e| format!("failed to hide browser webview before positioning: {e}"))?; let target_url = webview.url().map(|url| url.to_string()).unwrap_or_default(); - register_browser_target(webview.label(), &target_url, request.open_request_id); + register_browser_target( + webview.label(), + &target_url, + request.open_request_id, + request.conversation_id, + ); Ok(()) } @@ -321,13 +407,19 @@ pub async fn browser_webview_set_agent_target_state( ) -> Result<(), String> { validate_browser_label(&request.label)?; let mut registry = lock_browser_targets(); + if !registry.records.contains_key(&request.label) { + return Err(format!("Webview not found: {}", request.label)); + } if request.active { registry.next_seq = registry.next_seq.saturating_add(1); let next_seq = registry.next_seq; for record in registry.records.values_mut() { record.active = false; } - let record = registry.records.entry(request.label).or_default(); + let record = registry + .records + .get_mut(&request.label) + .expect("registered target"); record.active = true; record.last_active_seq = next_seq; if let Some(request_id) = request @@ -335,6 +427,9 @@ pub async fn browser_webview_set_agent_target_state( .as_deref() .map(str::trim) .filter(|request_id| !request_id.is_empty()) + // Session requests are registered by the host before presentation. + // A late callback from the old panel must not overwrite a newer open. + .filter(|_| record.conversation_id.is_none()) { record.open_request_id = Some(request_id.to_string()); } @@ -369,9 +464,21 @@ pub async fn browser_webview_navigate( scheme => return Err(format!("unsupported protocol: {scheme}")), } - find_browser_webview(&app, &request.label)? - .navigate(url) - .map_err(|e| format!("navigate failed: {e}"))?; + let webview = find_browser_webview(&app, &request.label)?; + let same_document = webview.url().ok().is_some_and(|mut current| { + let fragment_changed = current.fragment() != url.fragment(); + current.set_fragment(url.fragment()); + fragment_changed && current == url + }); + if let Some(record) = lock_browser_targets().records.get_mut(&request.label) { + record.navigation = (!same_document).then(BrowserNavigation::new); + } + if let Err(error) = webview.navigate(url) { + if let Some(record) = lock_browser_targets().records.get_mut(&request.label) { + record.navigation = None; + } + return Err(format!("navigate failed: {error}")); + } // A failed URL parse or native navigation must never acknowledge an open // request merely because an older target is already active. associate_browser_open_request(&request.label, request.open_request_id.as_deref()); @@ -495,6 +602,51 @@ pub async fn browser_get_url( mod tests { use super::*; + #[test] + fn replacement_navigation_does_not_accept_the_previous_document_finish() { + let mut navigation = BrowserNavigation::new(); + navigation.page_load("finished"); + assert!(!*navigation.completed.borrow()); + navigation.page_load("started"); + navigation.page_load("started"); // A redirect or a superseding navigation. + navigation.page_load("finished"); + assert!(*navigation.completed.borrow()); + } + + #[tokio::test] + async fn presentation_cannot_overwrite_a_session_hosts_newer_open_request() { + let label = "embedded-browser-panel-view-request-owner-test"; + register_browser_target( + label, + "https://example.com", + Some("new-request".into()), + Some("owner".into()), + ); + browser_webview_set_agent_target_state(BrowserAgentTargetStateRequest { + label: label.into(), + active: true, + open_request_id: Some("old-request".into()), + }) + .await + .unwrap(); + assert_eq!( + lock_browser_targets().label_for_open_request("new-request"), + Some(label.into()) + ); + unregister_browser_target(label); + } + + #[test] + fn legacy_create_payload_keeps_the_unscoped_browser_contract() { + let request: WebviewCreateRequest = serde_json::from_value(serde_json::json!({ + "label": "embedded-browser-panel-view-old", "url": "https://example.com", + "x": 0, "y": 0, "width": 800, "height": 600, + })) + .unwrap(); + assert!(request.conversation_id.is_none()); + assert!(request.open_request_id.is_none()); + } + #[test] fn preview_is_a_bounded_jpeg_and_rejects_invalid_data() { use base64::Engine as _; @@ -513,7 +665,7 @@ mod tests { } #[test] - fn open_request_correlation_requires_the_exact_active_target() { + fn open_request_correlation_accepts_the_exact_background_target() { let mut registry = BrowserTargetRegistry::default(); registry.records.insert( "embedded-browser-panel-view-old".to_string(), @@ -533,22 +685,13 @@ mod tests { ); assert_eq!( - registry - .active_label_for_open_request("old-request") - .as_deref(), + registry.label_for_open_request("old-request").as_deref(), Some("embedded-browser-panel-view-old") ); - assert_eq!(registry.active_label_for_open_request("new-request"), None); + assert_eq!(registry.label_for_open_request("missing-request"), None); - registry - .records - .get_mut("embedded-browser-panel-view-new") - .expect("new target") - .active = true; assert_eq!( - registry - .active_label_for_open_request("new-request") - .as_deref(), + registry.label_for_open_request("new-request").as_deref(), Some("embedded-browser-panel-view-new") ); } diff --git a/src/apps/desktop/src/builtin_browser_host.rs b/src/apps/desktop/src/builtin_browser_host.rs index 8fa2069a63..b55e5e706a 100644 --- a/src/apps/desktop/src/builtin_browser_host.rs +++ b/src/apps/desktop/src/builtin_browser_host.rs @@ -1,9 +1,8 @@ //! Desktop adapter for the built-in browser automation host port. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use std::time::Duration; use async_trait::async_trait; use openbitfun_core::agentic::tools::browser_control::builtin_browser::{ @@ -12,17 +11,17 @@ use openbitfun_core::agentic::tools::browser_control::builtin_browser::{ }; use openbitfun_core::infrastructure::events::{emit_global_event, BackendEvent}; use serde_json::{json, Value}; -use tauri::AppHandle; -use tokio::sync::RwLock; +use tauri::{AppHandle, Manager}; +use tokio::sync::{Mutex, RwLock}; use crate::api::browser_api; const OPEN_BUILT_IN_BROWSER_EVENT: &str = "agentic://open-built-in-browser"; -const OPEN_BROWSER_READY_TIMEOUT: Duration = Duration::from_secs(20); static NEXT_OPEN_REQUEST_ID: AtomicU64 = AtomicU64::new(1); struct DesktopBuiltInBrowserHost { app: AppHandle, + open_lock: Mutex<()>, drivers: RwLock>>, } @@ -30,6 +29,7 @@ impl DesktopBuiltInBrowserHost { fn new(app: AppHandle) -> Self { Self { app, + open_lock: Mutex::new(()), drivers: RwLock::new(HashMap::new()), } } @@ -42,69 +42,29 @@ impl DesktopBuiltInBrowserHost { if let Some(driver) = self.drivers.read().await.get(target_id).cloned() { return Ok(driver); } + let mut drivers = self.drivers.write().await; + if let Some(driver) = drivers.get(target_id).cloned() { + return Ok(driver); + } let driver = Arc::new( openbitfun_webdriver::EmbeddedWebviewAutomation::attach(self.app.clone(), target_id) .await?, ); - self.drivers - .write() - .await - .insert(target_id.to_string(), driver.clone()); + drivers.insert(target_id.to_string(), driver.clone()); Ok(driver) } -} - -#[async_trait] -impl BuiltInBrowserHost for DesktopBuiltInBrowserHost { - async fn open( + async fn discover_targets( &self, - request: BuiltInBrowserOpenRequest, - ) -> Result { - let request_id = format!( - "builtin-browser-open-{}-{}", - std::process::id(), - NEXT_OPEN_REQUEST_ID.fetch_add(1, Ordering::Relaxed) - ); - emit_global_event(BackendEvent::Custom { - event_name: OPEN_BUILT_IN_BROWSER_EVENT.to_string(), - payload: json!({ - "requestId": request_id.clone(), - "url": request.url, - "title": request.title, - "replaceExisting": request.replace_existing, - }), - }) - .await - .map_err(|error| format!("failed to present the built-in browser: {error}"))?; - - let deadline = tokio::time::Instant::now() + OPEN_BROWSER_READY_TIMEOUT; - loop { - if let Some(target) = - browser_api::browser_target_for_open_request(&self.app, &request_id) - { - return Ok(target); - } - if tokio::time::Instant::now() >= deadline { - return Err( - "the browser panel was requested, but its correlated native WebView did not become ready for Agent control" - .to_string(), - ); - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - } - - async fn list_targets(&self) -> Result, String> { + conversation_id: Option<&str>, + ) -> Result, String> { let mut targets = browser_api::list_browser_targets(&self.app); - let live_ids = targets - .iter() - .map(|target| target.id.clone()) - .collect::>(); let stale_drivers = { let mut drivers = self.drivers.write().await; let stale_ids = drivers .keys() - .filter(|target_id| !live_ids.contains(*target_id)) + // A driver may have attached after the inventory snapshot. + // Check current handles before dropping its automation session. + .filter(|target_id| self.app.get_webview(target_id).is_none()) .cloned() .collect::>(); stale_ids @@ -115,6 +75,11 @@ impl BuiltInBrowserHost for DesktopBuiltInBrowserHost { for driver in stale_drivers { driver.detach().await; } + // Do not evaluate other sessions while discovering this session: a slow + // background document must not block an unrelated conversation. + targets.retain(|target| { + conversation_id.is_none() || target.conversation_id.as_deref() == conversation_id + }); for target in &mut targets { if target.title.is_empty() { if let Ok(driver) = self.driver(&target.id).await { @@ -151,8 +116,95 @@ impl BuiltInBrowserHost for DesktopBuiltInBrowserHost { } Ok(targets) } +} + +#[async_trait] +impl BuiltInBrowserHost for DesktopBuiltInBrowserHost { + async fn open( + &self, + request: BuiltInBrowserOpenRequest, + ) -> Result { + let request_id = format!( + "builtin-browser-open-{}-{}", + std::process::id(), + NEXT_OPEN_REQUEST_ID.fetch_add(1, Ordering::Relaxed) + ); + // Native page lifetime belongs to the session, not to a mounted panel. + // Creating it hidden also works when another conversation is on screen. + let _open_guard = self.open_lock.lock().await; + let existing = request + .replace_existing + .then(|| { + browser_api::list_browser_targets(&self.app) + .into_iter() + .find(|target| target.conversation_id == request.conversation_id) + }) + .flatten(); + let target_id = if let Some(target) = existing { + browser_api::browser_webview_navigate( + self.app.clone(), + browser_api::WebviewNavigateRequest { + label: target.id.clone(), + url: request.url.clone(), + open_request_id: Some(request_id.clone()), + }, + ) + .await?; + target.id + } else { + let label = format!("embedded-browser-panel-view-{request_id}"); + browser_api::browser_webview_create( + self.app.clone(), + browser_api::WebviewCreateRequest { + label: label.clone(), + url: request.url.clone(), + // Stay outside the window even during add_child -> hide. The + // presenting panel supplies its bounds only when it is active. + x: -10000.0, + y: -10000.0, + width: 1024.0, + height: 768.0, + open_request_id: Some(request_id.clone()), + conversation_id: request.conversation_id.clone(), + }, + ) + .await?; + label + }; + browser_api::update_browser_target_metadata(&target_id, &request.url, &request.title); + let target = browser_api::browser_target_for_open_request(&self.app, &request_id) + .ok_or_else(|| "the session browser WebView was closed while opening".to_string())?; + emit_global_event(BackendEvent::Custom { + event_name: OPEN_BUILT_IN_BROWSER_EVENT.to_string(), + payload: json!({ + "requestId": request_id, + "sessionId": request.conversation_id, + "targetId": target.id, + "url": request.url, + "title": request.title, + "replaceExisting": request.replace_existing, + }), + }) + .await + .map_err(|error| format!("failed to present the built-in browser: {error}"))?; + Ok(target) + } + + async fn list_targets(&self) -> Result, String> { + self.discover_targets(None).await + } + + async fn list_targets_for_session( + &self, + conversation_id: Option<&str>, + ) -> Result, String> { + self.discover_targets(conversation_id).await + } async fn execute(&self, command: BuiltInBrowserCommand) -> Result { + if command.method == "Runtime.evaluate" { + browser_api::wait_for_browser_navigation(&command.target_id).await?; + } let driver = self.driver(&command.target_id).await?; let result = driver.execute(&command.method, command.params).await; if command.method == "Page.close" && result.is_ok() { diff --git a/src/apps/desktop/src/lib.rs b/src/apps/desktop/src/lib.rs index 7b0546e52b..265bcba307 100644 --- a/src/apps/desktop/src/lib.rs +++ b/src/apps/desktop/src/lib.rs @@ -822,7 +822,7 @@ pub async fn run() { tauri::webview::PageLoadEvent::Finished => "finished", }; let url = payload.url().to_string(); - api::browser_api::update_browser_target_url(label, &url); + api::browser_api::update_browser_target_page_load(label, event, &url); openbitfun_core::agentic::tools::browser_control::builtin_browser::publish_builtin_browser_page_load( label, event, diff --git a/src/crates/adapters/webdriver/AGENTS.md b/src/crates/adapters/webdriver/AGENTS.md index 92c6bde8e8..72e624200f 100644 --- a/src/crates/adapters/webdriver/AGENTS.md +++ b/src/crates/adapters/webdriver/AGENTS.md @@ -27,4 +27,8 @@ focused checks must not depend on feature unification from the Desktop host. For reveal/focus changes, build Desktop and use the real packaged UI regression documented in `tests/e2e/AGENTS.md` (`run-interaction-scroll.mjs`). +For script result serialization and background browser automation, build Desktop +and run `node tests/e2e/scripts/run-browser-session-isolation.mjs`. It checks +numeric-length object fields, DOM collection references and actual article reads. + For documentation-only changes, run `git diff --check`. diff --git a/src/crates/adapters/webdriver/src/runtime/script/core/store.rs b/src/crates/adapters/webdriver/src/runtime/script/core/store.rs index 8c477dbf9d..f2409b633b 100644 --- a/src/crates/adapters/webdriver/src/runtime/script/core/store.rs +++ b/src/crates/adapters/webdriver/src/runtime/script/core/store.rs @@ -51,7 +51,11 @@ pub(super) fn script() -> &'static str { if (isElementLike(value)) { return storeElement(value); } - if (value && typeof value === "object" && typeof value.length === "number" && typeof value !== "string") { + // A page result may have a numeric `length` field (for example article + // text metadata). Only native collections have array serialization. + const collectionTag = Object.prototype.toString.call(value); + if (Array.isArray(value) || ["[object NodeList]", "[object HTMLCollection]", "[object HTMLAllCollection]", "[object HTMLFormControlsCollection]", "[object HTMLOptionsCollection]"].includes(collectionTag) + || (ArrayBuffer.isView(value) && typeof value.length === "number")) { return Array.from(value).map((item) => serialize(item, seen)); } if (value && typeof value === "object" && "x" in value && "y" in value && "width" in value && "height" in value && "top" in value && "left" in value) { @@ -73,9 +77,6 @@ pub(super) fn script() -> &'static str { stack: value.stack }; } - if (Array.isArray(value)) { - return value.map((item) => serialize(item, seen)); - } if (typeof value === "object") { if (seen.has(value)) { return null; diff --git a/src/crates/assembly/core/src/agentic/tools/browser_control/builtin_browser.rs b/src/crates/assembly/core/src/agentic/tools/browser_control/builtin_browser.rs index 249d3ba7d8..d5e78bd18a 100644 --- a/src/crates/assembly/core/src/agentic/tools/browser_control/builtin_browser.rs +++ b/src/crates/assembly/core/src/agentic/tools/browser_control/builtin_browser.rs @@ -24,6 +24,9 @@ pub struct BuiltInBrowserTarget { pub url: String, pub title: String, pub active: bool, + /// Agent conversation that owns this page; visibility never changes ownership. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub conversation_id: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -34,18 +37,16 @@ pub struct BuiltInBrowserCommand { pub params: Option, } -/// Ask the Desktop host to present a built-in browser page and return only -/// after that exact native WebView has been registered for Agent control. -/// -/// Opening the product surface and registering its automation target are two -/// separate asynchronous frontend phases. Keeping their correlation inside -/// the host port prevents ControlHub from guessing readiness by URL or by a -/// process-wide target count. +/// Ask the host for a registered page owned by this conversation. Presentation +/// is independent: the page must be usable while its conversation is hidden or +/// has no mounted panel. A replacement can only reuse a page with the same owner. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct BuiltInBrowserOpenRequest { pub url: String, pub title: String, pub replace_existing: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub conversation_id: Option, } #[async_trait] @@ -57,6 +58,20 @@ pub trait BuiltInBrowserHost: Send + Sync { async fn list_targets(&self) -> Result, String>; + async fn list_targets_for_session( + &self, + conversation_id: Option<&str>, + ) -> Result, String> { + Ok(self + .list_targets() + .await? + .into_iter() + .filter(|target| { + conversation_id.is_none() || target.conversation_id.as_deref() == conversation_id + }) + .collect()) + } + async fn execute(&self, command: BuiltInBrowserCommand) -> Result; } @@ -112,13 +127,35 @@ pub async fn list_builtin_browser_targets() -> OpenBitFunResult, +) -> OpenBitFunResult> { + host()? + .list_targets_for_session(conversation_id) + .await + .map_err(|error| { + OpenBitFunError::tool(format!("Built-in browser target discovery failed: {error}")) + }) +} + pub async fn open_builtin_browser( request: BuiltInBrowserOpenRequest, ) -> OpenBitFunResult { + let conversation_id = request.conversation_id.clone(); let target = host()?.open(request).await.map_err(|error| { OpenBitFunError::tool(format!("Built-in browser surface failed to open: {error}")) })?; - set_default_builtin_browser_target(Some(target.id.clone())); + if target.conversation_id != conversation_id { + return Err(OpenBitFunError::tool( + "[WRONG_TAB] The browser host returned a page owned by another conversation" + .to_string(), + )); + } + if target.conversation_id.is_none() { + set_default_builtin_browser_target(Some(target.id.clone())); + } Ok(BuiltInBrowserClient { events: event_sender(&target.id), target, @@ -194,16 +231,31 @@ pub async fn connect_builtin_browser_matching( target_url: Option<&str>, target_title: Option<&str>, ) -> OpenBitFunResult { - let targets = list_builtin_browser_targets().await?; + connect_builtin_browser_for_session(requested_target_id, target_url, target_title, None).await +} + +pub async fn connect_builtin_browser_for_session( + requested_target_id: Option<&str>, + target_url: Option<&str>, + target_title: Option<&str>, + conversation_id: Option<&str>, +) -> OpenBitFunResult { + let targets = list_builtin_browser_targets_for_session(conversation_id).await?; + let default_id = conversation_id + .is_none() + .then(default_builtin_browser_target_id) + .flatten(); let target = select_builtin_browser_target( &targets, requested_target_id, target_url, target_title, - default_builtin_browser_target_id().as_deref(), + default_id.as_deref(), )?; - set_default_builtin_browser_target(Some(target.id.clone())); + if conversation_id.is_none() { + set_default_builtin_browser_target(Some(target.id.clone())); + } Ok(BuiltInBrowserClient { events: event_sender(&target.id), target, @@ -320,6 +372,21 @@ impl BrowserAutomationClient for BuiltInBrowserClient { mod tests { use super::*; + #[test] + fn legacy_browser_payloads_round_trip_without_an_owner() { + let request_json = + json!({"url":"https://example.com","title":"Example","replace_existing":true}); + let request: BuiltInBrowserOpenRequest = + serde_json::from_value(request_json.clone()).unwrap(); + assert_eq!(request.conversation_id, None); + assert_eq!(serde_json::to_value(request).unwrap(), request_json); + let target_json = + json!({"id":"old-page","url":"https://example.com","title":"Example","active":false}); + let target: BuiltInBrowserTarget = serde_json::from_value(target_json.clone()).unwrap(); + assert_eq!(target.conversation_id, None); + assert_eq!(serde_json::to_value(target).unwrap(), target_json); + } + #[test] fn page_load_events_use_the_shared_lifecycle_shape() { let mut receiver = event_sender("embedded-browser-test").subscribe(); @@ -357,6 +424,7 @@ mod tests { url: url.to_string(), title: title.to_string(), active, + conversation_id: None, } } diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_browser_tests.rs b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_browser_tests.rs index d0d280911e..f36ad18462 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_browser_tests.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_browser_tests.rs @@ -28,7 +28,8 @@ impl BuiltInBrowserHost for BrowserFixture { id: format!("embedded-browser-panel-view-fixture-{}", pages.len()), url: request.url.clone(), title: "Fixture".into(), - active: true, + active: false, + conversation_id: request.conversation_id.clone(), }; self.opens.lock().unwrap().push(request); pages.push(target.clone()); @@ -136,6 +137,10 @@ async fn cold_navigation_observes_once_and_keeps_conversations_on_their_pages() assert_eq!(first["data"]["loaded"], true); assert!(text.contains("BODY_AFTER_EXCERPT")); assert_eq!(host.opens.lock().unwrap().len(), 1); + assert_eq!( + host.opens.lock().unwrap()[0].conversation_id.as_deref(), + a.session_id.as_deref() + ); assert!(!host .commands .lock() @@ -188,6 +193,73 @@ async fn cold_navigation_observes_once_and_keeps_conversations_on_their_pages() release_browser_selection("browser-workflow-b").await; } +#[tokio::test] +async fn background_pages_are_private_to_their_conversations_even_with_an_explicit_foreign_id() { + let _guard = TEST_HOST_LOCK.lock().await; + let host = Arc::new(BrowserFixture::default()); + set_builtin_browser_host(host.clone()); + let a = context("isolated-browser-a"); + let b = context("isolated-browser-b"); + let ((first, _), (second, _)) = tokio::join!( + call( + "navigate", + json!({"url":"https://a.example","observe":"article"}), + &a + ), + call( + "navigate", + json!({"url":"https://b.example","observe":"article"}), + &b + ), + ); + assert_eq!(first["ok"], true, "{first}"); + assert_eq!(second["ok"], true, "{second}"); + let a_id = first["data"]["session_id"].as_str().unwrap(); + let b_id = second["data"]["session_id"].as_str().unwrap(); + assert_ne!(a_id, b_id); + // Simulate B being visible while A's native document remains hidden. + for page in host.pages.lock().unwrap().iter_mut() { + page.active = page.id == b_id; + } + for action in ["list_pages", "tab_query"] { + let (inventory, _) = call(action, json!({"target":"builtin"}), &a).await; + assert_eq!( + inventory["data"]["pages"].as_array().unwrap().len(), + 1, + "{inventory}" + ); + assert_eq!(inventory["data"]["pages"][0]["id"], a_id); + assert_eq!(inventory["data"]["default_session_id"], a_id); + } + let (article, _) = call("read_article", json!({}), &a).await; + assert_eq!(article["data"]["session_id"], a_id); + let before = host.commands.lock().unwrap().len(); + for action in [ + "navigate", + "connect", + "switch_page", + "read_article", + "close", + ] { + let mut params = json!({"target":"builtin","session_id":b_id,"page_id":b_id}); + if action == "navigate" { + params["url"] = json!("https://wrong.example"); + } + let (rejected, _) = call(action, params, &a).await; + assert_eq!(rejected["ok"], false, "{action}: {rejected}"); + } + assert_eq!( + host.commands.lock().unwrap().len(), + before, + "Foreign selections must fail before page IO" + ); + release_browser_selection("isolated-browser-a").await; + assert_eq!(host.pages.lock().unwrap().len(), 1); + let (remaining, _) = call("get_url", json!({}), &b).await; + assert_eq!(remaining["data"]["url"], "https://b.example"); + release_browser_selection("isolated-browser-b").await; +} + #[tokio::test] async fn navigation_observation_and_fetch_failures_keep_partial_evidence() { let _guard = TEST_HOST_LOCK.lock().await; diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs index 8f59a05229..f4b66c817a 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs @@ -16,9 +16,10 @@ use crate::agentic::tools::browser_control::browser_launcher::{ BrowserKind, BrowserLauncher, DEFAULT_CDP_PORT, }; use crate::agentic::tools::browser_control::builtin_browser::{ - builtin_browser_host_available, connect_builtin_browser, connect_builtin_browser_matching, - default_builtin_browser_target_id, list_builtin_browser_targets, open_builtin_browser, - set_default_builtin_browser_target, BuiltInBrowserClient, BuiltInBrowserOpenRequest, + builtin_browser_host_available, connect_builtin_browser_for_session, + default_builtin_browser_target_id, list_builtin_browser_targets_for_session, + open_builtin_browser, set_default_builtin_browser_target, BuiltInBrowserClient, + BuiltInBrowserOpenRequest, }; use crate::agentic::tools::browser_control::cdp_client::{CdpClient, CdpPageInfo, CdpVersionInfo}; use crate::agentic::tools::browser_control::session_registry::{ @@ -66,6 +67,21 @@ pub(crate) async fn release_browser_selection(owner: &str) { if let Some(registry) = BROWSER_SESSIONS.get() { registry.release_selection(owner).await; } + // Session deletion releases its native pages, including hidden panels. + if builtin_browser_host_available() { + if let Ok(targets) = list_builtin_browser_targets_for_session(Some(owner)).await { + for target in targets { + if let Ok(client) = + connect_builtin_browser_for_session(Some(&target.id), None, None, Some(owner)) + .await + { + if let Err(error) = client.send("Page.close", None).await { + log::warn!("Close session browser failed: conversation_id={}, target_id={}, error={}", owner, target.id, error); + } + } + } + } + } } /// Disconnect the user-selected external browser from OpenBitFun without closing @@ -226,6 +242,7 @@ impl ControlHubTool { fn builtin_target_json( target: &crate::agentic::tools::browser_control::BuiltInBrowserTarget, + selected_id: Option<&str>, ) -> Value { json!({ "id": target.id, @@ -235,7 +252,7 @@ impl ControlHubTool { "type": "page", "target": "builtin", "active": target.active, - "is_default_session": default_builtin_browser_target_id().as_deref() == Some(target.id.as_str()), + "is_default_session": selected_id == Some(target.id.as_str()), }) } @@ -627,7 +644,9 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o let browser_session_count = browser_session_ids.len(); let builtin_host_available = !is_remote && builtin_browser_host_available(); let (builtin_targets, discovery_error) = if builtin_host_available { - match list_builtin_browser_targets().await { + match list_builtin_browser_targets_for_session(context.session_id.as_deref()) + .await + { Ok(targets) => (targets, None), Err(error) => (Vec::new(), Some(error.to_string())), } @@ -635,17 +654,24 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o (Vec::new(), None) }; let builtin_target_count = builtin_targets.len(); - let builtin_default = if is_remote { - None - } else { - default_builtin_browser_target_id() - }; let selection = if let Some(owner) = context.session_id.as_deref().filter(|_| !is_remote) { browser_sessions().selection(owner).await } else { None }; + let builtin_default = selection + .as_ref() + .filter(|selection| selection.target == "builtin") + .map(|selection| selection.session_id.clone()) + .or_else(|| { + context + .session_id + .is_none() + .then(default_builtin_browser_target_id) + .flatten() + }) + .filter(|id| builtin_targets.iter().any(|target| &target.id == id)); let browser_default = selection .as_ref() .filter(|selection| selection.target == "external") @@ -1267,16 +1293,18 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o let target_title = params.get("target_title").and_then(Value::as_str); let exact_selection = requested_id.is_some() || target_url.is_some() || target_title.is_some(); - let pages = match list_builtin_browser_targets().await { - Ok(pages) => pages, - Err(error) => { - return Ok(err_response( - "browser", - action, - Self::builtin_open_error(error), - )) - } - }; + let pages = + match list_builtin_browser_targets_for_session(context.session_id.as_deref()).await + { + Ok(pages) => pages, + Err(error) => { + return Ok(err_response( + "browser", + action, + Self::builtin_open_error(error), + )) + } + }; // A new conversation gets its own page. Existing callers without // an owner can keep using the active page. Never recover a stale // explicit selection by opening or mutating some other page. @@ -1299,6 +1327,7 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o url: url.clone(), title: title.to_string(), replace_existing, + conversation_id: context.session_id.clone(), }) .await { @@ -1316,7 +1345,13 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o } } } else { - match connect_builtin_browser_matching(requested_id, target_url, target_title).await + match connect_builtin_browser_for_session( + requested_id, + target_url, + target_title, + context.session_id.as_deref(), + ) + .await { Ok(client) => client, Err(error) => { @@ -1702,10 +1737,11 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o let target_url = params.get("target_url").and_then(Value::as_str); let target_title = params.get("target_title").and_then(Value::as_str); let targeted = target_url.is_some() || target_title.is_some(); - let client = match connect_builtin_browser_matching( + let client = match connect_builtin_browser_for_session( session_id_param.as_deref(), target_url, target_title, + context.session_id.as_deref(), ) .await { @@ -1982,7 +2018,11 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o "list_pages" => { if browser_target_kind == BrowserTargetKind::BuiltIn { - let targets = match list_builtin_browser_targets().await { + let targets = match list_builtin_browser_targets_for_session( + context.session_id.as_deref(), + ) + .await + { Ok(targets) => targets, Err(error) => { return Ok(err_response( @@ -1994,7 +2034,14 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o }; let pages = targets .iter() - .map(Self::builtin_target_json) + .map(|target| { + Self::builtin_target_json( + target, + session_id_param + .as_deref() + .or(default_builtin_browser_target_id().as_deref()), + ) + }) .collect::>(); let default_id = page_inventory_default_id( &pages, @@ -2071,7 +2118,11 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o .max(1); let (pages, default_id) = if browser_target_kind == BrowserTargetKind::BuiltIn { - let targets = match list_builtin_browser_targets().await { + let targets = match list_builtin_browser_targets_for_session( + context.session_id.as_deref(), + ) + .await + { Ok(targets) => targets, Err(error) => { return Ok(err_response( @@ -2084,7 +2135,14 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o ( targets .iter() - .map(Self::builtin_target_json) + .map(|target| { + Self::builtin_target_json( + target, + session_id_param + .as_deref() + .or(default_builtin_browser_target_id().as_deref()), + ) + }) .collect::>(), default_builtin_browser_target_id(), ) @@ -2192,6 +2250,7 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o url: url.clone(), title: "Browser".to_string(), replace_existing: false, + conversation_id: context.session_id.clone(), }) .await { @@ -2212,7 +2271,7 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o "session_id": target.id, "page_url": target.url, "page_title": target.title, - "activated": true, + "activated": target.active, }), Some(format!( "New built-in browser page opened: {} (session {})", @@ -2269,7 +2328,14 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o OpenBitFunError::tool("switch_page requires 'page_id'".to_string()) })?; if browser_target_kind == BrowserTargetKind::BuiltIn { - let client = match connect_builtin_browser(Some(page_id)).await { + let client = match connect_builtin_browser_for_session( + Some(page_id), + None, + None, + context.session_id.as_deref(), + ) + .await + { Ok(client) => client, Err(error) => { return Ok(err_response( @@ -2279,7 +2345,9 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o )); } }; - set_default_builtin_browser_target(Some(page_id.to_string())); + if context.session_id.is_none() { + set_default_builtin_browser_target(Some(page_id.to_string())); + } return Ok(vec![ToolResult::ok( json!({ "success": true, @@ -2376,7 +2444,11 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o if action != "list_sessions" { return Ok(Self::builtin_extension_error(action)); } - let targets = match list_builtin_browser_targets().await { + let targets = match list_builtin_browser_targets_for_session( + context.session_id.as_deref(), + ) + .await + { Ok(targets) => targets, Err(error) => { return Ok(err_response( @@ -2598,7 +2670,14 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o // Resolve a target adapter, then run every portable operation // through the one shared BrowserActions implementation. let target = if browser_target_kind == BrowserTargetKind::BuiltIn { - match connect_builtin_browser(session_id_param.as_deref()).await { + match connect_builtin_browser_for_session( + session_id_param.as_deref(), + None, + None, + context.session_id.as_deref(), + ) + .await + { Ok(client) => ResolvedBrowserTarget::BuiltIn(client), Err(error) => { let code = if session_id_param.is_some() { @@ -4056,6 +4135,7 @@ mod control_hub_tests { url: request.url, title: request.title, active: true, + conversation_id: None, }) } @@ -4065,6 +4145,7 @@ mod control_hub_tests { url: "https://example.com".to_string(), title: "Example".to_string(), active: true, + conversation_id: None, }]) } diff --git a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json index 9bfee628f7..a2a2f05de9 100644 --- a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json +++ b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "digest": "fnv1a64:1f41015e75434c3c", + "digest": "fnv1a64:39cac84a6ae186a5", "retiredCommandPrefixes": [ { "prefix": "lsp_", @@ -666,11 +666,11 @@ "surface": "tauri_command", "remoteWorkspace": "LocalOnly", "peer": { - "kind": "proxied" + "kind": "controller_local" }, "cliPeer": { "kind": "unsupported", - "reason": "the CLI peer host has no handler for this command" + "reason": "the controller keeps this command; peer hosts refuse it before dispatch" } }, { diff --git a/src/crates/contracts/product-domains/src/remote_surface/table.rs b/src/crates/contracts/product-domains/src/remote_surface/table.rs index be8a063b89..2963eab2ba 100644 --- a/src/crates/contracts/product-domains/src/remote_surface/table.rs +++ b/src/crates/contracts/product-domains/src/remote_surface/table.rs @@ -131,7 +131,7 @@ pub(super) const OPERATIONS: &[OperationDefinition] = &[ op("browser_control_launch", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), op("browser_control_list_browsers", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), op("browser_control_restart_with_cdp", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), - op("browser_get_url", LocalOnly, Proxied, CLI_NOT_IMPLEMENTED), + op("browser_get_url", LocalOnly, ControllerLocal, REFUSED), op("browser_webview_capture_preview", LocalOnly, ControllerLocal, REFUSED), op("browser_webview_create", LocalOnly, ControllerLocal, REFUSED), op("browser_webview_eval", LocalOnly, ControllerLocal, REFUSED), diff --git a/src/shared/browser/blankTargetIntercept.js b/src/shared/browser/blankTargetIntercept.js new file mode 100644 index 0000000000..02570633f9 --- /dev/null +++ b/src/shared/browser/blankTargetIntercept.js @@ -0,0 +1,16 @@ +(function(){ + if(window.__openbitfun_blank_intercept){return;} + window.__openbitfun_blank_intercept=true; + document.addEventListener('click',function(e){ + var el=e.target; + while(el&&el.tagName!=='A'){el=el.parentElement;} + if(!el)return; + var t=(el.getAttribute('target')||'').toLowerCase(); + if(t==='_blank'){ + e.preventDefault(); + e.stopPropagation(); + var href=el.href; + if(href){location.href=href;} + } + },true); +})(); diff --git a/src/web-ui/src/app/components/panels/base/FlexiblePanel.tsx b/src/web-ui/src/app/components/panels/base/FlexiblePanel.tsx index a38dbc115e..e72aff0b9e 100644 --- a/src/web-ui/src/app/components/panels/base/FlexiblePanel.tsx +++ b/src/web-ui/src/app/components/panels/base/FlexiblePanel.tsx @@ -919,10 +919,12 @@ const FlexiblePanel: React.FC = memo(({ return ( {t('flexiblePanel.loading.terminal')}}> ); diff --git a/src/web-ui/src/app/components/panels/content-canvas/stores/canvasStore.ts b/src/web-ui/src/app/components/panels/content-canvas/stores/canvasStore.ts index a5a1ff89fc..180eab74f1 100644 --- a/src/web-ui/src/app/components/panels/content-canvas/stores/canvasStore.ts +++ b/src/web-ui/src/app/components/panels/content-canvas/stores/canvasStore.ts @@ -245,6 +245,13 @@ const createCanvasStoreHook = () => create()( } const newTab = createTab(content, state); + if (content.type === 'browser' && !content.data?.targetId + && draft.scopeKey && draft.scopeKey !== '__none__') { + newTab.content = { ...content, data: { ...content.data, + targetId: `embedded-browser-panel-view-${newTab.id}`, + conversationId: draft.scopeKey, + } }; + } insertTabRespectingPinnedBoundary(group, newTab); group.activeTabId = newTab.id; draft.activeGroupId = targetGroupId; @@ -1212,7 +1219,8 @@ function rememberAgentSnapshot(key: string, snapshot: CanvasStoreState): void { const evictIndex = agentSnapshotLruOrder.findIndex(scope => { const state = agentScopeSnapshots.get(scope); return !state || ![state.primaryGroup, state.secondaryGroup, state.tertiaryGroup] - .some(group => group.tabs.some(tab => tab.content.metadata?.discardSessionOnClose)); + .some(group => group.tabs.some(tab => tab.content.metadata?.discardSessionOnClose + || (tab.content.type === 'browser' && tab.content.data?.targetId))); }); if (evictIndex < 0) break; const [evict] = agentSnapshotLruOrder.splice(evictIndex, 1); @@ -1221,6 +1229,41 @@ function rememberAgentSnapshot(key: string, snapshot: CanvasStoreState): void { } } +/** Commit background output to its owner's snapshot without changing UI selection. */ +export function openAgentCanvasTabForSession(sessionId: string, content: PanelContent): void { + const key = normalizeAgentScopeKey(sessionId); + const live = useAgentCanvasStore.getState(); + const target = live.scopeKey === key ? useAgentCanvasStore : createCanvasStoreHook(); + if (target !== useAgentCanvasStore) { + const snapshot = agentScopeSnapshots.get(key); + if (snapshot) target.setState(structuredClone(snapshot)); + target.setState({ scopeKey: key }); + } + const state = target.getState(); + const duplicateKey = content.metadata?.duplicateCheckKey; + for (const groupId of ['primary', 'secondary', 'tertiary'] as const) { + const tab = state[`${groupId}Group`].tabs.find(candidate => duplicateKey + && candidate.content.metadata?.duplicateCheckKey === duplicateKey); + if (tab) { + state.updateTabContent(tab.id, groupId, content); + state.switchToTab(tab.id, groupId); + if (target !== useAgentCanvasStore) rememberAgentSnapshot(key, extractAgentPersistableState(target.getState())); + return; + } + } + state.addTab(content, 'active'); + if (target !== useAgentCanvasStore) rememberAgentSnapshot(key, extractAgentPersistableState(target.getState())); +} + +/** Session deletion also releases snapshots exempted from ordinary LRU eviction. */ +export function forgetAgentCanvasSession(sessionId: string): void { + const key = normalizeAgentScopeKey(sessionId); + agentScopeSnapshots.delete(key); + const index = agentSnapshotLruOrder.indexOf(key); + if (index >= 0) agentSnapshotLruOrder.splice(index, 1); + if (useAgentCanvasStore.getState().scopeKey === key) applyEmptyAgentCanvas(key); +} + /** Discard all views of an explicitly closed side conversation, including suspended canvases. */ export function removeBtwSessionFromAgentCanvas(sessionId: string): void { const ownsSession = (tab: CanvasTab) => diff --git a/src/web-ui/src/app/scenes/browser/BrowserPanel.tsx b/src/web-ui/src/app/scenes/browser/BrowserPanel.tsx index b44f155354..bc22fe5fa1 100644 --- a/src/web-ui/src/app/scenes/browser/BrowserPanel.tsx +++ b/src/web-ui/src/app/scenes/browser/BrowserPanel.tsx @@ -36,9 +36,12 @@ export interface BrowserPanelProps { initialUrl?: string; /** Correlates a host open request with this exact native WebView target. */ openRequestId?: string; + /** Native target retained by the session while this panel is unmounted. */ + targetId?: string; + conversationId?: string; } -const BrowserPanel: React.FC = ({ isActive, initialUrl, openRequestId }) => { +const BrowserPanel: React.FC = ({ isActive, initialUrl, openRequestId, targetId, conversationId }) => { const { t } = useTranslation('common'); const addContext = useContextStore((s) => s.addContext); const inspectorUnlistenRef = useRef<(() => void) | null>(null); @@ -51,6 +54,8 @@ const BrowserPanel: React.FC = ({ isActive, initialUrl, openR labelPrefix: 'embedded-browser-panel-view', log, openRequestId, + targetId, + conversationId, }); const { diff --git a/src/web-ui/src/app/scenes/browser/README.md b/src/web-ui/src/app/scenes/browser/README.md index 5aee36056b..44ea41f0bb 100644 --- a/src/web-ui/src/app/scenes/browser/README.md +++ b/src/web-ui/src/app/scenes/browser/README.md @@ -1,7 +1,32 @@ # Embedded browser occlusion -Both `BrowserPanel` and `BrowserScene` use `useEmbeddedBrowserWebview` to own the -native child view. All show/hide/focus paths go through `nativeWebviewVisibility`. +Session browser pages belong to the executing Desktop and carry an immutable +conversation ID. The host creates Agent-requested pages hidden, registers them, +then emits the owning session and exact native target ID. Page readiness never +depends on an active conversation, a mounted panel, or visible viewport bounds. +Tool discovery and attachment filter by conversation, including explicit page IDs; +legacy callers without a conversation retain their unscoped contract. +The shared new-window link interceptor runs at native document initialization, so +background navigation has the same behavior before a panel has ever mounted. +Replacement navigation gates tool evaluation on the next native load lifecycle, +so a slow response cannot expose the previous completed document as the new page. +Fragment-only transitions keep their existing document and do not await a load. + +The frontend commits each open event directly to that session's canvas snapshot +without activating the session or touching another session's selected tab. Browser +snapshots survive the ordinary canvas LRU eviction limit. Manually opened session +tabs also receive a stable target ID and conversation owner. + +Both `BrowserPanel` and `BrowserScene` use `useEmbeddedBrowserWebview`. Session +panels attach to the retained native target, read its current URL, and only hide +it on unmount; switching conversations never reloads the initial URL or closes +the document. Closing presentation leaves an Agent's page available until an +explicit browser close or session cleanup. Standalone, unowned browser surfaces +retain their component-owned lifetime. Native presentation calls share a per-target +queue across remounts so stale cleanup cannot hide the new presentation. WebKit's +background suspension is disabled for session-owned pages on supported systems. + +All show/hide/focus paths go through `nativeWebviewVisibility`. DOM floating surfaces declare `data-openbitfun-native-webview-occlusion`; their overlapping bounds suppress native visibility without destroying the page. Visible browser surfaces share one DOM observer. It tracks inserted/removed @@ -51,11 +76,15 @@ The command is controller-local in Peer Device Mode and never requests a screens from the peer. SSH/Docker workspaces do not change ownership of this local UI view. The web renderer keeps its existing iframe path. This command adds no preview transport to mobile/IM remote control or detached jobs; peer hosts refuse it. +Current-URL reads are controller-local too. Peer browser-open events never attach +their labels to local native pages. Agent browser tools retain the explicit remote +workspace gate, and CLI/headless dispatch hosts still report an unavailable native +browser; this lifetime change adds no remote browser transport. Focused verification: ```sh -pnpm --dir src/web-ui exec vitest run src/app/scenes/browser/browserPreviewCache.test.ts src/app/scenes/browser/nativeWebviewOcclusionObserver.test.ts src/app/scenes/browser/nativeWebviewVisibility.test.ts src/app/scenes/browser/useEmbeddedBrowserWebview.test.tsx src/app/scenes/browser/browserStreamPerformanceScript.test.ts src/infrastructure/api/adapters/peer-device-adapter.test.ts +pnpm --dir src/web-ui run test:run src/app/scenes/browser src/infrastructure/api/adapters/peer-device-adapter.test.ts src/flow_chat/services/flow-chat-manager/EventHandlerModule.test.ts src/app/scenes/session/sessionPaneLayoutStore.test.ts ``` Manual checks: after rebuilding the desktop host, open a browser and let it settle, @@ -63,3 +92,15 @@ then open session overview or a tab context menu over it. The last page frame sh remain in the uncovered region; closing the popup should restore the live page. Also check navigation, resizing, multiple browser tabs and a video page. The frozen image is not evidence that media playback or page execution has paused. + +For native multi-conversation acceptance after building Desktop and Web UI: + +```sh +node tests/e2e/scripts/run-browser-session-isolation.mjs +``` + +This isolated runner scripts only the model HTTP response. Real Agent tool calls, +native pages and production session controls exercise background opening, delayed +replacement, concurrent reads, repeated session switches, retained document/input +state, and native visibility. +It does not establish remote-host or cross-platform behavior. diff --git a/src/web-ui/src/app/scenes/browser/browserInspectorScript.ts b/src/web-ui/src/app/scenes/browser/browserInspectorScript.ts index 550c9494fc..c5fb226909 100644 --- a/src/web-ui/src/app/scenes/browser/browserInspectorScript.ts +++ b/src/web-ui/src/app/scenes/browser/browserInspectorScript.ts @@ -227,19 +227,4 @@ export const CANCEL_INSPECTOR_SCRIPT = * and navigate in-place instead (Tauri webviews cannot open new windows). * Re-injected after every page load since full navigations destroy JS state. */ -export const BLANK_TARGET_INTERCEPT_SCRIPT = `(function(){ - if(window.__openbitfun_blank_intercept){return;} - window.__openbitfun_blank_intercept=true; - document.addEventListener('click',function(e){ - var el=e.target; - while(el&&el.tagName!=='A'){el=el.parentElement;} - if(!el)return; - var t=(el.getAttribute('target')||'').toLowerCase(); - if(t==='_blank'){ - e.preventDefault(); - e.stopPropagation(); - var href=el.href; - if(href){location.href=href;} - } - },true); -})()`; +export { default as BLANK_TARGET_INTERCEPT_SCRIPT } from '../../../../../shared/browser/blankTargetIntercept.js?raw'; diff --git a/src/web-ui/src/app/scenes/browser/retainedBrowserPresentation.test.ts b/src/web-ui/src/app/scenes/browser/retainedBrowserPresentation.test.ts new file mode 100644 index 0000000000..a150015b0a --- /dev/null +++ b/src/web-ui/src/app/scenes/browser/retainedBrowserPresentation.test.ts @@ -0,0 +1,40 @@ +import { expect, it, vi } from 'vitest'; +import { claimBrowserPresentation } from './retainedBrowserPresentation'; + +it('drains in-flight native calls before a new panel presents the page and ignores stale cleanup', async () => { + const calls: string[] = []; + let finish!: () => void; + const old = claimBrowserPresentation('retained-page'); + const pending = old.run(async () => { + calls.push('old-start'); + await new Promise(resolve => { finish = resolve; }); + calls.push('old-finish'); + }); + await Promise.resolve(); + await Promise.resolve(); + const next = claimBrowserPresentation('retained-page'); + const present = next.run(async () => { calls.push('new-show'); }); + const staleHide = vi.fn(async () => {}); + const cleanup = old.run(staleHide); + old.release(); + expect(next.isCurrent()).toBe(true); + finish(); + await Promise.all([pending, present, cleanup]); + expect(calls).toEqual(['old-start', 'old-finish', 'new-show']); + expect(staleHide).not.toHaveBeenCalled(); + next.release(); +}); + +it('keeps different session pages independent when one native call is pending', async () => { + let finish!: () => void; + const a = claimBrowserPresentation('independent-a'); + const b = claimBrowserPresentation('independent-b'); + const pending = a.run(() => new Promise(resolve => { finish = resolve; })); + const showB = vi.fn(async () => {}); + await b.run(showB); + expect(showB).toHaveBeenCalledOnce(); + finish(); + await pending; + a.release(); + b.release(); +}); diff --git a/src/web-ui/src/app/scenes/browser/retainedBrowserPresentation.ts b/src/web-ui/src/app/scenes/browser/retainedBrowserPresentation.ts new file mode 100644 index 0000000000..eaa789fc64 --- /dev/null +++ b/src/web-ui/src/app/scenes/browser/retainedBrowserPresentation.ts @@ -0,0 +1,28 @@ +/** Native presentation calls share one queue across panel unmount/remount. */ +type Presentation = { owner?: symbol; pending: Promise }; +const presentations = new Map(); + +export function claimBrowserPresentation(label: string) { + const state = presentations.get(label) ?? { pending: Promise.resolve() }; + const owner = Symbol(label); + state.owner = owner; + presentations.set(label, state); + return { + isCurrent: () => state.owner === owner, + run(operation: () => Promise): Promise { + const pending = state.pending.catch(() => {}).then(async () => { + if (state.owner === owner) await operation(); + }); + state.pending = pending; + return pending; + }, + release() { + if (state.owner !== owner) return; + state.owner = undefined; + // Keep the queue until already-submitted native calls have finished. + void state.pending.catch(() => {}).then(() => { + if (!state.owner && presentations.get(label) === state) presentations.delete(label); + }); + }, + }; +} diff --git a/src/web-ui/src/app/scenes/browser/useEmbeddedBrowserWebview.test.tsx b/src/web-ui/src/app/scenes/browser/useEmbeddedBrowserWebview.test.tsx index e2f736a898..ef1ba17e56 100644 --- a/src/web-ui/src/app/scenes/browser/useEmbeddedBrowserWebview.test.tsx +++ b/src/web-ui/src/app/scenes/browser/useEmbeddedBrowserWebview.test.tsx @@ -15,6 +15,7 @@ const mocks = vi.hoisted(() => ({ hostListeners: new Set<(visible: boolean) => void>(), pageLoadListeners: new Set<(event: { payload: PageLoadPayload }) => void>(), documentReadyListeners: new Set<(event: { payload: BrowserDocumentReadyPayload }) => void>(), + getByLabel: vi.fn(), })); vi.mock('./browserPageLifecycleScript', async importOriginal => { const actual = await importOriginal(); @@ -28,7 +29,7 @@ vi.mock('@/infrastructure/runtime/windowVisibility', () => ({ }, })); vi.mock('@/infrastructure/api/service-api/ApiClient', () => ({ api: { invoke: mocks.invoke } })); -vi.mock('@tauri-apps/api/webview', () => ({ Webview: { getByLabel: async () => mocks.view } })); +vi.mock('@tauri-apps/api/webview', () => ({ Webview: { getByLabel: mocks.getByLabel } })); vi.mock('@tauri-apps/api/event', () => ({ listen: async (name: string, handler: (event: { payload: PageLoadPayload | BrowserDocumentReadyPayload }) => void) => { if (name === BROWSER_WEBVIEW_DOCUMENT_READY_EVENT) { @@ -43,8 +44,9 @@ vi.mock('@tauri-apps/api/event', () => ({ const log = { warn: vi.fn(), error: vi.fn() }; let browser: ReturnType; -function Harness({ active }: { active: boolean }) { - browser = useEmbeddedBrowserWebview({ defaultUrl: 'https://example.com', isVisible: active, labelPrefix: 'test', log }); +function Harness({ active, targetId, manual = false }: { active: boolean; targetId?: string; manual?: boolean }) { + browser = useEmbeddedBrowserWebview({ defaultUrl: 'https://example.com', isVisible: active, labelPrefix: 'test', log, + targetId, openRequestId: targetId && !manual ? 'open-request' : undefined, conversationId: targetId ? 'owner' : undefined }); return
; } @@ -58,6 +60,7 @@ beforeEach(() => { mocks.pageLoadListeners.clear(); mocks.documentReadyListeners.clear(); mocks.invoke.mockImplementation(async () => {}); + mocks.getByLabel.mockReset().mockResolvedValue(mocks.view); vi.stubGlobal('__TAURI__', {}); vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); vi.stubGlobal('ResizeObserver', class { observe() {} unobserve() {} disconnect() {} }); @@ -100,6 +103,86 @@ function overlay() { return node; } +it('reattaches the exact session page after unmount without closing, recreating or navigating it', async () => { + const label = mocks.view.label; + let currentUrl = 'https://example.com/form-in-progress'; + mocks.invoke.mockImplementation(async command => command === 'browser_get_url' ? currentUrl : undefined); + await act(async () => root.render()); + expect(browser.currentUrl).toBe(currentUrl); + await act(async () => root.render(null)); + expect(mocks.view.hide).toHaveBeenCalled(); + expect(mocks.view.close).not.toHaveBeenCalled(); + currentUrl = 'https://example.com/background-result'; + await act(async () => root.render()); + expect(browser.currentUrl).toBe(currentUrl); + expect(mocks.getByLabel).toHaveBeenCalledTimes(2); + expect(mocks.invoke.mock.calls.some(([command]) => command === 'browser_webview_create' || command === 'browser_webview_navigate')).toBe(false); + expect(mocks.view.show).toHaveBeenCalledTimes(2); +}); + +it('switches presentation between independent native pages while retaining both', async () => { + const a = { ...mocks.view, label: 'page-a', show: vi.fn(async () => {}), hide: vi.fn(async () => {}) }; + const b = { ...mocks.view, label: 'page-b', show: vi.fn(async () => {}), hide: vi.fn(async () => {}) }; + mocks.getByLabel.mockImplementation(async label => label === a.label ? a : b); + await act(async () => root.render()); + await act(async () => root.render()); + expect(a.hide).toHaveBeenCalled(); + expect(b.show).toHaveBeenCalledOnce(); + expect(mocks.view.close).not.toHaveBeenCalled(); + await act(async () => root.render()); + expect(b.hide).toHaveBeenCalled(); + expect(a.show).toHaveBeenCalledTimes(2); +}); + +it('ignores a late attachment from an unmounted conversation', async () => { + let resolveA!: (view: typeof mocks.view) => void; + mocks.getByLabel.mockImplementation(label => label === 'page-a' + ? new Promise(resolve => { resolveA = resolve; }) : Promise.resolve(mocks.view)); + await act(async () => root.render()); + await act(async () => root.render()); + const lateView = { ...mocks.view, show: vi.fn(async () => {}), hide: vi.fn(async () => {}) }; + await act(async () => resolveA(lateView)); + expect(lateView.show).not.toHaveBeenCalled(); + expect(lateView.hide).not.toHaveBeenCalled(); + expect(browser.webviewLabel).toBe('page-b'); +}); + +it('does not recreate a host-owned target that was explicitly closed', async () => { + mocks.getByLabel.mockResolvedValue(null); + await act(async () => root.render()); + expect(browser.error).toContain('Webview not found'); + expect(mocks.invoke.mock.calls.some(([command]) => command === 'browser_webview_create')).toBe(false); +}); + +it('creates a manually opened session page with its stable identity and keeps it on unmount', async () => { + mocks.getByLabel.mockResolvedValueOnce(null).mockResolvedValue(mocks.view); + await act(async () => root.render()); + expect(mocks.invoke).toHaveBeenCalledWith('browser_webview_create', expect.objectContaining({ + request: expect.objectContaining({ label: 'manual-page', conversationId: 'owner' }), + })); + await act(async () => root.render(null)); + expect(mocks.view.close).not.toHaveBeenCalled(); +}); + +it('reattaches a manual page whose first creation finishes after a rapid session switch', async () => { + let finishCreation!: () => void; + let created = false; + mocks.getByLabel.mockImplementation(async () => created ? mocks.view : null); + mocks.invoke.mockImplementation(async command => { + if (command === 'browser_webview_create') { + await new Promise(resolve => { finishCreation = resolve; }); + created = true; + } + }); + await act(async () => root.render()); + await act(async () => root.render()); + await act(async () => finishCreation()); + expect(mocks.invoke.mock.calls.filter(([command]) => command === 'browser_webview_create')).toHaveLength(1); + expect(browser.error).toBeNull(); + expect(mocks.view.show).toHaveBeenCalledOnce(); + expect(mocks.view.close).not.toHaveBeenCalled(); +}); + async function hostVisibility(visible: boolean) { await mutate(() => { mocks.hostVisible = visible; diff --git a/src/web-ui/src/app/scenes/browser/useEmbeddedBrowserWebview.ts b/src/web-ui/src/app/scenes/browser/useEmbeddedBrowserWebview.ts index 97609be6e0..dac6582b6e 100644 --- a/src/web-ui/src/app/scenes/browser/useEmbeddedBrowserWebview.ts +++ b/src/web-ui/src/app/scenes/browser/useEmbeddedBrowserWebview.ts @@ -1,3 +1,4 @@ +import { claimBrowserPresentation } from './retainedBrowserPresentation'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { BLANK_TARGET_INTERCEPT_SCRIPT } from './browserInspectorScript'; import { STREAM_RENDER_OPTIMIZATION_SCRIPT } from './browserStreamPerformanceScript'; @@ -60,6 +61,8 @@ export interface UseEmbeddedBrowserWebviewOptions { labelPrefix: string; log: BrowserLogger; openRequestId?: string; + targetId?: string; + conversationId?: string; } function isTauriEnvironment(): boolean { @@ -150,6 +153,7 @@ async function createBrowserWebview( url: string, bounds: WebviewBounds, openRequestId?: string, + conversationId?: string, ): Promise { const { Webview } = await import('@tauri-apps/api/webview'); await api.invoke('browser_webview_create', { @@ -161,6 +165,7 @@ async function createBrowserWebview( width: bounds.width, height: bounds.height, openRequestId, + conversationId, }, }); const handle = await Webview.getByLabel(label) as unknown as BrowserWebviewHandle | null; @@ -171,7 +176,7 @@ async function createBrowserWebview( } export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOptions) { - const { defaultUrl, initialUrl, isVisible, labelPrefix, log, openRequestId } = options; + const { defaultUrl, initialUrl, isVisible, labelPrefix, log, openRequestId, targetId, conversationId } = options; const isTauri = useMemo(() => isTauriEnvironment(), []); const startUrl = initialUrl ?? defaultUrl; @@ -187,6 +192,8 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti const activeRef = useRef(isVisible); activeRef.current = isVisible; const mountedRef = useRef(true); + const presentationRef = useRef | null>(null); + const initializationRef = useRef | null>(null); const toolbarSuspendedRef = useRef(false); const nativeVisibleRef = useRef(false); const hostVisibleRef = useRef(false); @@ -198,27 +205,38 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti onFrame: setPreviewUrl, onError: previewError => log.warn('Browser preview unavailable; using the default placeholder', previewError), }), [log]); - const visibility = useMemo(() => createNativeWebviewVisibility(target => { - const viewport = viewportRef.current; - if (!mountedRef.current || !hostVisibleRef.current || !activeRef.current || toolbarSuspendedRef.current - || target !== webviewRef.current || !viewport?.isConnected) return false; - const rect = viewport.getBoundingClientRect(); - // Keep the last valid geometry during transient layout swaps, but still - // check overlays against it rather than bypassing the visibility decision. - const last = lastBoundsRef.current; - const nativeBounds = last ? { - left: last.left, top: last.top, - right: last.left + last.width, bottom: last.top + last.height, - } : null; - const bounds = rect.width > 1 && rect.height > 1 ? rect : nativeBounds; - // Bounds updates are debounced; the native view may still occupy its old - // rectangle after the DOM container has moved. - return !!bounds && !hasNativeWebviewOccluder(viewport, bounds, nativeBounds ?? undefined); - }, (target, visible) => { - if (target !== webviewRef.current) return; - nativeVisibleRef.current = visible; - preview.setVisible(visible && hostVisibleRef.current); - }), [preview]); + const visibility = useMemo(() => { + const reconcile = createNativeWebviewVisibility(target => { + const viewport = viewportRef.current; + if (!mountedRef.current || (targetId && !presentationRef.current?.isCurrent()) + || !hostVisibleRef.current || !activeRef.current || toolbarSuspendedRef.current + || target !== webviewRef.current || !viewport?.isConnected) return false; + const rect = viewport.getBoundingClientRect(); + // Keep the last valid geometry during transient layout swaps, but still + // check overlays against it rather than bypassing the visibility decision. + const last = lastBoundsRef.current; + const nativeBounds = last ? { + left: last.left, top: last.top, + right: last.left + last.width, bottom: last.top + last.height, + } : null; + const bounds = rect.width > 1 && rect.height > 1 ? rect : nativeBounds; + // Bounds updates are debounced; the native view may still occupy its old + // rectangle after the DOM container has moved. + return !!bounds && !hasNativeWebviewOccluder(viewport, bounds, nativeBounds ?? undefined); + }, (target, visible) => { + if (target !== webviewRef.current) return; + nativeVisibleRef.current = visible; + preview.setVisible(visible && hostVisibleRef.current); + }); + return (target: BrowserWebviewHandle, focus = false) => targetId + ? presentationRef.current?.run(() => reconcile(target, focus)) ?? Promise.resolve() + : reconcile(target, focus); + }, [preview, targetId]); + + const updateAgentTargetState = useCallback((label: string) => { + const update = () => setAgentTargetState(label, mountedRef.current && activeRef.current, openRequestId); + return targetId ? presentationRef.current?.run(update) ?? Promise.resolve() : update(); + }, [openRequestId, targetId]); const syncVisibility = useCallback(async (focus = false) => { const target = webviewRef.current; @@ -279,7 +297,8 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti const syncWebviewBounds = useCallback(async (handle?: BrowserWebviewHandle | null) => { const target = handle ?? webviewRef.current; - if (!isTauri || !target || !viewportRef.current) return; + if (!isTauri || !target || !viewportRef.current || !mountedRef.current + || (targetId && !presentationRef.current?.isCurrent())) return; const nextBounds = readViewportBounds(); if (!nextBounds) { @@ -297,7 +316,14 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti if (!previous || previous.width !== nextBounds.width || previous.height !== nextBounds.height) { preview.invalidate(); } - await setWebviewBounds(target.label, nextBounds); + if (targetId) { + await presentationRef.current?.run(async () => { + if (mountedRef.current) await setWebviewBounds(target.label, nextBounds); + }); + } else { + await setWebviewBounds(target.label, nextBounds); + } + if (!mountedRef.current || (targetId && !presentationRef.current?.isCurrent())) return; lastBoundsRef.current = nextBounds; if (!previous || previous.width !== nextBounds.width || previous.height !== nextBounds.height) { preview.invalidate(); @@ -313,7 +339,7 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti setPreviewBounds(current => current && current.left === relative.left && current.top === relative.top && current.width === relative.width && current.height === relative.height ? current : relative); await visibility(target); - }, [isTauri, preview, readViewportBounds, visibility]); + }, [isTauri, preview, readViewportBounds, targetId, visibility]); const closeWebview = useCallback(async (handle?: BrowserWebviewHandle | null) => { const target = handle ?? webviewRef.current; @@ -488,17 +514,19 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti validateUrl(nextUrl); let handle = webviewRef.current; if (!handle) { + if (targetId) throw new Error(`Webview not found: ${targetId}`); handle = await createWebview(nextUrl); } else { const navigated = await navigateExistingWebview(nextUrl); if (!navigated) { + if (targetId) throw new Error(`Session browser navigation failed: ${targetId}`); handle = await createWebview(nextUrl); } } await syncWebviewBounds(handle); await visibility(handle, true); if (mountedRef.current && handle === webviewRef.current) { - await setAgentTargetState(handle.label, activeRef.current, openRequestId); + await updateAgentTargetState(handle.label); } } catch (loadError) { const message = formatUnknownError(loadError); @@ -506,7 +534,7 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti setError(message); markPageLoadSettled(); } - }, [createWebview, defaultUrl, isTauri, log, markPageLoadSettled, markPageLoadStarted, navigateExistingWebview, openRequestId, preview, syncWebviewBounds, visibility]); + }, [createWebview, defaultUrl, isTauri, log, markPageLoadSettled, markPageLoadStarted, navigateExistingWebview, preview, syncWebviewBounds, targetId, updateAgentTargetState, visibility]); const queueSync = useCallback(() => { if (resizeTimerRef.current !== null) window.clearTimeout(resizeTimerRef.current); @@ -521,9 +549,52 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti useEffect(() => { if (!isTauri) return; if (isVisible && !webviewRef.current) { - void loadUrl(currentUrlRef.current).catch(loadError => { + if (initializationRef.current) return; + const initialize = async () => { + if (!targetId) { + await loadUrl(currentUrlRef.current); + return; + } + const { Webview } = await import('@tauri-apps/api/webview'); + if (!mountedRef.current) return; + const presentation = claimBrowserPresentation(targetId); + presentationRef.current = presentation; + webviewLabelRef.current = targetId; + setWebviewLabel(targetId); + await startPageLoadListener(targetId); + let handle: BrowserWebviewHandle | null = null; + await presentation.run(async () => { + handle = await Webview.getByLabel(targetId) as unknown as BrowserWebviewHandle | null; + if (!mountedRef.current || !presentation.isCurrent()) return; + if (!handle && !openRequestId && conversationId) { + // Serialize first creation across rapid unmount/remount. A new + // panel waits for an in-flight creation and attaches the same page. + handle = await createBrowserWebview(targetId, currentUrlRef.current, + { left: -10000, top: -10000, width: 1024, height: 768 }, undefined, conversationId); + } + }); + if (!mountedRef.current || !presentation.isCurrent()) return; + if (!handle) throw new Error(`Webview not found: ${targetId}`); + webviewRef.current = handle; + preview.setTarget(targetId); + // Reattach the live document. Never reload the original URL on a session switch. + const url = await api.invoke('browser_get_url', { request: { label: targetId } }); + if (!mountedRef.current || !presentationRef.current.isCurrent()) return; + if (url) { + currentUrlRef.current = url; + setCurrentUrl(url); + setInputValue(url); + } + await evalWebview(targetId, createBrowserPageLifecycleScript(targetId, pageLoadSequenceRef.current)); + await injectBrowserPageScripts(targetId); + await syncWebviewBounds(handle); + await visibility(handle); + await updateAgentTargetState(targetId); + }; + initializationRef.current = initialize().catch(loadError => { log.warn('Restore browser webview failed', loadError); - }); + if (mountedRef.current) setError(formatUnknownError(loadError)); + }).finally(() => { initializationRef.current = null; }); return; } const handle = webviewRef.current; @@ -532,10 +603,10 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti if (activeRef.current) await syncWebviewBounds(handle); await visibility(handle, isVisible); if (mountedRef.current && handle === webviewRef.current) { - await setAgentTargetState(handle.label, activeRef.current, openRequestId); + await updateAgentTargetState(handle.label); } })().catch(syncError => log.warn('Update browser webview visibility failed', syncError)); - }, [isTauri, isVisible, loadUrl, log, openRequestId, syncWebviewBounds, visibility]); + }, [conversationId, isTauri, isVisible, loadUrl, log, openRequestId, preview, startPageLoadListener, syncWebviewBounds, targetId, updateAgentTargetState, visibility]); useEffect(() => { if (!isTauri) return; @@ -588,11 +659,21 @@ export function useEmbeddedBrowserWebview(options: UseEmbeddedBrowserWebviewOpti } const handle = webviewRef.current; if (handle) { - // Drain pending show/focus operations before closing the native view. - void visibility(handle).catch(() => {}).then(() => closeWebview(handle)); + // Drain pending presentation calls before hiding or closing the view. + if (targetId) { + const presentation = presentationRef.current; + void visibility(handle).catch(() => {}) + .then(() => presentation?.run(() => setAgentTargetState(handle.label, false))) + .catch(hideError => log.warn('Hide session browser failed', hideError)) + .finally(() => presentation?.release()); + } else { + void visibility(handle).catch(() => {}).then(() => closeWebview(handle)); + } + } else { + presentationRef.current?.release(); } }; - }, [clearPageLoadWatchdog, closeWebview, log, preview, visibility]); + }, [clearPageLoadWatchdog, closeWebview, log, preview, targetId, visibility]); useEffect(() => { if (!isTauri || !isVisible) return; diff --git a/src/web-ui/src/app/scenes/session/sessionPaneLayoutStore.test.ts b/src/web-ui/src/app/scenes/session/sessionPaneLayoutStore.test.ts index 4d5f01bf5f..74333eac2b 100644 --- a/src/web-ui/src/app/scenes/session/sessionPaneLayoutStore.test.ts +++ b/src/web-ui/src/app/scenes/session/sessionPaneLayoutStore.test.ts @@ -8,6 +8,19 @@ function preferences(values: Record = {}) { } describe('workbench pane layout compatibility', () => { + it('reveals background output only when its owning session is activated', () => { + const store = createSessionPaneLayoutStore(preferences()); + const pane = store.getState(); + pane.activate('local', 'b'); + const activeKey = store.getState().activeKey; + pane.showContentForSession('local', 'a'); + expect(store.getState().activeKey).toBe(activeKey); + expect(selectSessionPaneMode(store.getState())).toBe('chat-only'); + pane.activate('local', 'a'); + expect(selectSessionPaneMode(store.getState())).toBe('split'); + pane.activate('peer', 'a'); + expect(selectSessionPaneMode(store.getState())).toBe('chat-only'); + }); it('applies the local display preference immediately, retaining device modes and separate widths', () => { const storage = preferences({ [STORAGE_KEYS.RIGHT_PANEL_LAST_WIDTH]: '640' }); const store = createSessionPaneLayoutStore(storage); diff --git a/src/web-ui/src/app/workbench/workbenchPaneLayoutStore.ts b/src/web-ui/src/app/workbench/workbenchPaneLayoutStore.ts index 78112d5ad8..43e1a7a855 100644 --- a/src/web-ui/src/app/workbench/workbenchPaneLayoutStore.ts +++ b/src/web-ui/src/app/workbench/workbenchPaneLayoutStore.ts @@ -31,6 +31,7 @@ interface WorkbenchPaneLayoutState { preferredRightPaneWidth: number; activate: (surfaceId: string, sessionId: string | null) => void; showContent: () => void; + showContentForSession: (surfaceId: string, sessionId: string) => void; hideContent: () => void; collapseEmptyContent: () => void; maximizeContent: () => void; @@ -131,6 +132,13 @@ export function createWorkbenchPaneLayoutStore(storage = localPreferenceStorage( showContent: () => { if (selectWorkbenchPaneMode(get()) === 'chat-only') setMode('split'); }, + showContentForSession: (surfaceId, sessionId) => set(state => { + const key = scopeKey(surfaceId, sessionId); + if ((state.modes.get(key) ?? 'chat-only') !== 'chat-only') return state; + const modes = new Map(state.modes); + modes.set(key, 'split'); + return { modes }; + }), hideContent: () => setMode('chat-only'), collapseEmptyContent: () => { if (selectWorkbenchPaneMode(get()) === 'split') setMode('chat-only'); diff --git a/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.test.ts b/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.test.ts index f01d5c81da..2751bd9f95 100644 --- a/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.test.ts +++ b/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.test.ts @@ -28,6 +28,7 @@ import { chatInputSessionSubscriptionKey } from '../../utils/chatInputSessionSub import { selectInterruptedTurnRecovery } from '../../utils/interruptedTurnRecovery'; import { promptSuggestionKey, usePromptSuggestionStore } from '../../store/promptSuggestionStore'; import { clearAgentCanvasForPeerSwitch, switchAgentCanvasScope, useAgentCanvasStore } from '@/app/components/panels/content-canvas/stores'; +import { activateSurface } from '@/infrastructure/peer-device/deviceSurface'; const { buildBuiltInBrowserTabOptions, @@ -53,6 +54,19 @@ describe('host session deletion', () => { beforeEach(() => { resetFlowChatStore(); clearAgentCanvasForPeerSwitch(); }); afterEach(() => { resetFlowChatStore(); clearAgentCanvasForPeerSwitch(); }); + it.each([true, false])('releases retained browser snapshots when the deleted session is active=%s', active => { + switchAgentCanvasScope('browser-b'); + useAgentCanvasStore.getState().addTab({ type: 'text-viewer', title: 'B work' }, 'active'); + const bTab = useAgentCanvasStore.getState().primaryGroup.tabs[0].id; + __test_only__.handleOpenBuiltInBrowser({ sessionId: 'browser-a', targetId: 'page-a', url: 'https://a.example' }); + if (active) switchAgentCanvasScope('browser-a'); + __test_only__.handleSessionDeleted(createFlowChatContext(), { sessionId: 'browser-a' }); + switchAgentCanvasScope('browser-a'); + expect(useAgentCanvasStore.getState().primaryGroup.tabs).toEqual([]); + switchAgentCanvasScope('browser-b'); + expect(useAgentCanvasStore.getState().primaryGroup.tabs.map(tab => tab.id)).toEqual([bTab]); + }); + it.each([true, false])('retires an AgentControl-deleted child tab even when its record is present=%s', recordPresent => { createSessionWithTurn(createFinishingTurn()); const store = FlowChatStore.getInstance(); @@ -188,6 +202,55 @@ describe('prompt suggestion projection', () => { }); describe('built-in browser open request projection', () => { + beforeEach(() => clearAgentCanvasForPeerSwitch()); + afterEach(() => clearAgentCanvasForPeerSwitch()); + + it('binds a manually opened browser tab to its conversation and restores the same target', () => { + switchAgentCanvasScope('manual-owner'); + useAgentCanvasStore.getState().addTab({ type: 'browser', title: 'Browser', data: { url: 'https://example.com' } }, 'active'); + const data = useAgentCanvasStore.getState().primaryGroup.tabs[0].content.data; + expect(data.conversationId).toBe('manual-owner'); + expect(data.targetId).toMatch(/^embedded-browser-panel-view-/); + switchAgentCanvasScope('other-owner'); + switchAgentCanvasScope('manual-owner'); + expect(useAgentCanvasStore.getState().primaryGroup.tabs[0].content.data).toEqual(data); + }); + + it('never binds a peer-host browser label to a controller-local WebView', () => { + activateSurface('peer-browser'); + try { + switchAgentCanvasScope('same-session-id'); + __test_only__.handleOpenBuiltInBrowser({ sessionId: 'same-session-id', targetId: 'embedded-browser-panel-view-1', url: 'https://peer.example' }); + expect(useAgentCanvasStore.getState().primaryGroup.tabs).toEqual([]); + } finally { activateSurface('local'); } + }); + + it('opens A in its suspended canvas while B remains unchanged, and reuses each native target', () => { + switchAgentCanvasScope('browser-b'); + useAgentCanvasStore.getState().addTab({ type: 'text-viewer', title: 'B work', data: {} }, 'active'); + const b = useAgentCanvasStore.getState().primaryGroup; + const openA = { sessionId: 'browser-a', targetId: 'embedded-browser-panel-view-a', requestId: 'a-1', url: 'https://a.example' }; + __test_only__.handleOpenBuiltInBrowser(openA); + expect(useAgentCanvasStore.getState().scopeKey).toBe('browser-b'); + expect(useAgentCanvasStore.getState().primaryGroup).toEqual(b); + __test_only__.handleOpenBuiltInBrowser({ ...openA, requestId: 'a-2', url: 'https://a.example/next' }); + switchAgentCanvasScope('browser-a'); + expect(useAgentCanvasStore.getState().primaryGroup.tabs).toHaveLength(1); + expect(useAgentCanvasStore.getState().primaryGroup.tabs[0].content.data).toMatchObject({ + conversationId: 'browser-a', targetId: openA.targetId, url: 'https://a.example/next', + }); + switchAgentCanvasScope('browser-b'); + expect(useAgentCanvasStore.getState().primaryGroup).toEqual(b); + }); + + it('retains background browser tabs beyond the ordinary canvas LRU limit', () => { + const event = { sessionId: 'retained-a', targetId: 'embedded-browser-panel-view-retained', url: 'https://a.example' }; + __test_only__.handleOpenBuiltInBrowser(event); + for (let index = 0; index < 30; index++) switchAgentCanvasScope(`other-${index}`); + switchAgentCanvasScope('retained-a'); + expect(useAgentCanvasStore.getState().primaryGroup.tabs[0].content.data.targetId).toBe(event.targetId); + }); + it('correlates replacement and new-tab requests with distinct stable keys', () => { expect(buildBuiltInBrowserTabOptions({ url: ' https://openbitfun.com/ ', diff --git a/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.ts b/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.ts index 607aacdd5c..446f2cc12e 100644 --- a/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.ts +++ b/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.ts @@ -65,6 +65,8 @@ import { useBackgroundCommandActivityStore } from '../../store/backgroundCommand import { useBackgroundSubagentActivityStore } from '../../store/backgroundSubagentActivityStore'; import { clearPromptSuggestion, usePromptSuggestionStore } from '../../store/promptSuggestionStore'; import { createTab } from '@/shared/utils/tabUtils'; +import { openAgentCanvasTabForSession } from '@/app/components/panels/content-canvas/stores/canvasStore'; +import { workbenchPaneLayoutStore } from '@/app/workbench/workbenchPaneLayoutStore'; import type { TabCreationOptions } from '@/shared/utils/tabUtils'; import { interruptedTurnRecoveryGate } from '../interruptedTurnRecoveryGate'; import { @@ -220,6 +222,7 @@ export const __test_only__ = { handleDialogTurnRecovered, handleDialogTurnCancelled, buildBuiltInBrowserTabOptions, + handleOpenBuiltInBrowser, }; function eventOwnsLatestSessionTurn( @@ -2768,14 +2771,14 @@ function buildBuiltInBrowserTabOptions( const replaceExisting = event?.replaceExisting !== false; // Replacing uses one stable product surface. A true new-tab request gets a // request-scoped key so identical URLs can still open as distinct targets. - const duplicateCheckKey = replaceExisting + const duplicateCheckKey = event.targetId ? `browser-panel:${event.targetId}` : replaceExisting ? 'browser-panel' : `browser-panel:${requestId ?? url}`; return { type: 'browser', title, - data: { url, openRequestId: requestId }, + data: { url, openRequestId: requestId, targetId: event.targetId, conversationId: event.sessionId }, metadata: { duplicateCheckKey }, checkDuplicate: true, duplicateCheckKey, @@ -2785,12 +2788,22 @@ function buildBuiltInBrowserTabOptions( } function handleOpenBuiltInBrowser(event: OpenBuiltInBrowserEvent): void { + // These are native pages on the executing Desktop. A peer's target id must + // never be treated as a controller-local WebView (even if labels collide). + if (getActiveSurfaceScope().surfaceId !== 'local') return; const options = buildBuiltInBrowserTabOptions(event); if (!options) { log.warn('OpenBuiltInBrowser missing url', { event }); return; } - createTab(options); + if (event.sessionId) { + openAgentCanvasTabForSession(event.sessionId, { + type: 'browser', title: options.title, data: options.data, metadata: options.metadata, + }); + workbenchPaneLayoutStore.getState().showContentForSession('local', event.sessionId); + } else { + createTab(options); + } } export function handleDialogTurnComplete( diff --git a/src/web-ui/src/flow_chat/store/FlowChatStore.ts b/src/web-ui/src/flow_chat/store/FlowChatStore.ts index eca705db57..747ea212d5 100644 --- a/src/web-ui/src/flow_chat/store/FlowChatStore.ts +++ b/src/web-ui/src/flow_chat/store/FlowChatStore.ts @@ -1,5 +1,5 @@ import { workspaceManager } from '@/infrastructure/services/business/workspaceManager'; -import { removeBtwSessionFromAgentCanvas } from '@/app/components/panels/content-canvas/stores/canvasStore'; +import { forgetAgentCanvasSession, removeBtwSessionFromAgentCanvas } from '@/app/components/panels/content-canvas/stores/canvasStore'; import { resolveLegacySessionWorkspace } from '@/infrastructure/api/service-api/legacyWorkspaceCompatibility'; import { projectUserQuestionTiming } from '../utils/userQuestionTiming'; import { externalHistoryFromMetadata } from '../utils/externalHistory'; @@ -5421,7 +5421,10 @@ export class FlowChatStore { public removeDeletedSession(sessionId: string, options?: RemoveSessionOptions): string[] { const removedIds = this.removeSession(sessionId, options); // A deletion event may arrive after the record was already removed. - new Set([sessionId, ...removedIds]).forEach(removeBtwSessionFromAgentCanvas); + new Set([sessionId, ...removedIds]).forEach(id => { + removeBtwSessionFromAgentCanvas(id); + forgetAgentCanvasSession(id); + }); return removedIds; } diff --git a/src/web-ui/src/infrastructure/api/adapters/peer-device-adapter.test.ts b/src/web-ui/src/infrastructure/api/adapters/peer-device-adapter.test.ts index b9922f141c..554031b8d3 100644 --- a/src/web-ui/src/infrastructure/api/adapters/peer-device-adapter.test.ts +++ b/src/web-ui/src/infrastructure/api/adapters/peer-device-adapter.test.ts @@ -176,6 +176,7 @@ describe('isPeerLocalOnlyCommand', () => { // them to a peer would regress (peer host does not implement them, and // they drive the controller's own embedded surfaces). expect(isPeerLocalOnlyCommand('browser_webview_create')).toBe(true); + expect(isPeerLocalOnlyCommand('browser_get_url')).toBe(true); expect(isPeerLocalOnlyCommand('browser_webview_capture_preview')).toBe(true); expect(isPeerLocalOnlyCommand('browser_webview_eval')).toBe(true); expect(isPeerLocalOnlyCommand('browser_webview_navigate')).toBe(true); diff --git a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts index 2897f39baa..444530ff26 100644 --- a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts +++ b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts @@ -1,6 +1,6 @@ // Generated by scripts/generate-interactive-capabilities.mjs; do not edit. // Source: openbitfun_product_domains::remote_surface (Product Operation Registry). -export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:1f41015e75434c3c" as const; +export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:39cac84a6ae186a5" as const; /** * Registered Tauri commands the Peer Device controller keeps on the controller @@ -39,6 +39,7 @@ export const PEER_CONTROLLER_LOCAL_COMMANDS: ReadonlySet = new Set([ "appearance_market_submit_package", "appearance_market_withdraw_submission", "append_flow_chat_diagnostics", + "browser_get_url", "browser_webview_capture_preview", "browser_webview_create", "browser_webview_eval", diff --git a/src/web-ui/src/infrastructure/api/service-api/AgentAPI.ts b/src/web-ui/src/infrastructure/api/service-api/AgentAPI.ts index 4c2d24d62d..463e579735 100644 --- a/src/web-ui/src/infrastructure/api/service-api/AgentAPI.ts +++ b/src/web-ui/src/infrastructure/api/service-api/AgentAPI.ts @@ -646,6 +646,10 @@ export type DialogTurnStartedEvent = AgenticEvent; export interface OpenBuiltInBrowserEvent { requestId?: string; + /** Owning Agent conversation, independent of the currently displayed one. */ + sessionId?: string; + /** Host-owned native page; mounting its panel only attaches presentation. */ + targetId?: string; url: string; title?: string; replaceExisting?: boolean; diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 9e06faa0ea..a025ca95aa 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -63,6 +63,16 @@ and remote child-view ACL rejection with multiple browser children. The same binary/frontend snapshot overrides above are supported. This is native command coverage, not a pet-pointer or remote-session integration test. +For conversation-owned browser pages, build Desktop and Web UI and run +`node tests/e2e/scripts/run-browser-session-isolation.mjs`. The fixture uses +isolated storage and loopback model/page servers. It drives the actual session +controls and Agent tool dispatcher to check background opening, concurrent reads, +native visibility and document preservation across repeated session switches. +Inspect its `result.json`; local native coverage does not prove remote behavior. +Full visibility coverage needs an unlocked desktop. For a locked/headless display, +`OPENBITFUN_E2E_BACKGROUND_ONLY=1` runs background execution and panel binding; +the report explicitly excludes foreground native visibility in that mode. + Markdown editor browser interaction tests (no desktop binary required): ```bash diff --git a/tests/e2e/scripts/run-browser-session-isolation.mjs b/tests/e2e/scripts/run-browser-session-isolation.mjs new file mode 100644 index 0000000000..f86c7e9a7a --- /dev/null +++ b/tests/e2e/scripts/run-browser-session-isolation.mjs @@ -0,0 +1,343 @@ +/** Native multi-session browser regression. Only model HTTP responses are scripted. */ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { cp, mkdir, mkdtemp, open, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:net'; +import { createServer as createHttpServer } from 'node:http'; +import { randomUUID } from 'node:crypto'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..'); +const backgroundOnly = process.env.OPENBITFUN_E2E_BACKGROUND_ONLY === '1'; +const root = await mkdtemp(path.join(tmpdir(), 'openbitfun-browser-sessions-')); +const userRoot = path.join(root, 'user'); +const productHome = path.join(root, 'home'); +const frontend = path.join(root, 'frontend'); +const frontendSource = path.resolve(process.env.OPENBITFUN_E2E_FRONTEND_DIR || path.join(repo, 'dist')); +const manifest = await readFile(path.join(frontendSource, 'frontend-revision.json'), 'utf8'); +await cp(frontendSource, frontend, { recursive: true }); +assert.equal(await readFile(path.join(frontendSource, 'frontend-revision.json'), 'utf8'), manifest, + 'The frontend changed during the snapshot. Finish the build and rerun.'); +await mkdir(userRoot, { recursive: true }); +await mkdir(productHome, { recursive: true }); +const binaryName = 'openbitfun-desktop' + (process.platform === 'win32' ? '.exe' : ''); +const binarySource = path.resolve(process.env.OPENBITFUN_E2E_DESKTOP_BINARY || path.join(repo, 'target/debug', binaryName)); +const binary = path.join(root, binaryName); +const before = await stat(binarySource); +await cp(binarySource, binary); +const after = await stat(binarySource); +assert.equal(before.mtimeMs, after.mtimeMs, 'The Desktop binary changed during the snapshot. Rerun after the build.'); +assert.equal(before.size, after.size); + +const probe = createServer(); +probe.listen(0, '127.0.0.1'); +await once(probe, 'listening'); +const port = probe.address().port; +await new Promise(resolve => probe.close(resolve)); +const endpoint = 'http://127.0.0.1:' + port; +const log = await open(path.join(root, 'desktop.log'), 'w'); + +const workspace = path.join(root, 'workspace'); +await mkdir(workspace, { recursive: true }); +const pendingModels = new Map(); +const toolResults = new Map(); +const fixtureErrors = []; +let fixtureUrl; +function respondModel(res, marker, tool) { + const [owner, action] = marker.split('_'); + const input = { domain: 'browser', action: action === 'REOPEN' ? 'open_builtin' : action === 'OPEN' ? 'navigate' : action === 'CLICK' ? 'click' : 'read_article', + params: action === 'REOPEN' ? { url: `${fixtureUrl}/pages/${owner.toLowerCase()}?replacement=1`, replace_existing: true, observe: 'article' } + : action === 'OPEN' ? { target: 'builtin', url: `${fixtureUrl}/pages/${owner.toLowerCase()}`, observe: 'article' } + : action === 'CLICK' ? { selector: '#next' } : {} }; + const events = [ + { type: 'message_start', message: { id: marker, type: 'message', role: 'assistant', model: 'browser-fixture', usage: { input_tokens: 10, output_tokens: 0 } } }, + { type: 'content_block_start', index: 0, content_block: tool + ? { type: 'tool_use', id: `browser-${marker}`, name: 'ControlHub', input: {} } : { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: tool + ? { type: 'input_json_delta', partial_json: JSON.stringify(input) } : { type: 'text_delta', text: `BROWSER_${marker}_COMPLETE` } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: tool ? 'tool_use' : 'end_turn' }, usage: { output_tokens: 5 } }, + { type: 'message_stop' }, + ]; + if (!res.headersSent) res.writeHead(200, { 'content-type': 'text/event-stream' }); + for (const event of events) res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`); + res.end(); +} +const fixture = createHttpServer(async (req, res) => { + try { + const url = new URL(req.url, fixtureUrl); + if (url.pathname.startsWith('/pages/')) { + const owner = url.pathname.endsWith('/a') ? 'A' : 'B'; + const replacement = url.searchParams.has('replacement'); + if (replacement) await new Promise(resolve => setTimeout(resolve, 500)); + res.writeHead(200, { 'content-type': 'text/html' }); + res.end(`Browser ${owner}

SESSION_${owner}_PAGE

${replacement ? '

REPLACEMENT_DOCUMENT

' : ''}

${('Session ' + owner + ' content. ').repeat(100)}

`); + return; + } + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const body = JSON.parse(Buffer.concat(chunks).toString() || '{}'); + const messages = body.messages ?? []; + const markers = messages.filter(message => message.role === 'user').flatMap(message => { + const text = typeof message.content === 'string' ? message.content : (message.content ?? []).filter(block => block.type === 'text').map(block => block.text).join('\n'); + return [...text.matchAll(/BROWSER_([AB]_(?:OPEN|READ|CLICK|REOPEN))/g)].map(match => match[1]); + }); + const marker = markers.at(-1); + if (!marker) return respondModel(res, 'A_AUX', false); + const result = messages.flatMap(message => Array.isArray(message.content) ? message.content : []) + .find(block => block.type === 'tool_result' && block.tool_use_id === `browser-${marker}`); + if (result) { + toolResults.set(marker, result); + return respondModel(res, marker, false); + } + assert.ok((body.tools ?? []).some(tool => tool.name === 'ControlHub'), 'ControlHub must be available in the actual tool catalog'); + if (marker === 'A_OPEN' || marker.endsWith('_READ')) { + // Hold A's open until the UI switches to B, and release both reads together. + res.writeHead(200, { 'content-type': 'text/event-stream' }); + res.write(': awaiting conversation switch\n\n'); + pendingModels.set(marker, () => respondModel(res, marker, true)); + } else respondModel(res, marker, true); + } catch (error) { + fixtureErrors.push(String(error)); + res.writeHead(500, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: String(error) } })); + } +}); +fixture.listen(0, '127.0.0.1'); +await once(fixture, 'listening'); +fixtureUrl = `http://127.0.0.1:${fixture.address().port}`; + +const app = spawn(binary, [], { + cwd: repo, windowsHide: true, stdio: ['ignore', log.fd, log.fd], + env: { ...process.env, + OPENBITFUN_USER_ROOT: userRoot, OPENBITFUN_E2E_USER_ROOT: userRoot, + OPENBITFUN_HOME: productHome, OPENBITFUN_E2E_HOME: productHome, + OPENBITFUN_E2E_STORAGE_GUARD: '1', OPENBITFUN_E2E_PACKAGED_FRONTEND: '1', + OPENBITFUN_E2E_FRONTEND_DIR: frontend, OPENBITFUN_E2E_LOG_DIR: path.join(root, 'logs'), + OPENBITFUN_WEBDRIVER_PORT: String(port), OPENBITFUN_WEBDRIVER_LABEL: 'main', + WEBVIEW2_USER_DATA_FOLDER: path.join(root, 'webview'), + }, +}); +console.log('Native browser session evidence: ' + root); +let exitError; +app.on('error', error => { exitError = error; }); +let session; +const report = { passed: false, platform: process.platform, binarySource, + binaryModified: before.mtime.toISOString(), frontendRevision: JSON.parse(manifest).revision, + coverage: backgroundOnly ? 'background-and-panel-binding-only' : 'full-native-visibility', checks: [] }; +async function request(route, body, method = body === undefined ? 'GET' : 'POST') { + const response = await fetch(endpoint + route, { + method, signal: AbortSignal.timeout(15_000), + ...(body === undefined ? {} : { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }), + }); + const result = await response.json(); + if (!response.ok || (result.value?.error && result.value?.ok === undefined)) throw new Error(JSON.stringify(result)); + return result.value; +} +async function until(read, description, timeout = 45_000) { + const started = Date.now(); + let last; + while (Date.now() - started < timeout) { + if (exitError) throw exitError; + if (app.exitCode !== null || app.signalCode !== null) throw new Error('Desktop exited before ' + description); + try { if (await read()) return; } catch (error) { last = error; } + await new Promise(resolve => setTimeout(resolve, 250)); + } + throw new Error('Timed out waiting for ' + description + ': ' + (last ?? 'not ready')); +} +const execute = (fn, ...args) => request('/session/' + session + '/execute/sync', { script: fn.toString(), args }); +const invokeResult = (command, args = {}) => execute(async (command, args) => { + try { return { ok: true, value: await window.__TAURI_INTERNALS__.invoke(command, args) }; } + catch (error) { return { ok: false, error: String(error) }; } +}, command, args); +const invoke = (command, args = {}) => invokeResult(command, args).then(result => { + assert.equal(result.ok, true, command + ': ' + result.error); + return result.value; +}); + +const elementKey = 'element-6066-11e4-a52e-4f735466cecf'; +const locate = selector => request(`/session/${session}/element`, { using: 'css selector', value: selector }); +async function click(selector) { + const element = await locate(selector); + await request(`/session/${session}/element/${element[elementKey]}/click`, {}); +} +const selected = id => execute(id => Boolean(document.querySelector(`[data-testid="nav-session-item"][data-session-id="${id}"][data-session-active="true"]`)), id); +async function select(id) { + if (!backgroundOnly) { + await invoke('show_main_window'); + await until(() => execute(() => document.visibilityState === 'visible'), 'visible host window (unlock the desktop for full visibility coverage)'); + } + await click(`[data-testid="nav-session-item"][data-session-id="${id}"]`); + await until(() => selected(id), `selected conversation ${id}`); +} +async function tool(action, params = {}) { + const result = await invoke('execute_tool', { request: { toolName: 'ControlHub', input: { domain: 'browser', action, params } } }); + assert.equal(result.success, true, JSON.stringify(result)); + assert.equal(result.result.ok, true, JSON.stringify(result)); + return result.result.data; +} +async function pageState(id) { + const data = await tool('evaluate', { target: 'builtin', session_id: id, + expression: '({url:location.href,instance:window.pageInstance,ticks:window.ticks,draft:document.querySelector("#draft").value,visibility:document.visibilityState})' }); + return data.result.value; +} +async function presentation(activePage, hiddenPage) { + await until(() => execute(label => { + const panels = document.querySelectorAll('[data-testid=browser-panel]'); + return panels.length === 1 && panels[0].querySelector('[data-webview-label]')?.getAttribute('data-webview-label') === label; + }, activePage), 'active panel bound to its own native target'); + if (!backgroundOnly) { + await until(async () => (await pageState(activePage)).visibility === 'visible' + && (await pageState(hiddenPage)).visibility === 'hidden', 'only selected session browser visible'); + } else { + assert.equal((await pageState(hiddenPage)).visibility, 'hidden'); + } +} +async function turn(id, marker, workspaceId) { + await invoke('start_dialog_turn', { request: { sessionId: id, userInput: `BROWSER_${marker}`, agentType: 'agentic', workspaceId } }); +} +async function completed(marker) { + await until(async () => { + assert.deepEqual(fixtureErrors, []); + return toolResults.has(marker); + }, `browser tool result ${marker}`, 60_000); + const result = JSON.stringify(toolResults.get(marker)); + if (!result.includes(`SESSION_${marker[0]}_PAGE`)) { + console.log('Unexpected tool output', result); + console.log('Native diagnostic', JSON.stringify(await tool('evaluate', { target: 'builtin', + expression: '({url:location.href,ready:document.readyState,html:document.documentElement.outerHTML})' }))); + console.log('Article diagnostic', JSON.stringify(await tool('read_article', { target: 'builtin' }))); + } + assert.ok(result.includes(`SESSION_${marker[0]}_PAGE`), `Wrong page in ${marker}: ${result}`); +} +try { + await until(async () => (await request('/status')).ready, 'embedded driver'); + session = (await request('/session', {})).sessionId; + await request(`/session/${session}/timeouts`, { script: 40000 }); + await until(() => execute(() => !!document.querySelector('[data-testid=nav-workspace-new-session-btn]')), 'workspace navigation'); + assert.deepEqual(await execute(() => ({ length: 2, textContent: 'Article metadata', items: [1, 2] })), + { length: 2, textContent: 'Article metadata', items: [1, 2] }, 'Numeric length must not turn an object into a collection'); + const buttons = await execute(() => document.querySelectorAll('[data-testid=nav-workspace-new-session-btn]')); + assert.ok(Array.isArray(buttons) && buttons.length > 0 && buttons.every(button => button[elementKey]), + 'Native DOM collections must retain WebDriver element serialization'); + report.checks.push('WebDriver preserves article-shaped objects and native DOM collections'); + const models = await invoke('get_config', { request: { path: 'ai.models' } }); + await invoke('set_config', { request: { path: 'ai.models', expectedModels: models, value: [{ + id: 'browser-fixture', name: 'Browser fixture', provider: 'anthropic', model_name: 'browser-fixture', + base_url: fixtureUrl, request_url: `${fixtureUrl}/v1/messages`, api_key: 'isolated-fixture', + auth: { type: 'api_key' }, enabled: true, context_window: 128000, max_tokens: 1024, + capabilities: ['text_chat', 'function_calling'], + }] } }); + await invoke('set_config', { request: { path: 'ai.default_models', value: { primary: 'browser-fixture', fast: 'browser-fixture' } } }); + await invoke('set_config', { request: { path: 'ai.enable_deferred_tool_loading', value: false } }); + await invoke('set_config', { request: { path: 'app.ai_experience.enable_session_title_generation', value: false } }); + await invoke('set_config', { request: { path: 'app.ai_experience.enable_welcome_panel_ai_analysis', value: false } }); + const opened = await invoke('open_workspace', { request: { path: workspace, createLocal: true } }); + await invoke('set_active_workspace', { request: { workspaceId: opened.id } }); + const a = randomUUID(), b = randomUUID(); + for (const [sessionId, name] of [[a, 'Browser A'], [b, 'Browser B']]) { + await invoke('create_session', { request: { sessionId, sessionName: name, agentType: 'agentic', + workspacePath: workspace, workspaceId: opened.id, config: { modelName: 'browser-fixture', safeMode: false } } }); + } + await execute(() => { setTimeout(() => location.reload(), 0); return true; }); + await until(() => locate(`[data-testid="nav-session-item"][data-session-id="${a}"]`), 'persisted conversations'); + await select(a); + await turn(a, 'A_OPEN', opened.id); + await until(() => pendingModels.has('A_OPEN'), 'A waiting before its browser call'); + await select(b); + pendingModels.get('A_OPEN')(); + await completed('A_OPEN'); + assert.equal(await selected(b), true, 'Background A must not change the active conversation'); + assert.equal(await execute(() => document.querySelectorAll('[data-testid=browser-panel]').length), 0, 'A must not appear in B'); + let inventory = await tool('list_pages', { target: 'builtin' }); + const aPage = inventory.pages.find(page => page.url.includes('/pages/a')).id; + await turn(a, 'A_CLICK', opened.id); + await until(() => toolResults.has('A_CLICK'), 'A background link click'); + await until(async () => (await pageState(aPage)).url.endsWith('?step=2'), 'new-window link stays in its conversation page'); + assert.equal(await selected(b), true); + assert.equal(await execute(() => document.querySelectorAll('[data-testid=browser-panel]').length), 0); + report.checks.push('A follows a new-window link in its own hidden page before ever mounting a panel'); + await turn(a, 'A_REOPEN', opened.id); + await completed('A_REOPEN'); + assert.ok(JSON.stringify(toolResults.get('A_REOPEN')).includes('REPLACEMENT_DOCUMENT'), + 'A replacement read must observe the new document rather than the previous completed page'); + assert.equal((await tool('list_pages', { target: 'builtin' })).pages[0].id, aPage); + report.checks.push('Replacing a hidden page waits for the new document and retains its native identity'); + const before = await pageState(aPage); + assert.equal(before.visibility, 'hidden'); + report.checks.push('A opens and reads its native page in the background while B remains selected'); + await turn(b, 'B_OPEN', opened.id); + await completed('B_OPEN'); + if (!backgroundOnly) await invoke('show_main_window'); + inventory = await tool('list_pages', { target: 'builtin' }); + const bPage = inventory.pages.find(page => page.url.includes('/pages/b')).id; + assert.notEqual(aPage, bPage); + await presentation(bPage, aPage); + await tool('evaluate', { session_id: aPage, expression: 'document.querySelector("#draft").value="retained A draft"' }); + await turn(a, 'A_READ', opened.id); + await turn(b, 'B_READ', opened.id); + await until(() => pendingModels.has('A_READ') && pendingModels.has('B_READ'), 'both conversations ready for concurrent reads'); + pendingModels.get('A_READ')(); + pendingModels.get('B_READ')(); + await Promise.all([completed('A_READ'), completed('B_READ')]); + report.checks.push('Both real Agent conversations read their own pages concurrently'); + for (const id of [a, b, a, b, a]) { + await select(id); + const visibleId = id === a ? aPage : bPage; + const hiddenId = id === a ? bPage : aPage; + await presentation(visibleId, hiddenId); + } + const after = await pageState(aPage); + assert.equal(after.instance, before.instance, 'Switching must retain the original native document'); + assert.equal(after.draft, 'retained A draft'); + assert.ok(after.ticks > before.ticks, 'The hidden page must continue executing'); + const finalPages = await tool('list_pages', { target: 'builtin' }); + assert.equal(finalPages.pages.length, 2, 'Switching must not leak duplicate WebViews'); + report.checks.push('Repeated switches preserve page identity, edited DOM and background execution with exactly two WebViews'); + if (!backgroundOnly) report.checks.push('Only the active conversation native page is visible after every switch'); + await select(b); + const bState = await pageState(bPage); + await invoke('delete_session', { request: { sessionId: a, workspaceId: opened.id } }); + await until(async () => { + const pages = (await tool('list_pages', { target: 'builtin' })).pages; + return pages.length === 1 && pages[0].id === bPage; + }, 'only A native resources released on deletion'); + assert.equal((await pageState(bPage)).instance, bState.instance); + assert.equal(await selected(b), true); + await invoke('delete_session', { request: { sessionId: b, workspaceId: opened.id } }); + await until(async () => (await tool('list_pages', { target: 'builtin' })).pages.length === 0, + 'all native pages released after their conversations are deleted'); + report.checks.push('Deleting a conversation releases only its own page and leaves the other document intact'); + report.passed = true; +} catch (error) { + report.error = String(error); + if (session) { + report.dom = await execute(() => ({ text: document.body.innerText, + panels: [...document.querySelectorAll('[data-testid=browser-panel]')].map(panel => ({ + html: panel.outerHTML, rect: panel.getBoundingClientRect().toJSON(), + })), visibility: document.visibilityState })).catch(String); + const screenshot = await request(`/session/${session}/screenshot`).catch(() => null); + if (typeof screenshot === 'string') await writeFile(path.join(root, 'failure.png'), Buffer.from(screenshot, 'base64')); + } + throw error; +} finally { + report.fixtureErrors = fixtureErrors; + await writeFile(path.join(root, 'result.json'), JSON.stringify(report, null, 2)); + if (session) await request(`/session/${session}`, undefined, 'DELETE').catch(() => {}); + if (app.exitCode === null && app.signalCode === null) { + app.kill(); + await Promise.race([once(app, 'exit'), new Promise(resolve => setTimeout(resolve, 5000))]); + if (app.exitCode === null && app.signalCode === null) app.kill('SIGKILL'); + } + fixture.closeAllConnections(); + await new Promise(resolve => fixture.close(resolve)); + await log.close(); + if (process.env.OPENBITFUN_E2E_KEEP_SNAPSHOT !== '1') { + await rm(binary, { force: true }); + await rm(frontend, { recursive: true, force: true }); + } + console.log(JSON.stringify({ ...report, artifactRoot: root })); +}