Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,6 @@ Thumbs.db
*.profraw
*.profdata
coverage/

# Release test artifacts
release_test/
12 changes: 12 additions & 0 deletions crates/sandbox-cli/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
fn main() {
// Only apply on macOS
if cfg!(target_os = "macos") {
// Add Swift runtime rpath so screencapturekit can find
// libswift_Concurrency.dylib at runtime
println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift");

// Also add the Xcode Toolchain path as fallback
// This covers both Xcode and Command Line Tools installations
println!("cargo:rustc-link-arg=-Wl,-rpath,/Library/Developer/Toolchains/swift-latest.xctoolchain/usr/lib/swift/macosx");
}
}
9 changes: 6 additions & 3 deletions crates/sandbox-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ enum Commands {
/// Output file path
#[arg(short, long)]
output: Option<PathBuf>,
/// Window ID to capture (uses sandbox window if not specified)
#[arg(long)]
window_id: Option<u32>,
},

/// List windows in the sandbox
Expand Down Expand Up @@ -133,11 +136,11 @@ async fn main() -> anyhow::Result<()> {

axum::serve(listener, app).await?;
}
Commands::Screenshot { output } => {
Commands::Screenshot { output, window_id } => {
let path = output.unwrap_or_else(|| PathBuf::from("sandbox_screenshot.png"));
tracing::info!("Taking screenshot -> {path:?}");
tracing::info!("Taking screenshot -> {path:?} (window_id={window_id:?})");

let png_data = ScreenCapture::capture_sandbox()?;
let png_data = ScreenCapture::capture_sandbox_by_id(window_id)?;
std::fs::write(&path, &png_data)?;
println!("Screenshot saved to {path:?} ({} bytes)", png_data.len());
}
Expand Down
33 changes: 26 additions & 7 deletions crates/sandbox-cli/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,13 @@ struct RegionQuery {
height: u32,
}

/// Screenshot query params
#[derive(Deserialize)]
struct ScreenshotQuery {
#[serde(default)]
window_id: Option<u32>,
}

/// Build the HTTP API router
pub fn build_router(state: Arc<Mutex<AppState>>) -> Router {
Router::new()
Expand Down Expand Up @@ -235,8 +242,10 @@ async fn drag_handler(Json(req): Json<DragRequest>) -> Result<Json<serde_json::V
Ok(Json(serde_json::json!({"dragged": true})))
}

async fn screenshot_handler() -> Result<impl IntoResponse, AppError> {
let png_data = ScreenCapture::capture_sandbox()?;
async fn screenshot_handler(
Query(q): Query<ScreenshotQuery>,
) -> Result<impl IntoResponse, AppError> {
let png_data = ScreenCapture::capture_sandbox_by_id(q.window_id)?;
Ok((StatusCode::OK, [("content-type", "image/png")], png_data))
}

Expand All @@ -248,8 +257,10 @@ async fn screenshot_region_handler(
}

async fn ui_inspect_handler(Path(window_id): Path<u32>) -> Result<Json<UiElement>, AppError> {
let element = UiInspector::inspect_window(window_id)?;
Ok(Json(element))
let result = tokio::task::spawn_blocking(move || UiInspector::inspect_window(window_id))
.await
.map_err(|e| AppError::Accessibility(format!("UI inspect task failed: {e}")))?;
Ok(Json(result?))
}

#[derive(Deserialize)]
Expand All @@ -262,9 +273,15 @@ struct UiFindRequest {
}

async fn ui_find_handler(Json(req): Json<UiFindRequest>) -> Result<Json<Vec<UiElement>>, AppError> {
let elements =
UiInspector::find_elements(req.window_id, req.role.as_deref(), req.title.as_deref())?;
Ok(Json(elements))
let window_id = req.window_id;
let role = req.role;
let title = req.title;
let result = tokio::task::spawn_blocking(move || {
UiInspector::find_elements(window_id, role.as_deref(), title.as_deref())
})
.await
.map_err(|e| AppError::Accessibility(format!("UI find task failed: {e}")))?;
Ok(Json(result?))
}

#[derive(Deserialize)]
Expand Down Expand Up @@ -399,6 +416,7 @@ async fn diff_handler(Json(req): Json<DiffRequest>) -> Result<Json<DiffResult>,
enum AppError {
Core(sandbox_core::AppError),
BadRequest(String),
Accessibility(String),
}

impl From<sandbox_core::AppError> for AppError {
Expand All @@ -412,6 +430,7 @@ impl IntoResponse for AppError {
let (status, message) = match self {
AppError::Core(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()),
AppError::BadRequest(msg) => (StatusCode::BAD_REQUEST, msg),
AppError::Accessibility(msg) => (StatusCode::INTERNAL_SERVER_ERROR, msg),
};
(status, Json(serde_json::json!({"error": message}))).into_response()
}
Expand Down
8 changes: 8 additions & 0 deletions crates/sandbox-core/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
fn main() {
// Only apply on macOS
if cfg!(target_os = "macos") {
// Add Swift runtime rpath so screencapturekit can find
// libswift_Concurrency.dylib at runtime
println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift");
}
}
23 changes: 17 additions & 6 deletions crates/sandbox-core/src/automation/ax_ui.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,18 +151,29 @@ mod macos_impl {
}
}

const MAX_UI_DEPTH: usize = 30;

unsafe fn ax_to_ui_element(element: AXUIElementRef) -> UiElement {
ax_to_ui_element_inner(element, 0)
}

unsafe fn ax_to_ui_element_inner(element: AXUIElementRef, depth: usize) -> UiElement {
let role = ax_get_string(element, "AXRole").unwrap_or_else(|| "unknown".to_string());
let title = ax_get_string(element, "AXTitle");
let value = ax_get_string(element, "AXValue");
let description = ax_get_string(element, "AXDescription");

let children_elements = ax_get_children(element);
let children: Vec<UiElement> = children_elements
.iter()
.map(|&child| ax_to_ui_element(child))
.collect();
ax_release_all(&children_elements);
let children = if depth >= MAX_UI_DEPTH {
vec![]
} else {
let children_elements = ax_get_children(element);
let result: Vec<UiElement> = children_elements
.iter()
.map(|&child| ax_to_ui_element_inner(child, depth + 1))
.collect();
ax_release_all(&children_elements);
result
};

UiElement {
role,
Expand Down
48 changes: 38 additions & 10 deletions crates/sandbox-core/src/capture/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,25 +76,47 @@ mod macos_impl {

/// Capture the sandbox window by searching for it by title
pub fn capture_sandbox() -> Result<Vec<u8>> {
Self::capture_sandbox_by_id(None)
}

/// Capture the sandbox window, optionally by a specific window ID.
/// If window_id is None, searches for a window titled "System Test Sandbox".
pub fn capture_sandbox_by_id(window_id: Option<u32>) -> Result<Vec<u8>> {
let content = SCShareableContent::get().map_err(|e| {
AppError::Screenshot(format!("Failed to get shareable content: {e:?}"))
})?;

let window_list = content.windows();
let window = window_list
.iter()
.find(|w| {
w.title()
.map(|t| t.contains("System Test Sandbox"))
.unwrap_or(false)
})
.ok_or_else(|| AppError::WindowNotFound("Sandbox window not found".into()))?;

let window = if let Some(id) = window_id {
// Use the provided window ID directly
window_list
.iter()
.find(|w| w.window_id() == id)
.ok_or_else(|| AppError::WindowNotFound(format!("Window ID {id} not found")))?
} else {
// Fallback: search by title
window_list
.iter()
.find(|w| {
w.title()
.map(|t| t.contains("System Test Sandbox"))
.unwrap_or(false)
})
.ok_or_else(|| {
AppError::WindowNotFound(
"Sandbox window not found. In CLI mode, use capture_window(window_id) \
or start the Tauri app first."
.into(),
)
})?
};

let filter = SCContentFilter::create().with_window(window).build();

let config = SCStreamConfiguration::new()
.with_width(1280)
.with_height(800);
.with_width(window.frame().width as u32)
.with_height(window.frame().height as u32);

let image = SCScreenshotManager::capture_image(&filter, &config)
.map_err(|e| AppError::Screenshot(format!("Failed to capture sandbox: {e:?}")))?;
Expand Down Expand Up @@ -176,6 +198,12 @@ mod non_macos_impl {
))
}

pub fn capture_sandbox_by_id(_window_id: Option<u32>) -> Result<Vec<u8>> {
Err(AppError::Screenshot(
"ScreenCaptureKit only available on macOS".into(),
))
}

pub fn find_window_by_title(_title: &str) -> Result<u32> {
Err(AppError::Screenshot(
"ScreenCaptureKit only available on macOS".into(),
Expand Down
56 changes: 28 additions & 28 deletions crates/sandbox-core/src/process/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,39 +41,39 @@ static NEXT_ID: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new
pub struct ProcessManager;

impl ProcessManager {
/// Launch a macOS .app by path using NSWorkspace
/// Launch a macOS .app by path using the `open` command
/// This avoids ObjC NSExceptions that crash the Rust process
#[cfg(target_os = "macos")]
pub fn spawn_app(app_path: &str) -> Result<ProcessInfo> {
use objc::rc::autoreleasepool;
use objc::{class, msg_send, sel, sel_impl};
let path = std::path::Path::new(app_path);
if !path.exists() {
return Err(AppError::Process(format!(
"App path does not exist: {app_path}"
)));
}

autoreleasepool(|| unsafe {
let path: &objc::runtime::Object =
msg_send![class!(NSString), stringWithUTF8String: app_path.as_ptr() as *const i8];
let url: &objc::runtime::Object = msg_send![class!(NSURL), fileURLWithPath: path];
let workspace: &objc::runtime::Object = msg_send![class!(NSWorkspace), sharedWorkspace];
let config: &objc::runtime::Object =
msg_send![class!(NSWorkspaceOpenConfiguration), configuration];
let _: () = msg_send![config, setCreatesNewApplicationInstance: true as i8];
let success: bool = msg_send![workspace, openApplicationAtURL: url configuration: config error: std::ptr::null_mut::<*mut objc::runtime::Object>()];
let output = std::process::Command::new("open")
.arg(app_path)
.output()
.map_err(|e| AppError::Process(format!("Failed to run `open` command: {e}")))?;

if !success {
return Err(AppError::Process(format!(
"Failed to launch app: {app_path}"
)));
}
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(AppError::Process(format!(
"Failed to launch app: {app_path} ({stderr})"
)));
}

let id = NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(ProcessInfo {
pid: id,
name: std::path::Path::new(app_path)
.file_stem()
.unwrap_or_default()
.to_string_lossy()
.to_string(),
path: Some(app_path.to_string()),
is_running: true,
})
let id = NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(ProcessInfo {
pid: id,
name: std::path::Path::new(app_path)
.file_stem()
.unwrap_or_default()
.to_string_lossy()
.to_string(),
path: Some(app_path.to_string()),
is_running: true,
})
}

Expand Down
Loading
Loading