diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index d78fdaeab..70514479e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -313,7 +313,7 @@ jobs: uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master 2026-07-20; upstream has no releases with: toolchain: 1.96.0 - components: rustfmt + components: rustfmt, clippy - name: Enable shared compiler cache id: sccache continue-on-error: true diff --git a/.github/workflows/premerge.yml b/.github/workflows/premerge.yml index 62ad14561..2244a54aa 100644 --- a/.github/workflows/premerge.yml +++ b/.github/workflows/premerge.yml @@ -464,11 +464,22 @@ jobs: run: >- cargo test --locked -p colossus-cli --test worker_smoke --test windows_sandbox --test mcp_smoke --test release_install_smoke -- --nocapture + - name: Validate Windows computer protocol and compile native acceptance + id: computer_portable + continue-on-error: true + shell: pwsh + run: | + cargo test --locked --manifest-path native/computer/Cargo.toml --all-targets + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + cargo clippy --locked --manifest-path native/computer/Cargo.toml --all-targets --all-features -- -D warnings + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + cargo build --locked --manifest-path native/computer/Cargo.toml --features native-test-driver --examples - name: Require every Windows runtime acceptance check shell: bash env: CONTRACT_OUTCOME: ${{ steps.renderer_contracts.outcome }} CODEX_AUTH_OUTCOME: ${{ steps.windows_codex_auth.outcome }} + COMPUTER_OUTCOME: ${{ steps.computer_portable.outcome }} HOME_OUTCOME: ${{ steps.windows_home.outcome }} NATIVE_OUTCOME: ${{ steps.windows_native.outcome }} SDK_PATH_OUTCOME: ${{ steps.windows_sdk_path.outcome }} @@ -486,6 +497,7 @@ jobs: "windows-sdk-path=$SDK_PATH_OUTCOME" \ "windows-home=$HOME_OUTCOME" \ "windows-codex-auth=$CODEX_AUTH_OUTCOME" \ + "windows-computer-protocol-and-build=$COMPUTER_OUTCOME" \ "worker-and-sandbox=$WORKER_OUTCOME" do if test "${result#*=}" != success; then @@ -502,7 +514,7 @@ jobs: if: needs.eligibility.result == 'success' needs: eligibility runs-on: windows-latest-l - timeout-minutes: 75 + timeout-minutes: 90 env: CARGO_INCREMENTAL: "0" CARGO_TARGET_DIR: ${{ github.workspace }}\apps\desktop\src-tauri\target diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 471b4465b..bb7f3f144 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -193,7 +193,8 @@ jobs: environment: name: ${{ needs.validate.outputs.publish_draft == 'true' && vars.SCCACHE_R2_ENABLED == 'true' && 'sccache-r2-write' || 'release-validation' }} deployment: false - timeout-minutes: 60 + # Cold Intel macOS native acceptance plus optimized compilation can exceed one hour. + timeout-minutes: ${{ matrix.target == 'x86_64-apple-darwin' && 90 || 60 }} env: SCCACHE_GHA_ENABLED: "true" COLOSSUS_RELEASE_JOURNAL_KEY: "5555555555555555555555555555555555555555555555555555555555555555" diff --git a/CHANGELOG.md b/CHANGELOG.md index 567f0dc5b..5a8b72b6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,89 @@ include breaking changes while the public API is still settling. ## [Unreleased] +## [0.11.8-preview.2] - 2026-10-19 + +### Preview highlights + +- Desktop browser views stay visible during agent actions and retain the completed + page for inspection, with clear control indicators and inline screenshot previews. +- Browser settings and guidance, agent inbox navigation, secure provider setup, and + model selection are improved. +- Opt-in Windows computer control adds observation-bound input for existing apps, + visible Pause/Stop controls, and window screenshots. It requires an interactive user + session and enabled workspace capability. +- Windows includes an explicitly labeled functional Chromium preview. Production + browser containment and automatic Desktop updates remain unavailable. + + + +### Merged changes + +#### Added + +- **http:** add bounded redirects and Desktop settings ([#289](https://github.com/obscuritylabs/Colossus/pull/289)) +- **ui:** unify web workspace management and standalone Goal mode ([#287](https://github.com/obscuritylabs/Colossus/pull/287)) +- **agents:** add durable inboxes and A2A application edge ([#288](https://github.com/obscuritylabs/Colossus/pull/288)) +- **models:** switch models and reasoning in Desktop, VS Code and TUI ([#295](https://github.com/obscuritylabs/Colossus/pull/295)) +- **provider:** add native Anthropic Messages support ([#297](https://github.com/obscuritylabs/Colossus/pull/297)) +- **vscode:** bundle and manage local runtimes in the sidebar ([#296](https://github.com/obscuritylabs/Colossus/pull/296)) +- **images:** add local image viewing and activity previews ([#298](https://github.com/obscuritylabs/Colossus/pull/298)) +- **browser:** connect owned Chromium hosts to Core and Desktop ([#290](https://github.com/obscuritylabs/Colossus/pull/290)) +- **browser:** add runtime settings and bundled browser guidance ([637e25c](https://github.com/obscuritylabs/Colossus/commit/637e25ce8e326fb1c9b8e25d0c8ea3f7bd00cd71)) +- **computer:** add opt-in Windows desktop control ([c229164](https://github.com/obscuritylabs/Colossus/commit/c229164951faf8203c309dada2f1b6249935908b)) +- **windows:** expose read-only process job accounting ([c3e4a41](https://github.com/obscuritylabs/Colossus/commit/c3e4a4194b3729a3554b3a83a5989443d2273692)) +- **windows:** expose read-only process job accounting ([8c3d266](https://github.com/obscuritylabs/Colossus/commit/8c3d266f84c8967a29c800430f5e6985b795ef20)) +- **browser:** package Windows preview and preserve agent views ([c8d82a8](https://github.com/obscuritylabs/Colossus/commit/c8d82a825f4a2dd29bcbf99b30e7604f8b23a97a)) +- **browser:** package Windows preview and preserve agent views ([004ea37](https://github.com/obscuritylabs/Colossus/commit/004ea37e916574ebe19438d5777e0289dd87e959)) + +#### Fixed + +- **plugins:** avoid OCI packaging for local snapshots ([#293](https://github.com/obscuritylabs/Colossus/pull/293)) +- **desktop:** apply access settings without redundant prompts ([#294](https://github.com/obscuritylabs/Colossus/pull/294)) +- **browser:** preserve input gestures and agent viewer continuity ([b6c57f2](https://github.com/obscuritylabs/Colossus/commit/b6c57f23ddb3438253066e65bac75e4be0f33dd1)) +- **browser:** polish capability settings presentation ([492935a](https://github.com/obscuritylabs/Colossus/commit/492935a47ae59728e106512a82858f6b57bd3b8e)) +- **browser:** retain focus after inheriting settings ([5b39d62](https://github.com/obscuritylabs/Colossus/commit/5b39d62ff8597d71ffd4d9b963cc0a6e248c5904)) +- **browser:** load public pages and reveal agent tabs ([9989d72](https://github.com/obscuritylabs/Colossus/commit/9989d72d8e93150cd8b807b538c5c0d17f2508c0)) +- **desktop:** preserve browser views and clarify agent settings ([88cc3d1](https://github.com/obscuritylabs/Colossus/commit/88cc3d1847d27e08972b005a5bf1a4a250239dbe)) +- **desktop:** harden credential labels and native acceptance fixtures ([b3cc106](https://github.com/obscuritylabs/Colossus/commit/b3cc106fcce290adbc229e014117a772e3a2f584)) +- **desktop:** upgrade Tao to prevent Windows keyboard deadlocks ([e9a2b80](https://github.com/obscuritylabs/Colossus/commit/e9a2b8061f0e82cde8cb9fccdc15767fedb424bc)) +- **browser:** retain completed views and show agent control ([7111d76](https://github.com/obscuritylabs/Colossus/commit/7111d768d954309054ea7afef776571291d2f464)) +- **browser:** retain acknowledged leases during viewport recovery ([54ab8f3](https://github.com/obscuritylabs/Colossus/commit/54ab8f3759a7d7c5094c7620a1c4e0afa7e0607f)) +- **browser:** retire expired network sessions with clear recovery ([46444f4](https://github.com/obscuritylabs/Colossus/commit/46444f44c298c9bf575d747c69606e3516e68492)) +- **codex:** update protocol and retain stream error diagnostics ([4e63bed](https://github.com/obscuritylabs/Colossus/commit/4e63bed240ec22a8ab1a3cb4f10b930cb48524c9)) +- **computer:** retain verified Windows root for capture helpers ([6a5a730](https://github.com/obscuritylabs/Colossus/commit/6a5a7307fb05cc790ae41aa46521dfa040b05eb5)) +- **desktop:** include computer tools in managed application grants ([1dab1a4](https://github.com/obscuritylabs/Colossus/commit/1dab1a4d316e4ccbf5fcf17911d9a7ca8a1de128)) +- **tools:** accept attached search term prefixes ([3337a92](https://github.com/obscuritylabs/Colossus/commit/3337a924de5a408b6816837c1f726b7b28c5bc87)) +- **computer:** settle foreground activation and preserve input markers ([fcd03c5](https://github.com/obscuritylabs/Colossus/commit/fcd03c5f67e4be4ed8c9da77d52f261055400b6b)) +- **browser:** allow bounded slow metadata reads without disconnecting ([2a4c18a](https://github.com/obscuritylabs/Colossus/commit/2a4c18a1a1400410c7179a356df9892ab682261b)) +- **browser:** allow bounded slow metadata reads without disconnecting ([055a7c8](https://github.com/obscuritylabs/Colossus/commit/055a7c815807307272ae011e3726e59342e45c3d)) +- **desktop:** restore new-thread composer from inbox views ([2c824b4](https://github.com/obscuritylabs/Colossus/commit/2c824b49ce8c800d41c8ac6db4a2e4de6390f99a)) +- **desktop:** restore new-thread composer from inbox views ([ca16256](https://github.com/obscuritylabs/Colossus/commit/ca162568c9c8b784b907c9590b526a6e45e817d9)) + +#### Security + +- **dependencies:** replace vulnerable desktop and credential backends ([3cea322](https://github.com/obscuritylabs/Colossus/commit/3cea3224e398e78848c2315a039f44b1f0201a46)) +- **dependencies:** replace vulnerable desktop and credential backends ([dd3525a](https://github.com/obscuritylabs/Colossus/commit/dd3525ab58d1a17a6f74fb3db35cafb4455fa861)) + +#### Changed + +- **storage:** seek Redb cursors and batch projection catch-up ([#286](https://github.com/obscuritylabs/Colossus/pull/286)) + +#### Maintenance + +- **deps-dev:** update desktop and web Vitest to 4.1.11 ([#292](https://github.com/obscuritylabs/Colossus/pull/292)) +- **browser:** import expiry test clock for native builds ([afa8a99](https://github.com/obscuritylabs/Colossus/commit/afa8a99e60f689e84e6dbc77c3088c143fc9be72)) +- **computer:** trace native foreground eligibility ([e84d921](https://github.com/obscuritylabs/Colossus/commit/e84d9218cc411be051ebbab1daa8fea432147cb6)) +- **computer:** trace native input terminal receipts ([83e1f6f](https://github.com/obscuritylabs/Colossus/commit/83e1f6f67605ad0044bbfaca4ceb4aa3e82b42c8)) +- **computer:** trace native control admission refusals ([8572c0c](https://github.com/obscuritylabs/Colossus/commit/8572c0c30e379bcbe9fbc9406ca8f19e1acb169b)) +- **desktop:** verify authenticated computer tool discovery ([0bb3b9e](https://github.com/obscuritylabs/Colossus/commit/0bb3b9e0d6fd86e5ec6ff7806af24f1bf02afeb0)) + +**Source:** [8e750fd](https://github.com/obscuritylabs/Colossus/commit/8e750fd4294607486136eae953ce3dbab3b5a2f8) + +**Full changelog:** [v0.11.8...v0.11.8-preview.2](https://github.com/obscuritylabs/Colossus/compare/v0.11.8...v0.11.8-preview.2) + + + ## [0.11.8] - 2026-10-09 ### Added diff --git a/Cargo.lock b/Cargo.lock index b644469e9..56db52b17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1272,6 +1272,7 @@ dependencies = [ "colossus-grpc", "colossus-home", "colossus-journal-redb", + "colossus-native-computer", "colossus-native-dictation", "colossus-observability", "colossus-plugins", @@ -1858,6 +1859,24 @@ dependencies = [ "zeroize", ] +[[package]] +name = "colossus-native-computer" +version = "0.1.0" +dependencies = [ + "async-trait", + "colossus-contracts", + "colossus-ports", + "colossus-windows-native", + "getrandom 0.4.3", + "image", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "windows 0.62.2", + "windows-sys 0.61.2", +] + [[package]] name = "colossus-native-dictation" version = "0.1.0" @@ -2272,6 +2291,7 @@ dependencies = [ "colossus-grpc", "colossus-home", "colossus-mcp", + "colossus-native-computer", "colossus-network", "colossus-observability", "colossus-ports", diff --git a/Cargo.toml b/Cargo.toml index b500f5640..038d93c65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -exclude = ["native/browser/driver", "native/browser/pki", "native/browser/process-broker", "native/browser/tls-broker"] +exclude = ["native/browser/driver", "native/browser/pki", "native/browser/process-broker", "native/browser/tls-broker", "native/computer"] members = [ "crates/colossus-cloud", "crates/colossus-cloud-postgres", diff --git a/apps/desktop/src-tauri/src/managed_configuration_commands.rs b/apps/desktop/src-tauri/src/managed_configuration_commands.rs index 047c5b176..312dfa951 100644 --- a/apps/desktop/src-tauri/src/managed_configuration_commands.rs +++ b/apps/desktop/src-tauri/src/managed_configuration_commands.rs @@ -1289,6 +1289,15 @@ fn risky_field_authority_changed( .iter() .map(|field| (field.field_id.as_str(), &field.value)) .collect::>(); + let computer_enabled = |fields: &BTreeMap<&str, &Value>| { + fields + .get("computer.enabled") + .and_then(|value| value.as_bool()) + .unwrap_or(false) + }; + if !computer_enabled(&before) && computer_enabled(&after) { + return true; + } let browser_enabled = |fields: &BTreeMap<&str, &Value>, field: &str| { fields .get(field) @@ -1646,6 +1655,118 @@ fn effective_yaml( #[allow(clippy::too_many_lines)] fn field_descriptors() -> Vec { vec![ + descriptor( + "computer.enabled", + "Computer use", + "Computer access", + "Allow Windows computer tools and the bundled computer-use skill. Agents can operate normal user apps outside the workspace. Disabled by default; enabling requires Apply.", + "both", + "high", + "toggle", + false, + json!(false), + None, + None, + vec![], + ), + descriptor( + "computer.maxWindows", + "Computer use", + "Window list limit", + "Maximum windows returned when choosing an app.", + "both", + "low", + "number", + false, + json!(64), + Some(1), + Some(128), + vec![], + ), + descriptor( + "computer.maxSnapshotNodes", + "Computer use", + "Snapshot element limit", + "Maximum accessible elements returned from the selected window.", + "both", + "low", + "number", + false, + json!(512), + Some(1), + Some(1024), + vec![], + ), + descriptor( + "computer.maxScreenshotBytes", + "Computer use", + "Screenshot byte limit", + "Maximum PNG bytes released for one screenshot.", + "both", + "low", + "number", + false, + json!(8_388_608), + Some(1), + Some(16_777_216), + vec![], + ), + descriptor( + "computer.maxScreenshotPixels", + "Computer use", + "Screenshot pixel limit", + "Maximum pixels captured from the selected window.", + "both", + "low", + "number", + false, + json!(8_388_608), + Some(1), + Some(16_777_216), + vec![], + ), + descriptor( + "computer.actionTimeoutMs", + "Computer use", + "Action timeout (ms)", + "Maximum time for one native operation.", + "both", + "low", + "number", + false, + json!(5000), + Some(1), + Some(10000), + vec![], + ), + descriptor( + "computer.sessionTimeoutMs", + "Computer use", + "Session timeout (ms)", + "Maximum time before control expires and input stops.", + "both", + "low", + "number", + false, + json!(300_000), + Some(1), + Some(900_000), + vec![], + ), + descriptor( + "computer.observationMaxAgeMs", + "Computer use", + "Observation age limit (ms)", + "Maximum age of a screenshot or snapshot used for input.", + "both", + "low", + "number", + false, + json!(10000), + Some(1), + Some(30000), + vec![], + ), descriptor( "browser.enabled", "Browser", @@ -2404,6 +2525,27 @@ mod tests { settings } + #[test] + fn computer_access_starts_disabled_and_enabling_requires_explicit_apply() { + let disabled = resolved_for(&settings(), "space-one").expect("configuration"); + let mut enabled = disabled.clone(); + enabled.field_overrides.push(FieldOverrideSetting { + field_id: "computer.enabled".into(), + value: json!(true), + }); + assert!(requires_explicit_apply(&disabled, &enabled)); + assert!(!requires_explicit_apply(&enabled, &disabled)); + let mut explicitly_disabled = disabled.clone(); + explicitly_disabled + .field_overrides + .push(FieldOverrideSetting { + field_id: "computer.enabled".into(), + value: json!(false), + }); + assert!(!requires_explicit_apply(&disabled, &explicitly_disabled)); + assert!(requires_explicit_apply(&explicitly_disabled, &enabled)); + } + #[test] fn browser_disabling_can_apply_when_idle_but_reenabling_requires_apply() { let before = resolved_for(&settings(), "space-one").expect("configuration"); diff --git a/apps/desktop/src-tauri/src/managed_runtime.rs b/apps/desktop/src-tauri/src/managed_runtime.rs index 300452974..a11392af2 100644 --- a/apps/desktop/src-tauri/src/managed_runtime.rs +++ b/apps/desktop/src-tauri/src/managed_runtime.rs @@ -110,6 +110,19 @@ const TRUSTED_BUILTIN_TOOL_GRANT: &[&str] = &[ "browser.upload", "browser.download", "browser.close", + "computer.windows", + "computer.open", + "computer.status", + "computer.snapshot", + "computer.screenshot", + "computer.focus", + "computer.move", + "computer.click", + "computer.drag", + "computer.scroll", + "computer.type", + "computer.press", + "computer.close", "workflow.definition.list", "workflow.definition.get", "workflow.schedule.list", @@ -1786,11 +1799,51 @@ mod tests { #[test] fn minimal_grant_only_exposes_echo() { - let grant = application_grant(AccessProfileSetting::Minimal).expect("grant"); - let debug = format!("{grant:?}"); - assert!(debug.contains("echo")); - assert!(!debug.contains("filesystem.read")); - assert!(!debug.contains("browser.")); + for grant in [ + application_grant(AccessProfileSetting::Minimal).expect("Desktop grant"), + connector_application_grant(AccessProfileSetting::Minimal).expect("connector grant"), + ] { + let debug = format!("{grant:?}"); + assert!(debug.contains("echo")); + assert!(!debug.contains("filesystem.read")); + assert!(!debug.contains("browser.")); + assert!(!debug.contains("computer.")); + } + } + + #[test] + fn nonminimal_grants_admit_exact_computer_tools() { + let computer_tools = [ + "computer.windows", + "computer.open", + "computer.status", + "computer.snapshot", + "computer.screenshot", + "computer.focus", + "computer.move", + "computer.click", + "computer.drag", + "computer.scroll", + "computer.type", + "computer.press", + "computer.close", + ]; + for profile in [ + AccessProfileSetting::Pinned, + AccessProfileSetting::Development, + AccessProfileSetting::AllowAll, + ] { + for grant in [ + application_grant(profile).expect("Desktop grant"), + connector_application_grant(profile).expect("connector grant"), + ] { + let debug = format!("{grant:?}"); + for tool in computer_tools { + assert!(debug.contains(tool), "{profile:?} grant missing {tool}"); + } + assert!(!debug.contains('*'), "tool grants must remain explicit"); + } + } } #[test] diff --git a/apps/desktop/src-tauri/src/managed_runtime/computer_discovery_acceptance.rs b/apps/desktop/src-tauri/src/managed_runtime/computer_discovery_acceptance.rs new file mode 100644 index 000000000..adbd619c8 --- /dev/null +++ b/apps/desktop/src-tauri/src/managed_runtime/computer_discovery_acceptance.rs @@ -0,0 +1,410 @@ +//! Background Windows acceptance of Desktop grants, native availability and discovery. +//! Uses a fresh private sidecar and credential-free loopback provider. No computer +//! operation, capture, native input, UI activation or real profile is used. +//! Operator: set `COLOSSUS_ACCEPTANCE_SIDECAR` to the matching staged sidecar, then +//! `cargo test --locked --manifest-path apps/desktop/src-tauri/Cargo.toml --no-default-features --lib native_desktop_grant_discovers_computer_tools_without_computer_effects -- --ignored`. +use super::*; +use sha2::Digest as _; +use std::{ + io::Write as _, + net::{TcpListener, TcpStream}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + thread::JoinHandle, +}; + +const DISCOVERY_CHILD: &str = "managed_runtime::credential_acceptance::computer_discovery_acceptance::native_computer_discovery_child"; +const TOOLS: [&str; 13] = [ + "computer.windows", + "computer.open", + "computer.status", + "computer.snapshot", + "computer.screenshot", + "computer.focus", + "computer.move", + "computer.click", + "computer.drag", + "computer.scroll", + "computer.type", + "computer.press", + "computer.close", +]; + +#[test] +#[ignore = "requires ordinary unlocked Windows Default desktop, native credential storage, loopback and COLOSSUS_ACCEPTANCE_SIDECAR; no UI/input/capture"] +fn native_desktop_grant_discovers_computer_tools_without_computer_effects() { + let executable = sidecar_executable(); + let root = PrivateAcceptanceRoot::new(); + let workspace = PrivateAcceptanceRoot::new(); + let _keys = RuntimeKeys(instance_id(&root.0)); + let server = Server::start(); + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", DISCOVERY_CHILD, "--ignored", "--nocapture"]) + .env("COLOSSUS_DISCOVERY_ROOT", &root.0) + .env("COLOSSUS_DISCOVERY_WORKSPACE", &workspace.0) + .env("COLOSSUS_DISCOVERY_ORIGIN", &server.origin) + .env("COLOSSUS_ACCEPTANCE_SIDECAR", executable) + .stdin(Stdio::null()); + // Closing this exact owned job also closes the child's SDK-owned sidecar jobs. + let image = + colossus_windows_native::BoundPath::open_immutable_file(&std::env::current_exe().unwrap()) + .unwrap(); + colossus_windows_native::configure_suspended_process(&mut command); + let mut child = command.spawn().expect("spawn isolated discovery test"); + let job = colossus_windows_native::KillOnCloseJob::assign_verify_and_resume( + &child, + child.id(), + image.identity(), + ) + .unwrap_or_else(|_| { + child.kill().expect("stop unenrolled owned test child"); + child.wait().expect("reap unenrolled owned test child"); + panic!("verify and supervise test child"); + }); + let deadline = Instant::now() + Duration::from_secs(150); + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if Instant::now() >= deadline { + job.terminate().expect("terminate owned timed-out test job"); + child.wait().expect("reap owned test child"); + panic!("background discovery acceptance exceeded its deadline"); + } + std::thread::sleep(Duration::from_millis(25)); + }; + assert!(status.success(), "background discovery test child failed"); + server.finish(); + drop(job); +} + +#[tokio::test] +#[ignore = "owned subprocess for native_desktop_grant_discovers_computer_tools_without_computer_effects"] +async fn native_computer_discovery_child() { + let Some(root) = std::env::var_os("COLOSSUS_DISCOVERY_ROOT") else { + return; + }; + let root = PathBuf::from(root); + let workspace = PathBuf::from(std::env::var_os("COLOSSUS_DISCOVERY_WORKSPACE").unwrap()); + validate_fixture_root(&root); + validate_fixture_root(&workspace); + let confined = ConfinedRoot::bind(root.clone()).unwrap(); + let instance = confined.prepare_directory(Path::new("instance")).unwrap(); + let home = confined + .prepare_directory(Path::new("runtime-home")) + .unwrap(); + let workspace = crate::desktop_settings::validate_workspace(&workspace).unwrap(); + let instance_id = instance_id(&root); + let origin = std::env::var("COLOSSUS_DISCOVERY_ORIGIN").unwrap(); + let worker_authentication = TerminalWorkerAuthentication::random().unwrap(); + let worker_bootstrap_secret = worker_authentication.copy_secret(); + let bootstrap = managed_bootstrap( + &workspace.path, + workspace.identity.unwrap(), + &configuration(&origin), + Vec::new(), + approval_broker_grant().unwrap(), + worker_bootstrap_secret.as_ref(), + &ManagedBootstrapPaths { + ca_bundle: None, + codex_auth: None, + colossus_home: &home, + }, + None, + ) + .unwrap(); + let executable = sidecar_executable(); + let digest: [u8; 32] = Sha256::digest(std::fs::read(&executable).unwrap()).into(); + let options = SidecarOptions::new( + InstanceId::from_uuid(instance_id), + AppPrivateInstanceDir::new(instance).unwrap(), + VerifiedExecutable::new(executable, Sha256Digest::from_bytes(digest)).unwrap(), + ApiMajor::new(1).unwrap(), + ) + .unwrap(); + let lifecycle = NativeSidecarLifecycle::new(bootstrap); + let client = Colossus::start_sidecar(&lifecycle, options) + .await + .expect("verified Desktop sidecar"); + let result = std::panic::AssertUnwindSafe(async { + let created = client + .create_run(CreateRunRequest { + model_profile: None, + reasoning_effort: None, + plugin_skill_ids: Vec::new(), + input: vec![InputContentPart::Text( + "Discover +computer tools only. Do not control the computer.".into(), + )], + session_id: None, + end_user_id: None, + role: "primary".into(), + mode: RunMode::Execute, + goal_max_iterations: 0, + research_depth: None, + research_sources: Vec::new(), + plan_action: None, + branch: None, + max_turns: 2, + idempotency_key: IdempotencyKey::new(Uuid::now_v7().to_string()).unwrap(), + }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(90), async { + loop { + let run = client + .get_run(GetRunRequest { + run_id: created.run.run_id.clone(), + }) + .await + .unwrap() + .run; + if !matches!(run.status, RunStatus::Queued | RunStatus::Running) { + assert_eq!( + run.status, + RunStatus::Completed, + "discovery run must complete" + ); + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("bounded discovery run"); + }) + .catch_unwind() + .await; + client.close().await.expect("acknowledged sidecar cleanup"); + drop(client); + if let Err(panic) = result { + std::panic::resume_unwind(panic); + } +} + +struct RuntimeKeys(Uuid); +fn instance_id(root: &Path) -> Uuid { + let name = root.file_name().unwrap().to_string_lossy(); + Uuid::parse_str(name.strip_prefix(ROOT_PREFIX).unwrap()).unwrap() +} +impl Drop for RuntimeKeys { + fn drop(&mut self) { + let cleanup = std::panic::catch_unwind(|| { + for account in [ + format!("journal-key:journal-{}", self.0), + format!("signing-key:checkpoint-{}", self.0), + format!("journal-anchor:journal-{}", self.0), + ] { + let entry = colossus_credentials::SystemCredentialEntry::new( + "com.obscuritylabs.colossus.managed-runtime", + &account, + ) + .unwrap(); + entry.delete().expect("remove only owned runtime key"); + } + }); + if let Err(panic) = cleanup { + if std::thread::panicking() { + eprintln!("generated discovery runtime key cleanup incomplete"); + } else { + std::panic::resume_unwind(panic); + } + } + } +} + +fn configuration(origin: &str) -> ResolvedSpaceConfiguration { + // Reuse only the parent's synthetic nonstreaming config constructor. Never + // open its vault/MCP fixture; remove both credential references and MCP now. + let mut config = resolved_configuration(origin, 0); + config.access_profile = AccessProfileSetting::AllowAll; + config.providers[0].base_url = format!("{origin}/v1"); + config.providers[0].credential_id = None; + config.providers[0].timeout_ms = Some(15_000); + config.models[0].context_window_tokens = 131_072; + config.mcp_servers.clear(); + config.field_overrides = [("computer.enabled", true), ("browser.enabled", false)] + .into_iter() + .map( + |(field_id, value)| crate::managed_configuration::FieldOverrideSetting { + field_id: field_id.into(), + value: json!(value), + }, + ) + .collect(); + config +} + +struct Server { + origin: String, + stop: Arc, + task: Option>, +} +impl Server { + fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + listener.set_nonblocking(true).unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let cancelled = Arc::clone(&stop); + let task = std::thread::spawn(move || { + let deadline = Instant::now() + Duration::from_mins(2); + let mut calls = 0; + while calls < 2 && !cancelled.load(Ordering::Acquire) && Instant::now() < deadline { + let (mut stream, _) = match listener.accept() { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + continue; + } + Err(error) => panic!("loopback accept failed: {}", error.kind()), + }; + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let body = request(&mut stream); + let reply = completion(&body, calls); + let body = serde_json::to_vec(&reply).unwrap(); + write!(stream, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).unwrap(); + stream.write_all(&body).unwrap(); + calls += 1; + } + assert_eq!( + calls, 2, + "both real provider requests must reach the fixture" + ); + }); + Self { + origin, + stop, + task: Some(task), + } + } + fn finish(mut self) { + self.task + .take() + .unwrap() + .join() + .expect("provider discovery evidence"); + } +} +impl Drop for Server { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(task) = self.task.take() { + let _ = task.join(); + } + } +} + +fn request(stream: &mut TcpStream) -> Value { + let deadline = Instant::now() + Duration::from_secs(10); + let mut header = Vec::new(); + while !header.ends_with(b"\r\n\r\n") { + assert!(header.len() < 16 * 1024, "bounded HTTP headers"); + let mut byte = [0]; + read_bounded(stream, &mut byte, deadline); + header.push(byte[0]); + } + let header = String::from_utf8(header).unwrap(); + assert!(header.starts_with("POST /v1/chat/completions HTTP/1.1\r\n")); + assert!( + !header + .lines() + .any(|line| line.to_ascii_lowercase().starts_with("authorization:")), + "fixture must not receive provider credentials" + ); + let length = header + .lines() + .filter_map(|line| line.split_once(':')) + .find(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .unwrap() + .1 + .trim() + .parse::() + .unwrap(); + assert!( + length <= 2 * 1024 * 1024, + "bounded synthetic provider request" + ); + let mut body = vec![0; length]; + read_bounded(stream, &mut body, deadline); + serde_json::from_slice(&body).unwrap() +} + +fn read_bounded(stream: &mut TcpStream, mut bytes: &mut [u8], deadline: Instant) { + while !bytes.is_empty() { + let remaining = deadline.saturating_duration_since(Instant::now()); + assert!(!remaining.is_zero(), "synthetic request deadline exceeded"); + stream.set_read_timeout(Some(remaining)).unwrap(); + let count = stream.read(bytes).expect("bounded synthetic request read"); + assert!(count > 0, "synthetic request ended early"); + bytes = &mut bytes[count..]; + } +} + +fn completion(body: &Value, turn: usize) -> Value { + assert_ne!(body["stream"], true); + let definitions = body["tools"].as_array().unwrap(); + for tool in TOOLS { + assert!( + definitions + .iter() + .any(|definition| definition["function"]["name"] == tool.replace('.', "_")), + "authenticated Desktop model request omitted {tool}" + ); + } + let messages = body["messages"].as_array().unwrap(); + let results = messages + .iter() + .filter(|message| message["role"] == "tool") + .collect::>(); + let (finish, message) = if turn == 0 { + assert!(results.is_empty()); + assert!( + definitions + .iter() + .any(|tool| tool["function"]["name"] == "tool_search") + ); + ( + "tool_calls", + json!({"role":"assistant","content":null,"tool_calls":[{ + "id":"discovery-only","type":"function","function":{"name":"tool_search", + "arguments":json!({"query":"+computer","max_results":50}).to_string()}}]}), + ) + } else { + assert_eq!( + results.len(), + 1, + "only the requested discovery call may run" + ); + assert_eq!(results[0]["tool_call_id"], "discovery-only"); + let result: Value = serde_json::from_str(results[0]["content"].as_str().unwrap()).unwrap(); + assert_eq!(result["query"], "+computer"); + assert_eq!(result["truncated"], false); + let actual = result["tools"] + .as_array() + .unwrap() + .iter() + .map(|tool| tool["name"].as_str().unwrap()) + .filter(|name| name.starts_with("computer.")) + .collect::>(); + assert_eq!( + actual, + BTreeSet::from(TOOLS), + "released discovery result must contain all 13 tools" + ); + ( + "stop", + json!({"role":"assistant","content":"Computer discovery passed without desktop effects."}), + ) + }; + json!({"id":"synthetic-discovery","object":"chat.completion","created":0,"model":"synthetic-discovery", + "choices":[{"index":0,"message":message,"finish_reason":finish}], + "usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}) +} diff --git a/apps/desktop/src-tauri/src/managed_runtime/credential_acceptance.rs b/apps/desktop/src-tauri/src/managed_runtime/credential_acceptance.rs index 6b950cf2b..3354add43 100644 --- a/apps/desktop/src-tauri/src/managed_runtime/credential_acceptance.rs +++ b/apps/desktop/src-tauri/src/managed_runtime/credential_acceptance.rs @@ -24,6 +24,10 @@ mod cleanup; #[path = "credential_acceptance_fixture.rs"] mod fixture; +#[cfg(windows)] +#[path = "computer_discovery_acceptance.rs"] +mod computer_discovery_acceptance; + const CHILD: &str = "managed_runtime::credential_acceptance::native_backend_process_child"; const ROOT_PREFIX: &str = ".colossus-desktop-credential-acceptance-"; const CREDENTIAL_ID: &str = "synthetic-large-token"; diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index 43e861697..85d304194 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -925,7 +925,11 @@ export default function App() { : FIXTURE_SCENARIO === "activity-comparison" ? FIXTURE_QUERY.get("imageViews") === "1" ? developmentFixtures().buildImageViewFixture( - FIXTURE_QUERY.get("browserScreenshot") === "1", + FIXTURE_QUERY.get("computerScreenshot") === "1" + ? "computer" + : FIXTURE_QUERY.get("browserScreenshot") === "1" + ? "browser" + : undefined, ) : developmentFixtures().buildActivityComparisonFixture() : FIXTURE_SCENARIO === "plan-workflow" @@ -6103,7 +6107,11 @@ export default function App() { return Promise.resolve( developmentFixtures().buildImageViewActivityFixture( request, - FIXTURE_QUERY.get("browserScreenshot") === "1", + FIXTURE_QUERY.get("computerScreenshot") === "1" + ? "computer" + : FIXTURE_QUERY.get("browserScreenshot") === "1" + ? "browser" + : undefined, ), ); let liveActivityCount = 0; diff --git a/apps/desktop/src/components/BrowserCapabilityFields.tsx b/apps/desktop/src/components/BrowserCapabilityFields.tsx index 36e7b15dd..7b7f0d481 100644 --- a/apps/desktop/src/components/BrowserCapabilityFields.tsx +++ b/apps/desktop/src/components/BrowserCapabilityFields.tsx @@ -7,6 +7,7 @@ import type { ManagedFieldDescriptor } from "../types"; import "./browser-settings.css"; export function BrowserCapabilityFields({ + capability = "browser", descriptors, values, effective, @@ -16,6 +17,7 @@ export function BrowserCapabilityFields({ onChange, onInherit, }: { + capability?: "browser" | "computer"; descriptors: ManagedFieldDescriptor[]; values: Record; effective: Map; @@ -25,6 +27,7 @@ export function BrowserCapabilityFields({ onChange: (id: string, value: unknown) => void; onInherit: (id: string) => void; }) { + const computer = capability === "computer"; function field(descriptor: ManagedFieldDescriptor) { const overridden = Object.hasOwn(values, descriptor.id); const inherited = effective.get(descriptor.id); @@ -105,31 +108,53 @@ export function BrowserCapabilityFields({ ); } return ( -
+
{descriptors.filter(({ control }) => control === "toggle").map(field)}
+ {computer ? ( + + ) : null}

Resource limits

-

Keep each workspace’s browser usage within these bounds.

+

Keep each workspace’s {capability} usage within these bounds.

{descriptors.filter(({ control }) => control !== "toggle").map(field)}
-
); diff --git a/apps/desktop/src/components/ManagedSettingsPane.test.ts b/apps/desktop/src/components/ManagedSettingsPane.test.ts index b8feeba55..7048c9172 100644 --- a/apps/desktop/src/components/ManagedSettingsPane.test.ts +++ b/apps/desktop/src/components/ManagedSettingsPane.test.ts @@ -853,6 +853,17 @@ describe("ManagedSettingsPane", () => { tab: "sandbox", section: null, }); + const computer = descriptors.filter(({ id }) => id.startsWith("computer.")); + expect(computer).toHaveLength(8); + expect( + computer.find(({ id }) => id === "computer.enabled")?.defaultValue, + ).toBe(false); + for (const descriptor of computer) { + expect(managedFieldDestination(descriptor)).toEqual({ + tab: "computer", + section: null, + }); + } for (const descriptor of descriptors.filter(({ id }) => id.startsWith("plugins."), )) { diff --git a/apps/desktop/src/components/ManagedSettingsPane.tsx b/apps/desktop/src/components/ManagedSettingsPane.tsx index 3133b09d4..3f3cce85e 100644 --- a/apps/desktop/src/components/ManagedSettingsPane.tsx +++ b/apps/desktop/src/components/ManagedSettingsPane.tsx @@ -202,6 +202,13 @@ const DESKTOP_PAGES = [ keywords: "browser enabled disabled sessions tabs snapshot skill new tab home page url localhost preview", }, + { + id: "computer", + label: "Computer use", + description: "Control Windows desktop access and operation limits.", + keywords: + "computer desktop windows mouse keyboard screenshot skill automation enable disable", + }, { id: "terminal", label: "Terminal", @@ -237,6 +244,7 @@ type GlobalTab = | "defaults" | DesktopTab; type SpaceTab = + | "computer" | "browser" | "cloud" | "runtime" @@ -398,12 +406,13 @@ const GLOBAL_TABS: ReadonlyArray<{ { id: "mcp", label: "MCP", group: "Global settings" }, { id: "plugins", label: "Plugins", group: "Global settings" }, { id: "browser", label: "Browser", group: "Global settings" }, + { id: "computer", label: "Computer use", group: "Global settings" }, { id: "search", label: "Search", group: "Global settings" }, { id: "telemetry", label: "Telemetry", group: "Global settings" }, { id: "defaults", label: "Defaults", group: "Global settings" }, - ...DESKTOP_PAGES.filter(({ id }) => id !== "browser").map( - ({ id, label }) => ({ id, label, group: "Desktop" }), - ), + ...DESKTOP_PAGES.filter( + ({ id }) => id !== "browser" && id !== "computer", + ).map(({ id, label }) => ({ id, label, group: "Desktop" })), ]; const SPACE_TABS: ReadonlyArray<{ id: SpaceTab; label: string }> = [ @@ -415,6 +424,7 @@ const SPACE_TABS: ReadonlyArray<{ id: SpaceTab; label: string }> = [ { id: "mcp", label: "MCP" }, { id: "plugins", label: "Plugins" }, { id: "browser", label: "Browser" }, + { id: "computer", label: "Computer use" }, { id: "access", label: "Access" }, { id: "sandbox", label: "Sandbox" }, { id: "search", label: "Search" }, @@ -425,9 +435,19 @@ const SPACE_TABS: ReadonlyArray<{ id: SpaceTab; label: string }> = [ ]; export function managedFieldDestination(descriptor: ManagedFieldDescriptor): { - tab: "browser" | "sandbox" | "research" | "advanced" | "runtime" | "plugins"; + tab: + | "computer" + | "browser" + | "sandbox" + | "research" + | "advanced" + | "runtime" + | "plugins"; section: string | null; } { + if (descriptor.id.startsWith("computer.")) { + return { tab: "computer", section: null }; + } if (descriptor.id.startsWith("browser.")) { return { tab: "browser", section: null }; } @@ -538,6 +558,84 @@ const EMPTY_TELEMETRY_DRAFT: TelemetryEditorDraft = { }; const FIELD_DESCRIPTORS: ManagedFieldDescriptor[] = [ + toggleField( + "computer.enabled", + "Computer use", + "Computer access", + "Allow Windows computer tools and the bundled computer-use skill. Agents can operate normal user apps outside the workspace. Disabled by default; enabling requires Apply.", + false, + false, + ), + field( + "computer.maxWindows", + "Computer use", + "Window list limit", + "Maximum windows returned when choosing an app.", + 64, + false, + 1, + 128, + ), + field( + "computer.maxSnapshotNodes", + "Computer use", + "Snapshot element limit", + "Maximum accessible elements returned from the selected window.", + 512, + false, + 1, + 1024, + ), + field( + "computer.maxScreenshotBytes", + "Computer use", + "Screenshot byte limit", + "Maximum PNG bytes released for one screenshot.", + 8388608, + false, + 1, + 16777216, + ), + field( + "computer.maxScreenshotPixels", + "Computer use", + "Screenshot pixel limit", + "Maximum pixels captured from the selected window.", + 8388608, + false, + 1, + 16777216, + ), + field( + "computer.actionTimeoutMs", + "Computer use", + "Action timeout (ms)", + "Maximum time for one native operation.", + 5000, + false, + 1, + 10000, + ), + field( + "computer.sessionTimeoutMs", + "Computer use", + "Session timeout (ms)", + "Maximum time before control expires and input stops.", + 300000, + false, + 1, + 900000, + ), + field( + "computer.observationMaxAgeMs", + "Computer use", + "Observation age limit (ms)", + "Maximum age of a screenshot or snapshot used for input.", + 10000, + false, + 1, + 30000, + ), toggleField( "browser.enabled", "Browser", @@ -2992,7 +3090,7 @@ export function ManagedSettingsPane({ }, [descriptors, query, snapshot]); const desktopPage = - scope === "global" && globalTab !== "browser" + scope === "global" && globalTab !== "browser" && globalTab !== "computer" ? DESKTOP_PAGES.find((page) => page.id === globalTab) : undefined; const externalTargets = desktop.targets.filter( @@ -3874,37 +3972,56 @@ function GlobalSettingsBody({ ); } - if (tab === "browser") { - return ( - - - descriptor.id.startsWith("browser."), - )} - values={defaults.fields} - effective={new Map()} - scope="global" - fieldElementId={managedFieldElementId} - onChange={(id, value) => - setDefaults({ - ...defaults, - fields: { ...defaults.fields, [id]: value }, - }) - } - onInherit={(id) => { - const fields = { ...defaults.fields }; - delete fields[id]; - setDefaults({ ...defaults, fields }); - }} - /> - + if (tab === "browser" || tab === "computer") { + const controls = ( + + descriptor.id.startsWith(`${tab}.`), + )} + values={defaults.fields} + effective={new Map()} + scope="global" + fieldElementId={managedFieldElementId} + onChange={(id, value) => + setDefaults({ + ...defaults, + fields: { ...defaults.fields, [id]: value }, + }) + } + onInherit={(id) => { + const fields = { ...defaults.fields }; + delete fields[id]; + setDefaults({ ...defaults, fields }); + }} + /> + ); + return tab === "browser" ? ( + {controls} + ) : ( +
+
+
+

Computer use

+

+ Let agents work with Windows apps. Off until you enable and apply + it. +

+
+
+ {controls} +
); } if (tab === "defaults") { const defaultDescriptors = descriptors.filter( (descriptor) => !descriptor.id.startsWith("plugins.") && - !descriptor.id.startsWith("browser."), + !descriptor.id.startsWith("browser.") && + !descriptor.id.startsWith("computer."), ); return (
@@ -5104,6 +5221,7 @@ export function SpaceSettingsBody({ !descriptor.advanced && !descriptor.id.startsWith("plugins.") && !descriptor.id.startsWith("browser.") && + !descriptor.id.startsWith("computer.") && !descriptor.id.startsWith("research."), )} values={draft.fields} @@ -5329,8 +5447,13 @@ export function SpaceSettingsBody({ } const filtered = descriptors.filter((descriptor) => { if (descriptor.id.startsWith("plugins.")) return false; + if (tab === "computer") return descriptor.id.startsWith("computer."); if (tab === "browser") return descriptor.id.startsWith("browser."); - if (descriptor.id.startsWith("browser.")) return false; + if ( + descriptor.id.startsWith("browser.") || + descriptor.id.startsWith("computer.") + ) + return false; if (tab === "sandbox") return descriptor.id.startsWith("sandbox."); if (tab === "research") return descriptor.id.startsWith("research."); return descriptor.advanced; @@ -5340,7 +5463,11 @@ export function SpaceSettingsBody({

Workspace overrides

-

{tab[0]!.toUpperCase() + tab.slice(1)}

+

+ {tab === "computer" + ? "Computer use" + : tab[0]!.toUpperCase() + tab.slice(1)} +

{tab === "browser" ? (

Customize browser access and limits for this workspace. @@ -5354,8 +5481,9 @@ export function SpaceSettingsBody({ ) : null}

- {tab === "browser" ? ( + {tab === "browser" || tab === "computer" ? ( ); diff --git a/apps/desktop/src/components/ViewedImageActivity.tsx b/apps/desktop/src/components/ViewedImageActivity.tsx index 87bb128bc..692e1c7ef 100644 --- a/apps/desktop/src/components/ViewedImageActivity.tsx +++ b/apps/desktop/src/components/ViewedImageActivity.tsx @@ -49,6 +49,7 @@ function ImagePreview({ ref={dialog} className="settings-dialog viewed-image-dialog" aria-label={`Image preview: ${image.fileName}`} + data-browser-occluded onCancel={onClose} onClose={onClose} > @@ -179,14 +180,14 @@ export function ViewedImageActivity({ images, createdAt, load, - screenshot = false, + screenshot, }: { images: ViewedImage[]; createdAt: string; load?: ImagePreviewLoader | undefined; - screenshot?: boolean | undefined; + screenshot?: "browser" | "computer" | undefined; }) { - const [open, setOpen] = useState(screenshot); + const [open, setOpen] = useState(!!screenshot); return (
{screenshot - ? "Browser screenshot" + ? screenshot === "computer" + ? "Computer screenshot" + : "Browser screenshot" : `Viewed ${images.length} ${images.length === 1 ? "image" : "images"}`}