diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b46a083 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +* text=auto eol=lf +*.png binary +*.ico binary +*.icns binary +*.woff2 binary diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml new file mode 100644 index 0000000..33a1542 --- /dev/null +++ b/.github/workflows/windows.yml @@ -0,0 +1,67 @@ +name: Windows + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: windows-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: windows-2022 + env: + CARGO_BUILD_JOBS: 2 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - uses: Swatinem/rust-cache@v2 + with: + workspaces: | + . -> target + apps/desktop/src-tauri -> target + - uses: pnpm/action-setup@v4 + with: + version: 11 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: apps/desktop/pnpm-lock.yaml + - name: Install frontend dependencies + run: pnpm --dir apps/desktop install --frozen-lockfile + - name: Check daemon + run: | + cargo fmt --all --check + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + cargo clippy --locked --workspace --all-targets -- -D warnings + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + cargo test --locked --workspace --all-targets + - name: Check frontend + run: pnpm --dir apps/desktop check + - name: Stage daemon sidecar + run: ./scripts/prepare-sidecar.ps1 + - name: Check native shell + run: | + cargo fmt --manifest-path apps/desktop/src-tauri/Cargo.toml --check + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + cargo clippy --locked --manifest-path apps/desktop/src-tauri/Cargo.toml --all-targets -- -D warnings + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + cargo test --locked --manifest-path apps/desktop/src-tauri/Cargo.toml + - name: Build installer + run: pnpm --dir apps/desktop tauri build --bundles nsis + - name: Check installer lifecycle + run: ./scripts/test-windows-installer.ps1 -Installer (Get-ChildItem apps/desktop/src-tauri/target/release/bundle/nsis/*-setup.exe).FullName + - uses: actions/upload-artifact@v4 + with: + name: gravity-windows-x64 + path: apps/desktop/src-tauri/target/release/bundle/nsis/*-setup.exe + if-no-files-found: error diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea4d06c..aee6537 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,8 @@ to public issues. Security reports belong in the private channel described in Use a current stable Rust toolchain, Node.js 22, and pnpm 11. Building the native macOS app requires Xcode Command Line Tools. Official binaries target Apple -Silicon macOS; other platforms are not currently distributed. Docker is required +Silicon macOS; Windows x64 builds are available as CI artifacts. See +[Windows setup](docs/windows.md) for native prerequisites and PowerShell commands. Docker is required for the visual regression suite. ```sh @@ -35,7 +36,7 @@ cargo build --workspace ./scripts/dev.sh ``` -Real bots require your own Claude Code installation and authentication. Use +Real bots require your own Claude Code or Codex CLI installation and authentication. Use `GRAVITY_RUNTIME=double ./scripts/dev.sh` to run the deterministic test runtime without Claude credentials. Development state stays under `.dev/`. @@ -55,7 +56,8 @@ pnpm run verify This runs Rust and native-shell checks, desktop and marketing typechecking, linting, tests and builds, the file-length check, and visual regression tests. -It stops on the first failure. Full verification requires macOS and running Docker. +It stops on the first failure. Native-shell checks run on the host platform; +visual verification requires running Docker with Linux containers. Visual baselines must be generated in the pinned CI container; use the repository's visual regression workflow when accepting diff --git a/Cargo.lock b/Cargo.lock index 734722b..600e137 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -520,6 +520,7 @@ version = "0.13.0" dependencies = [ "anyhow", "axum", + "base64", "bus", "bytes", "chrono", diff --git a/README.md b/README.md index ccb9fce..55084a6 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,20 @@ # Gravity -Always-on, multi-bot desktop app with Claude Code as the agent runtime. A Rust -daemon (`gravityd`) runs on a Mac mini; the Tauri desktop client attaches +Always-on, multi-bot desktop app with Claude Code or Codex CLI as the agent runtime. +A Rust daemon (`gravityd`) runs on macOS or Windows; the Tauri desktop client attaches from any machine. See the [architecture](docs/architecture.md) for the system design and the [control-plane protocol](docs/protocol.md) for client APIs. +Native Windows x64 builds and local-daemon setup are described in +[Windows build instructions](docs/windows.md). Builds and tests require no Linear, PostHog, or Cloudflare account. Running real -bots requires your own Claude Code installation and authentication; the test +bots requires your own installation and authentication for the chosen CLI; the test runtime works without it. See [public-build configuration](docs/public-builds.md) for telemetry, releases, and secret scanning. +Choose each bot's CLI in **Bot info → Bot runtime**. See +[bot runtimes](docs/bot-runtimes.md) for setup, configuration and the native Codex terminal. + ## Layout ``` diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock index c049167..9b09d6e 100644 --- a/apps/desktop/src-tauri/Cargo.lock +++ b/apps/desktop/src-tauri/Cargo.lock @@ -1434,6 +1434,11 @@ dependencies = [ name = "gravity-desktop" version = "0.13.0" dependencies = [ + "block2", + "objc2", + "objc2-avf-audio", + "objc2-foundation", + "objc2-speech", "semver", "serde", "serde_json", @@ -1444,6 +1449,7 @@ dependencies = [ "tauri-plugin-updater", "toml 0.8.2", "ureq", + "windows", ] [[package]] @@ -2316,6 +2322,32 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.1", + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "bitflags 2.13.1", + "block2", + "objc2", + "objc2-audio-toolbox", + "objc2-core-audio-types", + "objc2-foundation", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -2327,6 +2359,28 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.1", + "objc2", +] + [[package]] name = "objc2-core-data" version = "0.3.2" @@ -2381,6 +2435,20 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + [[package]] name = "objc2-core-text" version = "0.3.2" @@ -2393,6 +2461,19 @@ dependencies = [ "objc2-core-graphics", ] +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-io-surface", +] + [[package]] name = "objc2-encode" version = "4.1.0" @@ -2456,6 +2537,19 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-speech" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9eb609f9d2a25e0f8b76954f3acaa58348ce2a91285fe867643b2224ac4d263" +dependencies = [ + "block2", + "objc2", + "objc2-avf-audio", + "objc2-core-media", + "objc2-foundation", +] + [[package]] name = "objc2-ui-kit" version = "0.3.2" diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index a9255e8..15d77ec 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -25,3 +25,15 @@ serde_json = "1" semver = "1" ureq = { version = "2", features = ["json"] } toml = "0.8" + +# On-device dictation for the chat composer (Apple's Speech framework). +[target.'cfg(target_os = "macos")'.dependencies] +objc2 = "0.6" +block2 = "0.6" +objc2-foundation = "0.3" +objc2-speech = "0.3" +objc2-avf-audio = "0.3" + +# On-device dictation on Windows (SAPI's in-process recognizer). +[target.'cfg(windows)'.dependencies] +windows = { version = "0.61", features = ["Win32_Foundation", "Win32_Media_Speech", "Win32_System_Com"] } diff --git a/apps/desktop/src-tauri/Entitlements.plist b/apps/desktop/src-tauri/Entitlements.plist new file mode 100644 index 0000000..82d6e63 --- /dev/null +++ b/apps/desktop/src-tauri/Entitlements.plist @@ -0,0 +1,10 @@ + + + + + + com.apple.security.device.audio-input + + + diff --git a/apps/desktop/src-tauri/Info.plist b/apps/desktop/src-tauri/Info.plist new file mode 100644 index 0000000..c7a8646 --- /dev/null +++ b/apps/desktop/src-tauri/Info.plist @@ -0,0 +1,11 @@ + + + + + + NSMicrophoneUsageDescription + Gravity listens while you dictate a message to a bot in the chat. + NSSpeechRecognitionUsageDescription + Gravity turns your dictation into text on this Mac, for messages to your bots. + + diff --git a/apps/desktop/src-tauri/src/daemon.rs b/apps/desktop/src-tauri/src/daemon.rs index 3fbac3f..1846d49 100644 --- a/apps/desktop/src-tauri/src/daemon.rs +++ b/apps/desktop/src-tauri/src/daemon.rs @@ -48,31 +48,47 @@ fn sidecar_path_for_exe(exe: &Path) -> Result { let dir = exe .parent() .ok_or_else(|| "the app path has no parent".to_string())?; - Ok(dir.join("gravityd")) + Ok(dir.join(format!("gravityd{}", std::env::consts::EXE_SUFFIX))) } /// Must match `LAUNCHD_LABEL` in `crates/gravityd/src/service.rs`. const LAUNCHD_LABEL: &str = "in.mikolajczuk.gravityd"; -fn user_home() -> Result { - let home = std::env::var("HOME").map_err(|err| format!("HOME is not set: {err}"))?; - Ok(PathBuf::from(home)) +pub(crate) fn user_home() -> Result { + let variable = if cfg!(windows) { "USERPROFILE" } else { "HOME" }; + std::env::var_os(variable) + .map(PathBuf::from) + .ok_or_else(|| format!("{variable} is not set")) } /// Mirrors `ServicePaths::bin_path` and `plist_path` in `crates/gravityd/src/service.rs`. fn managed_daemon_is_installed(home: &Path, user_home: &Path) -> bool { - home.join("bin/gravityd").is_file() - && user_home + home.join(format!("bin/gravityd{}", std::env::consts::EXE_SUFFIX)) + .is_file() + && managed_marker(home, user_home).is_file() +} + +fn managed_marker(home: &Path, user_home: &Path) -> PathBuf { + if cfg!(windows) { + home.join("gravityd-task.xml") + } else { + user_home .join("Library/LaunchAgents") .join(format!("{LAUNCHD_LABEL}.plist")) - .is_file() + } } /// Runs the bundled sidecar's own `service `, the same code path the /// CLI uses. fn run_bundled_service(action: &str, failure: &str) -> Result<(), String> { let sidecar = sidecar_path()?; - let out = std::process::Command::new(&sidecar) + let mut command = std::process::Command::new(&sidecar); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + command.creation_flags(0x08000000); // CREATE_NO_WINDOW + } + let out = command .args(["service", action]) .output() .map_err(|err| format!("failed to run {}: {err}", sidecar.display()))?; @@ -155,7 +171,7 @@ pub(crate) fn update_local_daemon_if_installed() -> Result<(), String> { const DEFAULT_PORT: u16 = 49777; /// Root of the daemon's state, mirroring `gravityd`'s own resolution. -fn daemon_home() -> Result { +pub(crate) fn daemon_home() -> Result { if let Some(home) = std::env::var_os("GRAVITY_HOME") { return Ok(PathBuf::from(home)); } @@ -255,7 +271,10 @@ mod tests { let exe = Path::new("/Applications/Gravity.app/Contents/MacOS/Gravity"); assert_eq!( sidecar_path_for_exe(exe).expect("sidecar path"), - PathBuf::from("/Applications/Gravity.app/Contents/MacOS/gravityd") + PathBuf::from(format!( + "/Applications/Gravity.app/Contents/MacOS/gravityd{}", + std::env::consts::EXE_SUFFIX + )) ); } @@ -270,10 +289,8 @@ mod tests { let _ = std::fs::remove_dir_all(&root); let home = root.join("gravity"); let user_home = root.join("user"); - let bin = home.join("bin/gravityd"); - let plist = user_home - .join("Library/LaunchAgents") - .join(format!("{LAUNCHD_LABEL}.plist")); + let bin = home.join(format!("bin/gravityd{}", std::env::consts::EXE_SUFFIX)); + let plist = managed_marker(&home, &user_home); std::fs::create_dir_all(bin.parent().expect("binary parent")).expect("binary directory"); std::fs::write(&bin, b"daemon").expect("daemon binary"); assert!(!managed_daemon_is_installed(&home, &user_home)); @@ -314,10 +331,8 @@ mod tests { let root = daemon_test_home("is-managed"); let home = root.join("gravity"); let user_home = root.join("user"); - let bin = home.join("bin/gravityd"); - let plist = user_home - .join("Library/LaunchAgents") - .join(format!("{LAUNCHD_LABEL}.plist")); + let bin = home.join(format!("bin/gravityd{}", std::env::consts::EXE_SUFFIX)); + let plist = managed_marker(&home, &user_home); std::fs::create_dir_all(bin.parent().expect("binary parent")).expect("binary directory"); std::fs::create_dir_all(plist.parent().expect("plist parent")).expect("plist directory"); std::fs::write(&bin, b"daemon").expect("daemon binary"); diff --git a/apps/desktop/src-tauri/src/dictation.rs b/apps/desktop/src-tauri/src/dictation.rs new file mode 100644 index 0000000..b2289ca --- /dev/null +++ b/apps/desktop/src-tauri/src/dictation.rs @@ -0,0 +1,82 @@ +//! On-device dictation for the chat composer. The operating system's own +//! speech recognizer turns the microphone into text, so audio never goes to +//! Gravity or a third party: Apple's Speech framework on macOS, set to +//! recognize on the device whenever the Mac supports it, and SAPI's +//! in-process recognizer on Windows, which always runs on the PC. +//! +//! The composer starts and stops a session; partial and final transcripts +//! arrive as `dictation` events. + +use serde::Serialize; +use tauri::{AppHandle, Emitter}; + +#[cfg(target_os = "macos")] +mod macos; +#[cfg(windows)] +mod sapi; + +/// The event the composer listens to. +const EVENT: &str = "dictation"; + +#[derive(Clone, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub(crate) enum DictationEvent { + /// The transcript so far; it may still change. + Partial { text: String }, + /// The settled transcript; the session has ended. + Final { text: String }, + /// The session ended without a transcript, or failed. + Ended { + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + }, +} + +pub(crate) fn emit(app: &AppHandle, event: DictationEvent) { + let _ = app.emit(EVENT, event); +} + +/// Whether this platform can dictate. +#[tauri::command] +pub fn dictation_available() -> bool { + #[cfg(windows)] + { + sapi::available() + } + #[cfg(not(windows))] + { + cfg!(target_os = "macos") + } +} + +/// Starts listening, asking for speech and microphone permission the first +/// time. Async so the permission prompt never waits on the main thread. +#[tauri::command] +pub async fn start_dictation(app: AppHandle) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + tauri::async_runtime::spawn_blocking(move || macos::start(app)) + .await + .map_err(|e| e.to_string())? + } + #[cfg(windows)] + { + tauri::async_runtime::spawn_blocking(move || sapi::start(app)) + .await + .map_err(|e| e.to_string())? + } + #[cfg(not(any(target_os = "macos", windows)))] + { + let _ = app; + Err("Dictation is not available on this platform yet.".to_string()) + } +} + +/// Stops listening; the final transcript follows as an event. +#[tauri::command] +pub fn stop_dictation() { + #[cfg(target_os = "macos")] + macos::stop(); + #[cfg(windows)] + sapi::stop(); +} diff --git a/apps/desktop/src-tauri/src/dictation/macos.rs b/apps/desktop/src-tauri/src/dictation/macos.rs new file mode 100644 index 0000000..daa09b5 --- /dev/null +++ b/apps/desktop/src-tauri/src/dictation/macos.rs @@ -0,0 +1,153 @@ +//! Dictation on macOS: `AVAudioEngine` feeds the microphone to an +//! `SFSpeechAudioBufferRecognitionRequest`, recognized on the device. + +use std::ptr::NonNull; +use std::sync::mpsc; +use std::sync::Mutex; +use std::time::Duration; + +use block2::RcBlock; +use objc2::rc::Retained; +use objc2_avf_audio::{AVAudioEngine, AVAudioPCMBuffer, AVAudioTime}; +use objc2_foundation::NSError; +use objc2_speech::{ + SFSpeechAudioBufferRecognitionRequest, SFSpeechRecognitionResult, SFSpeechRecognitionTask, + SFSpeechRecognizer, SFSpeechRecognizerAuthorizationStatus, +}; +use tauri::AppHandle; + +use super::{emit, DictationEvent}; + +/// Samples per microphone buffer handed to the recognizer. +const TAP_FRAMES: u32 = 1024; + +/// A running session's objects. They are only touched under `SESSION`'s +/// lock, and AVAudioEngine and the Speech objects accept calls from any +/// thread, so moving them between threads is sound. +struct Session { + engine: Retained, + request: Retained, + task: Retained, + _recognizer: Retained, +} + +// SAFETY: see `Session`; the objects are used from one thread at a time. +unsafe impl Send for Session {} + +static SESSION: Mutex> = Mutex::new(None); +/// A stopped session still finishing its transcript; kept alive until the +/// next one starts, so its final result is not cut off. +static FINISHING: Mutex> = Mutex::new(None); + +fn lock(slot: &'static Mutex>) -> std::sync::MutexGuard<'static, Option> { + slot.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// The speech permission, asking the first time. +fn authorize() -> Result<(), String> { + let status = unsafe { SFSpeechRecognizer::authorizationStatus() }; + let status = if status == SFSpeechRecognizerAuthorizationStatus::NotDetermined { + let (tx, rx) = mpsc::channel(); + let handler = RcBlock::new(move |answer: SFSpeechRecognizerAuthorizationStatus| { + let _ = tx.send(answer); + }); + unsafe { SFSpeechRecognizer::requestAuthorization(&handler) }; + rx.recv_timeout(Duration::from_secs(120)) + .map_err(|_| "No answer to the speech recognition permission prompt.".to_string())? + } else { + status + }; + if status == SFSpeechRecognizerAuthorizationStatus::Authorized { + Ok(()) + } else { + Err("Speech recognition is off for Gravity. Turn it on in System Settings › Privacy & Security › Speech Recognition.".to_string()) + } +} + +pub(super) fn start(app: AppHandle) -> Result<(), String> { + stop(); + authorize()?; + let recognizer = unsafe { SFSpeechRecognizer::new() }; + if !unsafe { recognizer.isAvailable() } { + return Err("Speech recognition is not available right now.".to_string()); + } + let request = unsafe { SFSpeechAudioBufferRecognitionRequest::new() }; + unsafe { + request.setShouldReportPartialResults(true); + request.setAddsPunctuation(true); + // Keep the audio on this Mac whenever it can recognize locally. + if recognizer.supportsOnDeviceRecognition() { + request.setRequiresOnDeviceRecognition(true); + } + } + + let engine = unsafe { AVAudioEngine::new() }; + let input = unsafe { engine.inputNode() }; + let format = unsafe { input.outputFormatForBus(0) }; + let feed = request.clone(); + let tap = RcBlock::new( + move |buffer: NonNull, _: NonNull| { + unsafe { feed.appendAudioPCMBuffer(buffer.as_ref()) }; + }, + ); + unsafe { + input.installTapOnBus_bufferSize_format_block( + 0, + TAP_FRAMES, + Some(&format), + RcBlock::as_ptr(&tap), + ); + engine.prepare(); + } + if let Err(error) = unsafe { engine.startAndReturnError() } { + unsafe { input.removeTapOnBus(0) }; + return Err(format!( + "The microphone could not start ({}). Check System Settings › Privacy & Security › Microphone.", + error.localizedDescription() + )); + } + + let results = app.clone(); + let handler = RcBlock::new( + move |result: *mut SFSpeechRecognitionResult, error: *mut NSError| { + if let Some(result) = unsafe { result.as_ref() } { + let text = unsafe { result.bestTranscription().formattedString() }.to_string(); + if unsafe { result.isFinal() } { + emit(&results, DictationEvent::Final { text }); + return; + } + emit(&results, DictationEvent::Partial { text }); + } + if let Some(error) = unsafe { error.as_ref() } { + emit( + &results, + DictationEvent::Ended { + error: Some(error.localizedDescription().to_string()), + }, + ); + } + }, + ); + let task = unsafe { recognizer.recognitionTaskWithRequest_resultHandler(&request, &handler) }; + *lock(&SESSION) = Some(Session { + engine, + request, + task, + _recognizer: recognizer, + }); + Ok(()) +} + +/// Stops the microphone and lets the recognizer settle the transcript. +pub(super) fn stop() { + let Some(session) = lock(&SESSION).take() else { + return; + }; + unsafe { + session.engine.stop(); + session.engine.inputNode().removeTapOnBus(0); + session.request.endAudio(); + session.task.finish(); + } + *lock(&FINISHING) = Some(session); +} diff --git a/apps/desktop/src-tauri/src/dictation/sapi.rs b/apps/desktop/src-tauri/src/dictation/sapi.rs new file mode 100644 index 0000000..0dd8013 --- /dev/null +++ b/apps/desktop/src-tauri/src/dictation/sapi.rs @@ -0,0 +1,320 @@ +//! Dictation on Windows: the SAPI in-process recognizer with its dictation +//! grammar, fed by the default microphone. SAPI's desktop recognizers run +//! entirely on this PC, unlike the WinRT recognizer, whose dictation needs +//! Windows' online speech service. +//! +//! Each session owns a thread that holds the COM objects, so nothing SAPI +//! hands out crosses threads. Phrases SAPI settles are kept; the phrase being +//! spoken shows as a hypothesis after them. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc, Mutex}; +use std::time::{Duration, Instant}; + +use tauri::AppHandle; +use windows::core::{IUnknown, Interface, PWSTR}; +use windows::Win32::Media::Speech::{ + ISpObjectToken, ISpObjectTokenCategory, ISpRecoContext, ISpRecoGrammar, ISpRecoResult, + ISpRecognizer, SpInprocRecognizer, SpMMAudioIn, SpObjectToken, SpObjectTokenCategory, + SPCAT_RECOGNIZERS, SPEI_END_SR_STREAM, SPEI_FALSE_RECOGNITION, SPEI_HYPOTHESIS, + SPEI_RECOGNITION, SPEI_RESERVED1, SPEI_RESERVED2, SPET_LPARAM_IS_OBJECT, + SPET_LPARAM_IS_POINTER, SPET_LPARAM_IS_STRING, SPET_LPARAM_IS_TOKEN, SPEVENT, SPEVENTENUM, + SPLO_STATIC, SPRST_ACTIVE, SPRST_INACTIVE, SPRS_ACTIVE, +}; +use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoTaskMemFree, CoUninitialize, CLSCTX_ALL, + COINIT_MULTITHREADED, +}; + +use super::{emit, DictationEvent}; + +/// How often the session thread looks for a stop request between events. +const POLL: Duration = Duration::from_millis(100); +/// After the microphone stops, how long to wait for the last phrase to +/// settle, and the most to wait in all. The composer stops listening to +/// events three seconds after it asks to stop. +const SETTLE_QUIET: Duration = Duration::from_millis(600); +const SETTLE_MAX: Duration = Duration::from_millis(1500); +/// `SP_GETWHOLEPHRASE` from sapi.h: the start and count that cover a phrase. +const WHOLE_PHRASE: u32 = u32::MAX; + +/// Asks the running session's thread to stop. +static STOP: Mutex>> = Mutex::new(None); + +fn stop_slot() -> std::sync::MutexGuard<'static, Option>> { + STOP.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Whether a SAPI recognizer is installed (Windows ships one per supported +/// display language). +pub(super) fn available() -> bool { + std::thread::spawn(|| { + with_com(|| unsafe { + default_recognizer()?; + Ok(()) + }) + .is_ok() + }) + .join() + .unwrap_or(false) +} + +pub(super) fn start(app: AppHandle) -> Result<(), String> { + stop(); + let stopping = Arc::new(AtomicBool::new(false)); + let (ready, started) = mpsc::channel(); + let flag = stopping.clone(); + std::thread::Builder::new() + .name("dictation".into()) + .spawn(move || { + let _ = with_com(|| { + let session = match unsafe { Session::open() } { + Ok(session) => session, + Err(error) => { + let _ = ready.send(Err(error)); + return Ok(()); + } + }; + let _ = ready.send(Ok(())); + session.run(&app, &flag); + Ok(()) + }); + }) + .map_err(|e| e.to_string())?; + started + .recv() + .map_err(|_| "Dictation stopped before it started.".to_string())??; + *stop_slot() = Some(stopping); + Ok(()) +} + +/// Stops the microphone; the session settles its transcript on its thread. +pub(super) fn stop() { + if let Some(stopping) = stop_slot().take() { + stopping.store(true, Ordering::SeqCst); + } +} + +/// Runs `work` with COM initialized on this thread. +fn with_com(work: impl FnOnce() -> Result) -> Result { + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) } + .ok() + .map_err(|e| e.message())?; + let result = work(); + unsafe { CoUninitialize() }; + result +} + +/// The installed recognizer the user picked in Speech settings. +unsafe fn default_recognizer() -> Result { + let category: ISpObjectTokenCategory = + CoCreateInstance(&SpObjectTokenCategory, None, CLSCTX_ALL).map_err(|e| e.message())?; + category + .SetId(SPCAT_RECOGNIZERS, false) + .map_err(|e| e.message())?; + let id = category + .GetDefaultTokenId() + .map_err(|_| "No speech recognizer is installed for this language.".to_string())?; + let token: Result = + CoCreateInstance(&SpObjectToken, None, CLSCTX_ALL).map_err(|e| e.message()); + let token = token.and_then(|token| { + token + .SetId(None, windows::core::PCWSTR(id.0), false) + .map_err(|e| e.message())?; + Ok(token) + }); + CoTaskMemFree(Some(id.0 as _)); + token +} + +/// The bit `SetInterest` takes for one event; sapi.h's `SPFEI`. +fn interest(event: SPEVENTENUM) -> u64 { + (1u64 << event.0) | (1u64 << SPEI_RESERVED1.0) | (1u64 << SPEI_RESERVED2.0) +} + +struct Session { + recognizer: ISpRecognizer, + context: ISpRecoContext, + _grammar: ISpRecoGrammar, +} + +/// What has been said so far. +#[derive(Default)] +struct Transcript { + settled: Vec, + speaking: Option, +} + +impl Transcript { + fn text(&self) -> String { + let mut parts: Vec<&str> = self.settled.iter().map(String::as_str).collect(); + if let Some(speaking) = &self.speaking { + parts.push(speaking); + } + parts.join(" ") + } +} + +impl Session { + unsafe fn open() -> Result { + let recognizer: ISpRecognizer = + CoCreateInstance(&SpInprocRecognizer, None, CLSCTX_ALL).map_err(|e| e.message())?; + recognizer + .SetRecognizer(&default_recognizer()?) + .map_err(|e| e.message())?; + let microphone: IUnknown = + CoCreateInstance(&SpMMAudioIn, None, CLSCTX_ALL).map_err(|e| e.message())?; + recognizer + .SetInput(µphone, true) + .map_err(|_| microphone_off())?; + let context = recognizer.CreateRecoContext().map_err(|e| e.message())?; + context.SetNotifyWin32Event().map_err(|e| e.message())?; + let events = [ + SPEI_RECOGNITION, + SPEI_HYPOTHESIS, + SPEI_FALSE_RECOGNITION, + SPEI_END_SR_STREAM, + ] + .into_iter() + .fold(0, |mask, event| mask | interest(event)); + context + .SetInterest(events, events) + .map_err(|e| e.message())?; + let grammar = context.CreateGrammar(0).map_err(|e| e.message())?; + grammar + .LoadDictation(None, SPLO_STATIC) + .map_err(|e| e.message())?; + grammar + .SetDictationState(SPRS_ACTIVE) + .map_err(|e| e.message())?; + recognizer + .SetRecoState(SPRST_ACTIVE) + .map_err(|_| microphone_off())?; + Ok(Self { + recognizer, + context, + _grammar: grammar, + }) + } + + /// Streams the transcript until asked to stop, then settles it. + fn run(self, app: &AppHandle, stopping: &AtomicBool) { + let mut transcript = Transcript::default(); + while !stopping.load(Ordering::SeqCst) { + unsafe { + let _ = self.context.WaitForNotifyEvent(POLL.as_millis() as u32); + } + match self.drain(&mut transcript) { + Ok(true) => emit( + app, + DictationEvent::Partial { + text: transcript.text(), + }, + ), + Ok(false) => {} + Err(error) => { + emit(app, DictationEvent::Ended { error: Some(error) }); + return; + } + } + } + + // The microphone stops; the phrase in progress may still settle. + unsafe { + let _ = self.recognizer.SetRecoState(SPRST_INACTIVE); + } + let began = Instant::now(); + let mut quiet_since = Instant::now(); + while began.elapsed() < SETTLE_MAX && quiet_since.elapsed() < SETTLE_QUIET { + unsafe { + let _ = self.context.WaitForNotifyEvent(POLL.as_millis() as u32); + } + if let Ok(true) = self.drain(&mut transcript) { + quiet_since = Instant::now(); + } + } + emit( + app, + DictationEvent::Final { + text: transcript.text(), + }, + ); + } + + /// Takes the queued events into `transcript`; true when it changed. + fn drain(&self, transcript: &mut Transcript) -> Result { + let mut changed = false; + loop { + let mut events = [SPEVENT::default(); 16]; + let mut fetched = 0u32; + unsafe { + self.context + .GetEvents(events.len() as u32, events.as_mut_ptr(), &mut fetched) + .map_err(|e| e.message())?; + } + if fetched == 0 { + return Ok(changed); + } + for event in &events[..fetched as usize] { + let id = SPEVENTENUM(event._bitfield & 0xffff); + if id == SPEI_END_SR_STREAM { + // lParam carries the audio stream's HRESULT. + let status = windows::core::HRESULT(event.lParam.0 as i32); + if status.is_err() { + return Err(microphone_off()); + } + continue; + } + let text = unsafe { take_result_text(event) }; + match id { + SPEI_RECOGNITION => { + transcript.speaking = None; + if let Some(text) = text.filter(|t| !t.trim().is_empty()) { + transcript.settled.push(text.trim().to_string()); + } + changed = true; + } + SPEI_HYPOTHESIS => { + transcript.speaking = text.map(|t| t.trim().to_string()); + changed = true; + } + SPEI_FALSE_RECOGNITION => { + changed |= transcript.speaking.take().is_some(); + } + _ => {} + } + } + } + } +} + +/// Reads a recognition event's phrase and releases what the event holds +/// (sapi.h's `SpClearEvent`). +unsafe fn take_result_text(event: &SPEVENT) -> Option { + let param = event.lParam.0 as *mut core::ffi::c_void; + if param.is_null() { + return None; + } + let kind = (event._bitfield >> 16) & 0xffff; + if kind == SPET_LPARAM_IS_OBJECT.0 || kind == SPET_LPARAM_IS_TOKEN.0 { + let object = IUnknown::from_raw(param); + let result: ISpRecoResult = object.cast().ok()?; + let mut text = PWSTR::null(); + result + .GetText(WHOLE_PHRASE, WHOLE_PHRASE, true, &mut text, None) + .ok()?; + let phrase = text.to_string().ok(); + CoTaskMemFree(Some(text.0 as _)); + return phrase; + } + if kind == SPET_LPARAM_IS_POINTER.0 || kind == SPET_LPARAM_IS_STRING.0 { + CoTaskMemFree(Some(param as _)); + } + None +} + +fn microphone_off() -> String { + "The microphone could not start. Check Settings › Privacy & security › Microphone, \ + including \"Let desktop apps access your microphone\"." + .to_string() +} diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 5956047..3a2b10d 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1,4 +1,5 @@ mod daemon; +mod dictation; mod shortcut; mod updater; @@ -9,8 +10,7 @@ use tauri::{AppHandle, Manager}; /// Reads the daemon client token from `~/.gravity/secrets/client.token`. #[tauri::command] fn read_client_token() -> Result { - let home = std::env::var("HOME").map_err(|err| format!("HOME is not set: {err}"))?; - let path = PathBuf::from(home).join(".gravity/secrets/client.token"); + let path = daemon::daemon_home()?.join("secrets/client.token"); let token = std::fs::read_to_string(&path) .map_err(|err| format!("failed to read {}: {err}", path.display()))?; Ok(token.trim().to_string()) @@ -92,10 +92,7 @@ fn reveal_project(dir_name: String) -> Result<(), String> { if dir_name.is_empty() || dir_name.contains('/') || dir_name.contains('\\') { return Err(format!("not a project directory name: {dir_name}")); } - let home = std::env::var("HOME").map_err(|err| format!("HOME is not set: {err}"))?; - let dir = PathBuf::from(home) - .join(".gravity/projects") - .join(&dir_name); + let dir = daemon::daemon_home()?.join("projects").join(&dir_name); if !dir.is_dir() { return Err(format!("no such project directory: {}", dir.display())); } @@ -190,6 +187,9 @@ pub fn run() { daemon::restart_local_daemon, daemon::local_daemon_port, daemon::daemon_log_tail, + dictation::dictation_available, + dictation::start_dictation, + dictation::stop_dictation, updater::check_for_update, updater::install_update, updater::relaunch_app diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json index 782e3d1..4e71875 100644 --- a/apps/desktop/src-tauri/tauri.conf.json +++ b/apps/desktop/src-tauri/tauri.conf.json @@ -46,6 +46,9 @@ "icons/128x128@2x.png", "icons/icon.icns", "icons/icon.ico" - ] + ], + "macOS": { + "entitlements": "Entitlements.plist" + } } } diff --git a/apps/desktop/src-tauri/tauri.windows.conf.json b/apps/desktop/src-tauri/tauri.windows.conf.json new file mode 100644 index 0000000..a354ad7 --- /dev/null +++ b/apps/desktop/src-tauri/tauri.windows.conf.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "app": { + "windows": [ + { + "title": "Gravity", + "width": 1280, + "height": 800, + "resizable": true, + "decorations": true + } + ] + }, + "bundle": { + "targets": ["nsis"], + "windows": { + "nsis": { + "installMode": "currentUser", + "installerHooks": "windows/installer-hooks.nsh" + } + } + } +} diff --git a/apps/desktop/src-tauri/windows/installer-hooks.nsh b/apps/desktop/src-tauri/windows/installer-hooks.nsh new file mode 100644 index 0000000..b860ec0 --- /dev/null +++ b/apps/desktop/src-tauri/windows/installer-hooks.nsh @@ -0,0 +1,33 @@ +; The desktop and its matching managed daemon are one installation. +!include "LogicLib.nsh" + +!macro GRAVITY_SERVICE ACTION + Push $0 + Push $1 + DetailPrint "Gravity background daemon: ${ACTION}" + nsExec::ExecToStack /TIMEOUT=60000 '"$INSTDIR\gravityd.exe" service ${ACTION}' + Pop $0 + Pop $1 + ${If} $0 != 0 + DetailPrint "$1" + MessageBox MB_OK|MB_ICONSTOP "Gravity could not ${ACTION} its background daemon.$\r$\n$\r$\n$1" /SD IDOK + Pop $1 + Pop $0 + SetErrorLevel 1 + Abort "Gravity background daemon ${ACTION} failed." + ${EndIf} + Pop $1 + Pop $0 +!macroend + +!macro NSIS_HOOK_POSTINSTALL + !insertmacro GRAVITY_SERVICE install +!macroend + +!macro NSIS_HOOK_PREUNINSTALL + ; Ask the user to close the desktop before stopping its background sessions. + !insertmacro CheckIfAppIsRunning "${MAINBINARYNAME}.exe" "${PRODUCTNAME}" + ${If} ${FileExists} "$INSTDIR\gravityd.exe" + !insertmacro GRAVITY_SERVICE uninstall + ${EndIf} +!macroend diff --git a/apps/desktop/src/components/BotView.chat.test.tsx b/apps/desktop/src/components/BotView.chat.test.tsx new file mode 100644 index 0000000..4e1d6be --- /dev/null +++ b/apps/desktop/src/components/BotView.chat.test.tsx @@ -0,0 +1,123 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { turn } from "../test/chatFixtures"; +import { FakeDaemon } from "../test/fakeDaemon"; +import * as fx from "../test/fixtures"; +import { actionToastSpy, botSpy, routinesSpy, stubLocalStorage } from "../test/spies"; +import BotView from "./BotView"; + +vi.mock("./TerminalPane", () => ({ + default: (): React.ReactElement =>
, +})); + +function chatDaemon(): FakeDaemon { + const client = new FakeDaemon() + .onRequest("list_routines", () => ({ type: "routines", req_id: "1", routines: [] })) + .onRequest("list_chat", () => ({ + type: "chat", + req_id: "2", + bot_id: "b1", + has_more: false, + turns: [ + turn({ + items: [ + { + type: "completed", + id: "c1", + task_id: "t1", + result: "Ported.", + artifacts: [{ path: "/p/artifacts/report.md", name: "report.md" }], + }, + ], + }), + ], + })) + .onRequest("read_file", () => ({ + type: "file", + req_id: "3", + file: { name: "report.md", mime: "text/markdown", text: "Report body", truncated: false }, + })) + .onRequest("list_conversations", () => ({ + type: "conversations", + req_id: "4", + conversations: [], + })); + client.capabilities = ["terminal_attach", "chat"]; + return client; +} + +function renderView(client: FakeDaemon, bot = fx.bot()): void { + render( + , + ); +} + +describe("BotView with chat", () => { + beforeEach(() => { + stubLocalStorage(); + }); + + it("opens on the chat and keeps the terminal a tab away", async () => { + renderView(chatDaemon()); + expect(screen.getByRole("button", { name: "Chat" })).toHaveClass("tab-active"); + expect(await screen.findByText("Ported.")).toBeInTheDocument(); + expect(screen.getByTestId("terminal")).toBeInTheDocument(); + + fireEvent.keyDown(window, { key: "2", metaKey: true }); + expect(screen.getByRole("button", { name: "Terminal" })).toHaveClass("tab-active"); + + fireEvent.keyDown(window, { key: "l", metaKey: true }); + expect(screen.getByRole("button", { name: "Chat" })).toHaveClass("tab-active"); + await waitFor(() => { + expect(screen.getByRole("textbox", { name: "Message" })).toHaveFocus(); + }); + }); + + it("opens a result's file in the Files panel", async () => { + renderView(chatDaemon()); + await userEvent.click(await screen.findByRole("button", { name: "report.md" })); + expect(screen.getByRole("button", { name: "Files" })).toHaveClass("tab-active"); + expect(await screen.findByText("Report body")).toBeInTheDocument(); + }); + + it("gives a linked bot only its chat", () => { + renderView(chatDaemon(), fx.bot({ peer: { id: "x", name: "win-pc", online: true } })); + expect(screen.getByRole("button", { name: "Chat" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Terminal" })).not.toBeInTheDocument(); + expect(screen.queryByTestId("terminal")).not.toBeInTheDocument(); + }); + + it("falls back to the terminal on a daemon without chat", () => { + const client = chatDaemon(); + client.capabilities = ["terminal_attach"]; + renderView(client); + expect(screen.queryByRole("button", { name: "Chat" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Terminal" })).toHaveClass("tab-active"); + }); +}); + +describe("BotView for a linked bot on a daemon with peer chat", () => { + beforeEach(() => { + stubLocalStorage(); + }); + + it("reads its real chat from its machine", async () => { + const client = chatDaemon(); + client.capabilities = ["chat", "peer_chat"]; + renderView(client, fx.bot({ peer: { id: "x", name: "win-pc", online: false } })); + expect(await screen.findByText("Ported.")).toBeInTheDocument(); + expect( + screen.getByText(/runs on win-pc; its chat is read from there\. It is offline/), + ).toBeInTheDocument(); + }); +}); diff --git a/apps/desktop/src/components/BotView.test.tsx b/apps/desktop/src/components/BotView.test.tsx index ebe3924..36a5313 100644 --- a/apps/desktop/src/components/BotView.test.tsx +++ b/apps/desktop/src/components/BotView.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { FakeDaemon } from "../test/fakeDaemon"; @@ -74,7 +74,10 @@ describe("BotView", () => { renderView(); expect(screen.getByTestId("terminal")).toBeInTheDocument(); expect(screen.getByRole("complementary", { name: "Bot info" })).toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Info" })).not.toBeInTheDocument(); + // Info is the side panel's tab, never one of the bot's own tabs. + const botTabs = screen.getByRole("button", { name: "Terminal" }).closest("nav"); + expect(botTabs).not.toBeNull(); + expect(within(botTabs ?? document.body).queryByRole("button", { name: "Info" })).toBeNull(); await user.click(screen.getByRole("button", { name: "Routines" })); expect(screen.getByText("Routines", { selector: ".panel-title" })).toBeInTheDocument(); diff --git a/apps/desktop/src/components/BotView.tsx b/apps/desktop/src/components/BotView.tsx index 5e68345..b677d5c 100644 --- a/apps/desktop/src/components/BotView.tsx +++ b/apps/desktop/src/components/BotView.tsx @@ -1,6 +1,5 @@ import { useEffect, useRef, useState } from "react"; import type { CSSProperties, KeyboardEvent, PointerEvent, ReactElement } from "react"; -import { isStopped } from "../app/bots"; import type { AddToast } from "../app/useToasts"; import type { DaemonApi } from "../protocol/api"; import type { Bot, Routine } from "../protocol/entities"; @@ -11,11 +10,12 @@ import { saveBotInfoPanel, } from "../settings"; import BotHeader from "./bot/BotHeader"; -import BotTabs from "./bot/BotTabs"; +import BotPanes from "./bot/BotPanes"; +import BotSidePanel from "./bot/BotSidePanel"; +import type { SideTab } from "./bot/BotSidePanel"; +import BotTabs, { botTabs } from "./bot/BotTabs"; import type { BotTab } from "./bot/BotTabs"; -import InfoPanel from "./InfoPanel"; -import RoutinesPanel from "./RoutinesPanel"; -import TerminalPane from "./TerminalPane"; +import { useBotKeys } from "./bot/useBotKeys"; interface BotViewProps { readonly client: DaemonApi; @@ -26,6 +26,7 @@ interface BotViewProps { readonly onBotUpdated: (bot: Bot) => void; readonly onRoutinesChanged: (botId: string, routines: readonly Routine[]) => void; readonly onToast: AddToast; + readonly onOpenDecision?: (decisionId: string) => void; } interface DragState { @@ -42,14 +43,21 @@ function clamp(value: number, minimum: number, maximum: number): number { export default function BotView(props: BotViewProps): ReactElement { const { client, bot, bots, connected, canControl, onToast } = props; - const [tab, setTab] = useState("terminal"); + const tabs = botTabs(client.capabilities.includes("chat"), bot.peer != null); + const [tab, setTab] = useState(tabs[0] ?? "terminal"); + const [side, setSide] = useState("info"); + const [openFile, setOpenFile] = useState(null); const [infoPanel, setInfoPanel] = useState(loadBotInfoPanel); const [maxInfoPanelWidth, setMaxInfoPanelWidth] = useState(FALLBACK_MAX_INFO_PANEL_WIDTH); const layoutRef = useRef(null); const dragRef = useRef(null); - // The terminal belongs to the user: any `control` connection may type while - // the bot runs. Bus deliveries arrive via the bot's inbox socket, never here. - const canWrite = canControl && !isStopped(bot); + useBotKeys(tabs, setTab); + + const showFile = (path: string): void => { + setSide("files"); + setOpenFile(path); + setInfoPanel((current) => ({ ...current, collapsed: false })); + }; useEffect(() => { saveBotInfoPanel(infoPanel); @@ -148,26 +156,20 @@ export default function BotView(props: BotViewProps): ReactElement { />
- -
- {/* Keep the terminal mounted across tab switches to preserve the buffer. */} -
- -
- {tab === "routines" ? ( -
- -
- ) : null} -
+ +
{infoPanel.collapsed ? null : (
- diff --git a/apps/desktop/src/components/InfoPanel.test.tsx b/apps/desktop/src/components/InfoPanel.test.tsx index ee33107..db62798 100644 --- a/apps/desktop/src/components/InfoPanel.test.tsx +++ b/apps/desktop/src/components/InfoPanel.test.tsx @@ -59,11 +59,11 @@ describe("InfoPanel", () => { expect(screen.getByText("ready — idle")).toBeInTheDocument(); }); - it("reveals the bot workspace in Finder", async () => { + it("opens the bot workspace folder", async () => { const user = userEvent.setup(); renderPanel(new FakeDaemon()); - await user.click(screen.getByRole("button", { name: "Reveal bot workspace in Finder" })); + await user.click(screen.getByRole("button", { name: "Open bot workspace folder" })); expect(revealBotWorkspace).toHaveBeenCalledWith("/tmp/alice"); }); diff --git a/apps/desktop/src/components/InfoPanel.tsx b/apps/desktop/src/components/InfoPanel.tsx index 2af1481..0900427 100644 --- a/apps/desktop/src/components/InfoPanel.tsx +++ b/apps/desktop/src/components/InfoPanel.tsx @@ -8,6 +8,7 @@ import { errText } from "../util"; import BotAvatar from "./BotAvatar"; import { BOT_ICONS } from "./botIcons"; import BotHistory from "./bot/BotHistory"; +import BotRuntimePicker from "./bot/BotRuntimePicker"; interface InfoPanelProps { readonly client: DaemonApi; @@ -132,8 +133,8 @@ export default function InfoPanel({
+ + { + daemon.select({ kind: "control", decisionId }); + }} /> ); } diff --git a/apps/desktop/src/components/Sidebar.test.tsx b/apps/desktop/src/components/Sidebar.test.tsx index 8a5e6cb..e849432 100644 --- a/apps/desktop/src/components/Sidebar.test.tsx +++ b/apps/desktop/src/components/Sidebar.test.tsx @@ -91,7 +91,7 @@ describe("Sidebar", () => { it("offers the same menu from the label's right-click and the ⋯ button", async () => { const user = userEvent.setup(); renderSidebar(); - const entries = ["New Bot", "Project settings", "Reveal in Finder", "Delete project"]; + const entries = ["New Bot", "Project settings", "Open folder", "Delete project"]; await user.pointer({ keys: "[MouseRight]", target: screen.getByText("Acme") }); expect(screen.getAllByRole("menuitem").map((item) => item.textContent)).toEqual(entries); diff --git a/apps/desktop/src/components/bot/BotPanes.tsx b/apps/desktop/src/components/bot/BotPanes.tsx new file mode 100644 index 0000000..3e56b4c --- /dev/null +++ b/apps/desktop/src/components/bot/BotPanes.tsx @@ -0,0 +1,115 @@ +import type { ReactElement } from "react"; +import { isStopped } from "../../app/bots"; +import type { AddToast } from "../../app/useToasts"; +import type { DaemonApi } from "../../protocol/api"; +import type { Bot, Routine } from "../../protocol/entities"; +import ChatPane from "../chat/ChatPane"; +import LinkedChat from "../chat/LinkedChat"; +import PermissionCards from "../permissions/PermissionCards"; +import { usePermissions } from "../permissions/usePermissions"; +import RoutinesPanel from "../RoutinesPanel"; +import TerminalPane from "../TerminalPane"; +import type { BotTab } from "./BotTabs"; + +interface BotPanesProps { + readonly client: DaemonApi; + readonly bot: Bot; + readonly bots: readonly Bot[]; + readonly tabs: readonly BotTab[]; + readonly active: BotTab; + readonly connected: boolean; + readonly canControl: boolean; + readonly onOpenFile: (path: string) => void; + readonly onOpenDecision?: (decisionId: string) => void; + readonly onRoutinesChanged: (botId: string, routines: readonly Routine[]) => void; + readonly onToast: AddToast; +} + +/** Why the owner cannot write to a bot right now, or null when they can. */ +function writeBlockedReason(connected: boolean, canControl: boolean, bot: Bot): string | null { + if (!connected) { + return "Not connected to the daemon"; + } + if (!canControl) { + return "Read-only connection"; + } + return isStopped(bot) ? `${bot.name} is stopped` : null; +} + +/** Where a linked bot runs, said above its chat. */ +function machineNote(bot: Bot): string { + const machine = bot.peer?.name ?? "another machine"; + const offline = + bot.peer?.online === false ? " It is offline, so its chat loads once it is back." : ""; + return `${bot.name} runs on ${machine}; its chat is read from there.${offline}`; +} + +function paneClass(shown: boolean): string { + return shown ? "tab-pane" : "tab-pane tab-pane-hidden"; +} + +/** + * The bot's tab bodies. Chat and terminal stay mounted when hidden, so + * switching back keeps the loaded conversation and the terminal buffer. + */ +export default function BotPanes(props: BotPanesProps): ReactElement { + const { client, bot, tabs, active, connected, canControl, onToast } = props; + const writeBlocked = writeBlockedReason(connected, canControl, bot); + const linked = bot.peer != null; + const permissions = usePermissions(client, bot.id, connected && !linked); + return ( + <> + {/* Above every tab: a prompt waits whether the owner reads the chat or the terminal. */} + +
+ {tabs.includes("chat") ? ( +
+ {linked && !client.capabilities.includes("peer_chat") ? ( + + ) : ( + + )} +
+ ) : null} + {tabs.includes("terminal") ? ( +
+ {/* The terminal belongs to the user: any `control` connection may type while the bot runs. */} + +
+ ) : null} + {active === "routines" ? ( +
+ +
+ ) : null} +
+ + ); +} diff --git a/apps/desktop/src/components/bot/BotRuntimePicker.test.tsx b/apps/desktop/src/components/bot/BotRuntimePicker.test.tsx new file mode 100644 index 0000000..3b08ad9 --- /dev/null +++ b/apps/desktop/src/components/bot/BotRuntimePicker.test.tsx @@ -0,0 +1,89 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; +import { FakeDaemon } from "../../test/fakeDaemon"; +import { bot } from "../../test/fixtures"; +import { botSpy, toastSpy } from "../../test/spies"; +import BotRuntimePicker from "./BotRuntimePicker"; + +function setup( + daemon = new FakeDaemon(), + connected = true, + canControl = true, +): ReturnType { + const updated = botSpy(); + render( + , + ); + return updated; +} + +describe("BotRuntimePicker", () => { + it("hides the picker for older daemons", () => { + setup(); + expect(screen.queryByRole("combobox")).not.toBeInTheDocument(); + }); + it("changes runtime only after an explicit save", async () => { + const user = userEvent.setup(); + const daemon = new FakeDaemon().onRequest("set_bot_runtime", () => ({ + type: "bot", + req_id: "1", + bot: bot({ runtime: "codex_cli" }), + })); + daemon.capabilities = ["bot_runtime"]; + const updated = setup(daemon); + expect(screen.getByRole("button", { name: "Change runtime" })).toBeDisabled(); + await user.selectOptions(screen.getByLabelText("Bot runtime"), "codex_cli"); + expect(daemon.requests).toHaveLength(0); + await user.click(screen.getByRole("button", { name: "Change runtime" })); + await waitFor(() => { + expect(updated).toHaveBeenCalledWith(expect.objectContaining({ runtime: "codex_cli" })); + }); + expect(daemon.requests[0]?.body).toEqual({ + type: "set_bot_runtime", + bot_id: "b1", + runtime: "codex_cli", + }); + }); + it.each([ + [false, true], + [true, false], + ])("disables changes without connection or control (%s, %s)", (connected, control) => { + const daemon = new FakeDaemon(); + daemon.capabilities = ["bot_runtime"]; + setup(daemon, connected, control); + expect(screen.getByRole("combobox")).toBeDisabled(); + expect(screen.getByRole("button", { name: "Change runtime" })).toBeDisabled(); + }); + it("reports failed switches and lets the user retry", async () => { + const user = userEvent.setup(); + const toast = toastSpy(); + const daemon = new FakeDaemon().onRequest("set_bot_runtime", () => { + throw new Error("CLI unavailable"); + }); + daemon.capabilities = ["bot_runtime"]; + render( + , + ); + await user.selectOptions(screen.getByRole("combobox"), "codex_cli"); + await user.click(screen.getByRole("button", { name: "Change runtime" })); + await waitFor(() => { + expect(toast).toHaveBeenCalledWith("error", "Runtime update failed", "CLI unavailable"); + }); + expect(screen.getByRole("button", { name: "Change runtime" })).toBeEnabled(); + }); +}); diff --git a/apps/desktop/src/components/bot/BotRuntimePicker.tsx b/apps/desktop/src/components/bot/BotRuntimePicker.tsx new file mode 100644 index 0000000..07b53ab --- /dev/null +++ b/apps/desktop/src/components/bot/BotRuntimePicker.tsx @@ -0,0 +1,86 @@ +import { useState } from "react"; +import type { ReactElement } from "react"; +import type { DaemonApi } from "../../protocol/api"; +import type { Bot, BotRuntime, NotifyLevel } from "../../protocol/entities"; +import { errText } from "../../util"; + +interface Props { + readonly client: DaemonApi; + readonly bot: Bot; + readonly connected: boolean; + readonly canControl: boolean; + readonly onBotUpdated: (bot: Bot) => void; + readonly onToast: (level: NotifyLevel, title: string, body: string) => void; +} + +export default function BotRuntimePicker({ + client, + bot, + connected, + canControl, + onBotUpdated, + onToast, +}: Props): ReactElement | null { + const current = bot.runtime ?? "claude_code"; + const [runtime, setRuntime] = useState(current); + const [saving, setSaving] = useState(false); + if (!client.capabilities.includes("bot_runtime")) { + return null; + } + const save = async (): Promise => { + setSaving(true); + try { + const reply = await client.request( + { type: "set_bot_runtime", bot_id: bot.id, runtime }, + "bot", + ); + onBotUpdated(reply.bot); + onToast( + "info", + "Runtime saved", + `${reply.bot.name} is restarting with ${runtime === "codex_cli" ? "Codex CLI" : "Claude Code"}. Startup progress appears in the terminal.`, + ); + } catch (error) { + onToast("error", "Runtime update failed", errText(error)); + } finally { + setSaving(false); + } + }; + return ( +
+ + + + Changing runtime interrupts the current turn. Both use the same workspace and saved facts; + each keeps its own conversation history. Install and sign in to the selected CLI first. + +
+ +
+
+ ); +} diff --git a/apps/desktop/src/components/bot/BotSidePanel.tsx b/apps/desktop/src/components/bot/BotSidePanel.tsx new file mode 100644 index 0000000..67ec62f --- /dev/null +++ b/apps/desktop/src/components/bot/BotSidePanel.tsx @@ -0,0 +1,73 @@ +import type { ReactElement } from "react"; +import type { AddToast } from "../../app/useToasts"; +import type { DaemonApi } from "../../protocol/api"; +import type { Bot } from "../../protocol/entities"; +import FilesPanel from "../files/FilesPanel"; +import InfoPanel from "../InfoPanel"; +import TasksPanel from "../tasks/TasksPanel"; + +export type SideTab = "info" | "tasks" | "files"; + +const SIDE_TABS: readonly SideTab[] = ["info", "tasks", "files"]; + +const SIDE_LABEL: Readonly> = { + info: "Info", + tasks: "Tasks", + files: "Files", +}; + +interface BotSidePanelProps { + readonly client: DaemonApi; + readonly bot: Bot; + readonly connected: boolean; + readonly canControl: boolean; + readonly side: SideTab; + readonly onSide: (side: SideTab) => void; + readonly openFile: string | null; + readonly onOpenFile: (path: string | null) => void; + readonly onBotUpdated: (bot: Bot) => void; + readonly onToast: AddToast; +} + +/** The right-hand panel: the bot's info, or the project's files. */ +export default function BotSidePanel(props: BotSidePanelProps): ReactElement { + const { client, bot, connected, side, onSide } = props; + return ( + <> + + {side === "info" ? ( + + ) : null} + {side === "tasks" ? : null} + {side === "files" ? ( + + ) : null} + + ); +} diff --git a/apps/desktop/src/components/bot/BotTabs.stories.tsx b/apps/desktop/src/components/bot/BotTabs.stories.tsx index 90a8b98..0a25caf 100644 --- a/apps/desktop/src/components/bot/BotTabs.stories.tsx +++ b/apps/desktop/src/components/bot/BotTabs.stories.tsx @@ -5,5 +5,5 @@ import BotTabs from "./BotTabs"; export const Interactive: Story = () => { const [active, setActive] = useState("terminal"); - return ; + return ; }; diff --git a/apps/desktop/src/components/bot/BotTabs.tsx b/apps/desktop/src/components/bot/BotTabs.tsx index 3fbe431..8368705 100644 --- a/apps/desktop/src/components/bot/BotTabs.tsx +++ b/apps/desktop/src/components/bot/BotTabs.tsx @@ -1,26 +1,36 @@ import type { ReactElement } from "react"; -const BOT_TABS = ["terminal", "routines"] as const; -export type BotTab = (typeof BOT_TABS)[number]; +export type BotTab = "chat" | "terminal" | "routines"; const TAB_LABEL: Readonly> = { + chat: "Chat", terminal: "Terminal", routines: "Routines", }; +/** The tabs a bot offers: chat needs a daemon that serves it, and a linked bot has only chat. */ +export function botTabs(chat: boolean, linked: boolean): readonly BotTab[] { + if (linked) { + return ["chat"]; + } + return chat ? ["chat", "terminal", "routines"] : ["terminal", "routines"]; +} + interface BotTabsProps { + readonly tabs: readonly BotTab[]; readonly active: BotTab; readonly onSelect: (tab: BotTab) => void; } -export default function BotTabs({ active, onSelect }: BotTabsProps): ReactElement { +export default function BotTabs({ tabs, active, onSelect }: BotTabsProps): ReactElement { return (