From 628b5c78061d8de94f6b2b6ada650953d3b19ed1 Mon Sep 17 00:00:00 2001 From: jli-software <123122329+jli-software@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:32:41 +0100 Subject: [PATCH 1/2] docs: claim 0.12.0 action per application (in progress) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014LfoKbq3S36AiYRW7j8QLK --- docs/plans/v1.0-roadmap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plans/v1.0-roadmap.md b/docs/plans/v1.0-roadmap.md index d517e6c..22ed24e 100644 --- a/docs/plans/v1.0-roadmap.md +++ b/docs/plans/v1.0-roadmap.md @@ -71,7 +71,7 @@ Proposals not yet confirmed by Jonas are marked **Proposal**. | 0.9.0 | Settings from scratch | merged, ships with 1.0 | | 0.10.0 | Codex login for text actions | merged, ships with 1.0 | | 0.11.0 | Context: selected text and clipboard | merged, ships with 1.0 | -| 0.12.0 | Action per application | to discuss first | +| 0.12.0 | Action per application | in progress (discussing with Jonas) | | 0.13.0 | Audio files | merged, ships with 1.0 | | 0.14.0 | First-run assistant | planned (was 0.8.0) | | 1.0.0 | Acceptance on Linux, Windows, macOS; release; WinGet | planned | From cb80c1c6c804a467f6b1b0355881e19ca84e675d Mon Sep 17 00:00:00 2001 From: Jonas <123122329+jli-software@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:18:19 +0100 Subject: [PATCH 2/2] feat: an action per application for the dictation key (0.12.0) Settings -> Context -> Per app gives an application its own action (Email in Thunderbird, Plain in the terminal). The dictation key reads the app in front (Hyprland, Sway, X11, Windows, macOS) and takes its action for that one recording; a notification names it. "+ App" offers the apps dictated into lately or a typed name. Empty by default; the app is never logged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014LfoKbq3S36AiYRW7j8QLK --- CHANGELOG.md | 11 + README.md | 17 +- docs/ARCHITECTURE.md | 42 +++ docs/DEVELOPMENT.md | 15 +- docs/TESTING.md | 22 ++ docs/plans/v1.0-roadmap.md | 5 +- docs/releases/v0.12.0.md | 30 ++ src-tauri/Cargo.lock | 1 + src-tauri/Cargo.toml | 2 +- src-tauri/src/commands.rs | 45 +++ src-tauri/src/domain.rs | 23 ++ src-tauri/src/foreground.rs | 500 +++++++++++++++++++++++++++++++ src-tauri/src/lib.rs | 4 + src-tauri/src/settings.rs | 23 ++ src/App.svelte | 88 +++++- src/App.test.ts | 33 +- src/app.css | 14 + src/lib/api.ts | 8 + src/lib/app-actions.test.ts | 46 +++ src/lib/app-actions.ts | 49 +++ src/lib/settings-catalog.test.ts | 5 +- src/lib/settings-catalog.ts | 3 +- src/lib/types.ts | 25 ++ tests/ui.e2e.ts | 79 ++++- 24 files changed, 1072 insertions(+), 18 deletions(-) create mode 100644 docs/releases/v0.12.0.md create mode 100644 src-tauri/src/foreground.rs create mode 100644 src/lib/app-actions.test.ts create mode 100644 src/lib/app-actions.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 385e625..17a0dab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ - Offer Utterform in "Open with" on Linux, Windows and macOS without becoming the default player for any audio type. See [0.13.0 release notes](docs/releases/v0.13.0.md). + +## 0.12.0 · The right action in every app (on `main`, ships with 1.0) + +- Add **Settings → Context → Per app**: give an application its own action — Email in Thunderbird, Plain in the terminal. Pressing the dictation key there takes that action for this one dictation; everywhere else the action chosen in the window applies. The list starts empty, so nothing changes until you add an app. +- Add an app with **+ App**, which lists the apps you dictated into lately, or type its name. Each app has its action menu and × to remove it. +- Say when an app's action takes over: a short notification "Utterform · Email — for Thunderbird", and the window shows the action with the app's name while it records. A recording started from Utterform's own window, and live dictation, never follow an app's action. +- Know the app in front on Linux under Hyprland and Sway (and X11 through `xprop`), on Windows and on macOS. GNOME and KDE on Wayland do not tell applications which window is focused; there the row says so. +- The app names are kept in `settings.json` on this device. The diagnostic log records only that an app was found and which action ran, never the app. + +See [0.12.0 release notes](docs/releases/v0.12.0.md). + ## 0.11.0 · Say what to do with the selected text (on `main`, ships with 1.0) - Add the **Context** action. Select an email, start dictating with Context, say "reply that Monday at ten works", and the reply arrives at the cursor. What you say is the instruction; the selected text is the material. The material is treated as data, so an instruction hidden in it is not followed. diff --git a/README.md b/README.md index 2f0ecd0..72ae0e3 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ and the matching rows from every category appear, ready to change right there. | Recording | Microphone, shortcut, sounds | | Transcription | Engine (Cloud, Live, Local), OpenAI key, language, vocabulary, topic, local model | | AI | Actions (built-in and your own prompts), provider (OpenAI key or ChatGPT), text model and thinking effort | -| Context | What the Context action may read: selection, clipboard | +| Context | What the Context action may read: selection, clipboard; an action per app | | Output | Typing method, keystroke delay, file folder, history | | App | Theme, start at login, diagnostics | @@ -283,6 +283,19 @@ Only the Context action reads the selection or the clipboard, never plain dictat another action. On Linux the primary selection is whatever you selected last, even in another window, so check what is selected before you use Context for something private. +## An action per app + +Under **Settings → Context → Per app** an app can have its own action: *Email* in +Thunderbird, *Plain* in the terminal. The dictation key then takes that action in that +app; everywhere else the action chosen in the window applies. **+ App** lists the apps +you dictated into lately, or type a name. When an app's action takes over, a short +notification says which, and the window shows it while recording. A recording started +from Utterform's own window, and live dictation, never follow an app's action. + +Utterform knows the app in front on Linux under Hyprland, Sway and X11 (`xprop`), on +Windows and on macOS. GNOME and KDE on Wayland do not say which window is focused, so +there the feature stays off. + ## Vocabulary Names, products and spellings a model would otherwise guess at go under **Settings → Transcription → Vocabulary**, one word at a time: type it and press Enter, remove it with ×. A pasted list keeps its lines. Write *Careum* there and it stops coming back as *Kareum*. @@ -306,6 +319,7 @@ Push/PR CI runs Linux validation without release compilation. For a test binary - Offline transcription through `whisper.cpp`, built for every supported desktop rather than for the machine that compiled it - One-click, SHA-256-verified downloads for Tiny, Base, and Small multilingual models - Plain, Clean, Polish, Summarize, Prompt, Email, and user-defined actions — every shipped prompt can be rewritten +- An action per app for the dictation key: Email in the mail client, Plain in the terminal - A vocabulary of your own terms, and a reasoning effort for the text step - Clipboard, TXT, Markdown, or combined output - Selectable microphone with a system-default fallback @@ -349,6 +363,7 @@ Global dictation is separate from these: see [Dictate from anywhere](#dictate-fr - **Local Whisper + Plain:** audio and text stay on the device. - **Local Whisper + transformed action:** audio stays local; transcribed text is sent to OpenAI. - **Context action:** the selected or copied text (up to 20,000 characters) is sent to the text model with your instruction. No other action reads the selection or the clipboard. +- **Action per app:** the names of up to eight apps you dictated into are kept in `settings.json` on this device, to choose from in Settings. They are not logged. - Temporary recordings are deleted after processing or cancellation. - The OpenAI API key is never written to `settings.json`; it is stored through the native OS keyring. - The last 100 completed texts are stored unencrypted on this device in `history.json`, including clipboard-only output. Titles are generated locally, without an AI call. Disable future storage or clear existing history in Settings; exported files and clipboard contents are not cleared. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index cce32e5..c877c9e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -224,6 +224,48 @@ as Plain with a note: it has no spoken instruction and nothing was read at its s **Diagnostics.** `context.captured` (source, characters, truncated) or `context.none` (which sources were on). The text itself is never logged, like a transcript. +## Action per application (0.12.0) + +**Rule.** `settings.app_actions` holds `{ app: { id, name }, action }` pairs, empty by +default. `action` is a built-in id or `custom:`. When the dictation key starts a +recording (`applyIntent` → `startRecording(true)`), the interface asks +`foreground_app` for the application in front, adds it to `settings.recent_apps` (newest +first, eight at most, written by the settings save the start already does) and looks for a +rule (`lib/app-actions.ts`: matched on the id or the name, case-insensitive, so a name +typed by hand matches as well; a rule whose action was removed is ignored). A match whose +action differs from the window's sets `appAction` for this one recording: +`effectiveAction` follows it for `start_recording`, `finish_recording` and a custom +action's prompt, the action menu shows it disabled, its label names the app, and +`announce_app_action` shows a notification. `showResult`, `setError` and +`cancelRecording` clear it. A start from the window's button or Space never asks, so the +window's choice holds there; live dictation stays Plain and only remembers the app. +Removing a custom action removes the rules that pointed at it. + +**Timing.** The query runs after `phase` is `starting`, so the existing guards against +double starts still hold, and before the microphone opens. On Hyprland it is one request +on the command socket (a few milliseconds); `spawn_blocking` keeps it off the main thread. + +**Which application** (`foreground.rs`). Utterform itself is never the answer: a window +whose process id is Utterform's own yields none. + +| Platform | Source | `id` | +| --- | --- | --- | +| Hyprland | command socket `j/activewindow` | `class`, else `initialClass` | +| Sway | `swaymsg -t get_tree -r`, the focused leaf | `app_id`, else X11 `class` | +| X11 | `xprop -root _NET_ACTIVE_WINDOW`, then `WM_CLASS` and `_NET_WM_PID` | class (second `WM_CLASS` string) | +| Windows | `GetForegroundWindow` → process → `QueryFullProcessImageNameW` | file name without `.exe` | +| macOS | `NSWorkspace.frontmostApplication` | bundle identifier | + +The compositor is asked before X11: on Wayland, `xprop` only sees XWayland windows. GNOME +and KDE on Wayland offer no such query; `foreground_support` says so and Settings disables +**+ App**. Child processes are killed after 400 ms. `display_name` turns an id into what +Settings shows (`org.mozilla.Thunderbird` → Thunderbird, `com.mitchellh.ghostty` → +Ghostty); macOS uses the localized name. + +**Diagnostics.** `foreground.read` (source, found, elapsed_ms) for every query and +`recording.app_action` (action) when a rule takes over. The application is never logged: +which program someone dictates into says as much as a window title. + ## What the local engine is compiled for whisper.cpp is built from source by `whisper-rs-sys`, and ggml's CMake targets the machine doing the compiling unless told otherwise. For a release built on someone else's server that is a trap rather than an optimization: the 0.7.2 Linux binary carried the runner's AVX-512 and AMX instructions and was stopped with SIGILL on an Intel N300 while loading a model — inside `ggml_backend_cpu_device_get_extra_buffers_type`, plain C++ frame code, which is before ggml's own runtime dispatch can choose a kernel and therefore beyond its help. The bug had been there since the first release; only the pairing of build machine and user machine decided whether it fired. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 8919d9f..ec83344 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -19,7 +19,20 @@ proposals are in [plans/v1.0-roadmap.md](plans/v1.0-roadmap.md). The plan of 202 (reliability, assistant, finish) is superseded; 0.7.13 shipped, 0.7.14 (`feat/simple-settings`) was rejected and will not be merged. -## Current handoff — 0.11.0 Context: selected text and clipboard +## Current handoff — 0.12.0 Action per application + +Stage 0.12.0 of [the roadmap](plans/v1.0-roadmap.md), branch `work/utterform-0.12.0-per-app`. +Jonas decided to build it small and accepted the mock-up: **Settings → Context → Per app**, +empty by default; **+ App** offers the apps dictated into lately (`settings.recent_apps`) or +a typed name; a rule applies only when the dictation key starts the recording; a +notification names the action and app only when a rule changes the action. `foreground.rs` +names the app in front (Hyprland, Sway, X11, Windows, macOS; none on GNOME/KDE Wayland), +`lib/app-actions.ts` matches the rule. Design in +[ARCHITECTURE.md → Action per application](ARCHITECTURE.md#action-per-application-0120), +validation in [TESTING.md](TESTING.md). Merged into `main` without tag or release; ships +with 1.0. Next stage: 0.14.0 first-run assistant. + +## Previous handoff — 0.11.0 Context: selected text and clipboard Stage 0.11.0 of [the roadmap](plans/v1.0-roadmap.md), branch `work/utterform-0.11.0-context`. A new built-in action **Context**: the transcript is the instruction, the selected text diff --git a/docs/TESTING.md b/docs/TESTING.md index 6baaf36..361799f 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -10,6 +10,28 @@ `PLAYWRIGHT_CHROMIUM_EXECUTABLE=/usr/bin/chromium npm run test:e2e`. - `cmake` is needed for whisper.cpp. Without root, `mise exec cmake@latest -- cargo test` works. +## 0.12.0 Action per application + +Unit tests in `foreground.rs`: display names from classes and program names, Utterform's +own process is never the answer, the compositor before X11 (none on GNOME/KDE Wayland), +the Hyprland answer (class, `initialClass`, an empty object), the focused Sway leaf also +among floating windows, and `xprop` output. `settings.rs`: a file without the new fields +gets none, rules and recent apps survive a reload. `lib/app-actions.test.ts`: matching by +id or typed name, a removed action ignored, recent apps newest first and capped, one rule +per app. `App.test.ts`: the key in a ruled app starts and finishes with the rule's custom +action and prompt, announces it and remembers the app; the window's own button asks +nothing and keeps Plain. Playwright: the rule taking over and handing back the window's +action, the notification only when it differs, recent apps saved; adding from the recent +list, choosing the action, typing a name, removing, all saved at once, at 920×720. + +Live on `asus16` (Hyprland), through +`dbus-run-session -- cargo test print_the_application_in_front -- --ignored --nocapture`: +the focused Herdr window was named `herdr-app`, supported, in a few milliseconds. + +Not exercised, to check at 1.0: the installed app with a rule (Thunderbird → Email via +Alt+Space, the notification), Sway and X11, the foreground program on Windows and the +frontmost app on macOS. + ## 0.13.0 Audio files Unit tests in `audio_file.rs`: the resampler (48 kHz averaged by three, odd ratios keep the diff --git a/docs/plans/v1.0-roadmap.md b/docs/plans/v1.0-roadmap.md index 22ed24e..5bbe4a3 100644 --- a/docs/plans/v1.0-roadmap.md +++ b/docs/plans/v1.0-roadmap.md @@ -58,6 +58,7 @@ and without its subscription: | 2026-11-07 | Jonas tests only once, at 1.0, everything together; until then no preview builds for him. Stages go through on their own: merge into `main` when tests and CI are green, no tag or release before 1.0. Style samples (mock-ups, images) are still welcome, and so are occasional questions. The two parallel sessions coordinate with each other. | | 2026-11-07 | 0.13.0: audio files open from a folder button next to Settings (and `O`) or by dropping on the window, which shows "Drop to transcribe" with action and engine; text to history and clipboard, never the cursor. Mock-up accepted. | | 2026-11-07 | Settings mock-up accepted: search field on top, categories as tabs below, one row per setting. The "Context" category appears only once 0.11.0 gives it rows. | +| 2026-11-08 | 0.12.0 is built, small: Settings → Context → "Per app", empty by default; "+ App" offers the apps dictated into lately or a typed name; a rule applies only to the dictation key, everywhere else the window's action; a notification "Utterform · Email — for Thunderbird" only when a rule changes the action. Mock-up accepted. | | 2026-11-07 | 0.10.0: ChatGPT is marked "Experimental" in Settings; **no fallback** to the OpenAI key — a ChatGPT failure is shown and the plain transcript delivered. Mock-up (AI → Provider: OpenAI key \| ChatGPT) accepted. | | 2026-11-07 | 0.11.0: context is its **own action "Context"**, no per-action switch; other actions never read context. **Linux first**: Linux reads the selection, Windows/macOS use the clipboard only — no simulated Ctrl+C, which would interrupt a terminal. | @@ -71,7 +72,7 @@ Proposals not yet confirmed by Jonas are marked **Proposal**. | 0.9.0 | Settings from scratch | merged, ships with 1.0 | | 0.10.0 | Codex login for text actions | merged, ships with 1.0 | | 0.11.0 | Context: selected text and clipboard | merged, ships with 1.0 | -| 0.12.0 | Action per application | in progress (discussing with Jonas) | +| 0.12.0 | Action per application | merged, ships with 1.0 | | 0.13.0 | Audio files | merged, ships with 1.0 | | 0.14.0 | First-run assistant | planned (was 0.8.0) | | 1.0.0 | Acceptance on Linux, Windows, macOS; release; WinGet | planned | @@ -154,7 +155,7 @@ select an email, say "reply that I have time on Monday", get the reply at the cu plain dictation and the other actions never read context. - Delivery replaces the selection where the application allows it. -### 0.12.0 — Action per application (discuss first) +### 0.12.0 — Action per application (decided 2026-11-08, built) Example: email action in the mail client, plain text in the terminal. Utterform already knows the active window class on Linux (`typing/linux.rs`). diff --git a/docs/releases/v0.12.0.md b/docs/releases/v0.12.0.md new file mode 100644 index 0000000..609bc49 --- /dev/null +++ b/docs/releases/v0.12.0.md @@ -0,0 +1,30 @@ +# Utterform 0.12.0 — The right action in every app + +*Stage of the road to 1.0, merged into `main`; not released on its own. It ships with 1.0.* + +Until now Utterform used one action everywhere, the one chosen in its window. Dictating a +mail meant switching to *Email* first, and back to *Plain* before the next command in the +terminal. Now an app can have an action of its own. + +**Per app.** Under **Settings → Context → Per app**, give Thunderbird *Email* and the +terminal *Plain*. Press the dictation key in Thunderbird and the text arrives as an email; +press it in the terminal and it arrives word for word. In every other app the action in +the window applies, as before. The list starts empty, so nothing changes until you add an +app. + +**Adding an app** takes one click: **+ App** lists the apps you dictated into lately — so +dictate into an app once, and it is there. An app that has not come up yet can be typed by +name. Each app has its action menu next to it and × to remove it. + +**Never a surprise.** When an app's action takes over, a short notification says so +("Utterform · Email — for Thunderbird"), and the window shows the action and the app while +it records. A recording started from Utterform's own window follows the window, and live +dictation stays word for word. + +**Where it works.** Linux under Hyprland (Omarchy) and Sway, X11 with `xprop`, Windows and +macOS. GNOME and KDE on Wayland do not tell other applications which window is focused; +there **Per app** says so and stays off. + +**Privacy.** The names of the apps you dictated into are kept in `settings.json` on this +device, at most eight. The diagnostic log notes that an app was found and which action +ran, never which app. diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index b40e3b4..09f4f8b 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -2988,6 +2988,7 @@ checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ "bitflags 2.13.1", "block2", + "libc", "objc2", "objc2-core-foundation", "objc2-core-graphics", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 97a4d78..8f21d0d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -74,5 +74,5 @@ winreg = "0.55" core-foundation = "0.10" core-graphics = "0.25" objc2 = "0.6" -objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSWorkspace", "NSRunningApplication"] } +objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSWorkspace", "NSRunningApplication"] } objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSString"] } diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index ce613b7..da37567 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -445,6 +445,51 @@ pub fn context_support() -> context::Support { context::support() } +#[tauri::command] +pub fn foreground_support() -> crate::foreground::Support { + crate::foreground::support() +} + +/// The application in front as the dictation key is pressed. Off the main +/// thread: the compositor is asked, and a slow answer must not freeze the +/// window. +#[tauri::command] +pub async fn foreground_app() -> Option { + let began = Instant::now(); + let app = tauri::async_runtime::spawn_blocking(crate::foreground::current) + .await + .ok() + .flatten(); + // Whether an app was named and how long it took — never which one. + diagnostics::info!( + "foreground.read", + source = crate::foreground::source(), + found = app.is_some(), + elapsed_ms = began.elapsed().as_millis() as u64 + ); + app +} + +/// Say that an application's own action is taking this dictation, so an +/// email where plain text was expected never comes as a surprise. Only the +/// action is logged, never the application. +#[tauri::command] +pub fn announce_app_action(app: AppHandle, action: String, action_name: String, app_name: String) { + diagnostics::info!( + "recording.app_action", + action = actions::diagnostic_id(&action) + ); + let shown = app + .notification() + .builder() + .title(format!("Utterform · {}", action_name.trim())) + .body(format!("for {}", app_name.trim())) + .show(); + if shown.is_err() { + diagnostics::warning!("recording.app_action_notification_failed"); + } +} + #[tauri::command] pub fn set_recording_paused( state: State<'_, AudioCaptureState>, diff --git a/src-tauri/src/domain.rs b/src-tauri/src/domain.rs index 7b51ce6..fe8f56a 100644 --- a/src-tauri/src/domain.rs +++ b/src-tauri/src/domain.rs @@ -181,6 +181,27 @@ pub struct AppSettings { pub context_selection: bool, /// The Context action may read the clipboard when nothing is selected. pub context_clipboard: bool, + /// The action a dictation key press takes in a given application. Empty + /// means the action chosen in the window applies everywhere. + pub app_actions: Vec, + /// The applications dictated into lately, newest first, to choose from + /// when adding one of the above. Kept on this device only. + pub recent_apps: Vec, +} + +/// An application as Settings knows it: the id rules match, and its name. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct KnownApp { + pub id: String, + pub name: String, +} + +/// "In this application, take this action." `action` is a built-in id or +/// `custom:`; a rule whose action no longer exists is ignored. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct AppAction { + pub app: KnownApp, + pub action: String, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -219,6 +240,8 @@ impl Default for AppSettings { global_hotkey: Some(crate::hotkey::DEFAULT.to_string()), context_selection: true, context_clipboard: true, + app_actions: Vec::new(), + recent_apps: Vec::new(), } } } diff --git a/src-tauri/src/foreground.rs b/src-tauri/src/foreground.rs new file mode 100644 index 0000000..cbc7879 --- /dev/null +++ b/src-tauri/src/foreground.rs @@ -0,0 +1,500 @@ +//! The application in front when a dictation starts, for the actions chosen +//! per application. +//! +//! Read once, when the dictation key is pressed, and never logged: an +//! application's name says what someone is doing as clearly as a window title. +//! Utterform itself is never the answer, so a recording started from its own +//! window follows the action chosen there. +//! +//! Linux asks the compositor (Hyprland, Sway) or, on X11, the window manager's +//! `_NET_ACTIVE_WINDOW` through `xprop`. GNOME and KDE on Wayland tell no +//! application which window is focused, so there the feature stays off. +//! Windows names the foreground window's program, macOS the frontmost +//! application. + +use serde::Serialize; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ForegroundApp { + /// What a rule is matched against: the Wayland/X11 class, the program's + /// file name on Windows, the bundle identifier on macOS. + pub id: String, + /// What Settings shows. + pub name: String, +} + +/// Whether this session can say which application is in front. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Support { + pub supported: bool, + /// Empty when supported. + pub explanation: String, +} + +pub fn support() -> Support { + #[cfg(target_os = "linux")] + { + if linux::backend().is_some() { + return Support { + supported: true, + explanation: String::new(), + }; + } + Support { + supported: false, + explanation: linux::unsupported_message().into(), + } + } + #[cfg(not(target_os = "linux"))] + { + Support { + supported: true, + explanation: String::new(), + } + } +} + +/// Where the answer comes from, as a diagnostic event may name it. +pub fn source() -> &'static str { + #[cfg(target_os = "linux")] + { + match linux::backend() { + Some(linux::Backend::Hyprland) => "hyprland", + Some(linux::Backend::Sway) => "sway", + Some(linux::Backend::X11) => "x11", + None => "none", + } + } + #[cfg(target_os = "windows")] + { + "windows" + } + #[cfg(target_os = "macos")] + { + "macos" + } +} + +/// The application in front, unless it is Utterform or cannot be told. +pub fn current() -> Option { + let own = std::process::id(); + #[cfg(target_os = "linux")] + { + linux::current(own) + } + #[cfg(target_os = "windows")] + { + windows::current(own) + } + #[cfg(target_os = "macos")] + { + macos::current(own) + } +} + +/// A Linux class or a program name as a person would write it: +/// `org.mozilla.Thunderbird` → `Thunderbird`, `com.mitchellh.ghostty` → +/// `Ghostty`, `thunderbird` → `Thunderbird`. A name with capitals of its own +/// (`OUTLOOK`, `WhatsApp`) is kept as it is. +pub fn display_name(id: &str) -> String { + let id = id.trim(); + let last = if id.contains('.') && !id.ends_with('.') { + id.rsplit('.').next().unwrap_or(id) + } else { + id + }; + if last.chars().any(char::is_uppercase) { + return last.to_string(); + } + let mut chars = last.chars(); + match chars.next() { + Some(first) => first.to_uppercase().chain(chars).collect(), + None => String::new(), + } +} + +/// An application from what the platform reported, unless it is Utterform. +fn found(id: Option<&str>, pid: Option, own: u32) -> Option { + if pid == Some(own) { + return None; + } + let id = id.map(str::trim).filter(|id| !id.is_empty())?; + Some(ForegroundApp { + id: id.to_string(), + name: display_name(id), + }) +} + +#[cfg(target_os = "linux")] +mod linux { + use std::{ + io::{Read, Write}, + os::unix::net::UnixStream, + path::PathBuf, + process::{Command, Stdio}, + time::{Duration, Instant}, + }; + + use serde_json::Value; + + use super::{ForegroundApp, found}; + + /// The key was just pressed; a compositor that does not answer by then + /// must not hold up the recording. + const TIMEOUT: Duration = Duration::from_millis(400); + const MAX_RESPONSE: u64 = 4 * 1024 * 1024; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Backend { + Hyprland, + Sway, + X11, + } + + pub fn backend() -> Option { + let set = |name: &str| std::env::var_os(name).is_some_and(|value| !value.is_empty()); + choose( + set("HYPRLAND_INSTANCE_SIGNATURE"), + set("SWAYSOCK"), + set("WAYLAND_DISPLAY"), + set("DISPLAY"), + installed("swaymsg"), + installed("xprop"), + ) + } + + fn installed(program: &str) -> bool { + std::env::var_os("PATH").is_some_and(|path| { + std::env::split_paths(&path).any(|directory| directory.join(program).is_file()) + }) + } + + /// The compositor first: on Wayland the X11 answer only knows XWayland + /// windows, and would mistake every native one for none at all. + pub fn choose( + hyprland: bool, + sway: bool, + wayland: bool, + x11: bool, + swaymsg: bool, + xprop: bool, + ) -> Option { + if hyprland { + Some(Backend::Hyprland) + } else if sway && swaymsg { + Some(Backend::Sway) + } else if !wayland && x11 && xprop { + Some(Backend::X11) + } else { + None + } + } + + pub fn unsupported_message() -> &'static str { + let wayland = std::env::var_os("WAYLAND_DISPLAY").is_some(); + if wayland { + "Works on Hyprland, Sway and X11. This desktop does not say which app is in front." + } else { + "Install xprop to use this." + } + } + + pub fn current(own: u32) -> Option { + match backend()? { + Backend::Hyprland => hyprland(own), + Backend::Sway => run("swaymsg", &["-t", "get_tree", "-r"]) + .and_then(|tree| serde_json::from_str::(&tree).ok()) + .and_then(|tree| sway_focused(&tree, own)), + Backend::X11 => { + let root = run("xprop", &["-root", "_NET_ACTIVE_WINDOW"])?; + let window = xprop_active_window(&root)?; + let props = run("xprop", &["-id", &window, "WM_CLASS", "_NET_WM_PID"])?; + let (class, pid) = xprop_class(&props); + found(class.as_deref(), pid, own) + } + } + } + + fn hyprland(own: u32) -> Option { + let signature = std::env::var("HYPRLAND_INSTANCE_SIGNATURE").ok()?; + if signature.is_empty() || signature.contains('/') || signature == "." || signature == ".." + { + return None; + } + let path = PathBuf::from(std::env::var_os("XDG_RUNTIME_DIR")?) + .join("hypr") + .join(signature) + .join(".socket.sock"); + // The command socket answers once and closes; it is never kept open. + let mut stream = UnixStream::connect(path).ok()?; + stream.set_read_timeout(Some(TIMEOUT)).ok()?; + stream.set_write_timeout(Some(TIMEOUT)).ok()?; + stream.write_all(b"j/activewindow").ok()?; + let mut response = Vec::new(); + stream.take(MAX_RESPONSE).read_to_end(&mut response).ok()?; + hyprland_window(&serde_json::from_slice(&response).ok()?, own) + } + + pub fn hyprland_window(window: &Value, own: u32) -> Option { + let class = ["class", "initialClass"] + .iter() + .filter_map(|key| window.get(key)?.as_str()) + .find(|class| !class.trim().is_empty()); + let pid = window + .get("pid") + .and_then(Value::as_u64) + .and_then(|pid| u32::try_from(pid).ok()); + found(class, pid, own) + } + + /// The focused leaf of Sway's tree: `app_id` for Wayland windows, the X11 + /// class for XWayland ones. + pub fn sway_focused(node: &Value, own: u32) -> Option { + let is_window = node.get("pid").is_some(); + if is_window && node.get("focused").and_then(Value::as_bool) == Some(true) { + let id = node + .get("app_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .or_else(|| node.pointer("/window_properties/class")?.as_str()); + let pid = node + .get("pid") + .and_then(Value::as_u64) + .and_then(|pid| u32::try_from(pid).ok()); + return found(id, pid, own); + } + ["nodes", "floating_nodes"] + .iter() + .filter_map(|key| node.get(key)?.as_array()) + .flatten() + .find_map(|child| sway_focused(child, own)) + } + + /// `_NET_ACTIVE_WINDOW(WINDOW): window id # 0x3a00007` → `0x3a00007`. + pub fn xprop_active_window(output: &str) -> Option { + let id = output.rsplit('#').next()?.trim(); + let id = id.split(|c: char| c == ',' || c.is_whitespace()).next()?; + (id.starts_with("0x") && id != "0x0").then(|| id.to_string()) + } + + /// `WM_CLASS(STRING) = "navigator", "firefox"` names the class second. + pub fn xprop_class(output: &str) -> (Option, Option) { + let mut class = None; + let mut pid = None; + for line in output.lines() { + if let Some(rest) = line.strip_prefix("WM_CLASS(STRING) = ") { + class = rest + .split(", ") + .map(|part| part.trim().trim_matches('"').to_string()) + .filter(|part| !part.is_empty()) + .last(); + } else if let Some(rest) = line.strip_prefix("_NET_WM_PID(CARDINAL) = ") { + pid = rest.trim().parse().ok(); + } + } + (class, pid) + } + + fn run(program: &str, args: &[&str]) -> Option { + let mut child = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + let mut stdout = child.stdout.take()?; + let reader = std::thread::spawn(move || { + let mut output = Vec::new(); + let _ = (&mut stdout).take(MAX_RESPONSE).read_to_end(&mut output); + output + }); + let started = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(status)) if status.success() => break, + Ok(Some(_)) | Err(_) => return None, + Ok(None) if started.elapsed() > TIMEOUT => { + let _ = child.kill(); + let _ = child.wait(); + return None; + } + Ok(None) => std::thread::sleep(Duration::from_millis(5)), + } + } + String::from_utf8(reader.join().ok()?).ok() + } +} + +#[cfg(target_os = "windows")] +mod windows { + use windows_sys::Win32::{ + Foundation::CloseHandle, + System::Threading::{ + OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, QueryFullProcessImageNameW, + }, + UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId}, + }; + + use super::{ForegroundApp, found}; + + /// The foreground window's program, by its file name without `.exe`. + pub fn current(own: u32) -> Option { + let window = unsafe { GetForegroundWindow() }; + if window.is_null() { + return None; + } + let mut pid = 0u32; + unsafe { GetWindowThreadProcessId(window, &mut pid) }; + if pid == 0 { + return None; + } + let process = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; + if process.is_null() { + return None; + } + let mut path = [0u16; 1024]; + let mut length = path.len() as u32; + let named = + unsafe { QueryFullProcessImageNameW(process, 0, path.as_mut_ptr(), &mut length) } != 0; + unsafe { CloseHandle(process) }; + if !named { + return None; + } + let path = String::from_utf16_lossy(&path[..length as usize]); + let stem = std::path::Path::new(&path) + .file_stem() + .map(|stem| stem.to_string_lossy().into_owned()); + found(stem.as_deref(), Some(pid), own) + } +} + +#[cfg(target_os = "macos")] +mod macos { + use objc2_app_kit::NSWorkspace; + + use super::{ForegroundApp, found}; + + /// The frontmost application, by bundle identifier, with the name macOS + /// shows for it. + pub fn current(own: u32) -> Option { + let application = NSWorkspace::sharedWorkspace().frontmostApplication()?; + let pid = u32::try_from(application.processIdentifier()).ok(); + let name = application.localizedName().map(|name| name.to_string()); + let id = application + .bundleIdentifier() + .map(|id| id.to_string()) + .or_else(|| name.clone()); + let app = found(id.as_deref(), pid, own)?; + Some(ForegroundApp { + name: name.unwrap_or(app.name), + id: app.id, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_read_the_way_people_write_them() { + assert_eq!(display_name("org.mozilla.Thunderbird"), "Thunderbird"); + assert_eq!(display_name("com.mitchellh.ghostty"), "Ghostty"); + assert_eq!(display_name("thunderbird"), "Thunderbird"); + assert_eq!(display_name("OUTLOOK"), "OUTLOOK"); + assert_eq!(display_name("WhatsApp"), "WhatsApp"); + assert_eq!(display_name("Alacritty"), "Alacritty"); + } + + #[test] + fn utterform_itself_is_never_the_application_in_front() { + assert_eq!(found(Some("Utterform"), Some(42), 42), None); + assert_eq!(found(Some(" "), Some(7), 42), None); + assert_eq!( + found(Some("thunderbird"), Some(7), 42), + Some(ForegroundApp { + id: "thunderbird".into(), + name: "Thunderbird".into() + }) + ); + } + + #[cfg(target_os = "linux")] + mod linux_parsing { + use super::super::linux::*; + use serde_json::json; + + #[test] + fn the_compositor_is_asked_before_x11() { + assert_eq!( + choose(true, false, true, true, false, true), + Some(Backend::Hyprland) + ); + assert_eq!( + choose(false, true, true, true, true, true), + Some(Backend::Sway) + ); + assert_eq!( + choose(false, false, false, true, false, true), + Some(Backend::X11) + ); + // GNOME or KDE on Wayland: XWayland would only know its own windows. + assert_eq!(choose(false, false, true, true, false, true), None); + assert_eq!(choose(false, true, true, false, false, false), None); + } + + #[test] + fn hyprland_names_the_class_and_skips_utterform() { + let window = json!({"class": "thunderbird", "initialClass": "thunderbird", "pid": 7}); + assert_eq!(hyprland_window(&window, 42).unwrap().id, "thunderbird"); + assert_eq!(hyprland_window(&window, 7), None); + let no_class = json!({"class": "", "initialClass": "Alacritty", "pid": 7}); + assert_eq!(hyprland_window(&no_class, 42).unwrap().name, "Alacritty"); + // No window focused: Hyprland answers with an empty object. + assert_eq!(hyprland_window(&json!({}), 42), None); + } + + #[test] + fn sway_finds_the_focused_window_also_among_floating_ones() { + let tree = json!({"focused": false, "nodes": [ + {"focused": false, "nodes": [ + {"focused": false, "pid": 3, "app_id": "foot", "nodes": []} + ], "floating_nodes": [ + {"focused": true, "pid": 5, "app_id": null, + "window_properties": {"class": "Slack"}, "nodes": []} + ]} + ]}); + assert_eq!(sway_focused(&tree, 42).unwrap().id, "Slack"); + let workspace_focused = json!({"focused": true, "nodes": []}); + assert_eq!(sway_focused(&workspace_focused, 42), None); + } + + #[test] + fn xprop_output_gives_window_class_and_pid() { + assert_eq!( + xprop_active_window("_NET_ACTIVE_WINDOW(WINDOW): window id # 0x3a00007\n"), + Some("0x3a00007".into()) + ); + assert_eq!( + xprop_active_window("_NET_ACTIVE_WINDOW(WINDOW): window id # 0x0\n"), + None + ); + let (class, pid) = xprop_class( + "WM_CLASS(STRING) = \"navigator\", \"firefox\"\n_NET_WM_PID(CARDINAL) = 1234\n", + ); + assert_eq!(class.as_deref(), Some("firefox")); + assert_eq!(pid, Some(1234)); + } + } +} +#[cfg(test)] +mod live_probe { + #[test] + #[ignore = "asks the running desktop"] + fn print_the_application_in_front() { + eprintln!("support: {:?}", super::support()); + eprintln!("in front: {:?}", super::current()); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index f0a80ca..bd70eb3 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -9,6 +9,7 @@ mod context; mod diagnostics; mod domain; mod feedback; +mod foreground; mod history; mod hotkey; mod live; @@ -304,6 +305,9 @@ pub fn run() { commands::list_built_in_actions, commands::typing_support, commands::context_support, + commands::foreground_support, + commands::foreground_app, + commands::announce_app_action, commands::get_settings, commands::save_settings, commands::list_input_devices, diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs index c09b664..a6d5763 100644 --- a/src-tauri/src/settings.rs +++ b/src-tauri/src/settings.rs @@ -164,6 +164,29 @@ mod tests { settings.global_hotkey.as_deref(), Some(crate::hotkey::DEFAULT) ); + assert!( + settings.app_actions.is_empty(), + "no application takes its own action until one is chosen" + ); + assert!(settings.recent_apps.is_empty()); + } + + #[test] + fn an_action_per_application_survives_a_save_and_a_reload() { + let mut settings = AppSettings::default(); + let thunderbird = crate::domain::KnownApp { + id: "thunderbird".into(), + name: "Thunderbird".into(), + }; + settings.app_actions.push(crate::domain::AppAction { + app: thunderbird.clone(), + action: "email".into(), + }); + settings.recent_apps.push(thunderbird); + let restored: AppSettings = + serde_json::from_str(&serde_json::to_string(&settings).unwrap()).unwrap(); + assert_eq!(restored.app_actions, settings.app_actions); + assert_eq!(restored.recent_apps, settings.recent_apps); } #[test] diff --git a/src/App.svelte b/src/App.svelte index 901a583..111cca3 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -28,6 +28,8 @@ LiveSupport, TypingSupport, ContextSupport, + ForegroundSupport, + KnownApp, ProcessResult, RemoteIntent, TextEffort, @@ -44,6 +46,7 @@ import { modalFocus } from "./lib/modal-focus"; import ChipInput from "./lib/ChipInput.svelte"; import { SETTINGS_CATEGORIES, categoryLabel, visibleRows } from "./lib/settings-catalog"; + import { actionFor, addable, rememberApp, removeRule, setRule, typedApp } from "./lib/app-actions"; import type { SettingsCategory, SettingsRowId } from "./lib/settings-catalog"; import { copiedSummary, createFailureReporter } from "./lib/diagnostics"; @@ -135,6 +138,12 @@ let liveSupport: LiveSupport = { supported: false, explanation: "Live dictation requires the Windows or Omarchy desktop app." }; let typingSupport: TypingSupport = { supported: true, explanation: "" }; let contextSupport: ContextSupport = { selection: true, explanation: "" }; + let foregroundSupport: ForegroundSupport = { supported: false, explanation: "" }; + /// The action an application's rule chose for the recording under way, shown + /// in place of the window's own choice until the recording is over. + let appAction: { action: string; app: KnownApp } | null = null; + let addingApp = false; + let typedAppName = ""; let liveStatusError = ""; let liveFailurePending = false; let polling = false; @@ -208,7 +217,7 @@ { value: "gpt_live_transcribe", label: "GPT Live Transcribe", hint: "Live at cursor · Windows and Omarchy" }, { value: "local_whisper", label: "Local Whisper", hint: localModelHint }, ]; - $: effectiveAction = liveMode ? "plain" : selectedAction; + $: effectiveAction = liveMode ? "plain" : appAction?.action ?? selectedAction; $: liveStartHint = !liveSupport.supported ? liveSupport.explanation : hotkeySupport.supported && settings.global_hotkey && !hotkeyMessage ? `Place the cursor in your text field, then press ${settings.global_hotkey} to start and finish live dictation.` @@ -286,7 +295,7 @@ return; } try { - [settings, devices, models, hasApiKey, hotkeySupport, builtInActions, liveSupport, typingSupport, contextSupport] = await Promise.all([ + [settings, devices, models, hasApiKey, hotkeySupport, builtInActions, liveSupport, typingSupport, contextSupport, foregroundSupport] = await Promise.all([ api.getSettings(), api.listInputDevices(), api.listLocalModels(), @@ -296,6 +305,7 @@ api.liveSupport().catch(() => ({ supported: false, explanation: "Live dictation support could not be checked. Restart Utterform to try again." })), api.typingSupport().catch(() => ({ supported: true, explanation: "" })), api.contextSupport().catch(() => ({ selection: false, explanation: "Could not be checked." })), + api.foregroundSupport().catch(() => ({ supported: false, explanation: "Could not be checked." })), ]); settings = { ...DEFAULT_SETTINGS, ...settings, cloud_model: settings.cloud_model ?? "gpt_transcribe" }; applyTheme(settings.theme); @@ -621,6 +631,8 @@ stopRequested = false; phase = "starting"; message = liveMode ? "Connecting GPT Live Transcribe…" : "Preparing the microphone…"; + // A dictation key press: the application in front may have its own action. + appAction = fromTarget ? await readForegroundApp() : null; // After `phase` is set, so a second press meanwhile is turned away; a // transcription choice still being written settles before this // recording is configured from it. @@ -634,6 +646,10 @@ settings.local_model_id, effectiveAction.startsWith("custom:") ? "custom" : effectiveAction, ); + if (appAction) { + const label = actionOptions.find((option) => option.value === appAction?.action)?.label ?? appAction.action; + void api.announceAppAction(appAction.action.startsWith("custom:") ? "custom" : appAction.action, label, appAction.app.name).catch(() => {}); + } elapsedSeconds = 0; phase = "recording"; message = ""; @@ -653,6 +669,20 @@ } } + /// Remember the application for Settings' "+ App", and take its action when + /// a rule names one other than the window's. Live dictation stays Plain. + async function readForegroundApp(): Promise<{ action: string; app: KnownApp } | null> { + if (!foregroundSupport.supported) return null; + const app = await api.foregroundApp().catch(() => null); + if (!app) return null; + const recent = rememberApp(settings.recent_apps, app); + // Written by the settings save that follows in startRecording. + if (JSON.stringify(recent) !== JSON.stringify(settings.recent_apps)) settings = { ...settings, recent_apps: recent }; + if (liveMode) return null; + const rule = actionFor(settings.app_actions, app, (action) => actionOptions.some((option) => option.value === action)); + return rule && rule.action !== selectedAction ? { action: rule.action, app: rule.app } : null; + } + function handleVisibilityChange() { if (!document.hidden) void pollRecording(); } @@ -695,7 +725,7 @@ const finished = await api.finishRecording({ action: effectiveAction.startsWith("custom:") ? "custom" : effectiveAction, customPrompt: effectiveAction.startsWith("custom:") - ? settings.custom_actions.find((action) => `custom:${action.id}` === selectedAction)?.prompt ?? null + ? settings.custom_actions.find((action) => `custom:${action.id}` === effectiveAction)?.prompt ?? null : null, copyToClipboard: settings.copy_to_clipboard, saveToFile: settings.save_to_file, @@ -726,7 +756,7 @@ showResult(await api.transcribeFile(path, source, { action: effectiveAction.startsWith("custom:") ? "custom" : effectiveAction, customPrompt: effectiveAction.startsWith("custom:") - ? settings.custom_actions.find((action) => `custom:${action.id}` === selectedAction)?.prompt ?? null + ? settings.custom_actions.find((action) => `custom:${action.id}` === effectiveAction)?.prompt ?? null : null, copyToClipboard: true, saveToFile: settings.save_to_file, @@ -788,6 +818,7 @@ } function showResult(value: ProcessResult) { + appAction = null; result = value; now = Date.now(); resultTimestamp = now; @@ -844,6 +875,7 @@ if ((phase !== "recording" && phase !== "paused") || changingPause) return; stopTimer(); phase = "starting"; + appAction = null; try { await api.cancelRecording(); if (liveMode) await recoverLiveText(); @@ -879,6 +911,7 @@ } function setError(error: unknown) { + appAction = null; stopTimer(); phase = "error"; message = error instanceof Error ? error.message : String(error); @@ -1333,8 +1366,18 @@ }, TYPING_SAVE_DELAY_MS); } + /// A new rule starts as Plain; its menu sits right next to it. + function addApp(app: KnownApp) { + change({ app_actions: setRule(settings.app_actions, app, "plain") }); + addingApp = false; + typedAppName = ""; + } + function removeCustomAction(id: string) { - change({ custom_actions: settings.custom_actions.filter((action) => action.id !== id) }); + change({ + custom_actions: settings.custom_actions.filter((action) => action.id !== id), + app_actions: settings.app_actions.filter((rule) => rule.action !== `custom:${id}`), + }); if (selectedAction === `custom:${id}`) selectedAction = "plain"; if (selectedPrompt === `custom:${id}`) selectPrompt(firstEditablePrompt()); settingsDetail = ""; @@ -1360,8 +1403,9 @@
- Action + Action{#if appAction}{` · ${appAction.app.name}`}{/if} {#if liveMode}
Plain
+ {:else if appAction} {:else}{/if}
Transcription @@ -1710,6 +1754,38 @@
{/if} + {#if shownRows.has("app-actions")} +
{@render rowHead("app-actions", "Per app")} +
+ {#each settings.app_actions as rule, index (rule.app.id)} +
+ {rule.app.name} + change({ app_actions: setRule(settings.app_actions, rule.app, action) })} /> + +
+ {/each} +
+ + {#if addingApp} + +
{ if (event.key === "Escape") { event.stopPropagation(); addingApp = false; } }}> + {#if addable(settings.recent_apps, settings.app_actions).length}Recently dictated in{/if} + {#each addable(settings.recent_apps, settings.app_actions) as app (app.id)} + + {/each} + { if (event.key === "Enter") { const app = typedApp(typedAppName); if (app) addApp(app); } }} /> +
+ {/if} +
+
+ {#if foregroundSupport.explanation}

{foregroundSupport.explanation}

{/if} +
+ {/if} + {#if shownRows.has("typing")}
{@render rowHead("typing", "Type at cursor")} diff --git a/src/App.test.ts b/src/App.test.ts index 9fbf8f2..34045d5 100644 --- a/src/App.test.ts +++ b/src/App.test.ts @@ -31,6 +31,8 @@ vi.mock("./lib/api", () => ({ api: { listBuiltInActions: vi.fn(), liveSupport: vi.fn(), getLiveStatus: vi.fn(), typingSupport: vi.fn(async () => ({ supported: true, explanation: "" })), contextSupport: vi.fn(async () => ({ selection: true, explanation: "" })), + foregroundSupport: vi.fn(async () => ({ supported: true, explanation: "" })), + foregroundApp: vi.fn(async () => null), announceAppAction: vi.fn(async () => {}), hasOpenAiApiKey: vi.fn(async () => true), codexStatus: vi.fn(), openCodexSetup: vi.fn(async () => {}), listHistory: vi.fn(), saveSettings: vi.fn(), globalHotkeySupport: vi.fn(async () => ({ supported: true, default: "Ctrl+Alt+D", explanation: "", failure: null })), applyGlobalHotkey: vi.fn(), @@ -369,6 +371,34 @@ describe("dictation hotkey", () => { expect(api.cancelRecording).not.toHaveBeenCalled(); }); + it("takes the action chosen for the app in front, with its prompt, and only from the key", async () => { + vi.mocked(api.getSettings).mockResolvedValue({ + ...structuredClone(DEFAULT_SETTINGS), + custom_actions: [{ id: "a", name: "Notes", prompt: "Bullets only" }], + app_actions: [{ app: { id: "thunderbird", name: "Thunderbird" }, action: "custom:a" }], + }); + vi.mocked(api.foregroundApp).mockResolvedValue({ id: "thunderbird", name: "Thunderbird" }); + vi.mocked(api.finishRecording).mockResolvedValue({ historyEntry: null, text: "- Bullet", savedPath: null, + copiedToClipboard: true, typedAtCursor: false, deliveryWarnings: [], durationMs: 900, engine: "open_ai" }); + const view = render(App); + await waitFor(() => expect(view.queryByRole("button", { name: "Start recording" })).not.toBeNull()); + + await remoteIntent("toggle"); + await waitFor(() => expect(api.startRecording).toHaveBeenCalledWith(null, "open_ai", "base", "custom")); + expect(api.announceAppAction).toHaveBeenCalledWith("custom", "Notes", "Thunderbird"); + await remoteIntent("toggle"); + await waitFor(() => expect(api.finishRecording).toHaveBeenCalledWith(expect.objectContaining({ action: "custom", customPrompt: "Bullets only" }))); + expect(vi.mocked(api.saveSettings).mock.calls.at(-1)?.[0].recent_apps).toEqual([{ id: "thunderbird", name: "Thunderbird" }]); + + // From the window itself the window's own action applies; nothing is asked. + await waitFor(() => expect(view.queryByRole("button", { name: "Start recording" })).not.toBeNull()); + await fireEvent.click(view.getByRole("button", { name: "Start recording" })); + await waitFor(() => expect(api.startRecording).toHaveBeenCalledTimes(2)); + expect(vi.mocked(api.startRecording).mock.calls[1][3]).toBe("plain"); + expect(api.foregroundApp).toHaveBeenCalledOnce(); + expect(api.announceAppAction).toHaveBeenCalledOnce(); + }); + it("records straight away when the hotkey had to start the app first", async () => { vi.mocked(api.takeStartupIntent).mockResolvedValue("toggle"); render(App); @@ -1400,7 +1430,8 @@ describe("the ready boundary after processing", () => { listeners.get("remote-intent")!({ payload: "start" }); await fireEvent.keyDown(window, { code: "Space" }); await fireEvent.click(view.getByRole("button", { name: "Start recording" })); - expect(api.startRecording).toHaveBeenCalledTimes(2); + // The key press first asks which app is in front; the start follows. + await waitFor(() => expect(api.startRecording).toHaveBeenCalledTimes(2)); started(); await waitFor(() => expect(view.queryByRole("button", { name: "Stop recording" })).not.toBeNull()); expect(api.startRecording).toHaveBeenCalledTimes(2); diff --git a/src/app.css b/src/app.css index 46c1e00..ab5afab 100644 --- a/src/app.css +++ b/src/app.css @@ -317,3 +317,17 @@ button:focus-visible, .transcript:focus-visible { outline:2px solid var(--accent /* A dropdown inside a settings row is as tall as the other row controls. */ .setting-row .select-trigger { min-height:34px; padding:6px 10px; } .setting-row .select-options { left:auto; right:0; } +/* Actions per application: one line per app, its action, and remove. */ +.app-actions-row .row-label { align-self:flex-start; padding-top:8px; } +.app-actions { flex:1; display:flex; flex-direction:column; align-items:flex-end; gap:8px; min-width:0; } +.app-action { display:flex; align-items:center; justify-content:flex-end; gap:8px; max-width:100%; } +.app-name { min-width:0; max-width:180px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; font-size:12px; text-align:right; } +.app-action .select-menu { width:150px; flex-shrink:0; } +.app-remove { width:24px; height:24px; padding:0; border:0; border-radius:50%; background:transparent; color:var(--muted); line-height:1; } +.app-remove:hover { background:var(--accent-soft); color:var(--text); } +.app-add { position:relative; } +.app-add-button { border-style:dashed; color:var(--muted); } +.app-add-button[aria-expanded="true"] { border-style:solid; color:var(--text); } +.app-picker { left:auto; right:0; width:240px; } +.app-picker-head { display:block; padding:6px 10px 4px; color:var(--muted); font:10px ui-monospace,monospace; text-transform:uppercase; letter-spacing:.1em; } +.app-picker-input { width:100%; min-height:36px; margin-top:4px; font-size:12px; } diff --git a/src/lib/api.ts b/src/lib/api.ts index 823b85f..62fd464 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -21,6 +21,8 @@ import type { RemoteIntent, TypingSupport, ContextSupport, + ForegroundSupport, + KnownApp, } from "./types"; export const api = { @@ -29,6 +31,12 @@ export const api = { // grant before the first delivery has to fail over it. typingSupport: () => invoke("typing_support"), contextSupport: () => invoke("context_support"), + foregroundSupport: () => invoke("foreground_support"), + // The application in front as the dictation key is pressed; `null` for + // Utterform itself or a desktop that does not say. + foregroundApp: () => invoke("foreground_app"), + announceAppAction: (action: string, actionName: string, appName: string) => + invoke("announce_app_action", { action, actionName, appName }), getLiveStatus: () => invoke("get_live_status"), // Delivered once: the intent Utterform was launched with, for a hotkey that // had to start the app first. diff --git a/src/lib/app-actions.test.ts b/src/lib/app-actions.test.ts new file mode 100644 index 0000000..79478ed --- /dev/null +++ b/src/lib/app-actions.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { actionFor, addable, rememberApp, removeRule, RECENT_APPS, sameApp, setRule, typedApp } from "./app-actions"; + +const thunderbird = { id: "org.mozilla.Thunderbird", name: "Thunderbird" }; +const ghostty = { id: "com.mitchellh.ghostty", name: "Ghostty" }; +const exists = (action: string) => ["plain", "email", "custom:a"].includes(action); + +describe("actions per application", () => { + it("matches a chosen application by id and a typed one by name", () => { + expect(sameApp(thunderbird, { id: "org.mozilla.thunderbird", name: "Thunderbird" })).toBe(true); + expect(sameApp(typedApp("thunderbird")!, thunderbird)).toBe(true); + expect(sameApp(typedApp("Thunder")!, thunderbird)).toBe(false); + expect(sameApp({ id: "", name: "" }, { id: "", name: "" })).toBe(false); + }); + + it("finds the rule for the application in front", () => { + const rules = [{ app: thunderbird, action: "email" }, { app: ghostty, action: "plain" }]; + expect(actionFor(rules, ghostty, exists)?.action).toBe("plain"); + expect(actionFor(rules, { id: "chromium", name: "Chromium" }, exists)).toBeNull(); + }); + + it("ignores a rule whose action was removed", () => { + expect(actionFor([{ app: thunderbird, action: "custom:gone" }], thunderbird, exists)).toBeNull(); + }); + + it("keeps recent applications newest first, once each, and few", () => { + let recent = rememberApp([], thunderbird); + recent = rememberApp(recent, ghostty); + recent = rememberApp(recent, { ...thunderbird }); + expect(recent.map((app) => app.name)).toEqual(["Thunderbird", "Ghostty"]); + for (let index = 0; index < 20; index++) recent = rememberApp(recent, { id: `app${index}`, name: `App ${index}` }); + expect(recent).toHaveLength(RECENT_APPS); + }); + + it("keeps one rule per application", () => { + let rules = setRule([], thunderbird, "plain"); + rules = setRule(rules, typedApp("Thunderbird")!, "email"); + expect(rules).toEqual([{ app: thunderbird, action: "email" }]); + expect(removeRule(rules, thunderbird)).toEqual([]); + }); + + it("offers only applications without a rule", () => { + expect(addable([thunderbird, ghostty], [{ app: thunderbird, action: "email" }])).toEqual([ghostty]); + expect(typedApp(" ")).toBeNull(); + }); +}); diff --git a/src/lib/app-actions.ts b/src/lib/app-actions.ts new file mode 100644 index 0000000..90480d7 --- /dev/null +++ b/src/lib/app-actions.ts @@ -0,0 +1,49 @@ +/// Actions per application: which rule a dictation key press meets, and the +/// short list of applications Settings offers when one is added. + +import type { AppAction, KnownApp } from "./types"; + +/// Enough to find the handful of programs someone dictates into. +export const RECENT_APPS = 8; + +const plain = (value: string) => value.trim().toLowerCase(); + +/// A rule written by hand names the application as a person would +/// ("Thunderbird"); one chosen from the list carries its id. Either matches. +export function sameApp(rule: KnownApp, app: KnownApp) { + const wanted = [plain(rule.id), plain(rule.name)].filter(Boolean); + return [plain(app.id), plain(app.name)].some((name) => name && wanted.includes(name)); +} + +/// The rule for the application in front, unless its action no longer exists +/// (a removed custom action): then the action chosen in the window applies. +export function actionFor(rules: AppAction[], app: KnownApp, exists: (action: string) => boolean): AppAction | null { + return rules.find((rule) => sameApp(rule.app, app) && exists(rule.action)) ?? null; +} + +/// Newest first, each application once. +export function rememberApp(recent: KnownApp[], app: KnownApp): KnownApp[] { + return [app, ...recent.filter((known) => plain(known.id) !== plain(app.id))].slice(0, RECENT_APPS); +} + +/// One rule per application: choosing it again changes its action. +export function setRule(rules: AppAction[], app: KnownApp, action: string): AppAction[] { + const index = rules.findIndex((rule) => sameApp(rule.app, app)); + if (index < 0) return [...rules, { app, action }]; + return rules.map((rule, at) => (at === index ? { ...rule, action } : rule)); +} + +export function removeRule(rules: AppAction[], app: KnownApp): AppAction[] { + return rules.filter((rule) => !sameApp(rule.app, app)); +} + +/// What "+ App" offers: the recent applications that have no rule yet. +export function addable(recent: KnownApp[], rules: AppAction[]): KnownApp[] { + return recent.filter((app) => !rules.some((rule) => sameApp(rule.app, app))); +} + +/// A name typed by hand, matched against the id or the name later. +export function typedApp(name: string): KnownApp | null { + const trimmed = name.trim(); + return trimmed ? { id: trimmed, name: trimmed } : null; +} diff --git a/src/lib/settings-catalog.test.ts b/src/lib/settings-catalog.test.ts index 5fcb20a..605e03f 100644 --- a/src/lib/settings-catalog.test.ts +++ b/src/lib/settings-catalog.test.ts @@ -22,8 +22,9 @@ describe("the settings catalog", () => { expect([...visibleRows("app", "selected")]).toEqual(["context-selection"]); }); - it("gives the Context action's sources a category of their own", () => { - expect([...visibleRows("context", "")]).toEqual(["context-selection", "context-clipboard"]); + it("gives the Context action's sources and the actions per app a category of their own", () => { + expect([...visibleRows("context", "")]).toEqual(["context-selection", "context-clipboard", "app-actions"]); + expect([...visibleRows("app", "per app")]).toEqual(["app-actions"]); expect([...visibleRows("app", "topic")]).toEqual(["context"]); }); diff --git a/src/lib/settings-catalog.ts b/src/lib/settings-catalog.ts index 444cfc0..86dccc0 100644 --- a/src/lib/settings-catalog.ts +++ b/src/lib/settings-catalog.ts @@ -17,7 +17,7 @@ export type SettingsRowId = | "microphone" | "shortcut" | "sounds" | "engine" | "api-key" | "language" | "vocabulary" | "context" | "local-model" | "actions" | "text-provider" | "text-model" - | "context-selection" | "context-clipboard" + | "context-selection" | "context-clipboard" | "app-actions" | "typing" | "typing-delay" | "folder" | "history" | "theme" | "autostart" | "diagnostics"; @@ -44,6 +44,7 @@ export const SETTINGS_ROWS: SettingsRow[] = [ { id: "text-model", category: "ai", label: "Text model", keywords: ["model", "gpt", "effort", "thinking", "reasoning", "llm"] }, { id: "context-selection", category: "context", label: "Selection", keywords: ["selected", "highlighted", "marked", "primary", "reply", "material"] }, { id: "context-clipboard", category: "context", label: "Clipboard", keywords: ["copied", "copy", "paste", "material"] }, + { id: "app-actions", category: "context", label: "Per app", keywords: ["application", "apps", "program", "window", "mail", "terminal", "automatic", "rule"] }, { id: "typing", category: "output", label: "Type at cursor", keywords: ["paste", "keystrokes", "typing", "insert"] }, { id: "typing-delay", category: "output", label: "Keystroke delay", keywords: ["delay", "milliseconds", "slow", "typing"] }, { id: "folder", category: "output", label: "Folder", keywords: ["file", "directory", "save", "export", "output"] }, diff --git a/src/lib/types.ts b/src/lib/types.ts index e9e5255..788a859 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -59,6 +59,29 @@ export interface AppSettings { context_selection: boolean; /// The Context action may read the clipboard when nothing is selected. context_clipboard: boolean; + /// The action a dictation key press takes in a given application. + app_actions: AppAction[]; + /// Applications dictated into lately, newest first. Kept on this device. + recent_apps: KnownApp[]; +} + +/// An application as the platform names it: `id` is what rules match (class, +/// program file, bundle id), `name` what Settings shows. +export interface KnownApp { + id: string; + name: string; +} + +/// `action` is a built-in id or `custom:`. +export interface AppAction { + app: KnownApp; + action: string; +} + +/// Whether this desktop says which application is in front. +export interface ForegroundSupport { + supported: boolean; + explanation: string; } /// What this session allows for a system-wide dictation key. @@ -178,6 +201,8 @@ export const DEFAULT_SETTINGS: AppSettings = { global_hotkey: "Ctrl+Alt+D", context_selection: true, context_clipboard: true, + app_actions: [], + recent_apps: [], }; /// The actions to show until the backend answers with the shipped prompts, and diff --git a/tests/ui.e2e.ts b/tests/ui.e2e.ts index 652bdc9..4dd8bf0 100644 --- a/tests/ui.e2e.ts +++ b/tests/ui.e2e.ts @@ -29,6 +29,9 @@ test.beforeEach(async ({ page }) => { __emitRemoteIntent: (intent: string) => callbacks.get(eventHandlers.get("remote-intent") ?? -1)?.({ payload: intent }), __emit: (event: string, payload: unknown) => callbacks.get(eventHandlers.get(event) ?? -1)?.({ payload }), __transcribedFiles: [] as unknown[], + __startedActions: [] as unknown[], + __finishRequests: [] as unknown[], + __announced: [] as unknown[], __finishCount: 0, __savedSettings: null, __appliedHotkey: undefined, @@ -65,7 +68,11 @@ test.beforeEach(async ({ page }) => { { id: "context", name: "Context", hint: "Apply your words to the selected text", prompt: "Carry out the spoken instruction on the material. Return only the resulting text." }, ]; case "context_support": return Reflect.get(window, "__contextSupport") ?? { selection: true, explanation: "" }; - case "get_settings": return settings; + case "foreground_support": return { supported: true, explanation: "" }; + // The application in front when the dictation key is pressed. + case "foreground_app": return Reflect.get(window, "__foreground") ?? null; + case "announce_app_action": (Reflect.get(window, "__announced") as unknown[]).push(args); return; + case "get_settings": return Reflect.get(window, "__settings") ?? settings; case "save_settings": Object.assign(window, { __savedSettings: args.value }); return; case "has_openai_api_key": return true; case "codex_status": return { signedIn: !Reflect.get(window, "__codexSignedOut"), models: [{ id: "gpt-6.1-sol", name: "GPT-6.1-Sol" }, { id: "gpt-6-luna", name: "GPT-6-Luna" }], defaultModel: "gpt-6-luna" }; @@ -80,14 +87,14 @@ test.beforeEach(async ({ page }) => { case "list_history": return history; case "clear_history": history = []; return; case "copy_text": Object.assign(window, { __copiedText: args.text }); return; - case "start_recording": started = Date.now(); pausedAt = 0; pausedMs = 0; return; + case "start_recording": (Reflect.get(window, "__startedActions") as unknown[]).push(args.action); started = Date.now(); pausedAt = 0; pausedMs = 0; return; case "set_recording_paused": if (args.paused && !pausedAt) pausedAt = Date.now(); else if (!args.paused && pausedAt) { pausedMs += Date.now() - pausedAt; pausedAt = 0; } return recordingStatus(); case "cancel_recording": started = 0; pausedAt = 0; return; case "get_recording_status": return recordingStatus(); - case "finish_recording": started = 0; Object.assign(window, { __finishCount: Reflect.get(window, "__finishCount") + 1 }); return { ...latest, historyEntry: latest, savedPath: null, copiedToClipboard: true, deliveryWarnings: [] }; + case "finish_recording": (Reflect.get(window, "__finishRequests") as unknown[]).push(args.request); started = 0; Object.assign(window, { __finishCount: Reflect.get(window, "__finishCount") + 1 }); return { ...latest, historyEntry: latest, savedPath: null, copiedToClipboard: true, deliveryWarnings: [] }; case "retained_recording": return Reflect.get(window, "__retained") ?? null; case "take_pending_file": return null; case "transcribe_file": { @@ -948,3 +955,69 @@ test("without a way to read the selection, Context says what to do and keeps the await page.waitForTimeout(300); await page.screenshot({ path: testInfo.outputPath("context-elsewhere.png") }); }); + +test("the dictation key takes the action chosen for the app in front, and says so", async ({ page }, testInfo) => { + await page.addInitScript((settings) => Object.assign(window, { + __settings: settings, + __foreground: { id: "org.mozilla.Thunderbird", name: "Thunderbird" }, + }), { ...DEFAULT_SETTINGS, theme: "dark", app_actions: [{ app: { id: "org.mozilla.Thunderbird", name: "Thunderbird" }, action: "email" }] }); + await page.goto("/"); + await expect(page.getByRole("combobox", { name: "Action" })).toContainText("Plain"); + await page.evaluate(() => Reflect.get(window, "__emitRemoteIntent")("toggle")); + await expect(page.getByRole("button", { name: "Pause recording" })).toBeVisible(); + await expect(page.getByRole("combobox", { name: "Action" })).toContainText("Email"); + await expect(page.locator(".action-control .control-label")).toHaveText("Action · Thunderbird"); + await page.screenshot({ path: testInfo.outputPath("app-action-recording.png") }); + expect(await page.evaluate(() => Reflect.get(window, "__startedActions"))).toEqual(["email"]); + expect(await page.evaluate(() => Reflect.get(window, "__announced"))).toEqual([{ action: "email", actionName: "Email", appName: "Thunderbird" }]); + await page.evaluate(() => Reflect.get(window, "__emitRemoteIntent")("toggle")); + await expect(page.locator(".copy-button")).toContainText("Copied"); + expect(await page.evaluate(() => (Reflect.get(window, "__finishRequests") as Array<{ action: string }>).map((request) => request.action))).toEqual(["email"]); + // Over: the window's own choice is back. + await expect(page.getByRole("combobox", { name: "Action" })).toContainText("Plain"); + await expect(page.locator(".action-control .control-label")).toHaveText("Action"); + + // Elsewhere, and from the window itself, the window's action applies, quietly. + await page.evaluate(() => Object.assign(window, { __foreground: { id: "chromium", name: "Chromium" } })); + await page.evaluate(() => Reflect.get(window, "__emitRemoteIntent")("toggle")); + await expect(page.getByRole("button", { name: "Pause recording" })).toBeVisible(); + await page.evaluate(() => Reflect.get(window, "__emitRemoteIntent")("toggle")); + await expect(page.locator(".copy-button")).toContainText("Copied"); + expect(await page.evaluate(() => Reflect.get(window, "__startedActions"))).toEqual(["email", "plain"]); + expect(await page.evaluate(() => (Reflect.get(window, "__announced") as unknown[]).length)).toBe(1); + const recent = await page.evaluate(() => (Reflect.get(window, "__savedSettings") as { recent_apps: Array<{ name: string }> }).recent_apps.map((app) => app.name)); + expect(recent).toEqual(["Chromium", "Thunderbird"]); +}); + +test("an app dictated into is added in Settings with its own action, at once", async ({ page }, testInfo) => { + await page.addInitScript((settings) => Object.assign(window, { __settings: settings }), { + ...DEFAULT_SETTINGS, theme: "dark", + recent_apps: [{ id: "org.mozilla.Thunderbird", name: "Thunderbird" }, { id: "com.mitchellh.ghostty", name: "Ghostty" }], + }); + await page.goto("/"); + await page.getByRole("button", { name: "Open settings" }).click(); + await page.getByRole("tab", { name: "Context", exact: true }).click(); + const saved = () => page.evaluate(() => (Reflect.get(window, "__savedSettings") as { app_actions?: unknown } | null)?.app_actions); + await page.getByRole("button", { name: "+ App" }).click(); + await expect(page.getByRole("group", { name: "Add an app" })).toBeVisible(); + await page.mouse.move(0, 0); + await page.waitForTimeout(300); // The menu has settled. + await page.screenshot({ path: testInfo.outputPath("app-actions-add.png") }); + await page.getByRole("button", { name: "Thunderbird" }).click(); + await expect.poll(saved).toEqual([{ app: { id: "org.mozilla.Thunderbird", name: "Thunderbird" }, action: "plain" }]); + await page.getByRole("combobox", { name: "Action in Thunderbird" }).click(); + await page.getByRole("option", { name: /Email/ }).click(); + await expect.poll(saved).toEqual([{ app: { id: "org.mozilla.Thunderbird", name: "Thunderbird" }, action: "email" }]); + // One already chosen is not offered again; a name can be typed instead. + await page.getByRole("button", { name: "+ App" }).click(); + await expect(page.getByRole("group", { name: "Add an app" }).getByRole("button", { name: "Thunderbird" })).toHaveCount(0); + await page.getByRole("textbox", { name: "App name" }).fill("Obsidian"); + await page.getByRole("textbox", { name: "App name" }).press("Enter"); + await expect.poll(saved).toHaveLength(2); + await page.mouse.move(0, 0); + await page.waitForTimeout(300); + await page.screenshot({ path: testInfo.outputPath("app-actions.png") }); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true); + await page.getByRole("button", { name: "Remove Thunderbird" }).click(); + await expect.poll(saved).toEqual([{ app: { id: "Obsidian", name: "Obsidian" }, action: "plain" }]); +});