diff --git a/CHANGELOG.md b/CHANGELOG.md index c5f333f..c86ceb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.13.0 · Voice messages and recordings (on `main`, ships with 1.0) + +- Transcribe an audio file: the new folder button next to Settings or `O` opens one, dropping it on the window shows **Drop to transcribe** with the action and engine it will use, `utterform --file ` and **Open with → Utterform** in the file manager hand one over. Several files are done one after another. +- The text goes to history and the clipboard, and to a file when File is on; it is never typed at the cursor. The chosen action applies as for a dictation. +- Read WhatsApp and Telegram voice messages (Opus in Ogg), M4A/AAC from a phone, MP3, WAV, FLAC, Vorbis, WebM and AIFF, up to two hours. Opus is decoded by libopus, compiled into Utterform; nothing has to be installed. +- Send audio longer than twelve minutes to GPT Transcribe in parts of about ten minutes, each cut at the quietest moment before the mark, so the 25 MB upload limit no longer applies. Local Whisper takes the whole file. +- Offer Utterform in "Open with" on Linux, Windows and macOS without becoming the default player for any audio type. + +See [0.13.0 release notes](docs/releases/v0.13.0.md). + ## 0.10.0 · Text actions on your ChatGPT subscription (on `main`, ships with 1.0) - Add **Settings → AI → Provider**: OpenAI key or **ChatGPT**. With ChatGPT, Clean, Polish, Summarize, Prompt, Email and your own actions run on your ChatGPT subscription through the Codex CLI login (`codex login`) at no extra cost. Transcription keeps using the OpenAI key or the local model. diff --git a/README.md b/README.md index 02d8483..12dc211 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,12 @@ Under **Settings → AI → Text model**: the model that runs these prompts (`gp OpenAI does not officially support other apps on this path and may block it, which is why it is marked *Experimental*. Utterform does not quietly fall back to your API key: if ChatGPT refuses, the reason is shown and the plain transcript is delivered. +## Voice messages and recordings + +Drop an audio file on the window, open one with the folder button next to Settings (or `O`), run `utterform --file `, or choose **Open with → Utterform** in your file manager. The text goes to the history and the clipboard — and to a file when File is on — with the chosen action applied, the same as a dictation; it is never typed at the cursor. + +WhatsApp and Telegram voice messages (`.opus`, `.ogg`), phone recordings (`.m4a`), MP3, WAV, FLAC, WebM and AIFF work, up to two hours. With GPT Transcribe, audio longer than twelve minutes is sent in parts of about ten minutes, cut at pauses, so OpenAI's 25 MB upload limit does not stop it. Utterform only adds itself to "Open with"; your default player stays as it is. + ## Vocabulary Names, products and spellings a model would otherwise guess at go under **Settings → Transcription → Vocabulary**, one word at a time: type it and press Enter, remove it with ×. A pasted list keeps its lines. Write *Careum* there and it stops coming back as *Kareum*. @@ -277,6 +283,7 @@ Push/PR CI runs Linux validation without release compilation. For a test binary ## Features - Batch transcription with `gpt-transcribe` — no realtime session required +- Audio files: voice messages and recordings by drop, Open, `--file` or "Open with" - Offline transcription through `whisper.cpp`, built for every supported desktop rather than for the machine that compiled it - One-click, SHA-256-verified downloads for Tiny, Base, and Small multilingual models - Plain, Clean, Polish, Summarize, Prompt, Email, and user-defined actions — every shipped prompt can be rewritten @@ -312,13 +319,14 @@ Shortcuts work while the Utterform window is focused. | `1`–`6` | Select Plain, Clean, Polish, Summarize, Prompt, or Email | | `C` | Toggle clipboard output | | `F` | Toggle file output | +| `O` | Open an audio file to transcribe | | `Ctrl+Shift+C` / `Cmd+Shift+C` | Copy the displayed text again (latest by default) | Global dictation is separate from these: see [Dictate from anywhere](#dictate-from-anywhere). **An already-started recording continues when you switch apps, minimize, or close the window to tray** (including Omarchy's `Super+W`). Reopen from the tray to pause or finish it, or let the ten-minute active-recording limit stop capture. A paused recording remains paused across app switches and close-to-tray. Paused audio is discarded, not stored or sent; the microphone device stays open so resuming works consistently across platforms. Processing resumes when the WebView is available. Tray **Quit** discards active audio and exits; simply launching Utterform does not start recording. ## Privacy model -- **GPT Transcribe:** the temporary audio recording is sent to OpenAI. +- **GPT Transcribe:** the temporary audio recording is sent to OpenAI — for an opened audio file, a decoded copy of its audio. - **Local Whisper + Plain:** audio and text stay on the device. - **Local Whisper + transformed action:** audio stays local; transcribed text is sent to OpenAI. - Temporary recordings are deleted after processing or cancellation. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ad79454..9bb6476 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -70,6 +70,57 @@ The audio is the one thing that cannot be recreated, so failures keep it. `keyring` holds one Secret Service session for the process lifetime and that session ends with the daemon. macOS and Windows have no such session. +## Audio files (0.13.0) + +A file is turned into what a recording leaves behind and then takes the recording's path. +`audio_file::decode` probes the content with Symphonia (Ogg, Matroska/WebM, MP4/M4A, MP3, +WAV, FLAC, AIFF, CAF; AAC, MP3, FLAC, Vorbis, PCM, ADPCM, ALAC) and decodes Opus through +`symphonia-adapter-libopus`, which compiles libopus into the binary with CMake — Symphonia +has no Opus decoder of its own. Each packet is mixed to mono and streamed through a small +resampler (box average when going down, linear interpolation when going up) into a +temporary 16 kHz mono 16-bit WAV from `audio::new_temporary_wav`, named like a recording so +the stale cleanup knows it. The result is an ordinary `RecordingArtifact`; nothing +downstream knows where the audio came from, and dropping it deletes the copy. Undecodable +packets are skipped and counted; a cut-off file keeps what came before the cut; more than +two hours is refused while decoding. + +`commands::transcribe_file` takes a path, a source (`drop`, `open`, `command_line`, +`open_with`) and the usual `ProcessRequest`, forces clipboard on and cursor off, refuses +while a recording or Live dictation runs or another file is in progress (`FileBusy`), and +calls `process_recording` with `Origin::File`. That origin logs `mode=file` and keeps no +copy when transcription fails: the original is still there, so opening it again is the +retry. + +Files from outside queue in `PendingFiles` (at most 20): the launch's own arguments, +a second instance's arguments resolved against its working directory, and macOS' +`RunEvent::Opened`. `cli::files_from` reads `--file `, `--file=`, `file://` +URLs and a bare argument that names an existing file — the form "Open with" and the +Windows registry use — so `utterform toggle` keeps meaning what it did. The interface +hears `file-waiting`, takes paths with `take_pending_file` one at a time while it is free, +and also drains the queue once at start. Drag and drop uses the webview's +`tauri://drag-enter/-leave/-drop` events. + +Uploads: `transcription::openai` cuts audio longer than twelve minutes into parts of about +ten (`split_at_pauses`), moving each cut to the quietest 100 ms window in the 30 s before +the mark, uploads them in order and joins the texts with a space. A silent part adds +nothing; any other failure fails the whole file. Local Whisper takes the whole buffer. +`RecordingArtifact::whisper_pcm` now reads frame by frame and skips resampling at 16 kHz, +so a two-hour file costs one buffer of about 460 MB rather than three. + +"Open with" registration: Linux's desktop entry lists audio MIME types with `Exec=… %F`; +Windows' NSIS hook writes `Utterform.AudioFile` into each type's `OpenWithProgids` and an +`Applications\utterform.exe` entry, under HKCU, and removes them on a real uninstall — +Tauri's own `fileAssociations` is not used because its NSIS macro replaces the type's +default program; macOS' `Info.plist` declares `CFBundleDocumentTypes` for `public.audio` +with rank `Alternate`. + +Events: `file.requested` (source, count), `file.started` (recording, source), +`file.decoded` (codec, source rate and channels, skipped packets, audio and decode ms), +`file.failed` (class, stage `start` or `decode`), `file.packets_skipped`, +`file.requests_dropped`, `transcription.split` (parts), `transcription.part_failed`, then +the recording's own `transcription.*` → `recording.completed mode=file`. Names, paths and +extensions of files are never logged. + ## Background capture & feedback Capture is owned by Rust/CPAL, not the WebView. Focus changes, minimization, and closing the window to tray do not stop a recording. Explicit Stop processes it, Escape discards it, and tray Quit discards active audio before exiting. No microphone capture starts merely by launching the app. The in-window keys stay focused-window shortcuts; global dictation is a separate path described below. @@ -199,11 +250,11 @@ Diagnostics are local, bounded and free of user content, and `diagnostics.rs` is **Storage.** `tauri-plugin-log` 2.9 supplies the filtered logger and writes to stderr; Utterform's bounded writer stores `utterform.log` in Tauri's log directory (`app_log_dir`: `$XDG_DATA_HOME/software.jli.utterform/logs` on Linux, `~/Library/Logs/software.jli.utterform` on macOS, `%LOCALAPPDATA%\software.jli.utterform\logs` on Windows). It checks the size after every record and rotates at 2 MiB to `utterform.1.log` and `utterform.2.log`, so the disk budget is about 6 MiB. It buffers at most one already-bounded record and drops that record after a write or rotation failure, preventing a full disk or a removed/locked log directory from growing process memory without limit. The level is `INFO` in release builds and `DEBUG` in debug builds, and a filter admits only the `utterform`/`utterform_lib` targets and the frontend bridge's `utterform::frontend` — dependency output is never written. The logger is built with the plugin's `split` and attached in `setup` before anything else runs, rather than registered as a plugin: its only other content is a JavaScript `log` command the WebView must not reach, and a log directory that cannot be created then costs the file instead of the launch (events still reach stderr and `diagnostics.file_unavailable` says why). The 0.7.8 `utterform.log` in app-local-data is not moved, deleted or written. -**Events.** One line per event: `2026-09-16T08:00:00.123Z INFO utterform run=1a2b3c4d recording.opened recording=3 device="Studio microphone" open_ms=42`. Names are stable and dotted (`app.*`, `recording.*`, `transcription.*`, `transform.*`, `history.*`, `delivery.*`, `live.*`, `cue.*`, `settings.*`, `hotkey.*`, `models.*`, `secrets.*`, `support.*`, `window.*`, `tray.*`, `typing.*`, `frontend.error`); fields are `key=value`, quoted and escaped when they contain whitespace, quotes, `=` or backslashes. Every line carries `run`, eight hex digits for this process, so a file shared by several runs stays attributable. Every batch and Live recording gets a process-local `recording` id from one counter, carried through `StartedRecording`, `ActiveRecording`, `RecordingArtifact` and the Live `Session` — separate from the `Instant` that identifies a capture session. A run of a recording therefore reads as `recording.requested` → `recording.opened` → `recording.armed` → `recording.stopped` → `transcription.started/completed` → `transform.started/completed` or `transform.fallback` → `history.saved` → `delivery.completed` → `recording.completed`, and a Live one adds `live.input_ready`, `live.connected`, `live.streaming`, `live.stream_completed` and the input worker's events. +**Events.** One line per event: `2026-09-16T08:00:00.123Z INFO utterform run=1a2b3c4d recording.opened recording=3 device="Studio microphone" open_ms=42`. Names are stable and dotted (`app.*`, `recording.*`, `file.*`, `transcription.*`, `transform.*`, `history.*`, `delivery.*`, `live.*`, `cue.*`, `settings.*`, `hotkey.*`, `models.*`, `secrets.*`, `support.*`, `window.*`, `tray.*`, `typing.*`, `frontend.error`); fields are `key=value`, quoted and escaped when they contain whitespace, quotes, `=` or backslashes. Every line carries `run`, eight hex digits for this process, so a file shared by several runs stays attributable. Every batch and Live recording gets a process-local `recording` id from one counter, carried through `StartedRecording`, `ActiveRecording`, `RecordingArtifact` and the Live `Session` — separate from the `Instant` that identifies a capture session. A run of a recording therefore reads as `recording.requested` → `recording.opened` → `recording.armed` → `recording.stopped` → `transcription.started/completed` → `transform.started/completed` or `transform.fallback` → `history.saved` → `delivery.completed` → `recording.completed`, and a Live one adds `live.input_ready`, `live.connected`, `live.streaming`, `live.stream_completed` and the input worker's events. **Failures and references.** Low-level code returns a `diagnostics::Failure`: the message the user reads, kept apart from a closed-vocabulary `class`, an optional HTTP `status` and an optional `detail` (an `io::ErrorKind`, a keyring variant, an audio backend's error). `Failure` has no `Display`, so its message — which can quote an OpenAI error body — cannot become a log field by accident. The code that knows the context owns the one event: `failure!` logs `ERROR`, `fallback!` logs `WARN` for work that continued without that part, and both append `(ref -)` to the message; the same reference is the event's `ref` field. Guidance the user acts on (add a key, choose an output, a model not yet downloaded, install `wtype`, grant Accessibility, start Live from another application's text field) is logged at `INFO` with `outcome=guidance` and no reference; `typing::is_guidance` and `typing::is_capture_guidance` recognise the platform modules' own refusal sentences for that. Commands that only pass a failure on do not log it again. -**Privacy.** Allowed: version, OS/architecture, desktop and session kind, run/recording/reference ids, durations, counts, enabled modes, model and action ids (a custom action is only `custom`), HTTP statuses, microphone and output device names, program/application names, window classes and bundle ids, error classes, and the platform typing modules' own refusal sentences, which name tools, programs and counts. The text model is named only when it looks like a model id; anything else typed into that field is `custom`. Never: API keys or authorization material, transcripts, history, clipboard or output-file contents, audio, prompts and prompt overrides, vocabulary, language hints, recording context, window titles, request/response bodies, WebSocket deltas, raw third-party errors, full settings or history JSON, output folders (a saved file is named by its basename). Linux Live focus events keep their existing rendering: event name, window address, workspace id. Every record is still normalized by `redact::line` — one physical line, no control or direction-override characters, home directories reduced to `$HOME`, `sk-…` keys, bearer tokens and `Authorization:` values replaced — and bounded to 16 KiB; a copied report applies the same again, legacy lines included. The sanitizer cannot recognise dictated prose; the protection is that it never reaches diagnostics. +**Privacy.** Allowed: version, OS/architecture, desktop and session kind, run/recording/reference ids, durations, counts, enabled modes, model and action ids (a custom action is only `custom`), HTTP statuses, microphone and output device names, program/application names, window classes and bundle ids, error classes, and the platform typing modules' own refusal sentences, which name tools, programs and counts. The text model is named only when it looks like a model id; anything else typed into that field is `custom`. Never: API keys or authorization material, transcripts, history, clipboard or output-file contents, audio, prompts and prompt overrides, vocabulary, language hints, recording context, window titles, names and paths of opened audio files, request/response bodies, WebSocket deltas, raw third-party errors, full settings or history JSON, output folders (a saved file is named by its basename). Linux Live focus events keep their existing rendering: event name, window address, workspace id. Every record is still normalized by `redact::line` — one physical line, no control or direction-override characters, home directories reduced to `$HOME`, `sk-…` keys, bearer tokens and `Authorization:` values replaced — and bounded to 16 KiB; a copied report applies the same again, legacy lines included. The sanitizer cannot recognise dictated prose; the protection is that it never reaches diagnostics. **Bounds.** Nothing is logged from the audio callback, the writer loop, per delta, per keystroke, per animation frame or per status poll: glitches, delivered chunks, WebSocket events and audio bytes are counted and reported once. `Limiter` keeps Linux Live focus events at 200 per session and reports `live.focus_events_suppressed`; the frontend bridge admits 50 reports per run (20 per mounted interface on its own side); a failing `get_recording_status` is reported once per run; undelivered model-download progress events are counted and reported once. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 2263d0a..ac31990 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -19,7 +19,20 @@ proposals are in [plans/v1.0-roadmap.md](plans/v1.0-roadmap.md). The plan of 202 (reliability, assistant, finish) is superseded; 0.7.13 shipped, 0.7.14 (`feat/simple-settings`) was rejected and will not be merged. -## Current handoff — 0.10.0 Codex login for text actions +## Current handoff — 0.13.0 Audio files + +Stage 0.13.0 of [the roadmap](plans/v1.0-roadmap.md), branch +`work/utterform-0.13.0-audio-files`. Voice messages and recordings are decoded by +`audio_file.rs` (Symphonia plus bundled libopus) into a temporary 16 kHz WAV and run +through `process_recording` with `Origin::File`; GPT Transcribe uploads longer than twelve +minutes in parts cut at pauses. In: folder button / `O`, drop, `--file`, "Open with" +(desktop entry, NSIS `OpenWithProgids`, macOS document types — none takes the default). +Jonas accepted the mock-up. Design in +[ARCHITECTURE.md → Audio files](ARCHITECTURE.md#audio-files-0130), live results and what +is open for 1.0 (Windows/macOS "Open with", drag and drop in a real window) in +[TESTING.md](TESTING.md). Merged into `main` without tag or release; ships with 1.0. + +## Previous handoff — 0.10.0 Codex login for text actions Stage 0.10.0 of [the roadmap](plans/v1.0-roadmap.md), branch `work/utterform-0.10.0-codex`. Settings → AI → Provider chooses OpenAI key or ChatGPT; with ChatGPT the text actions run diff --git a/docs/TESTING.md b/docs/TESTING.md index 890ef28..ae382ab 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -10,6 +10,41 @@ `PLAYWRIGHT_CHROMIUM_EXECUTABLE=/usr/bin/chromium npm run test:e2e`. - `cmake` is needed for whisper.cpp. Without root, `mise exec cmake@latest -- cargo test` works. +## 0.13.0 Audio files + +Unit tests in `audio_file.rs`: the resampler (48 kHz averaged by three, odd ratios keep the +length, upsampling interpolates, a 12 kHz tone at 48 kHz is attenuated), a stereo 44.1 kHz +WAV, the Opus, M4A and MP3 fixtures in `src-tauri/tests/fixtures/` (1.5 s of tone, made with +ffmpeg), a missing file, a text file, a folder and an empty WAV, none of which put the file +name in the message. `transcription/openai.rs`: twelve minutes go up whole, thirty minutes +become three parts that each fit 25 MB, a cut lands in a pause. `cli.rs`: `--file`, +`--file=`, `file://`, a bare existing path relative to the caller's directory, and +`utterform toggle` still not a file. Interface: Open button and `O` with the dialog filter, +the chosen action, drop with overlay, a drag that leaves, nothing taken while recording, the +queue from outside, a failure message. Playwright drops a file at 920×720. + +Live on `asus16` (2026-11-07), with an invented German voice message made by Piper TTS and +encoded like WhatsApp (`ffmpeg -c:a libopus -b:a 16k -ar 16000 -ac 1 -application voip`): + +- GPT Transcribe, through + `UTTERFORM_LIVE_AUDIO= cargo test --lib live_openai -- --ignored --nocapture`: + 11 s message word for word in 1.6 s. A 13-minute file (the message 52 times with pauses): + two parts, 9:34 and 3:37, 17 s, both ending on a full sentence. The model returned 47 of + the 52 identical repetitions — it drops some of a verbatim repeat within one part; the + cut itself lost nothing. +- The built app, isolated (own D-Bus session, `XDG_*` in a scratch folder, no window via + `--autostart`), local Whisper Base: `--file ` → `file.decoded codec=opus` → + history → clipboard in 4.1 s, correct text. A second instance with a relative bare path + (what a file manager runs) handed the file to the running one, 2.5 s. No path in the log. +- `install-linux.sh` into a scratch prefix: the desktop entry has `Exec=… %F` and the + MIME types, passes `desktop-file-validate`, and `mimeinfo.cache` lists it. + +Not exercised, to check at 1.0: drag and drop and the Open dialog in the real window on +Linux, Windows and macOS; "Open with" on Windows (registry entries from the NSIS hook, +Explorer listing Utterform without changing the default player) and on macOS (Finder's +Open With, `RunEvent::Opened`); a real WhatsApp file from a phone; libopus building on the +Windows and macOS runners (only Linux CI builds it on pull requests). + ## 0.10.0 Text actions on ChatGPT Unit tests in `transcription/codex.rs`: renewal only near expiry, a renewal replacing the diff --git a/docs/plans/v1.0-roadmap.md b/docs/plans/v1.0-roadmap.md index 3abf857..3fe80a1 100644 --- a/docs/plans/v1.0-roadmap.md +++ b/docs/plans/v1.0-roadmap.md @@ -56,6 +56,7 @@ and without its subscription: | 2026-11-07 | Settings are redesigned from scratch. 0.7.14 (`feat/simple-settings`) is rejected as well and will not be merged; the branch stays as a reference only. | | 2026-11-07 | Settings rules: as little text as possible, **no Save button** (every change applies at once), as simple as possible but clear, everything findable. Do not spend long on it — it has to work. | | 2026-11-07 | Jonas tests only once, at 1.0, everything together; until then no preview builds for him. Stages go through on their own: merge into `main` when tests and CI are green, no tag or release before 1.0. Style samples (mock-ups, images) are still welcome, and so are occasional questions. The two parallel sessions coordinate with each other. | +| 2026-11-07 | 0.13.0: audio files open from a folder button next to Settings (and `O`) or by dropping on the window, which shows "Drop to transcribe" with action and engine; text to history and clipboard, never the cursor. Mock-up accepted. | | 2026-11-07 | Settings mock-up accepted: search field on top, categories as tabs below, one row per setting. The "Context" category appears only once 0.11.0 gives it rows. | | 2026-11-07 | 0.10.0: ChatGPT is marked "Experimental" in Settings; **no fallback** to the OpenAI key — a ChatGPT failure is shown and the plain transcript delivered. Mock-up (AI → Provider: OpenAI key \| ChatGPT) accepted. | @@ -70,7 +71,7 @@ Proposals not yet confirmed by Jonas are marked **Proposal**. | 0.10.0 | Codex login for text actions | merged, ships with 1.0 | | 0.11.0 | Context: selected text and clipboard | planned | | 0.12.0 | Action per application | to discuss first | -| 0.13.0 | Audio files | planned | +| 0.13.0 | Audio files | merged, ships with 1.0 | | 0.14.0 | First-run assistant | planned (was 0.8.0) | | 1.0.0 | Acceptance on Linux, Windows, macOS; release; WinGet | planned | diff --git a/docs/releases/v0.13.0.md b/docs/releases/v0.13.0.md new file mode 100644 index 0000000..506d9dd --- /dev/null +++ b/docs/releases/v0.13.0.md @@ -0,0 +1,36 @@ +# Utterform 0.13.0 — Voice messages and recordings + +*Stage of the road to 1.0, merged into `main`; not released on its own. It ships with 1.0.* + +Someone sends a two-minute voice message and you would rather read it. Or a meeting was +recorded on the phone. Utterform now takes the file and gives back the text, the same way +it does for a dictation. + +**Four ways in.** The folder button next to Settings, or `O` in the window. Drag the file +onto the window: it shows **Drop to transcribe** with the action and engine that will be +used. `utterform --file ` from a terminal or a script. And **Open with → Utterform** +in the file manager, on Linux, Windows and macOS — Utterform is only added to that list, +it never becomes the default player for MP3 or WAV. + +**Same result as a dictation.** The text lands in the history and on the clipboard, and +in a file when File is on. The chosen action runs on it — Clean, Summarize, Email, your +own. It is never typed at the cursor: no text field is waiting for a file. Several files +dropped or opened at once are done one after another. + +**The formats people actually get.** WhatsApp and Telegram voice messages (`.opus`, +`.ogg`), iPhone and Android recordings (`.m4a`), MP3, WAV, FLAC, WebM, AIFF. Opus is +decoded by libopus, built into Utterform; nothing else has to be installed. + +**Long recordings.** OpenAI accepts at most 25 MB per upload. Audio up to twelve minutes +goes up whole, as dictations always did; anything longer is sent in parts of about ten +minutes, each cut at the quietest moment in the half minute before the mark, and the +texts are joined. Local Whisper reads the whole file at once. The limit is two hours. + +**Privacy.** With GPT Transcribe the decoded audio is sent to OpenAI, as for a recording; +with local Whisper it stays on the device. Utterform works on a temporary copy that is +deleted afterwards and never touches the original. The log records the codec, length and +outcome — never the file's name, path or contents. + +**Measured on `asus16`.** An 11-second WhatsApp-style message (Opus, 16 kb/s): GPT +Transcribe 1.6 s, local Whisper Base 4 s, both word for word. A 13-minute file went up in +two parts (9:34 and 3:37) in 17 s, both parts ending on a full sentence. diff --git a/packaging/windows/installer-hooks.nsh b/packaging/windows/installer-hooks.nsh index 7b1f74f..a9f4afe 100644 --- a/packaging/windows/installer-hooks.nsh +++ b/packaging/windows/installer-hooks.nsh @@ -3,6 +3,54 @@ ; no cache files are deleted and Explorer is never killed or restarted. ; Tauri's stock NSIS installer retains ownership of upgrade, running-app checks, ; shortcut removal and the explicit opt-in checkbox for deleting app data. +; "Open with > Utterform" for voice messages and recordings (0.13.0), per user. +; Only the Open-with list is extended: each type gains an OpenWithProgids value +; and Utterform an Applications entry. The type's default program is never +; read or written, unlike Tauri's own fileAssociations, which would make +; Utterform the default player for MP3 and WAV. +!define UTTERFORM_PROGID "Utterform.AudioFile" +!macro UTTERFORM_OPEN_WITH EXT + WriteRegStr HKCU "Software\Classes\.${EXT}\OpenWithProgids" "${UTTERFORM_PROGID}" "" + WriteRegStr HKCU "Software\Classes\Applications\utterform.exe\SupportedTypes" ".${EXT}" "" +!macroend +!macro UTTERFORM_OPEN_WITH_REMOVE EXT + DeleteRegValue HKCU "Software\Classes\.${EXT}\OpenWithProgids" "${UTTERFORM_PROGID}" +!macroend +!macro UTTERFORM_REGISTER_OPEN_WITH + WriteRegStr HKCU "Software\Classes\${UTTERFORM_PROGID}" "" "Audio file" + WriteRegStr HKCU "Software\Classes\${UTTERFORM_PROGID}\DefaultIcon" "" '"$INSTDIR\utterform.exe",0' + WriteRegStr HKCU "Software\Classes\${UTTERFORM_PROGID}\shell\open" "FriendlyAppName" "Utterform" + WriteRegStr HKCU "Software\Classes\${UTTERFORM_PROGID}\shell\open\command" "" '"$INSTDIR\utterform.exe" --file "%1"' + WriteRegStr HKCU "Software\Classes\Applications\utterform.exe" "FriendlyAppName" "Utterform" + WriteRegStr HKCU "Software\Classes\Applications\utterform.exe\shell\open\command" "" '"$INSTDIR\utterform.exe" --file "%1"' + !insertmacro UTTERFORM_OPEN_WITH "opus" + !insertmacro UTTERFORM_OPEN_WITH "ogg" + !insertmacro UTTERFORM_OPEN_WITH "oga" + !insertmacro UTTERFORM_OPEN_WITH "m4a" + !insertmacro UTTERFORM_OPEN_WITH "mp3" + !insertmacro UTTERFORM_OPEN_WITH "wav" + !insertmacro UTTERFORM_OPEN_WITH "flac" + !insertmacro UTTERFORM_OPEN_WITH "aac" + !insertmacro UTTERFORM_OPEN_WITH "webm" + !insertmacro UTTERFORM_OPEN_WITH "mka" + !insertmacro UTTERFORM_OPEN_WITH "aiff" +!macroend +!macro UTTERFORM_UNREGISTER_OPEN_WITH + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "opus" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "ogg" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "oga" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "m4a" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "mp3" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "wav" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "flac" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "aac" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "webm" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "mka" + !insertmacro UTTERFORM_OPEN_WITH_REMOVE "aiff" + DeleteRegKey HKCU "Software\Classes\${UTTERFORM_PROGID}" + DeleteRegKey HKCU "Software\Classes\Applications\utterform.exe" +!macroend + !macro NSIS_HOOK_POSTINSTALL ; Preserve an existing opt-in across upgrades, but repair the command from ; auto-launch 0.5.0: an unquoted per-user path can look registered while @@ -13,6 +61,7 @@ WriteRegStr HKCU "Software\Microsoft\Windows\CurrentVersion\Run" "Utterform" '"$INSTDIR\utterform.exe" --autostart' ${EndIf} Pop $0 + !insertmacro UTTERFORM_REGISTER_OPEN_WITH System::Call 'shell32::SHChangeNotify(i 0x08000000, i 0, p 0, p 0)' !macroend @@ -23,6 +72,7 @@ ${If} $UpdateMode <> 1 DeleteRegValue HKCU "Software\Microsoft\Windows\CurrentVersion\Run" "Utterform" DeleteRegValue HKCU "Software\Microsoft\Windows\CurrentVersion\Explorer\StartupApproved\Run" "Utterform" + !insertmacro UTTERFORM_UNREGISTER_OPEN_WITH ${EndIf} System::Call 'shell32::SHChangeNotify(i 0x08000000, i 0, p 0, p 0)' !macroend diff --git a/scripts/install-linux.sh b/scripts/install-linux.sh index 12c3d63..1a86ee9 100755 --- a/scripts/install-linux.sh +++ b/scripts/install-linux.sh @@ -98,11 +98,15 @@ desktop_tmp="${desktop_file}.tmp.$$" printf '%s\n' 'Type=Application' printf '%s\n' 'Name=Utterform' printf '%s\n' 'Comment=Turn speech into useful text' - printf 'Exec=%s\n' "$launcher" + # %F: "Open with Utterform" from a file manager hands over voice messages + # and recordings to transcribe. Listing the types only adds Utterform to that + # menu; the desktop's default player for them stays what it was. + printf 'Exec=%s %%F\n' "$launcher" printf '%s\n' 'Icon=utterform' printf '%s\n' 'Terminal=false' printf '%s\n' 'Categories=Utility;AudioVideo;' printf '%s\n' 'StartupNotify=true' + printf '%s\n' 'MimeType=audio/ogg;audio/opus;audio/x-opus+ogg;audio/x-vorbis+ogg;audio/mpeg;audio/mp4;audio/x-m4a;audio/aac;audio/wav;audio/x-wav;audio/flac;audio/x-flac;audio/webm;audio/x-matroska;audio/x-aiff;' # Utterform runs as a single instance; the window class lets desktops match # the running window to this entry instead of offering another launch. printf '%s\n' 'StartupWMClass=Utterform' diff --git a/scripts/test-linux-system-package.sh b/scripts/test-linux-system-package.sh index e650091..8c4f563 100755 --- a/scripts/test-linux-system-package.sh +++ b/scripts/test-linux-system-package.sh @@ -126,6 +126,16 @@ cmp "$installed_icon" "$repo_root/src-tauri/icons/128x128@2x.png" || { exit 1 } grep -Fxq 'Icon=utterform' "$desktop_file" +# "Open with Utterform" for voice messages: the entry takes files and names +# Opus among its types. +grep -Eq '^Exec=.* %F$' "$desktop_file" || { + printf 'Utterform package test: desktop entry does not accept files\n' >&2 + exit 1 +} +grep -Eq '^MimeType=.*audio/ogg;' "$desktop_file" || { + printf 'Utterform package test: desktop entry does not offer to open audio files\n' >&2 + exit 1 +} printf 'stale reinstalled logo\n' > "$installed_icon" UTTERFORM_VERSION=ci \ diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index fd4253b..b40e3b4 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -1337,6 +1337,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fastrand" version = "2.5.0" @@ -2538,6 +2544,12 @@ dependencies = [ "zbus", ] +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + [[package]] name = "libappindicator" version = "0.9.0" @@ -3229,6 +3241,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "opusic-sys" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986d749e9dc12035f11bf3ecc83a2a696ff7534d4143be9469a4a0faa25b9fd1" +dependencies = [ + "cmake", +] + [[package]] name = "ordered-stream" version = "0.2.0" @@ -3510,6 +3531,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -3805,6 +3835,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" @@ -3940,6 +3976,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.4" @@ -4467,6 +4517,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.9.0" @@ -4514,6 +4570,215 @@ dependencies = [ "serde_json", ] +[[package]] +name = "symphonia" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-caf", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-adapter-libopus" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6febe6f88f9a9483db7e5b72a2dad916d8f8eb588d18905a39c861319fc7fa1" +dependencies = [ + "log", + "opusic-sys", + "symphonia-core", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6405d5c34ff6f8f7ca08a4101efe66d21e180f7322ef9359900a0e55698a7892" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ea5ffc8716bff677dfb3b01b420c7b758de901a72b8c330bf2040ab74b4add" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5bf8e39552d34a3c4c98333370e62f48c92456d2e814f273b5c3ad7c4a5f45c" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445932ecb0c59362fde9c082dd63a75380650c5077fdb8b80b8cac850442a74c" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "920a78f96f3cf62932d0c497959c0cc2100ef36b03c7ca1417b260f432a482d8" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04ba75686acbe43542fdd374571195f0530c0b7785ca25cc6840e9c6c4b6eea" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73d90b4fcf796137cc683c538282804ff9629f8ad9dbfd881fcbba331ac4e986" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" +dependencies = [ + "bitflags 2.13.1", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "rustfft", + "smallvec", +] + +[[package]] +name = "symphonia-format-caf" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab64327ee1920531c5bf86cf7e9cd1a599d57013ddc30691da62683cefc65191" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e681a70e1870d34e02abf1dbc51e4267c3f1827801474e8870be8c689fc4dc3" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d015c5c0558864665894b3f4cbd95e10abb01b9c868e751c72670f326a56360e" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5495e7f7e3c7035328d82b6d6e377eef289bb0c4105bdeb557fc93a833f994" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" @@ -5443,6 +5708,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.24.2" @@ -5650,6 +5925,8 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "symphonia", + "symphonia-adapter-libopus", "tauri", "tauri-build", "tauri-plugin-autostart", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 27e5ac6..97a4d78 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -40,6 +40,8 @@ tempfile = "3.23.0" whisper-rs = "0.16.0" tauri-plugin-global-shortcut = "2" tauri-plugin-autostart = "2.5.1" +symphonia = { version = "0.6.1", default-features = false, features = ["ogg", "mkv", "wav", "aiff", "flac", "mp3", "aac", "isomp4", "caf", "vorbis", "pcm", "adpcm", "alac", "opt-simd"] } +symphonia-adapter-libopus = "0.3.0" [target.'cfg(target_os = "linux")'.dependencies] # The same store `keyring` uses, named directly so its session can be replaced diff --git a/src-tauri/Info.plist b/src-tauri/Info.plist index 204e122..d54c78c 100644 --- a/src-tauri/Info.plist +++ b/src-tauri/Info.plist @@ -6,6 +6,26 @@ way in. lib.rs sets the matching activation policy at startup. --> LSUIElement + + CFBundleDocumentTypes + + + CFBundleTypeName + Audio file + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + LSItemContentTypes + + public.audio + public.mpeg-4-audio + org.xiph.opus + org.xiph.ogg-audio + + + NSMicrophoneUsageDescription Utterform needs microphone access to record speech for transcription. diff --git a/src-tauri/src/audio.rs b/src-tauri/src/audio.rs index 96b3514..99ecaf8 100644 --- a/src-tauri/src/audio.rs +++ b/src-tauri/src/audio.rs @@ -24,7 +24,7 @@ use crate::{ use serde::Serialize; const CHANNEL_CAPACITY: usize = 64; -const WHISPER_SAMPLE_RATE: u32 = 16_000; +pub(crate) const WHISPER_SAMPLE_RATE: u32 = 16_000; const LIVE_SAMPLE_RATE: u32 = 24_000; const LIVE_CHUNK_SAMPLES: usize = 480; // 20 ms, independent of microphone callbacks. const RECORDING_DIRECTORY: &str = "utterform"; @@ -140,7 +140,7 @@ pub struct RecordingArtifact { pub interrupted: bool, } -struct TemporaryRecording { +pub(crate) struct TemporaryRecording { path: Option, } @@ -149,7 +149,7 @@ impl TemporaryRecording { Self { path: Some(path) } } - fn into_path(mut self) -> Result { + pub(crate) fn into_path(mut self) -> Result { self.path .take() .ok_or_else(|| Failure::new("audio_writer", "Temporary recording path is unavailable")) @@ -164,6 +164,39 @@ impl Drop for TemporaryRecording { } } +/// An empty temporary WAV in Utterform's own folder, deleted again unless it +/// is handed on with [`TemporaryRecording::into_path`]. Named so the stale +/// cleanup at the next start recognises it. +pub(crate) fn new_temporary_wav() -> Result<(std::fs::File, TemporaryRecording), Failure> { + let directory = recording_directory(); + std::fs::create_dir_all(&directory).map_err(|error| { + Failure::io( + "audio_writer", + &error, + format!("Could not create the recording directory: {error}"), + ) + })?; + let temporary = tempfile::Builder::new() + .prefix("utterform-") + .suffix(".wav") + .tempfile_in(directory) + .map_err(|error| { + Failure::io( + "audio_writer", + &error, + format!("Could not create a temporary recording: {error}"), + ) + })?; + let (file, path) = temporary.keep().map_err(|error| { + Failure::io( + "audio_writer", + &error.error, + format!("Could not retain the temporary recording: {error}"), + ) + })?; + Ok((file, TemporaryRecording::new(path))) +} + /// Deletes a temporary WAV. A file that stays behind holds audio, so a /// failure is recorded — its kind, never its path. The next start's stale /// cleanup retries it after a day. @@ -185,6 +218,10 @@ impl RecordingArtifact { frames.saturating_mul(1_000) / u64::from(self.sample_rate.max(1)) } + /// The audio as Whisper wants it: mono, 16 kHz, `f32`. + /// + /// Read frame by frame, so a long audio file costs one buffer of its + /// length and not three. pub fn whisper_pcm(&self) -> Result, Failure> { let mut reader = hound::WavReader::open(&self.path).map_err(|error| { Failure::new( @@ -193,22 +230,28 @@ impl RecordingArtifact { ) })?; let channels = usize::from(reader.spec().channels.max(1)); - let samples: Result, _> = reader.samples::().collect(); - let samples = samples.map_err(|error| { + let decode = |error: hound::Error| { Failure::new( "recording_decode", format!("Could not decode the recording: {error}"), ) - })?; - - let mono = samples - .chunks(channels) - .map(|frame| { - let sum = frame.iter().map(|sample| f32::from(*sample)).sum::(); - sum / (frame.len() as f32 * f32::from(i16::MAX)) - }) - .collect::>(); - + }; + let scale = channels as f32 * f32::from(i16::MAX); + let mut mono = Vec::with_capacity(reader.duration() as usize); + let mut frame_sum = 0.0_f32; + let mut in_frame = 0; + for sample in reader.samples::() { + frame_sum += f32::from(sample.map_err(decode)?); + in_frame += 1; + if in_frame == channels { + mono.push(frame_sum / scale); + frame_sum = 0.0; + in_frame = 0; + } + } + if self.sample_rate == WHISPER_SAMPLE_RATE { + return Ok(mono); + } Ok(resample_linear( &mono, self.sample_rate, @@ -461,33 +504,7 @@ fn start_recording_inner( let mut live = live_sender.map(|sender| { LiveTap::new(sender, writer_config.sample_rate, writer_config.channels) }); - let directory = recording_directory(); - std::fs::create_dir_all(&directory).map_err(|error| { - Failure::io( - "audio_writer", - &error, - format!("Could not create the recording directory: {error}"), - ) - })?; - let temporary = tempfile::Builder::new() - .prefix("utterform-") - .suffix(".wav") - .tempfile_in(directory) - .map_err(|error| { - Failure::io( - "audio_writer", - &error, - format!("Could not create a temporary recording: {error}"), - ) - })?; - let (file, path) = temporary.keep().map_err(|error| { - Failure::io( - "audio_writer", - &error.error, - format!("Could not retain the temporary recording: {error}"), - ) - })?; - let temporary = TemporaryRecording::new(path); + let (file, temporary) = new_temporary_wav()?; let spec = hound::WavSpec { channels: writer_config.channels, sample_rate: writer_config.sample_rate, diff --git a/src-tauri/src/audio_file.rs b/src-tauri/src/audio_file.rs new file mode 100644 index 0000000..a19dbbf --- /dev/null +++ b/src-tauri/src/audio_file.rs @@ -0,0 +1,547 @@ +//! Audio files the user hands Utterform: a voice message, a recording. +//! +//! A file is decoded once into the same kind of temporary WAV a microphone +//! recording leaves behind — 16 kHz, mono, 16-bit — and from there takes the +//! recording's own path: transcription, the quiet retry, the kept recording, +//! history and delivery. Nothing downstream has to know where the audio came +//! from, and the copy is deleted like every other recording. +//! +//! Symphonia reads the containers and most codecs in pure Rust. Opus, which +//! WhatsApp and Telegram voice messages use, comes from libopus, compiled into +//! the binary, because Symphonia has no Opus decoder of its own. +//! +//! The file's name and path are the user's and never reach the log. + +use std::{ + fs::File, + io::BufWriter, + path::Path, + time::{Duration, Instant}, +}; + +use symphonia::core::{ + codecs::{audio::AudioDecoderOptions, registry::CodecRegistry}, + errors::Error as SymphoniaError, + formats::{FormatOptions, TrackType, probe::Hint}, + io::MediaSourceStream, + meta::MetadataOptions, +}; + +use crate::{ + audio::{self, RecordingArtifact, WHISPER_SAMPLE_RATE}, + diagnostics::{self, Failure}, +}; + +/// Long enough for a meeting, short enough that the decoded audio Whisper +/// reads stays within a few hundred megabytes. +pub const MAX_DURATION: Duration = Duration::from_secs(2 * 60 * 60); + +/// The decoded copy, and what the log may say about where it came from. +pub struct Decoded { + pub artifact: RecordingArtifact, + /// The codec, as Symphonia names it — `opus`, `aac`, `mp3`… + pub codec: String, + pub source_rate: u32, + pub source_channels: usize, + /// Packets that could not be decoded and were left out. + pub skipped_packets: u64, + pub decode_ms: u128, +} + +/// Decode `path` into a temporary 16 kHz mono WAV for recording `recording`. +/// +/// Blocking: run it off the async runtime. +pub fn decode(path: &Path, recording: u64) -> Result { + let began = Instant::now(); + let file = File::open(path).map_err(|error| { + let message = if error.kind() == std::io::ErrorKind::NotFound { + "The audio file could not be found." + } else { + "The audio file could not be opened." + }; + Failure::io("file_open", &error, message) + })?; + if file.metadata().is_ok_and(|metadata| metadata.is_dir()) { + return Err(Failure::guidance( + "file_not_audio", + "That is a folder, not an audio file.", + )); + } + + let mut hint = Hint::new(); + if let Some(extension) = path.extension().and_then(|value| value.to_str()) { + hint.with_extension(extension); + } + let source = MediaSourceStream::new(Box::new(file), Default::default()); + let mut format = symphonia::default::get_probe() + .probe( + &hint, + source, + FormatOptions::default(), + MetadataOptions::default(), + ) + .map_err(|error| { + Failure::guidance("file_not_audio", unreadable_message()).detail(error_kind(&error)) + })?; + let track = format + .default_track(TrackType::Audio) + .ok_or_else(|| Failure::guidance("file_no_audio", "The file contains no audio."))?; + let track_id = track.id; + let parameters = track + .codec_params + .as_ref() + .and_then(|parameters| parameters.audio()) + .ok_or_else(|| Failure::guidance("file_no_audio", "The file contains no audio."))? + .clone(); + let mut decoder = codecs() + .make_audio_decoder(¶meters, &AudioDecoderOptions::default()) + .map_err(|error| { + Failure::guidance("file_codec", unreadable_message()).detail(error_kind(&error)) + })?; + let codec = codecs() + .get_audio_decoder(parameters.codec) + .map(|decoder| decoder.codec.info.short_name.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + let (output, temporary) = audio::new_temporary_wav()?; + let spec = hound::WavSpec { + channels: 1, + sample_rate: WHISPER_SAMPLE_RATE, + bits_per_sample: 16, + sample_format: hound::SampleFormat::Int, + }; + let write_failed = |error: hound::Error| { + Failure::new( + "audio_writer", + format!("Could not write the decoded audio: {error}"), + ) + }; + let mut wav = hound::WavWriter::new(BufWriter::new(output), spec).map_err(write_failed)?; + + let limit = MAX_DURATION.as_secs() * u64::from(WHISPER_SAMPLE_RATE); + let mut written = 0_u64; + let mut resampler: Option = None; + let mut source_rate = 0; + let mut source_channels = 0; + let mut interleaved = Vec::::new(); + let mut decoded_packets = 0_u64; + let mut skipped_packets = 0_u64; + let mut write_error = None; + + loop { + let packet = match format.next_packet() { + Ok(Some(packet)) => packet, + // A chained Ogg stream starts over with new tracks; what came + // before is a whole message already. + Ok(None) | Err(SymphoniaError::ResetRequired) => break, + // A file cut off mid-packet still holds everything before the cut. + Err(SymphoniaError::IoError(error)) + if error.kind() == std::io::ErrorKind::UnexpectedEof => + { + break; + } + Err(error) => { + return Err( + Failure::new("file_decode", unreadable_message()).detail(error_kind(&error)) + ); + } + }; + if packet.track_id != track_id { + continue; + } + let buffer = match decoder.decode(&packet) { + Ok(buffer) => buffer, + Err(SymphoniaError::DecodeError(_) | SymphoniaError::IoError(_)) => { + skipped_packets += 1; + continue; + } + Err(error) => { + return Err( + Failure::new("file_decode", unreadable_message()).detail(error_kind(&error)) + ); + } + }; + decoded_packets += 1; + let channels = buffer.spec().channels().count().max(1); + let rate = buffer.spec().rate(); + if buffer.frames() == 0 || rate == 0 { + continue; + } + let resampler = resampler.get_or_insert_with(|| { + source_rate = rate; + source_channels = channels; + Resampler::new(rate, WHISPER_SAMPLE_RATE) + }); + interleaved.resize(buffer.samples_interleaved(), 0.0); + buffer.copy_to_slice_interleaved(&mut interleaved); + for frame in interleaved.chunks(channels) { + let mono = frame.iter().sum::() / frame.len() as f32; + resampler.push(mono, |sample| { + if write_error.is_none() { + if let Err(error) = wav.write_sample(to_pcm(sample)) { + write_error = Some(error); + } + written += 1; + } + }); + } + if let Some(error) = write_error.take() { + return Err(write_failed(error)); + } + if written > limit { + return Err(Failure::guidance( + "file_too_long", + format!( + "The audio file is longer than {} hours.", + MAX_DURATION.as_secs() / 3600 + ), + )); + } + } + if let Some(resampler) = resampler.as_mut() { + resampler.finish(|sample| { + if write_error.is_none() { + if let Err(error) = wav.write_sample(to_pcm(sample)) { + write_error = Some(error); + } + written += 1; + } + }); + } + if let Some(error) = write_error { + return Err(write_failed(error)); + } + wav.finalize().map_err(write_failed)?; + + if decoded_packets == 0 && skipped_packets > 0 { + return Err(Failure::new("file_decode", unreadable_message()) + .detail(format!("{skipped_packets} packets undecodable"))); + } + if written == 0 { + return Err(Failure::guidance( + "empty_recording", + "The audio file contains no sound.", + )); + } + if skipped_packets > 0 { + diagnostics::warning!( + "file.packets_skipped", + recording = recording, + skipped = skipped_packets, + decoded = decoded_packets + ); + } + + Ok(Decoded { + artifact: RecordingArtifact { + recording, + path: temporary.into_path()?, + sample_rate: WHISPER_SAMPLE_RATE, + channels: 1, + sample_count: written, + interrupted: false, + }, + codec, + source_rate, + source_channels, + skipped_packets, + decode_ms: began.elapsed().as_millis(), + }) +} + +fn unreadable_message() -> String { + "Utterform cannot read this file. Voice messages (Opus, Ogg), M4A, MP3, WAV and FLAC work." + .to_string() +} + +/// What kind of error Symphonia met, without anything it may have quoted. +fn error_kind(error: &SymphoniaError) -> &'static str { + match error { + SymphoniaError::IoError(_) => "io", + SymphoniaError::DecodeError(_) => "decode", + SymphoniaError::SeekError(_) => "seek", + SymphoniaError::Unsupported(_) => "unsupported", + SymphoniaError::LimitError(_) => "limit", + SymphoniaError::ResetRequired => "reset", + _ => "other", + } +} + +fn to_pcm(sample: f32) -> i16 { + (sample.clamp(-1.0, 1.0) * f32::from(i16::MAX)).round() as i16 +} + +/// Symphonia's decoders plus libopus. +fn codecs() -> &'static CodecRegistry { + static REGISTRY: std::sync::OnceLock = std::sync::OnceLock::new(); + REGISTRY.get_or_init(|| { + let mut registry = CodecRegistry::new(); + symphonia::default::register_enabled_codecs(&mut registry); + registry.register_audio_decoder::(); + registry + }) +} + +/// Streaming sample-rate conversion of one mono channel. +/// +/// Going down — 48 kHz Opus, 44.1 kHz MP3 — every output sample is the mean of +/// the input samples it covers, which keeps what lies above the new Nyquist +/// frequency from folding back into speech. Going up — 8 kHz telephone audio — +/// it interpolates linearly. Either way it holds a few samples, never the file. +struct Resampler { + /// Input samples per output sample. + ratio: f64, + seen: u64, + next_output: u64, + sum: f32, + count: u32, + previous: Option, +} + +impl Resampler { + fn new(source_rate: u32, target_rate: u32) -> Self { + Self { + ratio: f64::from(source_rate) / f64::from(target_rate), + seen: 0, + next_output: 0, + sum: 0.0, + count: 0, + previous: None, + } + } + + fn push(&mut self, sample: f32, mut emit: impl FnMut(f32)) { + if self.ratio >= 1.0 { + let bin = (self.seen as f64 / self.ratio) as u64; + if bin != self.next_output && self.count > 0 { + emit(self.sum / self.count as f32); + self.sum = 0.0; + self.count = 0; + } + self.next_output = bin; + self.sum += sample; + self.count += 1; + } else { + match self.previous { + None => { + emit(sample); + self.next_output = 1; + } + Some(previous) => { + let left = (self.seen - 1) as f64; + loop { + let position = self.next_output as f64 * self.ratio; + if position > self.seen as f64 { + break; + } + let fraction = (position - left) as f32; + emit(previous + (sample - previous) * fraction); + self.next_output += 1; + } + } + } + self.previous = Some(sample); + } + self.seen += 1; + } + + fn finish(&mut self, mut emit: impl FnMut(f32)) { + if self.ratio >= 1.0 && self.count > 0 { + emit(self.sum / self.count as f32); + self.count = 0; + self.sum = 0.0; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn resample(input: &[f32], from: u32, to: u32) -> Vec { + let mut resampler = Resampler::new(from, to); + let mut output = Vec::new(); + for sample in input { + resampler.push(*sample, |value| output.push(value)); + } + resampler.finish(|value| output.push(value)); + output + } + + #[test] + fn opus_rate_becomes_whisper_rate_by_averaging_each_three_samples() { + let input = [0.3, 0.6, 0.9, -0.3, -0.6, -0.9]; + let output = resample(&input, 48_000, 16_000); + assert_eq!(output.len(), 2); + assert!((output[0] - 0.6).abs() < 1e-6); + assert!((output[1] + 0.6).abs() < 1e-6); + } + + #[test] + fn the_length_follows_the_rate_for_odd_ratios() { + for (from, seconds) in [ + (44_100, 3), + (22_050, 2), + (16_000, 1), + (8_000, 2), + (11_025, 1), + ] { + let input = vec![0.1; (from * seconds) as usize]; + let output = resample(&input, from, 16_000); + let expected = 16_000 * seconds as usize; + assert!( + output.len().abs_diff(expected) <= 1, + "{from} Hz: {} instead of {expected}", + output.len() + ); + assert!(output.iter().all(|sample| (sample - 0.1).abs() < 1e-6)); + } + } + + #[test] + fn upsampling_interpolates_between_neighbours() { + let output = resample(&[0.0, 1.0], 8_000, 16_000); + assert_eq!(output, vec![0.0, 0.5, 1.0]); + } + + #[test] + fn a_tone_above_the_new_nyquist_does_not_fold_back_at_full_strength() { + // 12 kHz at 48 kHz: four samples per cycle. Averaging three of them + // leaves a third of the amplitude rather than all of it. + let input = (0..48_000) + .map(|index| (index as f32 * std::f32::consts::FRAC_PI_2).sin()) + .collect::>(); + let output = resample(&input, 48_000, 16_000); + let peak = output + .iter() + .fold(0.0_f32, |peak, sample| peak.max(sample.abs())); + assert!(peak < 0.4, "{peak}"); + } + + fn write_wav(path: &Path, rate: u32, channels: u16, frames: usize) { + let spec = hound::WavSpec { + channels, + sample_rate: rate, + bits_per_sample: 16, + sample_format: hound::SampleFormat::Int, + }; + let mut writer = hound::WavWriter::create(path, spec).unwrap(); + for index in 0..frames { + let value = ((index as f32 / rate as f32) * 440.0 * std::f32::consts::TAU).sin(); + for _ in 0..channels { + writer.write_sample((value * 8_000.0) as i16).unwrap(); + } + } + writer.finalize().unwrap(); + } + + #[test] + fn a_stereo_wav_becomes_whisper_audio_of_the_same_length() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("Sprachnachricht von Anna.wav"); + write_wav(&path, 44_100, 2, 44_100 * 2); + let decoded = decode(&path, 7).unwrap(); + assert_eq!(decoded.source_rate, 44_100); + assert_eq!(decoded.source_channels, 2); + assert_eq!(decoded.codec, "pcm_s16le"); + let artifact = &decoded.artifact; + assert_eq!(artifact.recording, 7); + assert_eq!(artifact.sample_rate, 16_000); + assert_eq!(artifact.channels, 1); + assert!(artifact.duration_ms().abs_diff(2_000) <= 1); + let pcm = artifact.whisper_pcm().unwrap(); + assert!(pcm.len().abs_diff(32_000) <= 1); + assert!(pcm.iter().any(|sample| sample.abs() > 0.1)); + // The copy is Utterform's own temporary file, never the user's. + assert_ne!(artifact.path, path); + let copy = artifact.path.clone(); + drop(decoded); + assert!(!copy.exists()); + assert!(path.exists()); + } + + #[test] + fn a_missing_file_says_so_without_naming_it() { + let failure = match decode(Path::new("/nowhere/Geheim.opus"), 1) { + Err(failure) => failure, + Ok(_) => panic!("decoded a missing file"), + }; + assert_eq!(failure.class(), "file_open"); + assert!(!failure.message().contains("Geheim")); + } + + #[test] + fn a_text_file_is_not_audio() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("notes.txt"); + std::fs::write(&path, "Dies ist keine Sprachnachricht.\n".repeat(200)).unwrap(); + let failure = match decode(&path, 1) { + Err(failure) => failure, + Ok(_) => panic!("decoded text as audio"), + }; + assert_eq!(failure.class(), "file_not_audio"); + assert!(!failure.message().contains("notes")); + } + + #[test] + fn a_folder_is_not_audio() { + let directory = tempfile::tempdir().unwrap(); + let failure = match decode(directory.path(), 1) { + Err(failure) => failure, + Ok(_) => panic!("decoded a folder"), + }; + assert!(matches!(failure.class(), "file_not_audio" | "file_open")); + } + + #[test] + fn silence_of_zero_length_is_an_empty_recording() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("empty.wav"); + write_wav(&path, 16_000, 1, 0); + let failure = match decode(&path, 1) { + Err(failure) => failure, + Ok(_) => panic!("decoded nothing into something"), + }; + assert!(matches!( + failure.class(), + "empty_recording" | "file_no_audio" | "file_not_audio" + )); + } + + /// A real WhatsApp-style voice message: Ogg Opus, mono, 48 kHz decode rate. + /// Generated once with `ffmpeg -f lavfi -i sine=440:d=1.5 -ac 1 -c:a libopus + /// -b:a 24k -application voip voice.opus`; 1.5 s of tone at ffmpeg's default + /// amplitude of 1/8, no voice. + #[test] + fn a_whatsapp_voice_message_decodes_through_libopus() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/voice.opus"); + let decoded = decode(&path, 3).unwrap(); + assert_eq!(decoded.codec, "opus"); + assert_eq!(decoded.source_rate, 48_000); + assert_eq!(decoded.source_channels, 1); + assert_eq!(decoded.skipped_packets, 0); + let duration = decoded.artifact.duration_ms(); + assert!(duration.abs_diff(1_500) <= 30, "{duration} ms"); + let pcm = decoded.artifact.whisper_pcm().unwrap(); + let peak = pcm + .iter() + .fold(0.0_f32, |peak, sample| peak.max(sample.abs())); + assert!((0.1..0.16).contains(&peak), "{peak}"); + } + + #[test] + fn an_m4a_from_a_phone_decodes() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/voice.m4a"); + let decoded = decode(&path, 4).unwrap(); + assert_eq!(decoded.codec, "aac"); + assert!(decoded.artifact.duration_ms().abs_diff(1_500) <= 60); + } + + #[test] + fn an_mp3_decodes() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/voice.mp3"); + let decoded = decode(&path, 5).unwrap(); + assert_eq!(decoded.codec, "mp3"); + assert!(decoded.artifact.duration_ms().abs_diff(1_500) <= 80); + } +} diff --git a/src-tauri/src/cli.rs b/src-tauri/src/cli.rs index 791be0e..3cfe05f 100644 --- a/src-tauri/src/cli.rs +++ b/src-tauri/src/cli.rs @@ -6,10 +6,15 @@ //! already running app. `utterform --toggle` from a compositor binding is //! therefore a real global hotkey, with no extra permissions or daemon. +use std::path::{Path, PathBuf}; + /// The argument the operating system's autostart entry carries. Internal: it /// exists so a start nobody asked for can be told apart from one that was. pub const AUTOSTART_FLAG: &str = "--autostart"; +/// `utterform --file ` transcribes an audio file. +pub const FILE_FLAG: &str = "--file"; + /// A request that arrives from a second process, or from the command line the /// app itself was started with. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize)] @@ -85,6 +90,84 @@ where .unwrap_or_default() } +/// The audio files a command line asks Utterform to transcribe, resolved +/// against the directory it was given in. +/// +/// `--file ` (or `--file=`) is the documented form. A bare +/// argument counts too, but only when it names an existing file: that is how +/// a file manager's "Open with" and Windows' file association hand a file +/// over, and it keeps `utterform toggle` meaning what it always did. +pub fn files_from(args: I, cwd: &Path) -> Vec +where + I: IntoIterator, + S: AsRef, +{ + let resolve = |value: &str| { + let path = PathBuf::from(value); + if path.is_absolute() { + path + } else { + cwd.join(path) + } + }; + let mut files = Vec::new(); + let mut args = args.into_iter().skip(1); + while let Some(arg) = args.next() { + let arg = arg.as_ref(); + if arg == FILE_FLAG { + if let Some(value) = args.next() { + files.push(resolve(value.as_ref())); + } + } else if let Some(value) = arg.strip_prefix("--file=") { + files.push(resolve(value)); + } else if let Some(url) = arg.strip_prefix("file://") { + // Some file managers pass a URL even for `%f`. + if let Some(path) = file_url_path(url) { + files.push(path); + } + } else if !arg.starts_with('-') && !arg.is_empty() { + let path = resolve(arg); + if path.is_file() { + files.push(path); + } + } + } + files +} + +/// The path of a `file://` URL's remainder, percent-decoded. A host other +/// than none or `localhost` is not a local file. +pub fn file_url_path(rest: &str) -> Option { + let path = rest.strip_prefix("localhost").unwrap_or(rest); + if !path.starts_with('/') { + return None; + } + let bytes = path.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' + && let Some(hex) = path.get(index + 1..index + 3) + && let Ok(byte) = u8::from_str_radix(hex, 16) + { + decoded.push(byte); + index += 3; + continue; + } + decoded.push(bytes[index]); + index += 1; + } + let text = String::from_utf8(decoded).ok()?; + // `file:///C:/Users/...` on Windows. + #[cfg(windows)] + let text = text + .strip_prefix('/') + .filter(|rest| rest.as_bytes().get(1) == Some(&b':')) + .map(str::to_string) + .unwrap_or(text); + Some(PathBuf::from(text)) +} + #[cfg(test)] mod tests { use super::*; @@ -164,4 +247,57 @@ mod tests { assert!(intent.reaches_the_interface(), "{intent:?}"); } } + + #[test] + fn a_file_flag_names_a_file_relative_to_where_it_was_typed() { + let cwd = Path::new("/home/anna/Downloads"); + assert_eq!( + files_from(["utterform", "--file", "voice.opus"], cwd), + vec![PathBuf::from("/home/anna/Downloads/voice.opus")] + ); + assert_eq!( + files_from(["utterform", "--file=/tmp/a b.m4a"], cwd), + vec![PathBuf::from("/tmp/a b.m4a")] + ); + assert!(files_from(["utterform", "--file"], cwd).is_empty()); + } + + #[test] + fn a_file_is_not_a_recording_action() { + assert_eq!( + intent_from(["utterform", "--file", "voice.opus"]), + Intent::Show + ); + } + + #[test] + fn a_bare_argument_is_a_file_only_when_one_exists() { + let directory = tempfile::tempdir().unwrap(); + let voice = directory.path().join("Sprachnachricht.opus"); + std::fs::write(&voice, b"OggS").unwrap(); + assert_eq!( + files_from(["utterform", "Sprachnachricht.opus"], directory.path()), + vec![voice.clone()] + ); + assert!(files_from(["utterform", "toggle"], directory.path()).is_empty()); + assert!(files_from(["utterform", "--toggle"], directory.path()).is_empty()); + // The executable itself is never taken for a file. + assert!(files_from([voice.to_str().unwrap()], directory.path()).is_empty()); + } + + #[test] + fn a_file_url_becomes_a_path() { + assert_eq!( + files_from( + ["utterform", "file:///home/anna/Voice%20note.ogg"], + Path::new("/") + ), + vec![PathBuf::from("/home/anna/Voice note.ogg")] + ); + assert_eq!( + file_url_path("localhost/tmp/x.opus"), + Some(PathBuf::from("/tmp/x.opus")) + ); + assert_eq!(file_url_path("server/share/x.opus"), None); + } } diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 7a8a6e0..58c9756 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -11,6 +11,7 @@ use crate::{ actions::{self, BuiltInAction}, activation::reveal_main_window, audio::{self, AudioCaptureState, RecordingArtifact}, + audio_file, diagnostics::{self, Failure}, domain::{ AppSettings, AudioDeviceInfo, LocalModelInfo, ProcessRequest, ProcessResult, TextProvider, @@ -479,7 +480,15 @@ pub async fn finish_recording( let artifact = artifact.map_err(|failure| { diagnostics::failure!("recording.failed", &failure, stage = "capture") })?; - process_recording(&app, artifact, request, current_settings, began).await + process_recording( + &app, + artifact, + request, + current_settings, + began, + Origin::Microphone, + ) + .await } /// Worth one quiet second attempt: the connection dropped or the service said @@ -516,12 +525,22 @@ async fn transcribe_with_retry( } } +/// Where the audio being processed came from. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Origin { + Microphone, + /// An audio file the user opened. It stays where it is, so a failed + /// transcription keeps no copy: opening the file again is the retry. + File, +} + async fn process_recording( app: &AppHandle, artifact: RecordingArtifact, request: ProcessRequest, current_settings: AppSettings, began: Instant, + origin: Origin, ) -> Result { let recording = artifact.recording; let duration_ms = artifact.duration_ms(); @@ -560,7 +579,7 @@ async fn process_recording( audio_ms = duration_ms, elapsed_ms = transcribing.elapsed().as_millis() ); - if !worth_keeping(&failure) { + if !worth_keeping(&failure) || origin == Origin::File { return Err(message); } app.state::() @@ -690,7 +709,11 @@ async fn process_recording( diagnostics::info!( "recording.completed", recording = recording, - mode = "batch", + mode = if origin == Origin::File { + "file" + } else { + "batch" + }, total_ms = began.elapsed().as_millis(), warnings = warnings.len(), done_cue = done_cue @@ -708,6 +731,127 @@ async fn process_recording( }) } +/// How an audio file reached Utterform, for the log. Anything the interface +/// did not name is `other`, so the field stays a short closed set. +fn file_source(source: &str) -> &'static str { + match source { + "drop" => "drop", + "open" => "open", + "command_line" => "command_line", + "open_with" => "open_with", + _ => "other", + } +} + +/// Only one file at a time: a second request while one is decoding or being +/// transcribed is refused rather than queued behind minutes of work. +#[derive(Default)] +pub struct FileBusy(AtomicBool); + +struct FileClaim<'a>(&'a AtomicBool); + +impl Drop for FileClaim<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +/// Transcribe an audio file — a voice message, a recording — the way a +/// dictation is: same engine, same action, history first, then the clipboard. +/// +/// Never typed at the cursor: the file was chosen in Utterform or a file +/// manager, so no text field is waiting for it. The path is the user's and is +/// never logged; the decoded copy is deleted like any recording. +#[tauri::command] +pub async fn transcribe_file( + app: AppHandle, + capture: State<'_, AudioCaptureState>, + busy: State<'_, FileBusy>, + path: String, + source: String, + request: ProcessRequest, +) -> Result { + let source = file_source(&source); + let recording = diagnostics::next_recording_id(); + let refuse = |failure: Failure| { + diagnostics::failure!( + "file.failed", + &failure, + recording = recording, + source = source, + stage = "start" + ) + }; + let capturing = app.state::().active() + || audio::status(&capture).is_ok_and(|status| status.recording); + if capturing { + return Err(refuse(Failure::guidance( + "recording_active", + "Finish the recording first, then open the file again.", + ))); + } + if busy.0.swap(true, Ordering::AcqRel) { + return Err(refuse(Failure::guidance( + "file_busy", + "Another audio file is being transcribed. Try again when it is done.", + ))); + } + let _claim = FileClaim(&busy.0); + let began = Instant::now(); + let current_settings = settings::load(&app).map_err(|failure| { + diagnostics::failure!("settings.load_failed", &failure, during = "file.start") + })?; + diagnostics::info!("file.started", recording = recording, source = source); + tray::adopt(&app, recording); + let path = std::path::PathBuf::from(path); + let decoded = + tauri::async_runtime::spawn_blocking(move || audio_file::decode(&path, recording)) + .await + .map_err(|error| Failure::new("worker", format!("Could not read the file: {error}"))) + .and_then(|outcome| outcome) + .map_err(|failure| { + diagnostics::failure!( + "file.failed", + &failure, + recording = recording, + source = source, + stage = "decode", + elapsed_ms = began.elapsed().as_millis() + ) + })?; + diagnostics::info!( + "file.decoded", + recording = recording, + codec = decoded.codec, + source_rate = decoded.source_rate, + source_channels = decoded.source_channels, + skipped_packets = decoded.skipped_packets, + audio_ms = decoded.artifact.duration_ms(), + decode_ms = decoded.decode_ms + ); + let request = ProcessRequest { + type_at_cursor: false, + copy_to_clipboard: true, + ..request + }; + process_recording( + &app, + decoded.artifact, + request, + current_settings, + began, + Origin::File, + ) + .await +} + +/// The next audio file handed in from outside, if any. A query: the request +/// was logged when it arrived. +#[tauri::command] +pub fn take_pending_file(state: State<'_, crate::PendingFiles>) -> Option { + state.take().map(|path| path.to_string_lossy().into_owned()) +} + /// The kept recording next in line, if any. A query: nothing is logged. #[tauri::command] pub fn retained_recording(state: State<'_, RetainedRecordings>) -> Option { @@ -747,7 +891,15 @@ pub async fn retry_recording(app: AppHandle) -> Result { copy_to_clipboard: true, ..request }; - let outcome = process_recording(&app, artifact, request, current_settings, began).await; + let outcome = process_recording( + &app, + artifact, + request, + current_settings, + began, + Origin::Microphone, + ) + .await; tray::set_kept(&app, app.state::().waiting().is_some()); outcome } @@ -930,6 +1082,14 @@ mod tests { } } + #[test] + fn a_file_source_is_one_of_a_few_words() { + assert_eq!(file_source("drop"), "drop"); + assert_eq!(file_source("open"), "open"); + assert_eq!(file_source("command_line"), "command_line"); + assert_eq!(file_source("/home/anna/Sprachnachricht.opus"), "other"); + } + #[test] fn only_a_dropped_connection_or_a_busy_service_earns_the_quiet_retry() { assert!(is_transient(&Failure::new("connect", ""))); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index e0ec4ea..ae976b1 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,6 +1,7 @@ mod actions; mod activation; mod audio; +mod audio_file; mod autostart; mod cli; mod commands; @@ -50,6 +51,57 @@ impl StartupIntent { } } +/// Audio files handed to Utterform from outside — the command line, a file +/// manager's "Open with", macOS' open event — waiting for the interface to +/// transcribe them one after another. The interface is told with a +/// `file-waiting` event and takes them; one that is not ready yet takes them +/// when it starts. +#[derive(Default)] +pub struct PendingFiles(std::sync::Mutex>); + +impl PendingFiles { + /// More than this at once is a mistaken selection, not a stack of voice + /// messages; the rest is dropped and said so in the log. + const LIMIT: usize = 20; + + fn queue(&self) -> std::sync::MutexGuard<'_, std::collections::VecDeque> { + self.0.lock().unwrap_or_else(|poison| poison.into_inner()) + } + + pub fn take(&self) -> Option { + self.queue().pop_front() + } + + /// Queue `files` and tell the interface. The paths are never logged. + fn offer( + &self, + app: &tauri::AppHandle, + files: Vec, + source: &'static str, + ) { + if files.is_empty() { + return; + } + let count = files.len(); + let dropped = { + let mut queue = self.queue(); + let room = Self::LIMIT.saturating_sub(queue.len()); + let dropped = count.saturating_sub(room); + queue.extend(files.into_iter().take(room)); + dropped + }; + diagnostics::info!("file.requested", source = source, files = count); + if dropped > 0 { + diagnostics::warning!( + "file.requests_dropped", + dropped = dropped, + limit = Self::LIMIT + ); + } + diagnostics::notify_interface(app, "file-waiting", ()); + } +} + /// The operating system's own "start when I sign in" entry. /// /// Nothing about it is stored in `settings.json`: the entry itself is the only @@ -162,6 +214,9 @@ pub fn run() { // Read before the window and the event loop exist: on Linux an autostart // launch has to settle its environment before GTK is initialized. let startup_intent = cli::intent_from(std::env::args()); + let startup_files = std::env::current_dir() + .map(|cwd| cli::files_from(std::env::args(), &cwd)) + .unwrap_or_default(); #[cfg(target_os = "linux")] prepare_autostart_environment(startup_intent); @@ -169,7 +224,8 @@ pub fn run() { // Registered first so a launcher entry or a second `utterform` process // hands its arguments to the running app instead of starting a rival // instance with its own tray icon and recording state. - .plugin(tauri_plugin_single_instance::init(|app, args, _cwd| { + .plugin(tauri_plugin_single_instance::init(|app, args, cwd| { + let files = cli::files_from(&args, std::path::Path::new(&cwd)); let intent = cli::intent_from(args); diagnostics::info!("app.second_instance", intent = intent.as_str()); if intent.raises_window() { @@ -181,6 +237,8 @@ pub fn run() { if intent.reaches_the_interface() { diagnostics::notify_interface(app, "remote-intent", intent); } + app.state::() + .offer(app, files, "command_line"); })) .plugin(autostart()) .plugin(tauri_plugin_dialog::init()) @@ -197,10 +255,12 @@ pub fn run() { ) .manage(audio::AudioCaptureState::default()) .manage(retained::RetainedRecordings::default()) + .manage(commands::FileBusy::default()) .manage(feedback::DoneCues::default()) .manage(tray::TrayState::default()) .manage(live::LiveState::default()) .manage(StartupIntent::default()) + .manage(PendingFiles::default()) .manage(history::HistoryState::default()) .manage(hotkey::Failure::default()) .setup(move |app| { @@ -209,6 +269,8 @@ pub fn run() { diagnostics::install(app.handle()); let began = std::time::Instant::now(); let outcome = setup(app, startup_intent); + app.state::() + .offer(app.handle(), startup_files, "command_line"); if outcome.is_ok() { diagnostics::info!( "app.setup_completed", @@ -255,6 +317,8 @@ pub fn run() { commands::set_recording_paused, commands::cancel_recording, commands::finish_recording, + commands::transcribe_file, + commands::take_pending_file, commands::retained_recording, commands::retry_recording, commands::discard_recording, @@ -283,6 +347,20 @@ pub fn run() { if let tauri::RunEvent::Reopen { .. } = &event { reveal_main_window(app, "reopen"); } + // "Open with" and a file dropped on the app's icon arrive as an + // open event on macOS, never as arguments. + #[cfg(target_os = "macos")] + if let tauri::RunEvent::Opened { urls } = &event { + let files = urls + .iter() + .filter(|url| url.scheme() == "file") + .filter_map(|url| url.to_file_path().ok()) + .collect::>(); + if !files.is_empty() { + reveal_main_window(app, "open_file"); + } + app.state::().offer(app, files, "open_with"); + } // The one clean ending: tray Quit arrives here after capture was // discarded. Anything that never gets here leaves the run marker. if let tauri::RunEvent::Exit = &event { diff --git a/src-tauri/src/transcription/openai.rs b/src-tauri/src/transcription/openai.rs index 8d637df..782fdab 100644 --- a/src-tauri/src/transcription/openai.rs +++ b/src-tauri/src/transcription/openai.rs @@ -4,12 +4,21 @@ use reqwest::{Client, StatusCode, multipart}; use serde::Deserialize; use crate::{ - actions, audio::RecordingArtifact, diagnostics::Failure, domain::AppSettings, secrets, + actions, + audio::RecordingArtifact, + diagnostics::{self, Failure}, + domain::AppSettings, + secrets, }; const TRANSCRIPTIONS_URL: &str = "https://api.openai.com/v1/audio/transcriptions"; const RESPONSES_URL: &str = "https://api.openai.com/v1/responses"; const MAX_UPLOAD_BYTES: u64 = 25 * 1024 * 1024; +/// 16 kHz, 16-bit mono is 32 kB a second: twelve minutes are 23 MB. +const SINGLE_UPLOAD_SAMPLES: usize = 12 * 60 * 16_000; +const PART_SAMPLES: usize = 10 * 60 * 16_000; +const CUT_SEARCH_SAMPLES: usize = 30 * 16_000; +const CUT_WINDOW_SAMPLES: usize = 1_600; const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); // A ten-minute recording is a ~19 MB upload that the model then has to work // through, so a two-minute cap on the whole exchange cut off legitimate long @@ -56,7 +65,44 @@ pub async fn transcribe( artifact: &RecordingArtifact, settings: &AppSettings, ) -> Result { - let audio = normalized_wav(artifact)?; + let pcm = artifact.whisper_pcm()?; + let parts = split_at_pauses(&pcm); + if parts.len() > 1 { + diagnostics::info!( + "transcription.split", + recording = artifact.recording, + parts = parts.len() + ); + } + let mut texts = Vec::with_capacity(parts.len()); + for (index, part) in parts.iter().enumerate() { + match transcribe_part(part, settings).await { + Ok(text) => texts.push(text), + // A long file can hold minutes of silence; that part simply adds + // nothing. A single part that is silent is the whole answer. + Err(failure) if failure.class() == "empty_result" && parts.len() > 1 => {} + Err(failure) => { + if parts.len() > 1 { + diagnostics::warning!( + "transcription.part_failed", + recording = artifact.recording, + part = index + 1, + parts = parts.len(), + class = failure.class() + ); + } + return Err(failure); + } + } + } + if texts.is_empty() { + return Err(Failure::guidance("empty_result", super::NOTHING_HEARD)); + } + Ok(texts.join(" ")) +} + +async fn transcribe_part(samples: &[f32], settings: &AppSettings) -> Result { + let audio = wav_bytes(samples)?; if audio.len() as u64 > MAX_UPLOAD_BYTES { return Err(Failure::guidance( "upload_too_large", @@ -214,8 +260,46 @@ pub(super) fn transport_class(error: &reqwest::Error) -> &'static str { } } -fn normalized_wav(artifact: &RecordingArtifact) -> Result, Failure> { - let samples = artifact.whisper_pcm()?; +/// Where a long recording is cut for upload, as `[start, end)` ranges of +/// 16 kHz samples. +/// +/// Anything up to [`SINGLE_UPLOAD_SAMPLES`] goes up whole, as every microphone +/// recording does. Longer audio — a file — is cut roughly every +/// [`PART_SAMPLES`], each cut moved to the quietest tenth of a second in the +/// half minute before it, so a word is rarely split in two. +fn split_at_pauses(samples: &[f32]) -> Vec<&[f32]> { + if samples.len() <= SINGLE_UPLOAD_SAMPLES { + return vec![samples]; + } + let mut parts = Vec::new(); + let mut start = 0; + while samples.len() - start > SINGLE_UPLOAD_SAMPLES { + let target = start + PART_SAMPLES; + let cut = quietest_window(samples, target - CUT_SEARCH_SAMPLES, target); + parts.push(&samples[start..cut]); + start = cut; + } + parts.push(&samples[start..]); + parts +} + +/// The start of the quietest [`CUT_WINDOW_SAMPLES`] window in `from..to`. +fn quietest_window(samples: &[f32], from: usize, to: usize) -> usize { + (from..to.saturating_sub(CUT_WINDOW_SAMPLES)) + .step_by(CUT_WINDOW_SAMPLES / 2) + .min_by(|left, right| { + let energy = |at: usize| { + samples[at..at + CUT_WINDOW_SAMPLES] + .iter() + .map(|sample| sample * sample) + .sum::() + }; + energy(*left).total_cmp(&energy(*right)) + }) + .map_or(to, |at| at + CUT_WINDOW_SAMPLES / 2) +} + +fn wav_bytes(samples: &[f32]) -> Result, Failure> { let mut cursor = Cursor::new(Vec::new()); { let spec = hound::WavSpec { @@ -299,6 +383,73 @@ async fn api_error(status: StatusCode, response: reqwest::Response) -> Failure { mod tests { use super::*; + /// The whole cloud path for a real file with the stored key: + /// `UTTERFORM_LIVE_AUDIO= cargo test --lib live_openai -- --ignored --nocapture`. + /// Costs a few cents per ten minutes of audio; never run by CI. + #[tokio::test] + #[ignore = "uploads audio to OpenAI with the stored key"] + async fn live_openai_transcribes_an_audio_file() { + let path = std::env::var("UTTERFORM_LIVE_AUDIO").expect("set UTTERFORM_LIVE_AUDIO"); + let decoded = crate::audio_file::decode(std::path::Path::new(&path), 1).unwrap(); + let settings = AppSettings { + language_hints: vec!["de".into()], + ..AppSettings::default() + }; + let pcm = decoded.artifact.whisper_pcm().unwrap(); + for (index, part) in split_at_pauses(&pcm).iter().enumerate() { + let began = std::time::Instant::now(); + let text = transcribe_part(part, &settings).await.unwrap(); + println!( + "part {}: {} s audio, {} ms, {} characters: {text}", + index + 1, + part.len() / 16_000, + began.elapsed().as_millis(), + text.chars().count() + ); + } + let text = transcribe(&decoded.artifact, &settings).await.unwrap(); + assert!(!text.trim().is_empty()); + } + + #[test] + fn a_microphone_recording_goes_up_whole() { + let ten_minutes = vec![0.1; 10 * 60 * 16_000]; + assert_eq!(split_at_pauses(&ten_minutes).len(), 1); + let twelve_minutes = vec![0.1; SINGLE_UPLOAD_SAMPLES]; + assert_eq!(split_at_pauses(&twelve_minutes).len(), 1); + } + + #[test] + fn a_long_file_is_cut_into_parts_that_each_fit_one_upload() { + let half_hour = vec![0.1; 30 * 60 * 16_000 + 123]; + let parts = split_at_pauses(&half_hour); + assert_eq!(parts.len(), 3); + assert_eq!( + parts.iter().map(|part| part.len()).sum::(), + half_hour.len() + ); + for part in &parts { + assert!(part.len() <= SINGLE_UPLOAD_SAMPLES); + assert!(wav_bytes(part).unwrap().len() as u64 <= MAX_UPLOAD_BYTES); + } + } + + #[test] + fn a_cut_lands_in_the_pause_before_the_ten_minute_mark() { + let mut audio = vec![0.4; 25 * 60 * 16_000]; + // A second of silence twenty seconds before the mark. + let pause = PART_SAMPLES - 20 * 16_000; + audio[pause..pause + 16_000].fill(0.0); + let parts = split_at_pauses(&audio); + let cut = parts[0].len(); + assert!(cut > pause && cut < pause + 16_000, "cut at {cut}"); + assert!( + audio[cut - 400..cut + 400] + .iter() + .all(|sample| *sample == 0.0) + ); + } + #[test] fn plain_is_not_a_transform_action() { assert!(action_instructions("plain", None, &AppSettings::default()).is_err()); diff --git a/src-tauri/tests/fixtures/voice.m4a b/src-tauri/tests/fixtures/voice.m4a new file mode 100644 index 0000000..9a610b6 Binary files /dev/null and b/src-tauri/tests/fixtures/voice.m4a differ diff --git a/src-tauri/tests/fixtures/voice.mp3 b/src-tauri/tests/fixtures/voice.mp3 new file mode 100644 index 0000000..853e522 Binary files /dev/null and b/src-tauri/tests/fixtures/voice.mp3 differ diff --git a/src-tauri/tests/fixtures/voice.opus b/src-tauri/tests/fixtures/voice.opus new file mode 100644 index 0000000..745fc01 Binary files /dev/null and b/src-tauri/tests/fixtures/voice.opus differ diff --git a/src/App.svelte b/src/App.svelte index 1a12a5f..a796dae 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -4,6 +4,7 @@ import { isTauri } from "@tauri-apps/api/core"; import { open } from "@tauri-apps/plugin-dialog"; import { + AUDIO_FILE_EXTENSIONS, BUILT_IN_ACTIONS, DEFAULT_SETTINGS, FORBIDDEN_IN_KEYWORD, @@ -19,6 +20,7 @@ CloudModel, CodexStatus, DownloadProgress, + FileSource, HistoryEntry, HotkeySupport, LocalModel, @@ -163,6 +165,12 @@ // that fails has already recorded itself natively. const failures = createFailureReporter(() => phase); let unlistenCueTest: UnlistenFn | undefined; + let unlistenFiles: UnlistenFn[] = []; + /// A file is being dragged over the window and would be taken if dropped. + let dragActive = false; + /// Files from the command line or "Open with" are waiting in the backend. + let filesWaiting = false; + let takingFiles = false; const apple = isApplePlatform(); const copyShortcut = apple ? "⌘⇧C" : "Ctrl+Shift+C"; let message = ""; @@ -256,6 +264,11 @@ $: settingsEmpty = settingsSearching && shownRows.size === 0; $: todaySeconds = Math.round(history.filter((entry) => new Date(historyTimestamp(entry) ?? NaN).toDateString() === new Date(now).toDateString()).reduce((total, entry) => total + entry.durationMs, 0) / 1000); $: controlsLocked = phase === "starting" || recordingActive || phase === "processing"; + $: canTakeFile = !controlsLocked && !showSettings && !transcriptionSaving; + $: fileActionLabel = actionOptions.find((option) => option.value === effectiveAction)?.label ?? "Plain"; + // Live dictation has no file mode: a file always goes to GPT Transcribe. + $: fileEngineLabel = settings.engine === "open_ai" ? "GPT Transcribe" : "Local"; + $: if (filesWaiting && canTakeFile) void takeWaitingFiles(); onMount(async () => { applyTheme(settings.theme); @@ -316,12 +329,25 @@ cueTestPending = false; cueTestMessage = event.payload; }); + unlistenFiles = await Promise.all([ + listen("file-waiting", () => { filesWaiting = true; }), + listen<{ paths: string[] }>("tauri://drag-enter", (event) => { + dragActive = canTakeFile && event.payload.paths.length > 0; + }), + listen("tauri://drag-leave", () => { dragActive = false; }), + listen<{ paths: string[] }>("tauri://drag-drop", (event) => { + const taken = dragActive; + dragActive = false; + if (taken) void transcribeFiles(event.payload.paths, "drop"); + }), + ]); if (destroyed) { unlistenProgress?.(); unlistenLimit?.(); unlistenLiveFailed?.(); unlistenIntent?.(); unlistenCueTest?.(); + unlistenFiles.forEach((unlisten) => unlisten()); return; } await refreshDiagnosticsInfo(); @@ -332,6 +358,8 @@ } catch (error) { setError(error); } + // A file Utterform was started with waits in the same queue. + filesWaiting = true; }); onDestroy(() => { @@ -344,6 +372,7 @@ unlistenLiveFailed?.(); unlistenIntent?.(); unlistenCueTest?.(); + unlistenFiles.forEach((unlisten) => unlisten()); document.removeEventListener("visibilitychange", handleVisibilityChange); window.removeEventListener("keydown", handleKeyDown); window.removeEventListener("error", failures.onError); @@ -510,6 +539,10 @@ if (!liveMode && event.key.toLowerCase() === "t") { settings = { ...settings, type_at_cursor: !settings.type_at_cursor }; } + if (event.key.toLowerCase() === "o" && !event.repeat) { + event.preventDefault(); + void openAudioFile(); + } } } @@ -677,6 +710,80 @@ } } + /// A voice message or recording, with the action and engine shown in the + /// window. The text goes to history and the clipboard, never to the cursor: + /// no text field is waiting for it. + async function transcribeFile(path: string, source: FileSource): Promise { + if (!canTakeFile) return false; + stopTimer(); + elapsedSeconds = 0; + phase = "processing"; + message = settings.engine === "open_ai" ? "Transcribing the file with GPT Transcribe…" : "Transcribing the file locally…"; + try { + showResult(await api.transcribeFile(path, source, { + action: effectiveAction.startsWith("custom:") ? "custom" : effectiveAction, + customPrompt: effectiveAction.startsWith("custom:") + ? settings.custom_actions.find((action) => `custom:${action.id}` === selectedAction)?.prompt ?? null + : null, + copyToClipboard: true, + saveToFile: settings.save_to_file, + typeAtCursor: false, + outputFormat: settings.output_format, + })); + return true; + } catch (error) { + setError(error); + return false; + } + } + + /// Several files are done one after another; the first that fails stops the + /// rest, so its message stays on screen. + async function transcribeFiles(paths: string[], source: FileSource) { + for (const path of paths) { + if (destroyed || !(await transcribeFile(path, source))) return; + } + } + + async function openAudioFile() { + if (!canTakeFile || !isTauri()) return; + let selected: string | string[] | null; + try { + selected = await open({ + multiple: false, + directory: false, + title: "Open audio file", + filters: [{ name: "Audio", extensions: AUDIO_FILE_EXTENSIONS }], + }); + } catch (error) { + // The dialog plugin answers the interface alone; no command saw this. + void failures.report("dialog", error); + return; + } + if (typeof selected === "string") await transcribeFile(selected, "open"); + } + + /// Drain the files handed in from outside while nothing else is running. + async function takeWaitingFiles() { + if (takingFiles) return; + takingFiles = true; + try { + while (!destroyed && canTakeFile) { + const path = await api.takePendingFile(); + if (!path) { + filesWaiting = false; + break; + } + if (!(await transcribeFile(path, "command_line"))) break; + } + } catch (error) { + filesWaiting = false; + setError(error); + } finally { + takingFiles = false; + } + } + function showResult(value: ProcessResult) { result = value; now = Date.now(); @@ -1240,6 +1347,9 @@ utterform
{version} +
@@ -1360,7 +1470,14 @@ {/if} {#if historyMessage}

{historyMessage}

{/if} -
{recordingActive ? "P " + (phase === "paused" ? "resumes" : "pauses") + (liveMode ? " · Esc stops" : " · Esc discards") : liveMode ? "Live dictation · Plain only" : "1–6 select an action"}{formatTime(todaySeconds)} saved audio today
+
{recordingActive ? "P " + (phase === "paused" ? "resumes" : "pauses") + (liveMode ? " · Esc stops" : " · Esc discards") : liveMode ? "Live dictation · Plain only" : "1–6 select an action · O open audio file"}{formatTime(todaySeconds)} saved audio today
+ {#if dragActive} +
+ + Drop to transcribe + {fileActionLabel} · {fileEngineLabel} → Clipboard +
+ {/if} {#snippet rowHead(id: SettingsRowId, label: string, forId: string | null = null)} diff --git a/src/App.test.ts b/src/App.test.ts index 1cc2802..4cde8ca 100644 --- a/src/App.test.ts +++ b/src/App.test.ts @@ -10,6 +10,7 @@ import { DEFAULT_SETTINGS } from "./lib/types"; import type { BuiltInAction, HistoryEntry } from "./lib/types"; import App from "./App.svelte"; import { api } from "./lib/api"; +import { open } from "@tauri-apps/plugin-dialog"; vi.mock("@tauri-apps/api/core", () => ({ isTauri: () => true })); const listeners = new Map void>(); @@ -41,6 +42,7 @@ vi.mock("./lib/api", () => ({ api: { playTestCues: vi.fn(), diagnosticsInfo: vi.fn(), copyDiagnostics: vi.fn(), openLogFile: vi.fn(), openLogFolder: vi.fn(), reportFrontendError: vi.fn(), + transcribeFile: vi.fn(), takePendingFile: vi.fn(async () => null), } })); // What the backend answers with: the prompts live in Rust, so the interface @@ -1648,3 +1650,109 @@ describe("failures only the interface sees", () => { }); }); }); + +describe("an audio file", () => { + const fromFile = { historyEntry: { ...latest, id: "3", text: "Bring the blue folder on Monday." }, text: "Bring the blue folder on Monday.", savedPath: null, copiedToClipboard: true, typedAtCursor: false, deliveryWarnings: [], durationMs: 63_000, engine: "open_ai" as const }; + const voiceMessage = "/home/tester/Downloads/PTT-20261107-WA0003.opus"; + + beforeEach(() => { + vi.mocked(api.transcribeFile).mockReset().mockResolvedValue(fromFile); + vi.mocked(api.takePendingFile).mockReset().mockResolvedValue(null); + vi.mocked(open).mockReset(); + }); + + async function ready() { + const view = render(App); + await waitFor(() => expect(view.queryByRole("button", { name: /Latest text/ })).not.toBeNull()); + await waitFor(() => expect(listeners.has("tauri://drag-drop")).toBe(true)); + return view; + } + + it("opens from the button and goes to history and the clipboard, never the cursor", async () => { + vi.mocked(open).mockResolvedValueOnce(voiceMessage); + const view = await ready(); + await fireEvent.click(view.getByRole("button", { name: "Open audio file" })); + await waitFor(() => expect(api.transcribeFile).toHaveBeenCalledTimes(1)); + expect(vi.mocked(open).mock.calls[0][0]).toMatchObject({ multiple: false, directory: false }); + expect(vi.mocked(open).mock.calls[0][0]?.filters?.[0].extensions).toContain("opus"); + const [path, source, request] = vi.mocked(api.transcribeFile).mock.calls[0]; + expect(path).toBe(voiceMessage); + expect(source).toBe("open"); + expect(request).toMatchObject({ action: "plain", copyToClipboard: true, typeAtCursor: false }); + await waitFor(() => expect(view.queryByText("Copied to clipboard")).not.toBeNull()); + await fireEvent.click(view.getByRole("button", { name: /Latest text/ })); + expect(view.getByRole("region", { name: "Transcript" }).textContent).toContain("blue folder"); + }); + + it("opens with the O key, and a cancelled dialog changes nothing", async () => { + vi.mocked(open).mockResolvedValueOnce(null); + const view = await ready(); + await fireEvent.keyDown(window, { key: "o", code: "KeyO" }); + await waitFor(() => expect(open).toHaveBeenCalledTimes(1)); + expect(api.transcribeFile).not.toHaveBeenCalled(); + expect(view.queryByText("Attention")).toBeNull(); + }); + + it("applies the chosen action", async () => { + vi.mocked(open).mockResolvedValueOnce(voiceMessage); + const view = await ready(); + await fireEvent.keyDown(window, { key: "2" }); + await fireEvent.click(view.getByRole("button", { name: "Open audio file" })); + await waitFor(() => expect(api.transcribeFile).toHaveBeenCalledTimes(1)); + expect(vi.mocked(api.transcribeFile).mock.calls[0][2].action).toBe("clean"); + }); + + it("shows where a dragged file will go and transcribes it when dropped", async () => { + const view = await ready(); + listeners.get("tauri://drag-enter")!({ payload: { paths: [voiceMessage], position: { x: 1, y: 1 } } }); + await waitFor(() => expect(view.queryByText("Drop to transcribe")).not.toBeNull()); + expect(view.getByText(/Plain · GPT Transcribe → Clipboard/)).not.toBeNull(); + listeners.get("tauri://drag-drop")!({ payload: { paths: [voiceMessage], position: { x: 1, y: 1 } } }); + await waitFor(() => expect(api.transcribeFile).toHaveBeenCalledWith(voiceMessage, "drop", expect.anything())); + expect(view.queryByText("Drop to transcribe")).toBeNull(); + }); + + it("lets a drag that leaves again do nothing", async () => { + const view = await ready(); + listeners.get("tauri://drag-enter")!({ payload: { paths: [voiceMessage], position: { x: 1, y: 1 } } }); + await waitFor(() => expect(view.queryByText("Drop to transcribe")).not.toBeNull()); + listeners.get("tauri://drag-leave")!({ payload: null }); + await waitFor(() => expect(view.queryByText("Drop to transcribe")).toBeNull()); + expect(api.transcribeFile).not.toHaveBeenCalled(); + }); + + it("takes no file while a recording runs", async () => { + const view = await ready(); + await fireEvent.click(view.getByRole("button", { name: "Start recording" })); + await waitFor(() => expect(view.queryByRole("button", { name: "Stop recording" })).not.toBeNull()); + expect((view.getByRole("button", { name: "Open audio file" }) as HTMLButtonElement).disabled).toBe(true); + listeners.get("tauri://drag-enter")!({ payload: { paths: [voiceMessage], position: { x: 1, y: 1 } } }); + listeners.get("tauri://drag-drop")!({ payload: { paths: [voiceMessage], position: { x: 1, y: 1 } } }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(view.queryByText("Drop to transcribe")).toBeNull(); + expect(api.transcribeFile).not.toHaveBeenCalled(); + }); + + it("works through files handed in from outside, one after another", async () => { + const queue = ["/tmp/one.opus", "/tmp/two.m4a"]; + vi.mocked(api.takePendingFile).mockImplementation(async () => queue.shift() ?? null); + await ready(); + await waitFor(() => expect(api.transcribeFile).toHaveBeenCalledTimes(2)); + expect(vi.mocked(api.transcribeFile).mock.calls.map((call) => [call[0], call[1]])).toEqual([ + ["/tmp/one.opus", "command_line"], + ["/tmp/two.m4a", "command_line"], + ]); + queue.push("/tmp/three.ogg"); + listeners.get("file-waiting")!({ payload: null }); + await waitFor(() => expect(api.transcribeFile).toHaveBeenCalledTimes(3)); + }); + + it("says why a file could not be read", async () => { + vi.mocked(open).mockResolvedValueOnce("/tmp/notes.txt"); + vi.mocked(api.transcribeFile).mockRejectedValueOnce("Utterform cannot read this file. (ref 0a1b2c3d-9)"); + const view = await ready(); + await fireEvent.click(view.getByRole("button", { name: "Open audio file" })); + await waitFor(() => expect(view.queryByText(/cannot read this file/)).not.toBeNull()); + expect(api.notifyIfUnattended).toHaveBeenCalledWith("Utterform cannot read this file. (ref 0a1b2c3d-9)", true); + }); +}); diff --git a/src/app.css b/src/app.css index 99fc1fb..46c1e00 100644 --- a/src/app.css +++ b/src/app.css @@ -35,6 +35,11 @@ header { height:81px; flex-shrink:0; display:flex; align-items:center; justify-c .app-version { color:var(--muted); font:10px ui-monospace,monospace; } .icon-button { width:34px; height:34px; border:1px solid var(--line); background:transparent; border-radius:5px; display:grid; place-items:center; } .icon-button svg { width:18px; } +.header-tools .file-trigger { margin-right:-7px; } +.drop-overlay { position:fixed; inset:12px; z-index:40; display:flex; flex-direction:column; align-items:center; justify-content:center; gap:10px; border:2px dashed var(--muted); border-radius:14px; background:color-mix(in srgb, var(--bg) 88%, transparent); color:var(--text); pointer-events:none; } +.drop-overlay svg { width:44px; fill:none; stroke:currentColor; stroke-width:1.6; } +.drop-overlay strong { font-size:20px; font-weight:500; } +.drop-overlay small { font-family:var(--mono, ui-monospace, monospace); font-size:12px; color:var(--muted); } .icon-button .settings-glyph, .icon-button .line-icon { fill:none; stroke:currentColor; stroke-width:1.5; } .icon-button:hover:not(:disabled) { background:var(--accent-soft); border-color:var(--muted); } .controls { width:min(620px,100%); margin:24px auto 0; display:grid; grid-template-columns:1.2fr 1fr; align-items:end; gap:16px; } diff --git a/src/lib/api.ts b/src/lib/api.ts index 958c15e..95ef7d1 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -15,6 +15,7 @@ import type { LiveStatus, LiveSupport, OutputFormat, + FileSource, ProcessResult, RecordingStatus, RemoteIntent, @@ -84,6 +85,17 @@ export const api = { typeAtCursor: boolean; outputFormat: OutputFormat; }) => invoke("finish_recording", { request }), + // An audio file: decoded, transcribed, then the same path as a recording. + transcribeFile: (path: string, source: FileSource, request: { + action: string; + customPrompt: string | null; + copyToClipboard: boolean; + saveToFile: boolean; + typeAtCursor: boolean; + outputFormat: OutputFormat; + }) => invoke("transcribe_file", { path, source, request }), + // Files handed in from the command line or "Open with", one at a time. + takePendingFile: () => invoke("take_pending_file"), // Whether `codex login` signed the Codex CLI in to ChatGPT, and its models. codexStatus: () => invoke("codex_status"), openCodexSetup: () => invoke("open_codex_setup"), diff --git a/src/lib/types.ts b/src/lib/types.ts index decd108..74396ed 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -21,6 +21,13 @@ export interface CodexStatus { export type TypingMethod = "paste" | "keystrokes"; /// What a compositor hotkey or a second launch asks the running app to do. export type RemoteIntent = "show" | "toggle" | "start" | "stop" | "cancel"; +/// How an audio file reached Utterform; named in the log, never the path. +export type FileSource = "drop" | "open" | "command_line"; +/// What the Open dialog offers. The backend reads the content, not the name, +/// so a dropped file outside this list is still tried. +export const AUDIO_FILE_EXTENSIONS = [ + "opus", "ogg", "oga", "m4a", "mp4", "aac", "mp3", "wav", "flac", "webm", "mka", "mkv", "caf", "aif", "aiff", +]; export interface AppSettings { engine: Engine; diff --git a/tests/ui.e2e.ts b/tests/ui.e2e.ts index 04dab00..f4ec37c 100644 --- a/tests/ui.e2e.ts +++ b/tests/ui.e2e.ts @@ -27,6 +27,8 @@ test.beforeEach(async ({ page }) => { isTauri: true, __copiedText: "", __emitRemoteIntent: (intent: string) => callbacks.get(eventHandlers.get("remote-intent") ?? -1)?.({ payload: intent }), + __emit: (event: string, payload: unknown) => callbacks.get(eventHandlers.get(event) ?? -1)?.({ payload }), + __transcribedFiles: [] as unknown[], __finishCount: 0, __savedSettings: null, __appliedHotkey: undefined, @@ -85,6 +87,13 @@ test.beforeEach(async ({ page }) => { case "get_recording_status": return recordingStatus(); case "finish_recording": started = 0; Object.assign(window, { __finishCount: Reflect.get(window, "__finishCount") + 1 }); return { ...latest, historyEntry: latest, savedPath: null, copiedToClipboard: true, deliveryWarnings: [] }; case "retained_recording": return Reflect.get(window, "__retained") ?? null; + case "take_pending_file": return null; + case "transcribe_file": { + (Reflect.get(window, "__transcribedFiles") as unknown[]).push({ path: args.path, source: args.source, request: args.request }); + const entry = { ...latest, id: "3", title: "Bring the blue folder", text: "Bring the blue folder on Monday.", durationMs: 63_000 }; + history = [entry, ...history]; + return { historyEntry: entry, text: entry.text, savedPath: null, copiedToClipboard: true, typedAtCursor: false, deliveryWarnings: [], durationMs: entry.durationMs, engine: "open_ai" }; + } case "notify_if_unattended": return; case "play_test_cues": return; case "diagnostics_info": return { logPath: "/home/tester/.local/share/software.jli.utterform/logs/utterform.log" }; @@ -861,6 +870,23 @@ for (const viewport of [{ width: 920, height: 720 }, { width: 360, height: 400 } }); } +test("a voice message dropped on the window goes to the clipboard", async ({ page }) => { + await page.goto("/"); + await expect(page.getByRole("button", { name: "Open audio file" })).toBeEnabled(); + const emit = (event: string, payload: unknown) => page.evaluate(([event, payload]) => (Reflect.get(window, "__emit") as (event: string, payload: unknown) => void)(event as string, payload), [event, payload] as const); + await expect.poll(() => page.evaluate(() => typeof Reflect.get(window, "__emit"))).toBe("function"); + await emit("tauri://drag-enter", { paths: ["/home/tester/PTT-20261107-WA0003.opus"], position: { x: 400, y: 300 } }); + await expect(page.getByText("Drop to transcribe")).toBeVisible(); + await expect(page.getByText("Plain · GPT Transcribe → Clipboard")).toBeVisible(); + await page.screenshot({ path: "test-results/audio-file-drop.png" }); + await emit("tauri://drag-drop", { paths: ["/home/tester/PTT-20261107-WA0003.opus"], position: { x: 400, y: 300 } }); + await expect(page.getByText("Drop to transcribe")).toBeHidden(); + await expect(page.getByText("Copied to clipboard")).toBeVisible(); + const calls = await page.evaluate(() => Reflect.get(window, "__transcribedFiles")); + expect(calls).toEqual([{ path: "/home/tester/PTT-20261107-WA0003.opus", source: "drop", request: expect.objectContaining({ copyToClipboard: true, typeAtCursor: false }) }]); + await page.screenshot({ path: "test-results/audio-file-done.png" }); +}); + test("text actions can run on the ChatGPT subscription, marked experimental", async ({ page }, testInfo) => { await page.addInitScript(() => Object.assign(window, { __codexSignedOut: true })); await page.goto("/");