diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b991935..adbe05e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,6 +132,10 @@ jobs: -p:FileVersion=${{ steps.version.outputs.file_version }} -p:InformationalVersion=${{ steps.version.outputs.informational_version }} + - name: Test + if: github.event_name != 'workflow_dispatch' || !inputs.submit_winget_only + run: dotnet test tests/PrimeDictate.Core.Tests -c Release + - name: Publish Windows payloads if: github.event_name != 'workflow_dispatch' || !inputs.submit_winget_only shell: pwsh diff --git a/AGENTS.md b/AGENTS.md index 7ac4773..d1f642f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This file orients coding agents and future maintainers. It is not an end-user ma ## Purpose -**PrimeDictate** is a .NET 10 **Windows** app with a **WPF tray shell** and **first-run onboarding**. It is what the installers ship (6.0.0 and later). A cross-platform successor lives under `src/` (see the layout table) and is not packaged yet. The WPF app: +**PrimeDictate** is one .NET 10 app for hotkey dictation and for transcription and meetings. From 6.1.0 the installers ship the Avalonia app in `src/PrimeDictate.Desktop` as `PrimeDictate.exe`. The WPF app at the repository root (what 6.0.0 shipped) is legacy: it still builds, and it is not packaged. The Qualcomm QNN/AI Hub models are in both apps (the new app's code is under `src/PrimeDictate.Platforms/Speech/Qualcomm`; see `docs/architecture/qualcomm-npu.md`). The dictation behavior below is shared by both; the WPF file names in the Layout table are the legacy implementation. The behavior: 1. Listens for a **global** hotkey (`Ctrl+Shift+Space` / SharpHook) to start and stop capture. 2. Records from the **default** Windows input device using **WASAPI** (NAudio `WasapiCapture`), normalizing to **16 kHz, 16-bit, mono** PCM. @@ -24,14 +24,14 @@ This file orients coding agents and future maintainers. It is not an end-user ma | `WhisperTextInjectionPipeline.cs` | Transcription orchestration, logging, and final-only text injection. | | `WindowsInputHelpers.cs` | Foreground-window guard for final injection and optional Windows Mouse Sonar pulse. | | `TranscriptionOverlayWindow.xaml` | Non-activating live transcript overlay; placement is user-configurable. | -| `PrimeDictate.csproj` | The WPF app. Target `net10.0-windows` (SDK pinned in `global.json`); NAudio, SharpHook, sherpa-onnx; references `src/PrimeDictate.Core`. | +| `PrimeDictate.csproj` | The legacy WPF app (6.0.0; not packaged since 6.1.0). Target `net10.0-windows` (SDK pinned in `global.json`); NAudio, SharpHook, sherpa-onnx; references `src/PrimeDictate.Core`. | | `Directory.Build.props` | Shared assembly/file `Version` (installers read this too). | -| `scripts/Publish-Windows.ps1` | Self-contained publish of the WPF app for one runtime (`win-x64` or `win-arm64`) to `artifacts//publish`. | +| `scripts/Publish-Windows.ps1` | Self-contained publish of `src/PrimeDictate.Desktop` (PrimeDictate.exe) for one runtime (`win-x64` or `win-arm64`) to `artifacts//publish`. | | `scripts/Build-Installers.ps1` | Publishes then builds the online WiX `.wixproj` (NuGet `WixToolset.Sdk`) to x64 and ARM64 MSIs in `artifacts/installer`. | | `installer/wix/` | WiX: `online/` is the shipped MSI (app payload only; models are downloaded in the app), with Start Menu and optional launch-at-login shortcuts; `offline/` is not built. `Branding.wxs` + `PrimeDictate.ico` for ARP/exe icon. | | `src/PrimeDictate.Core` | Portable core used by both apps: transcript model, SQLite session store, pipeline, dictation controller, audio and provider contracts. | -| `src/PrimeDictate.Platforms` | Platform adapters: microphone capture (miniaudio; WASAPI loopback for system audio), speech providers (sherpa-onnx, Nemotron), hotkeys, typing, the meeting final pass. | -| `src/PrimeDictate.Desktop` | The cross-platform Avalonia app: transcription workspace, meetings, dictation. Not in the installers yet. | +| `src/PrimeDictate.Platforms` | Platform adapters: microphone capture (miniaudio; WASAPI loopback for system audio), speech providers (sherpa-onnx, Nemotron), hotkeys, typing, launch at login (`Startup/`), the GitHub updater (`Updates/`), the meeting final pass. | +| `src/PrimeDictate.Desktop` | The shipped app (6.1.0+), assembly name `PrimeDictate`: dictation, tray, overlay, transcription workspace and meetings, updater (`Updates/`). Avalonia; runs on Windows, macOS and Linux, installers are Windows only. | | `tests/PrimeDictate.Core.Tests` | xUnit tests for Core and Platforms: `dotnet test tests/PrimeDictate.Core.Tests`. | | `docs/architecture/` | Design notes and status per stage; `meeting-v1-acceptance.md` is the Meeting v1 test. | @@ -61,9 +61,16 @@ This file orients coding agents and future maintainers. It is not an end-user ma - **Whisper options**: Whisper uses sherpa-onnx `OfflineRecognizerConfig.ModelConfig.Whisper`; add provider/thread/language controls there when needed. - **Non-Windows audio**: `DefaultMicrophoneRecorder` is Windows-centric (`WasapiCapture`); a cross-platform build would need an abstraction and platform-specific capture. -## Shipped shell + onboarding notes +## Shipped app (6.1.0+) -The tray/onboarding milestone is now implemented: +- **Output**: `src/PrimeDictate.Desktop` builds `PrimeDictate.exe` (AssemblyName `PrimeDictate`, RootNamespace stays `PrimeDictate.Desktop`). The version comes from `Directory.Build.props`. +- **Launch at login is one mechanism** (`Platforms/Startup/LaunchAtLogin.cs`): the MSI installs the all-users Startup shortcut `PrimeDictate.lnk` running `--background`, and the Settings checkbox turns that shortcut on or off per user through the Explorer StartupApproved switch. Only when the shortcut is absent (`LAUNCHATLOGIN=0`, portable copy) does it use the per-user Run value `PrimeDictate`. Never both. `--from-login` (the 6.0.0 shortcut argument) is treated as `--background`. The first start removes the 6.0.0 per-user shortcut and stale Run values. +- **Updater** (`Platforms/Updates`, `Desktop/Updates`): GitHub Releases, asset `PrimeDictate-Setup-vX.Y.Z-.msi` plus its `.sha256`, msiexec handoff after the clean exit path (`App.ExitAsync`). Windows only; installing is always user-initiated. +- **Single instance** keeps the name `PrimeDictate.Desktop.`; it does not detect the legacy WPF app. + +## Legacy WPF shell + onboarding notes (6.0.0) + +The WPF tray/onboarding milestone (legacy, not packaged since 6.1.0): 1. **Host process**: WPF tray host (`App.xaml`) with notification icon, Settings/Exit menu, live transcript overlay, and Ready/Listening/Processing tooltip state. 2. **User settings**: Persisted under `%LocalAppData%\PrimeDictate\settings.json`; loaded at startup and applied to `GlobalHotkeyListener`. @@ -73,15 +80,15 @@ The tray/onboarding milestone is now implemented: ## Build and test hints -- **Windows installers**: See [installer/README.md](installer/README.md). CI builds the online x64 and ARM64 MSIs; users download a model in the app's first-run setup or Settings. -- Run from the **repository root** so `./models/whisper/` can be discovered during development when models are staged in the repo-local `models` tree. +- **Windows installers**: See [installer/README.md](installer/README.md). `scripts/Build-Installers.ps1` publishes `src/PrimeDictate.Desktop` and builds the MSIs locally (no signing inputs needed); CI builds the online x64 and ARM64 MSIs; users download a model in the app's first-run setup or Settings. +- The legacy WPF app (`PrimeDictate.csproj`) must keep building (`dotnet build PrimeDictate.sln`). Run from the **repository root** so `./models/whisper/` can be discovered during development when models are staged in the repo-local `models` tree. - A running `PrimeDictate.exe` from `dotnet run` can **lock the apphost**; stop the process if `MSB3021` / copy-to-output fails. - Linter: project should build with **0 warnings** under default SDK analysis when possible; platform-specific API use should stay behind `OperatingSystem` checks or documented trade-offs. ## Releases - Merging into `main` publishes nothing. A release is a "Release X.Y.Z" commit on `main` that bumps `Directory.Build.props` (`Version`, `AssemblyVersion`, `FileVersion`), then a pushed `vX.Y.Z` tag. -- The tag run of `.github/workflows/build.yml` builds and signs the WPF app, publishes the GitHub Release (x64 and ARM64 MSIs with checksums), pushes the Chocolatey package (then Chocolatey moderation) and opens a winget-pkgs PR (then winget validation and moderation). Details and fixes: [installer/README.md](installer/README.md). +- The tag run of `.github/workflows/build.yml` builds and signs `PrimeDictate.exe` (the Avalonia app), publishes the GitHub Release (x64 and ARM64 MSIs with checksums), pushes the Chocolatey package (then Chocolatey moderation) and opens a winget-pkgs PR (then winget validation and moderation). Details and fixes: [installer/README.md](installer/README.md). - Never merge to `main`, tag a release or change signing without the owner's explicit go-ahead. ## Out of scope (unless explicitly requested) diff --git a/Directory.Build.props b/Directory.Build.props index c0057f9..1794520 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,5 +1,7 @@ + + 1.24.4 6.0.0 6.0.0.0 6.0.0.0 diff --git a/README.md b/README.md index 2d6d935..d294eef 100644 --- a/README.md +++ b/README.md @@ -281,16 +281,19 @@ PrimeDictate normalizes unsupported saved Whisper.net hardware settings at start - GPU appears only when supported local Whisper.net GPU runtime support is present. - NPU appears only for supported GGML models with the required sidecars. -### Experimental Qualcomm QNN path +### Experimental Qualcomm QNN path (native Windows ARM64; in the ARM64 installer) -The Qualcomm backend is experimental and intended for native Windows ARM64 builds on supported Snapdragon X devices. +The Qualcomm backend is experimental and intended for native Windows ARM64 builds on supported Snapdragon X devices. It is in the app that the installers ship (6.1.0 and later) as well as in the legacy WPF app. The models are offered in Settings and first-run setup only where they can run; on any other PC they are hidden, and a saved Qualcomm choice falls back to Whisper or Moonshine on the CPU. It requires: - Windows ARM64 process/runtime -- QNN runtime assets in the build output +- QNN runtime assets in the build output (the ARM64 installer carries them) - Qualcomm AI Hub Whisper wrapper package files, including `encoder.onnx`, `decoder.onnx`, context binaries, metadata, and tokenizer files - Successful EPContext session creation without CPU fallback when strict validation is requested +- For Moonshine on the NPU, the prepared `qnn` subfolder of the Moonshine model (see `scripts/qnn/quantize_moonshine_for_qnn.py`); such a model is then listed again as "(Qualcomm NPU)" + +Developer validation commands (print a JSON report, optionally also write it to a file; they never start the app): `--qnn-aihub-whisper-transcribe [out]`, `--qnn-whisper-smoke [out]`, `--qnn-whisper-proof [true|false] [out]`, `--qnn-smoke [out]`, `--qnn-proof [true|false] [out]`. Maintainer environment knobs: @@ -345,7 +348,7 @@ For installer-specific details, see `installer/README.md`. Tagged pushes that match `vX.Y.Z` build the app, produce release assets, and publish to GitHub Releases. Merging into `main` alone publishes nothing. -The release pipeline packages the Windows WPF app (`PrimeDictate.exe`). The cross-platform app under `src/` is in the repository but not in the installers yet. +From 6.1.0 the release pipeline packages the Avalonia app in `src/PrimeDictate.Desktop` as `PrimeDictate.exe`: one app for dictation and for transcription and meetings. The WPF app at the repository root is legacy: it still builds but is not packaged. The Qualcomm QNN path described above is in both apps. In practice, the release pipeline keeps three public distribution channels aligned: diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 7a39c92..84b6210 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,29 @@ +# PrimeDictate 6.1.0 + +## Highlights + +- Transcription and meetings are a **preview** in 6.1.0: they work, but have had far less real-world use than dictation. Dictation is the same daily-use feature set as 6.0.0. +- One app. The installers now ship the new PrimeDictate that does both jobs in one process: hotkey dictation from the tray, and the transcription and meeting workspace (recordings, speaker timeline, sessions). It is still `PrimeDictate.exe`, so shortcuts, winget, Chocolatey and the updater keep working, and 6.1.0 upgrades 6.0.0 in place. +- Your setup carries over. The new app reads the 6.0.0 settings and history once (it never rewrites them), keeps the same model folders under `%LocalAppData%\PrimeDictate\models`, and keeps the lifetime stats. +- Launch at login is one thing again. The installer's Startup shortcut now starts PrimeDictate in the tray (`--background`), and the app's "Start PrimeDictate when I sign in" checkbox turns that same shortcut on or off for you without administrator rights. `LAUNCHATLOGIN=0` still leaves it out; the checkbox then uses a per-user startup entry instead. The 6.0.0 per-user startup entries are cleaned up on first start. +- Record a meeting from the tray. "Record meeting" starts a microphone and system-audio recording in the workspace; the same item stops it. The main window shows dictation status and opens history, stats and Settings, and there is one Settings window for dictation and for transcription and meetings. +- Same models everywhere. Dictation and transcription pick from the same installed models, including Whisper.net (ggml) models on the GPU (CUDA or Vulkan); one loaded copy of a model is shared by both. +- NPU models carry over. On Snapdragon (Windows ARM64) PCs the ARM64 installer includes the Qualcomm QNN runtime, and Settings and first-run setup offer the Qualcomm AI Hub Whisper Small download and Moonshine on the NPU, with a saved 6.0.0 Qualcomm choice picked up as it was. On Intel PCs the Whisper.net device list now has the NPU (OpenVINO) choice, the large-v3 download fetches Intel's OpenVINO bundle, and "Auto" means the GPU, else the NPU when the model has its OpenVINO files, else the CPU. Choices the PC cannot run are not listed, and a saved choice that does not fit the PC falls back to what does. +- Voice shell commands run again in dictation (never in transcription). Commands from 6.0.0 keep their on/off state; new ones start off. +- Text delivery as in 6.0.0 on Windows: the final text goes straight into the focused edit control when there is one (keystrokes otherwise, never the clipboard), "return to the starting window" inserts into that window without bringing it forward unless coding-mode Enter is on, the Windows Mouse Sonar pulse marks start and stop, and "Request exclusive microphone access while dictating" is back (shared if the microphone refuses). +- Updates: "Check for updates..." in the tray menu, plus an automatic check at most once a day that you can turn off in Settings (the 6.0.0 choice and last-check time carry over). Nothing installs without your OK, the MSI is verified against its published SHA-256 first, and PrimeDictate closes the normal way (an active meeting recording is saved) before Windows Installer starts. +- The overlay has its 6.0.0 controls back: pin (keep it on screen), copy (the last transcript, only when you click it), settings, collapse and expand, the elapsed time, the "Local only" badge and the model in its header, and the compact microphone has its ripple animation and a Settings option keeps it on screen while idle, as 6.0.0 did (by default it shows only while dictating). +- The tray icon: single or double click (or neither) opens PrimeDictate, as chosen in Settings; it shows "needs attention" for 10 seconds after an error and when the wake word could not start, and its tooltip names the model and the wake phrase. +- Settings: start at sign-in for just you or for everyone on this PC (asks for administrator permission, as in 6.0.0; the `--enable-launch-at-login` and `--disable-launch-at-login` command-line switches work again), the typing speed that time saved is compared with (also in the stats window), a color scheme (dark by default, as in 6.0.0, or light, or follow the system), and the tray click choice. +- History window: filters for typed or not typed, app and window, "Clear filters", and "Copy details"; the stats window shows the achievements and daily word counts as before. +- The wake word listens with a small model of the same kind as your dictation model (Whisper.net tiny or base, Moonshine tiny or base, or a small Whisper), as in 6.0.0, and falls back to the dictation model. + +## Not in 6.1.0 yet + +Nothing from the 6.0.0 app is left out. + +The old WPF app stays in the repository and still builds; it is just not shipped. + # PrimeDictate 6.0.0 ## Highlights diff --git a/TranscriptionPreferencesSeed.cs b/TranscriptionPreferencesSeed.cs deleted file mode 100644 index c54f2c6..0000000 --- a/TranscriptionPreferencesSeed.cs +++ /dev/null @@ -1,39 +0,0 @@ -using PrimeDictate.Core.Settings; - -namespace PrimeDictate; - -/// -/// First-run transcription defaults copied from the dictation settings. Dictation's -/// settings.json is only read here, never rewritten. -/// -internal static class TranscriptionPreferencesSeed -{ - public static TranscriptionPreferences FromAppSettings(AppSettings settings) => new() - { - AsrModelId = LegacyModelIds.From(settings.TranscriptionBackend, settings.SelectedModelId), - RequestedBackend = settings.TranscriptionComputeInterface switch - { - TranscriptionComputeInterface.Gpu => "gpu", - TranscriptionComputeInterface.Npu => "npu", - _ => "cpu" - }, - InputDeviceId = settings.SelectedInputDeviceId - }; -} - -/// Stable model IDs for the existing backends, used until the model registry owns them. -internal static class LegacyModelIds -{ - public static string From(TranscriptionBackendKind backend, string? selectedModelId) - { - var family = backend switch - { - TranscriptionBackendKind.Parakeet => "parakeet-onnx", - TranscriptionBackendKind.Moonshine => "moonshine", - TranscriptionBackendKind.WhisperNet => "whisper-net", - TranscriptionBackendKind.QualcommQnn => "qualcomm-qnn", - _ => "whisper-onnx" - }; - return string.IsNullOrWhiteSpace(selectedModelId) ? $"{family}:auto" : $"{family}:{selectedModelId.Trim()}"; - } -} diff --git a/docs/architecture/qualcomm-npu.md b/docs/architecture/qualcomm-npu.md new file mode 100644 index 0000000..4ce3b2d --- /dev/null +++ b/docs/architecture/qualcomm-npu.md @@ -0,0 +1,31 @@ +# Qualcomm NPU (QNN) and the onnxruntime arrangement + +Status of what ran where: `stage-7-dictation-status.md`, section "NPU". This note records the design and the rules that are easy to break. + +## What is offered, and where + +| Model | Runs on | Offered when | +|---|---|---| +| Qualcomm AI Hub Whisper Small (`qualcomm-qnn:qaihub-whisper-small-snapdragon-x-elite`) | QNN HTP, CPU fallback disabled | A native Windows ARM64 process with `QnnHtp.dll`, `QnnSystem.dll`, `onnxruntime_providers_qnn.dll` next to the app, and the package installed | +| An installed Moonshine model with a prepared `qnn` folder (`qualcomm-qnn:`, listed as "(Qualcomm NPU)") | QNN HTP; sherpa-onnx on the CPU if the NPU session cannot be created or fails, unless `PRIMEDICTATE_QNN_STRICT=1` | Same, and `qnn/` holds every stage (see `MoonshineQnnArtifacts`) | + +Everything else about the gate is `MachineSupport` (one record: Whisper.net CUDA, Vulkan, OpenVINO and the Qualcomm availability). Nothing outside `Platforms/Speech` decides what hardware can run. + +The provider for a `qualcomm-qnn` model is chosen as the WPF engine chose: the AI Hub package by its catalog id, otherwise Moonshine (`SpeechProviders.Create`). + +## One onnxruntime.dll per process + +sherpa-onnx (Whisper, Parakeet, Moonshine on the CPU) and Microsoft.ML.OnnxRuntime.QNN each ship an `onnxruntime.dll` for win-arm64, both 1.24.4. A process loads one. As in the WPF app, the build and publish of `PrimeDictate.Desktop` copy the QNN package's natives (`onnxruntime.dll`, `onnxruntime_providers_qnn.dll`, `onnxruntime_providers_shared.dll`, `QnnHtp.dll`, `QnnSystem.dll`, the HTP stubs and skels) over the output for `win-arm64`, so sherpa-onnx and the managed ONNX Runtime API share the QNN build. + +- `PrimeDictate.Platforms` references only `Microsoft.ML.OnnxRuntime.QNN`. It pulls the managed API (`Microsoft.ML.OnnxRuntime.Managed`) for every RID and natives for win-arm64 only, so it cannot collide with sherpa-onnx's library on x64, Linux or macOS. (The WPF app also referenced the full `Microsoft.ML.OnnxRuntime`, which brings natives for every RID; the new app does not need it.) +- Keep `OnnxRuntimeVersion` (`Directory.Build.props`) equal to the version inside the sherpa-onnx package. +- `scripts/Publish-Windows.ps1` fails a win-arm64 publish that lacks the QNN natives and any other publish that has them. +- On win-x64 nothing changes: sherpa-onnx's `onnxruntime.dll` stays, and the optional CUDA drop-in (`OnnxRuntimeDevice`) still replaces it as before. + +## Do not probe by loading whisper.dll + +Whisper.net's native builds (`runtimes/{cuda,vulkan,openvino}/win-x64/whisper.dll`) cannot be probed by loading and freeing them. Loading one variant, freeing it, then loading another (which Whisper.net does for its real runtime) aborts the process with `GGML_ASSERT(prev != ggml_uncaught_exception)`. `MachineSupport` therefore only looks for files: the variant's `whisper.dll`, the NVIDIA or Vulkan driver DLL, and for OpenVINO the `openvino.dll` the OpenVINO build imports, on the app folder, system folder and PATH. Whisper.net's own library order still falls through to the CPU if a variant fails to load. + +## What cannot be tested here + +The ONNX Runtime sessions and decode loops (`QualcommAihubWhisperTranscriber`, `MoonshineV1QnnTranscriber`, `MoonshineV2QnnTranscriber`) need a Snapdragon PC. On such a PC, `PrimeDictate.exe --qnn-aihub-whisper-transcribe <16 kHz mono wav> out.json` is the quickest proof that the NPU runs; the report lists the loaded `onnxruntime` and `Qnn*` modules. diff --git a/docs/architecture/stage-7-dictation-status.md b/docs/architecture/stage-7-dictation-status.md index 5d2b181..a7af700 100644 --- a/docs/architecture/stage-7-dictation-status.md +++ b/docs/architecture/stage-7-dictation-status.md @@ -2,7 +2,7 @@ Branch `claude/dictation-parity-4k0tl9`, following `stage-7-parity-plan.md`. "Ran" means executed on Linux in a container; "Compiled only" means it builds but has not been run on that OS. -**Status (2026-09-30):** merged into `main` (PR #3). Release 6.0.0 ships the WPF app; this app is not in the installers yet. Hotkey and wake-word dictation, the overlay and the settings window were first run on Windows the same day (see the Windows notes below). Before it can replace the WPF app it still needs Whisper.net models (the WPF backend Justin uses), an updater and its own installer. +**Status (2026-09-30):** merged into `main` (PR #3). Release 6.0.0 ships the WPF app; this app is not in the installers yet. Hotkey and wake-word dictation, the overlay and the settings window were first run on Windows the same day (see the Windows notes below). Whisper.net models (the WPF backend Justin uses) are now in this app (see the Whisper.net row). Before it can replace the WPF app it still needs an updater and its own installer. ## Done @@ -31,12 +31,14 @@ Branch `claude/dictation-parity-4k0tl9`, following `stage-7-parity-plan.md`. "Ra | Audio cues (synthesized WAV; Windows `PlaySound`, macOS `afplay`, Linux `paplay`/`aplay`) | `AudioCues.cs`, `ProcessAudioCuePlayer.cs` | WAV generation ran; playback not heard on any OS | | Tray icon: the WPF "voice wire" icon redrawn in Avalonia, five states | `TrayIconRenderer.cs` | Rendered under Xvfb and viewed (`--render-tray-icons `); not seen in a real tray | | `--show` / `--workspace` (accepted; the window is the default start) and new `--background` (tray only, Windows and macOS) | `App.axaml.cs` | Not run on Windows | -| Parakeet TDT v2/v3 and Moonshine v1/v2 (CPU) as dictation models, with download (same ids, folders, archives and file rules as the WPF catalogs) | `Platforms/Speech/SpeechModels.cs`, `SherpaWhisperProvider.cs` (Parakeet and Moonshine providers), `DictationHost.cs` | Ran, here on Linux with the real models: downloaded through the app's downloader, validated, and transcribed the sherpa test clips correctly (Moonshine tiny v2, Moonshine base v1, Parakeet v3), plus unit tests for discovery and settings. Not run: Windows, live dictation with these models, GPU. Moonshine QNN/NPU and Whisper.net are not ported. The wake listener still prefers Whisper tiny/base | +| Parakeet TDT v2/v3 and Moonshine v1/v2 (CPU) as dictation models, with download (same ids, folders, archives and file rules as the WPF catalogs) | `Platforms/Speech/SpeechModels.cs`, `SherpaWhisperProvider.cs` (Parakeet and Moonshine providers), `DictationHost.cs` | Ran, here on Linux with the real models: downloaded through the app's downloader, validated, and transcribed the sherpa test clips correctly (Moonshine tiny v2, Moonshine base v1, Parakeet v3), plus unit tests for discovery and settings. Not run: Windows, live dictation with these models, GPU. Moonshine on the Qualcomm NPU is in the Qualcomm row below. The wake listener still prefers Whisper tiny/base | +| Whisper.net (ggml) models for dictation and the transcription workspace: same ids (large-v3-turbo, large-v3, base.en, tiny.en), file names, `models/whisper.net/` folder, Hugging Face URLs and sizes as the WPF catalog, so an installed file is shared. Single-file download (temp file, size check, then move into place) in Settings and first run. `WhisperNetProvider` shares one loaded model per file (ref-counted), so dictation and the workspace load large-v3-turbo on the GPU once; one inference at a time per loaded model. English-only models are forced to "en"; other models use "en" unless a language or "auto" is given, as in WPF. Device: the WPF `TranscriptionComputeInterface` (Cpu/Gpu/Npu) is read from `settings.json`, plus a new Auto/CPU/GPU choice in Settings (`WhisperNetDevice`, applies after restart; `PRIMEDICTATE_WHISPERNET_DEVICE` overrides). Library order as WPF (CPU: Cpu, CpuNoAvx; GPU: Cuda, Vulkan, Cpu, CpuNoAvx; NPU: OpenVINO first only when the model has its OpenVINO files, else CPU; Auto is resolved first, see the NPU row), applied once before the first model loads; the loaded library is logged and a notice is shown when the GPU was wanted but not loaded. A WPF file with `TranscriptionBackend=WhisperNet` now selects `whisper-net:` (before it looked for a Whisper ONNX model of that name). The workspace picker starts on the dictation model, else large-v3-turbo. Wake word still prefers small sherpa Whisper models | Ran, on Windows 11 with an RTX 5070 and the real `ggml-large-v3-turbo.bin`: a 6.2 s synthetic clip through the provider loaded the **Cuda** runtime, 2.5 s for the first call including model load, 0.24 s warm; a second provider on the same file reused the loaded model (one load, two leases). CPU for comparison: 16 s per call. Unit tests: catalog, locator, download, id mapping, settings migration (WPF file byte-identical), library order, cache ref-counting. Publish: `win-x64` and `win-arm64` self-contained both build; the win-x64 output has `runtimes/{cuda,vulkan,openvino}/win-x64` and `runtimes/win-x64` whisper natives (Whisper.net also copies its linux and macOS natives; 344 MB of output in total). Not run: Vulkan, OpenVINO/NPU, the download through the app UI, dictation by hotkey with this model, the new settings control looked at | | ONNX models on CUDA (Auto/CPU/CUDA setting, ONNX Runtime GPU pack, checked fallback with a note, fp32 Whisper, fp16 Parakeet v2) | `Platforms/Speech/OnnxRuntimeDevice.cs`, `docs/architecture/onnx-cuda.md`, `scripts/Install-OnnxGpuRuntime.ps1` | Ran: unit tests with a fake GPU probe. Not run: any real GPU, the GPU DLLs, Windows | | Clean exit (tray Exit, `--quit`, one running instance, recording saved and dictation dropped on exit) | `Platforms/Startup/SingleInstance.cs`, `App.axaml.cs`, `Program.cs` | Ran: unit tests; on Linux under Xvfb `--quit` shut the app down, a second launch exited without a second copy, `--quit` with nothing running exits 0. Not run: on Windows, with a real recording | | Whisper model download (same 7 models, GitHub release URLs and install folders as WPF, so installs are shared; staged unpack, validated before it is moved into place; progress and cancel in Settings and first run) | `Platforms/Speech/WhisperModelDownloader.cs`, `ModelDownloadPanel.cs` | Ran: unit tests with a fake HTTP server and real `tar` (good archive, incomplete archive, HTTP error, broken archive, already installed). Confirmed from this container that the real tiny.en archive is reachable, 118,071,777 bytes as in the catalog, and holds both full and int8 files (the locator prefers int8). Not run: a full real download through the app, the progress UI, `tar` on Windows | -| Launch at login (current user only, tray-only `--background`; Windows Run key value `PrimeDictate.Desktop`, macOS LaunchAgent, Linux XDG autostart; separate from WPF's startup shortcut, so running both starts both; refuses under the `dotnet` host) | `Platforms/Startup/LaunchAtLogin.cs`, checkbox in Settings | Ran: unit tests for the file writers on Linux and the Windows branch against a fake Run key. Not run: the real registry, a real sign-in on any OS. No all-users scope (WPF's needed elevation) | -| Stats and milestones (words, time saved vs a typing baseline, speaking WPM, 14-day bars, 1k/10k/100k/1M word milestones with a tray notice; same `stats.json` and JSON shape as WPF, so lifetime totals carry over; built from history on first run) | `Core/Dictation/DictationStats.cs`, `DictationStatsWindow.cs`, tray menu | Ran: unit tests (counting, milestones once, WPF file read, corrupt file, day gaps). Window compiles and is not looked at. The baseline-WPM setting is read from the WPF file but has no editor in the new Settings yet. WPF and the new app writing the same file at once could lose an update | +| Launch at login (current user only, tray-only `--background`; on Windows it is the MSI's Startup shortcut switched per user through StartupApproved, or the Run value `PrimeDictate` when the shortcut is absent, never both; 6.0.0 per-user entries migrated on first start; macOS LaunchAgent, Linux XDG autostart; refuses under the `dotnet` host) | `Platforms/Startup/LaunchAtLogin.cs`, checkbox in Settings | Ran: unit tests for the file writers on Linux and the Windows decision logic and migration against fakes. Not run: the real registry and StartupApproved switch, a real sign-in on any OS | +| Updater (GitHub Releases check at most daily and from the tray, SHA-256 verified MSI download, asks before installing, quits through `App.ExitAsync` then msiexec; same asset names as WPF) | `Platforms/Updates/`, `Desktop/Updates/` | Ran: unit tests for version compare, asset selection, checksum parsing, check interval. Not run: a real release download, the dialogs, the handoff | +| Stats and milestones (words, time saved vs a typing baseline, speaking WPM, 14-day bars, 1k/10k/100k/1M word milestones with a tray notice; same `stats.json` and JSON shape as WPF, so lifetime totals carry over; built from history on first run) | `Core/Dictation/DictationStats.cs`, `DictationStatsWindow.cs`, tray menu | Ran: unit tests (counting, milestones once, WPF file read, corrupt file, day gaps). Window compiles and is not looked at. The baseline-WPM setting is read from the WPF file and now has an editor (see the parity table below). WPF and the new app writing the same file at once could lose an update | | First-run check (mic, model, hotkey, typing guard, model choice, model download) | `DictationOnboardingWindow.cs` | Starts under Xvfb without errors; not looked at | ## Rules kept from AGENTS.md @@ -58,9 +60,78 @@ Final-only typing, no clipboard, no live retyping into the target, Enter only af ## Not done yet -- Voice shell commands: matching is ported, running is not. Off until an explicit opt-in with a visible warning exists. Never in transcription mode. -- Updater, installers. -- Wake word uses a Tiny/Base Whisper model when installed, else the dictation model (as WPF does). Moonshine/Whisper.net wake models are not ported. -- The WPF app's focused-edit-control insertion (`WindowsFocusedTextControl`) and direct injection into the original target. -- Compact-mode ripple animation and the copy/pin buttons of the WPF overlay, macOS `NSPanel`, Linux X11 hints. -- First-run onboarding, launch at login, updater, packaging. +- Installers ship this app from 6.1.0 (see `installer/README.md`); Qualcomm QNN/AI Hub models stay WPF-only. +- macOS `NSPanel` and Linux X11 hints for the overlay. +- First-run onboarding (now the three-step wizard, see the parity table below) has been started on Windows with a fresh data folder but not clicked through. + +## NPU: Qualcomm QNN and Whisper.net OpenVINO (6.1.0 work) + +Design and the onnxruntime arrangement: `qualcomm-npu.md`. This dev box is Windows x64 with no NPU, so the NPU itself has not run here. Everything around it has. + +| Piece | Where | Verified | +|---|---|---| +| Qualcomm AI Hub Whisper Small (`qaihub-whisper-small-snapdragon-x-elite`, `models/qualcomm-aihub-whisper/`, same zip and tokenizer URLs and file rules as WPF) as a catalog model with download (unzip in a scratch folder, add the tokenizer, move into place only when all six files exist) | `Platforms/Speech/Qualcomm/QualcommAihubWhisperCatalog.cs`, `QualcommAihubWhisperInstaller.cs`, `SpeechModels.cs`, `ModelDownloader.cs` | Ran: unit tests with a fake server and real zips (good package, incomplete package, tokenizer failure, HTTP error, already installed) | +| Providers: AI Hub Whisper (strict QNN HTP, CPU fallback disabled; managed log-mel front end and greedy decode, ported unchanged except that it no longer logs token ids) and Moonshine v1/v2 on the NPU from the model's prepared `qnn` folder (falls back to sherpa-onnx on the CPU unless `PRIMEDICTATE_QNN_STRICT=1`, as WPF). One loaded copy per package shared by dictation and the workspace | `QnnProviders.cs`, `QualcommAihubWhisperTranscriber.cs`, `QualcommAihubWhisperFeatures.cs`, `MoonshineQnnTranscriber.cs`, `MoonshineV1QnnTranscriber.cs`, `MoonshineV2QnnTranscriber.cs`, `MoonshineQnnArtifacts.cs` | Ran, with a fake in place of the NPU session: windows, sharing and unload, disposal, the fallback rules, prepared-file resolution (v1 and v2), front-end shape, tokenizers. Compiled only: the ONNX Runtime sessions and decode loops | +| Gating: Qualcomm models exist only in a native Windows ARM64 process with `QnnHtp.dll`, `QnnSystem.dll` and `onnxruntime_providers_qnn.dll` present (WPF `QnnRuntimeSupport`). Not listed in Settings, first run, the workspace picker or the download list anywhere else. A Moonshine model with prepared QNN files is listed a second time as "(Qualcomm NPU)" | `QnnRuntimeSupport.cs`, `MachineSupport.cs`, `SpeechModelLocator.Discover`, `SpeechModelCatalog.AvailableOptions` | Ran: unit tests for x64, ARM64 with and without natives, non-Windows | +| Settings migration: a WPF file with `TranscriptionBackend=QualcommQnn` selects `qualcomm-qnn:` (before it mapped to a Whisper ONNX id); a Qualcomm Moonshine id that cannot run on the NPU uses the same Moonshine model on the CPU. The WPF file stays byte-identical. A custom `ModelPath` works with the Qualcomm backend as in WPF (`SpeechModelLocator.TryResolveCustom`): a folder with a runnable AI Hub package, else a Moonshine folder (NPU with prepared `qnn` files, CPU otherwise); a raw `qnn_context_binary` folder is refused with WPF's explanation; used only where the NPU can run | `DictationSettings.cs` (prefix), `SpeechModelLocator.Resolve`, `DictationHost` | Ran: unit tests | +| Startup normalization like WPF `NormalizeSettingsForCurrentMachine`, in memory only: a Qualcomm setup on a PC without the NPU becomes Whisper (package) or Moonshine (Moonshine id) on the CPU; a GPU or NPU choice the PC cannot honour moves to the best installed hardware setup, else the best compute. Applied before the Whisper.net runtime is chosen and again in `DictationHost` (shown as a notice) | `HardwareNormalization.cs`, `DictationHost.cs`, `App.axaml.cs` | Ran: unit tests | +| Whisper.net NPU, matching WPF `GetComputeChoices`: an explicit NPU choice in Settings; GPU and NPU listed only where the machine supports them; OpenVINO runs only for the NPU choice and only with the model's OpenVINO files; **Auto** now means GPU if CUDA or Vulkan can run, else NPU if OpenVINO can run and the model has its files, else CPU (before: OpenVINO on the CPU device whenever the files existed, never the GPU), with a label that says which. An OpenVINO choice that did not load gives a notice | `MachineSupport.cs`, `WhisperNetRuntime.cs`, `WhisperNetDeviceChoices.cs`, `WhisperNetProvider.cs`, `DictationSettingsWindow.cs` | Ran: unit tests for support rules, Auto resolution, library order, device list, saving. Not run: any OpenVINO or NPU use | +| OpenVINO bundle download: models with an Intel bundle (large-v3, `ggml-large-v3-models.zip`) download the zip on a PC that can use OpenVINO, take the model and its encoder `.xml` and `.bin` out as `.download` files, check, and move all three together; a model installed without its files is completed the same way | `ModelDownloader.cs`, `SpeechModels.cs` | Ran: unit tests with a fake server (good bundle, missing file, too-small model, HTTP error, completes an existing model, plain download elsewhere) | +| `--qnn-aihub-whisper-transcribe`, `--qnn-whisper-smoke`, `--qnn-whisper-proof`, `--qnn-smoke`, `--qnn-proof` (same arguments and JSON report as WPF; they never start the app) | `QnnValidation.cs`, `Desktop/Program.cs` | Ran: argument parsing in unit tests; the published win-x64 exe prints usage and writes a failure report for a missing model. Compiled only: the NPU runs themselves | +| Packaging: the app project copies the QNN package's natives over sherpa-onnx's `onnxruntime.dll` after build and publish on win-arm64 only (the WPF arrangement); the publish script fails if they are missing from win-arm64 or present in another publish; installer harvests the whole publish folder | `PrimeDictate.Desktop.csproj`, `PrimeDictate.Platforms.csproj`, `scripts/Publish-Windows.ps1` | Ran: `win-arm64` publish has the package's 15 native files at the root with the QNN package's 31.7 MB `onnxruntime.dll` (sherpa's is 15.7 MB), the ARM64 MSI builds (142 MB); `win-x64` publish has sherpa's `onnxruntime.dll` and no QNN files, unchanged layout. Not run: the ARM64 app on Windows ARM64 | + +Found on the way: the WPF app proved a Whisper.net runtime by loading its `whisper.dll` and freeing it again. Doing that for two variants in one process aborts the process (ggml asserts when a second whisper.cpp build loads after the first was freed), including the later real load. The new app only looks for files (and for `openvino.dll` on the search path) and never loads a probe. + +## One app: tray, window and settings (6.1.0 work) + +| Piece | Where | Verified | +|---|---|---| +| Voice shell commands run in dictation only: matched in the final transcript (never in the live preview, never in transcription), the command string comes only from settings, Stop/Continue as in WPF (Continue types the words that remain), history row "Voice command: phrase". Imported WPF commands keep their On state; commands added here start Off; the Settings editor (On / Say / After / Command) shows a warning | `DictationController.cs`, `VoiceShellCommandRunner.cs` (interface), `ProcessVoiceShellCommandRunner.cs` (cmd.exe /d /c, /bin/sh -c), `DictationSettingsWindow.cs` | Ran: unit tests with a fake runner. Real process start and the editor not exercised by hand | +| Tray menu: Start dictation, Record meeting (flips to Stop meeting recording; same code as the Record button, so either can stop the other; opens the window), Open PrimeDictate, Dictation history, Stats, Settings, Check for updates, Exit. Tray icon and tooltip show recording during a meeting | `DictationShell.cs`, `MainWindow.axaml.cs` | Compiled and started on Windows; menu clicks and a real recording not exercised | +| Main window header: dictation state, model, hotkey, buttons for history, stats, settings | `MainWindow.axaml` | Started on Windows and looked at | +| Transcription model defaults to the transcription pick, else the dictation model, else large-v3-turbo | `MainWindow.axaml.cs` | Not exercised | +| One Settings window with a "Transcription & meetings" section (source, live text, speaker labels, boost), kept in sync with the main window; stored in `transcription-settings.json`, the old `meeting-live-text.txt` is carried over once | `TranscriptionPrefsService.cs`, `TranscriptionPreferences.cs` | Defaults and load: unit tests. Sync not exercised by hand | +| Dictation activity (WPF Workspace diagnostics: sessions list with time range, status typed / not typed / error / voice command / discarded and app name; per-session log; app-wide activity log with repeat aggregation "(xN)", capped at 600 (300 per session, 100 sessions); Copy errors, Copy all activity, Clear). In memory, fed by the same `AppLog` calls and by the controller. **Policy change from WPF: no recognized text anywhere in it** (the voice-command phrase is the one saved in Settings). Opened from the header "Activity" button and the tray item "Dictation activity…" | `Core/Diagnostics/ActivityFeed.cs`, `ActivityText.cs`, `AppLog.cs`, `DictationController.cs`, `Desktop/Dictation/DictationActivityWindow.cs` | Ran: unit tests (aggregation, 600 cap, session grouping, session cap, copy text, AppLog levels, a full dictation whose recognized words are absent from the feed). App starts on Windows with a temp data dir. Window not looked at, Copy buttons not clicked | + +## Windows text delivery parity (6.1.0 work) + +| Piece | Where | Verified | +|---|---|---| +| Focused-control insertion first, SendInput second (same order as WPF `WindowsUnicodeInput.SendText`): `EM_REPLACESEL` into the focused control when its class name contains "Edit", else virtual-key and unicode `SendInput`. No clipboard. The route taken is logged, never the text | `Platforms/Input/WindowsFocusedTextControl.cs`, `WindowsTextEntry.cs`, `WindowsSendInput.cs`, `SharpHookTextInjector.cs` | Ran: decision unit tests with fakes. Real run on Windows 11: own Notepad, both routes read back with `WM_GETTEXT` (see below) | +| Direct injection into the original target without reactivating it (`IForegroundTarget.TryInjectDirectly`): when focus moved and "return to the starting window" is on and coding-mode Enter is off, insert into the original window's focused edit control; else restore the window and type; Enter only after a successful injection | `Core/Dictation/Delivery.cs`, `WindowsForegroundGuard.cs` | Ran: unit tests for every branch with fakes. The real run checks the target's direct injection while Notepad is in front, not with another window in front (that would mean using someone else's window) | +| Mouse Sonar pulse (Ctrl tap 150 ms after recording or processing starts, only when the Windows "show pointer location" setting is on, no setting of ours) | `Platforms/Input/WindowsMousePointerIndicator.cs`, `DictationShell.OnState` | Compiled and started; the pulse itself was not looked at | +| Exclusive microphone access while dictating (WPF `ExclusiveMicAccessWhileDictating`, imported by name; checkbox in Settings on Windows; miniaudio WASAPI exclusive share mode on a concrete endpoint; falls back to shared and logs; the wake word and meetings always open shared; tray tooltip shows "[Exclusive]" or "[Shared]" while listening) | `IAudioSource.OpenAsync(.., MicAccessMode, ..)`, `MiniAudioCaptureSource.cs`, `DictationController.ActiveMicAccess`, `DictationSettingsWindow.cs` | Ran: unit tests (request only when set, granted mode reported, fallback passes it through, WPF setting imports). Real run on this PC: exclusive was refused by the Arctis endpoint (`ShareModeNotSupported`), the NAudio exclusive path the WPF app uses also fails on both microphones here (`Unsupported Wave Format`), and capture fell back to shared with frames flowing. A real exclusive grant was not seen on any device | + +Env-gated real runs (skipped by default): `PRIMEDICTATE_REAL_INPUT=1` (`WindowsTextDeliveryRealRunTests`, starts and closes its own Notepad, takes focus for a few seconds) and `PRIMEDICTATE_REAL_MIC=1` (`ExclusiveMicRealRunTests`). Keyboard route note: a sentence with shifted punctuation (a colon) came out as `;` in about half the runs while another app was running its own input hooks, so the test sentence uses letters, digits and an accented letter only; the SendInput code is the WPF code unchanged. + +## WPF parity: overlay, tray, settings, history (6.1.0 work) + +Checked against the WPF code first. "Ran" = unit tests (`tests/PrimeDictate.Core.Tests/ShellParityTests.cs`); the window code compiles and the app starts and quits cleanly with a temporary `PRIMEDICTATE_DATA_DIR`, but none of the new UI was looked at or clicked. + +| Item | What WPF had | Now | +|---|---|---| +| Overlay controls | Pin (toggle `IsOverlaySticky`, saved), settings, minimize, close, copy (enabled only with real text), elapsed timer, "Local only" badge, header `Listening [Whisper.net (GGML)]` / `Processing` / `Wake listening` / `Ready`, "Waiting for hotkey..." and "Waiting for wake phrase..." text, click compact to expand, close/minimize on an expanded compact collapses | All ported in `DictationOverlayWindow.cs` (pin saves `IsOverlaySticky`; copy puts the last transcript on the clipboard on click only, never typed; expand and collapse are for the session, like WPF `ExpandOverlayPanel`/`CollapseOverlayPanel`). Wording and rules are pure code in `OverlayRules.cs` (ran). Still non-activating (`Win32Overlay`). Placement stays the new bottom-centre/drag-and-remember (WPF had fixed lower-right/lower-centre) | +| Compact ripple | Three rings growing out of the 46 px microphone, glow inside, state color | `MicRippleControl.cs` draws it, the numbers are `CompactRipple.cs` (ran: ring offsets, ranges, voice response, wrap). The compact overlay is now the ripple microphone beside a status word and the buttons; the waveform and particles are full-panel only (as in WPF) | +| Overlay persistence | `ShouldPersistOverlay`: the compact microphone stays on screen, and so does a pinned overlay | Restored by default (`OverlayRules.ShouldShow`, ran). New setting `HideOverlayWhenIdle` brings back the earlier new-app behavior of showing it only while dictating. This reverses the Windows-run change recorded above; confirm with Justin | +| Updater setting | `CheckForUpdatesAutomatically` (default on), `LastUpdateCheckUtc` | Setting read from the WPF file and editable in Settings; honored on launch (`UpdateController.CheckOnLaunchAsync`, re-read after the 8 s delay); the tray command always works. `UpdateCheckState` falls back to the WPF `LastUpdateCheckUtc` until this app writes its own record (ran). WPF had no explicit UAC-cancel text in the updater (the prompt comes from the detached PowerShell step after the app has exited); a Win32 1223 message was added anyway where the app itself starts a process, and in the launch-at-login helper | +| Stats / Impact | Four figures, baseline editor (20 to 120), 14-day bars with counts, achievements Unlocked/Locked with percent | Baseline editor in the stats window and in Settings (`NormalizeBaselineWpm`, ran), counts above the bars, achievements with Unlocked/Locked and percent. Layout matches in content, not looked at | +| Launch at login scope | CurrentUser / AllUsers / Disabled, elevated helper for AllUsers, headless switches with exit codes | `LoginScope` (`LaunchAtLoginScope.cs`): Off, current user (no admin; the 6.1.0 shortcut or Run mechanism), all users (the machine-wide `HKLM...\Run` value; changed by relaunching the app elevated with `--launch-at-login-machine=on` or `=off`, never started as the app). Settings: the checkbox plus "for everyone" (Windows). `LaunchAtLoginCli.cs` handles `--enable-launch-at-login`/`--launch-at-login`/`/EnableLaunchAtLogin`, the disable forms and the scope flags with WPF exit codes (0, 1, 2) before the app starts. Ran with fake registry and elevation. Not run: the real HKLM write or a UAC prompt. WPF's AllUsers made a Startup-folder shortcut; here it is a Run value because the all-users shortcut belongs to the MSI | +| Wake word models | Same-family small model: Whisper.net tiny.en/base.en, Moonshine tiny-v2/base, Whisper ONNX smallest of tiny.en/base.en/tiny/base/distil-small.en, else the dictation model | `WakeModelChooser.cs` (ran). Same family first, then a small Whisper ONNX (a deliberate extra: lighter than a Parakeet or large GGML model), then the dictation model. WPF `TryPrepareWakeTranscription` failure state: the tray shows "wake listening failed" when the listener cannot open the microphone (`WakeWordListener.HasFailed`) | +| History window | Search (every word, final and original), status filter (All, Typed, Not typed), app filter, window filter (within the app, title parsed from the display name), Clear, Copy transcript, Copy details, full detail pane, no re-insert | Same, in `DictationHistoryWindow.cs`; filters and the details text are `DictationHistoryFilter.cs` (ran). Reads up to 1,000 entries (was 200). The WPF window had no re-insert, so none was added (it would be injection from history). The history hotkey and voice phrase already open it | +| Tray | Single or double click opens the workspace (`TrayClickBehavior`), error icon for 10 s on any error log and on wake restart failure, tooltips with backend, mode, wake phrase | `TrayClickDecider` (a double click is two clicks within 500 ms; ran) and a new "does nothing" choice; `AppLog.ErrorLogged` raised by `AppLog.Fault` and `AppLog.Error` turns the icon to attention for 10 s; WPF icon priority (recording, processing, attention, wake, ready); tooltips from `OverlayRules.TrayTooltip` (63 character limit, ran). WPF "Exclusive/Shared" mic mode in the tooltip has no equivalent here | +| Theme | `Themes/DarkTheme.xaml`, always dark, no setting | New `Theme` setting (system, light, dark) applied through `RequestedThemeVariant`; default follows the system, so a WPF user who liked the dark look picks Dark. Avalonia's own theme, not a port of the WPF palette | + +Run with `PRIMEDICTATE_DATA_DIR` set, the app no longer touches the real startup entries or the update record (it skips `StartUpdates`). + +## WPF parity: settings and first run (6.1.0 work) + +| Piece | Where | Verified | +|---|---|---| +| Ollama block in Settings: enable, endpoint, model, the seven WPF modes, and a checkbox for `OllamaAllowRemoteEndpoint` (off: loopback only; a note appears when the endpoint is remote and not allowed). With rewriting on, the endpoint must be http(s) and a model must be named | `Desktop/Dictation/OllamaPanel.cs`, `DictationSettingsValidator.ValidateOllama` | Validation: unit tests. Panel not clicked | +| Custom model path (`ModelPath`, same JSON name as WPF so it is imported): a path box under the model list with Browse folder and Browse file (Whisper.net `.bin`). The family is detected with the same file rules as an installed model (`SpeechModelLocator.TryResolveCustom`, wording of the WPF messages); a path that is a catalog install keeps its catalog id. A valid path is the model dictation uses (`SpeechModelLocator.WantedModelId`), listed first as "Custom: ..." in Settings, first run and the transcription workspace (`DiscoverFor`); an invalid one blocks Save with the reason. Qualcomm QNN paths are not resolved here | `CustomModelPanel.cs`, `Platforms/Speech/SpeechModels.cs`, `DictationHost.InstalledModels` | Locator, import and fallback: unit tests. Browse dialogs not clicked | +| Repo-local and app-folder discovery: `models` next to the app and in the working directory and its parents (8 levels), as `WhisperModelCatalog` did, for every family (`SpeechModelLocator.Discover(root, extraRoots)`, `DevelopmentRoots`). The managed folder wins; used by the host, Settings and the workspace | `SpeechModels.cs`, `TranscriptionWorkspaceService.cs` | Unit tests (all four families, managed wins, roots list) | +| Validation, blocking Save with red messages next to the field: invalid or duplicate shortcuts; voice commands need at least one phrase; blank phrases default; duplicate phrases (compared as words, so "Thank you!" equals "thank you"); computer-command phrases against the three phrases, each other and, with the wake word on, the wake phrase; wake phrase against the three phrases; incomplete command rows; Ollama; custom model. Nothing is applied until all of it passes. The WPF startup repair (`NormalizeShortcutSettings`) runs on every load | `Core/Dictation/DictationSettingsValidator.cs`, `CommandsPanel.cs`, `DictationSettingsWindow.cs`, `DictationSettingsStore.Load` | Unit tests for every rule and the repair. Window not clicked | +| `FirstRunCompleted`: WPF's flag is honoured on import (false shows setup); the new app saves its own; a `dictation-settings.json` from before the flag counts as finished. `DictationHost.IsFirstRun` reads it | `DictationSettings.FirstRunCompleted`, `DictationHost` | Unit tests (WPF false/true, old new-app file, round trip) | +| First-run wizard in three steps like WPF: Welcome (what works here, checklist, commit delay, coding-mode Enter, tones), Model (installed list, download, custom path; moving on from Welcome starts the first download when nothing is installed, as WPF did), Commands (shortcuts and voice phrases, "More settings..."). Back, Next, Finish; Finish applies the Settings validation; closing early asks "Finish setup later?" and setup shows again next start (an app shutdown is not blocked) | `DictationOnboardingWindow.cs`, `CommandsPanel.cs`, `CustomModelPanel.cs`, `ConfirmDialog.cs` | Started on Windows with a fresh `PRIMEDICTATE_DATA_DIR`: no errors, `--quit` exited cleanly. Not clicked through | +| Replacements as a grid: Find, Replace with, Remove per row, Add rule (blank Find rows dropped on Save). Replaces the `a => b` text box; the stored list is unchanged | `ReplacementsEditor.cs` | Not clicked | +| Settings window: Save and Cancel (Cancel and the close button discard edits; nothing is written until Save passes validation) and a History button in the footer | `DictationSettingsWindow.cs`, `DictationHost.RequestHistory` | Not clicked | diff --git a/docs/architecture/stage-7-parity-plan.md b/docs/architecture/stage-7-parity-plan.md index 4f9034f..35500b3 100644 --- a/docs/architecture/stage-7-parity-plan.md +++ b/docs/architecture/stage-7-parity-plan.md @@ -25,7 +25,7 @@ Inventory is from the code on this branch. Justin has uncommitted edits (includi | Settings (`AppSettings`, `SettingsWindow`) | ~30 settings, 5 backend tabs, stats | **Redo.** Read the old `settings.json` unchanged (migration, never rewrite it), then own settings. Avalonia settings window built from a per-backend adapter, not five copies. | | Model catalogs (Whisper, Parakeet, Moonshine, Whisper.net) | Folder rules and downloads | **Move to Core/Platforms** as one catalog behind `IModelCatalog`, keeping the same folders so installed models keep working. | | Engines: Whisper ONNX, Parakeet, Moonshine (sherpa) | Local ONNX recognition | **Port.** Whisper is done; Parakeet and Moonshine come next as providers. | -| Engines: Whisper.net, Qualcomm QNN/AI Hub | Windows-specific runtimes, ARM64 NPU | **Keep in the WPF app for now.** Re-add to the new app as Windows-only providers after the rest reaches parity, if still wanted. | +| Engines: Whisper.net, Qualcomm QNN/AI Hub | Windows-specific runtimes, ARM64 NPU | **Ported** as Windows-only providers: Whisper.net (CPU, CUDA, Vulkan, OpenVINO NPU) and the Qualcomm NPU models (AI Hub Whisper Small, Moonshine) behind machine gating. See `qualcomm-npu.md` and the status doc. | | Tray (`App.xaml.cs`) | Notification icon, Settings/Exit, state tooltip | **Redo** with Avalonia `TrayIcon` (Windows, macOS, most Linux desktops). | | Launch at login (`LaunchAtLoginManager`) | Registry / scope handling | **Redo per OS:** registry on Windows, LaunchAgent on macOS, autostart `.desktop` on Linux. | | Updater (`GitHubUpdateService`) | Checks Releases, downloads MSI, verifies checksum | **Redo per OS, last.** Windows keeps the MSI flow. Checks are read-only; installing is always user-initiated. No signing or release changes without Justin. | diff --git a/installer/README.md b/installer/README.md index a1eab7d..d081772 100644 --- a/installer/README.md +++ b/installer/README.md @@ -2,24 +2,25 @@ Installers are native **x64** and **ARM64** Windows Installer packages (`.msi`) built with the [WiX Toolset](https://wixtoolset.org/) **through NuGet** (`WixToolset.Sdk`). You only need the **.NET 10 SDK** (pinned in `global.json`); you do **not** install WiX separately. -The installers package the Windows WPF app (`PrimeDictate.csproj`, `PrimeDictate.exe`). The cross-platform app under `src/` is not in them yet. +From 6.1.0 the installers package the Avalonia app in `src/PrimeDictate.Desktop` (self-contained, output `PrimeDictate.exe`: dictation, tray, transcription workspace and meetings). The WPF app at the repository root (`PrimeDictate.csproj`, what 6.0.0 shipped) is legacy: it still builds but is not packaged. The MSI keeps the same `UpgradeCode`, so 6.1.0 upgrades 6.0.0 in place. The **online** MSI installs the PrimeDictate app payload only. Model acquisition happens inside PrimeDictate's first-run setup and Settings window. | MSI | Contents | |-----|----------| -| **Online x64** (`PrimeDictate-*-Windows-x64-Online.msi`) | x64 app under `Program Files\PrimeDictate`, **Start Menu** shortcut, all-users launch-at-login Startup shortcut by default, and **Add/Remove Programs** icon. The x64 MSI is blocked on ARM64 Windows so Copilot+ PCs use the native ARM64/QNN build. | -| **Online ARM64** (`PrimeDictate-*-Windows-arm64-Online.msi`) | Native ARM64 app under `Program Files\PrimeDictate` with the same installer behavior and QNN-capable ARM64 runtime payload. | +| **Online x64** (`PrimeDictate-*-Windows-x64-Online.msi`) | x64 app under `Program Files\PrimeDictate`, **Start Menu** shortcut, all-users launch-at-login Startup shortcut by default, and **Add/Remove Programs** icon. The x64 MSI is blocked on ARM64 Windows so ARM64 PCs use the native ARM64 build. | +| **Online ARM64** (`PrimeDictate-*-Windows-arm64-Online.msi`) | Native ARM64 app under `Program Files\PrimeDictate` with the same installer behavior. It also carries the ONNX Runtime QNN natives (`QnnHtp.dll`, `onnxruntime_providers_qnn.dll` and friends), so Snapdragon PCs can run the Qualcomm AI Hub Whisper and Moonshine NPU models. | After install, users open PrimeDictate and choose a model in first-run setup or Settings. The app can download supported models itself or browse to an existing local model folder. ## Installer UX - **Online MSI**: Uses WiX UI to install the app payload only. The MSI does not run external download commands or launch PrimeDictate from the finish dialog. -- **Launch at login**: Setup installs `PrimeDictate.lnk` in the all-users Windows Startup folder by default so PrimeDictate runs after users sign in. Silent installs can opt out with `LAUNCHATLOGIN=0`. +- **Launch at login**: Setup installs `PrimeDictate.lnk` in the all-users Windows Startup folder by default; it runs `PrimeDictate.exe --background` (tray only) after users sign in. Silent installs can opt out with `LAUNCHATLOGIN=0`. The app's "Start PrimeDictate when I sign in" checkbox controls that same shortcut per user (Explorer's StartupApproved switch, no administrator rights); when the shortcut is absent it uses a per-user Run value named `PrimeDictate` instead, never both. "Start it for everyone on this PC" (or `--enable-launch-at-login --scope=all-users`) writes a machine-wide Run value through an elevated helper (UAC prompt); turning it off for all users, in Settings or with `--disable-launch-at-login --scope=all-users`, removes that value and also deletes the MSI's all-users `PrimeDictate.lnk`, as the 6.0.0 app did (reinstall or enable launch at login for yourself to bring startup back). On first start the app removes the 6.0.0 per-user startup shortcut and stale Run values (and keeps launch at login on for a user who had it). - **First-run app entry**: Open PrimeDictate from the Start Menu or Startup shortcut after install to complete first-run setup, including model selection or download. - **Branding continuity**: ARP metadata, MSI names, and Start Menu shortcut text align with the app’s branded status language (**Ready=Blue, Recording=Red, Error=Yellow**). -- **Upgrade continuity**: The online MSI keeps the existing product identity (`Name` + `UpgradeCode`) for clean upgrades. +- **Upgrade continuity**: The online MSI keeps the existing product identity (`Name` + `UpgradeCode`) for clean upgrades. The 6.0.0 shortcut (argument `--from-login`) is replaced in place by the same component with `--background`, and all WPF files are removed with the old product. +- **Updates**: the app checks GitHub Releases (tray menu "Check for updates...", and automatically at most once a day), verifies the `PrimeDictate-Setup-vX.Y.Z-.msi` download against its `.sha256`, asks before installing, quits through its clean exit path (a meeting recording is saved), then runs the MSI with `LAUNCHATLOGIN=1` only if the installer shortcut exists. - **Language**: The installer is pinned to `en-US` UI resources for consistent English setup dialogs. ## Prerequisites (maintainer) @@ -96,7 +97,7 @@ Use this when winget reviewers request metadata changes for an existing version, | Path | Role | |------|------| -| `wix/shared/AppPayload.wxs` | `Program Files\PrimeDictate` tree and harvested publish payload | +| `wix/shared/AppPayload.wxs` | `Program Files\PrimeDictate` tree and harvested publish payload (`src/PrimeDictate.Desktop` publish output) | | `wix/shared/Branding.wxs` | ARP icon + common Add/Remove Programs metadata | | `wix/shared/StartMenuShortcuts.wxs` | Shared Start Menu shortcut component used by the online installer | | `wix/shared/LaunchAtLogin.wxs` | Optional all-users Startup-folder shortcut controlled by `LAUNCHATLOGIN` | diff --git a/installer/chocolatey/primedictate.nuspec b/installer/chocolatey/primedictate.nuspec index 0094005..09fd378 100644 --- a/installer/chocolatey/primedictate.nuspec +++ b/installer/chocolatey/primedictate.nuspec @@ -13,9 +13,9 @@ https://raw.githubusercontent.com/flowdevs-io/PrimeDictate/main/installer/wix/assets/PrimeDictate-chocolatey-icon.png https://github.com/flowdevs-io/PrimeDictate?tab=readme-ov-file#license https://github.com/flowdevs-io/PrimeDictate/releases - dictation speech-to-text whisper local-ai wpf windows + dictation speech-to-text whisper local-ai windows PrimeDictate is a fast, local speech-to-text application for Windows. - PrimeDictate is a Windows tray dictation app built with .NET and WPF. It uses local Whisper transcription for privacy-first speech-to-text, global hotkey capture, live overlay preview, and final-only text injection into the active app. + PrimeDictate is a Windows tray dictation app built with .NET. It uses local Whisper transcription for privacy-first speech-to-text, global hotkey capture, live overlay preview, and final-only text injection into the active app. diff --git a/installer/wix/online/Package.wxs b/installer/wix/online/Package.wxs index a9bc3d7..79a6ee8 100644 --- a/installer/wix/online/Package.wxs +++ b/installer/wix/online/Package.wxs @@ -22,7 +22,7 @@ diff --git a/installer/wix/online/PrimeDictate.Online.wixproj b/installer/wix/online/PrimeDictate.Online.wixproj index 72945dd..9400b88 100644 --- a/installer/wix/online/PrimeDictate.Online.wixproj +++ b/installer/wix/online/PrimeDictate.Online.wixproj @@ -7,6 +7,8 @@ obj\$(InstallerPlatform)\$(Configuration)\ PrimeDictate-$(Version)-Windows-$(InstallerPlatform)-Online en-US + + ICE60 $([MSBuild]::NormalizePath('$(MSBuildThisFileDirectory)', '..\..\..\artifacts\$(RuntimeIdentifier)\publish')) $(Version).0 ProductVersion=$(ProductVersion) diff --git a/installer/wix/shared/Branding.wxs b/installer/wix/shared/Branding.wxs index a5b6484..1f245c5 100644 --- a/installer/wix/shared/Branding.wxs +++ b/installer/wix/shared/Branding.wxs @@ -3,7 +3,7 @@ - + diff --git a/installer/wix/shared/LaunchAtLogin.wxs b/installer/wix/shared/LaunchAtLogin.wxs index 55a9d18..8ca888f 100644 --- a/installer/wix/shared/LaunchAtLogin.wxs +++ b/installer/wix/shared/LaunchAtLogin.wxs @@ -21,10 +21,10 @@ Id="ScPrimeDictateStartup" Name="PrimeDictate" Target="[INSTALLFOLDER]PrimeDictate.exe" - Arguments="--from-login" + Arguments="--background" WorkingDirectory="INSTALLFOLDER" Icon="AppIcon" - Description="Start PrimeDictate when Windows starts." /> + Description="Start PrimeDictate in the tray when you sign in." /> + Description="Local dictation, transcription and meetings with a color-coded tray state." /> \publish directories without running dotnet publish. .NOTES - Requires .NET 8 SDK. WiX Toolset is restored via NuGet (WixToolset.Sdk); no separate WiX install needed. + Requires the .NET SDK pinned in global.json. WiX Toolset is restored via NuGet (WixToolset.Sdk); no separate WiX install needed. #> param( [ValidateSet("Online")] @@ -78,8 +78,44 @@ function Get-InstallerPlatform { } } +# What the self-contained publish of the Avalonia app (src\PrimeDictate.Desktop) must contain besides the .NET runtime: +# Avalonia and its Win32 backend, Skia and ANGLE, then the natives the app loads at runtime (miniaudio capture, ONNX Runtime and +# sherpa-onnx speech, the SharpHook global hook, SQLite). A missing one only fails on the machine that lacks it, so check here. +$script:AppPayloadFiles = @( + "Avalonia.Base.dll", + "Avalonia.Controls.dll", + "Avalonia.Win32.dll", + "libSkiaSharp.dll", + "av_libglesv2.dll", + "miniaudio.dll", + "onnxruntime.dll", + "sherpa-onnx-c-api.dll", + "uiohook.dll", + "e_sqlite3.dll" +) + +# Files that depend on the target: Whisper.net (managed library and its whisper.cpp native, published under runtimes\) on both, +# and on arm64 the Qualcomm QNN natives (copied over the output at publish, replacing the ONNX Runtime that sherpa-onnx brought) and the +# Microsoft.ML.OnnxRuntime managed assembly the QNN code calls. +function Get-RidPayloadFiles { + param([string] $Rid) + + $files = @("Whisper.net.dll", "runtimes\$Rid\whisper.dll") + if ($Rid -eq "win-arm64") { + $files += @( + "Microsoft.ML.OnnxRuntime.dll", + "onnxruntime_providers_qnn.dll", + "onnxruntime_providers_shared.dll", + "QnnHtp.dll", + "QnnSystem.dll" + ) + } + + return $files +} + function Test-SelfContainedPublishOutput { - param([string] $PublishDir) + param([string] $PublishDir, [string] $Rid) $requiredFiles = @( "PrimeDictate.exe", @@ -88,10 +124,8 @@ function Test-SelfContainedPublishOutput { "hostfxr.dll", "hostpolicy.dll", "coreclr.dll", - "System.Private.CoreLib.dll", - "PresentationFramework.dll", - "WindowsBase.dll" - ) + "System.Private.CoreLib.dll" + ) + $script:AppPayloadFiles + (Get-RidPayloadFiles -Rid $Rid) foreach ($file in $requiredFiles) { $path = Join-Path $PublishDir $file @@ -108,18 +142,18 @@ function Test-SelfContainedPublishOutput { } function Test-MsiContainsSelfContainedRuntime { - param([string] $MsiPath) + param([string] $MsiPath, [string] $Rid) $requiredFiles = @( "hostfxr.dll", "hostpolicy.dll", "coreclr.dll", "System.Private.CoreLib.dll", - "PresentationFramework.dll", - "WindowsBase.dll" - ) + "PrimeDictate.exe", + "PrimeDictate.dll" + ) + $script:AppPayloadFiles + ((Get-RidPayloadFiles -Rid $Rid) | ForEach-Object { Split-Path -Leaf $_ }) - $installer = New-Object -ComObject WindowsInstaller.Installer + $installer =New-Object -ComObject WindowsInstaller.Installer $database = $null $view = $null try { @@ -195,7 +229,7 @@ foreach ($rid in $requestedRids) { throw "Publish output missing PrimeDictate.exe at $publishDir. Run without -SkipPublish." } - Test-SelfContainedPublishOutput -PublishDir $publishDir + Test-SelfContainedPublishOutput -PublishDir $publishDir -Rid $rid $publishDirFull = (Resolve-Path $publishDir).Path Write-Host "Building online MSI for $rid..." @@ -215,7 +249,7 @@ foreach ($rid in $requestedRids) { throw "Expected built MSI not found: $builtOnlineMsi" } - Test-MsiContainsSelfContainedRuntime -MsiPath $builtOnlineMsi + Test-MsiContainsSelfContainedRuntime -MsiPath $builtOnlineMsi -Rid $rid Copy-Item -Force $builtOnlineMsi $publishedOnlineMsi } diff --git a/scripts/Publish-Windows.ps1 b/scripts/Publish-Windows.ps1 index 1c9ea1d..fe200ca 100644 --- a/scripts/Publish-Windows.ps1 +++ b/scripts/Publish-Windows.ps1 @@ -1,7 +1,8 @@ #Requires -Version 5.1 <# .SYNOPSIS - Publishes a self-contained Windows build to artifacts\\publish. + Publishes the self-contained Windows build of the Avalonia app (src\PrimeDictate.Desktop, output PrimeDictate.exe) + to artifacts\\publish. The WPF app at the repo root is legacy and is not published here. #> param( [string] $Configuration = "Release", @@ -39,7 +40,7 @@ try { $msbuildProps += "-p:InformationalVersion=$InformationalVersion" } - dotnet publish .\PrimeDictate.csproj ` + dotnet publish .\src\PrimeDictate.Desktop\PrimeDictate.Desktop.csproj ` -c $Configuration ` -r $RuntimeIdentifier ` --self-contained true ` @@ -52,6 +53,32 @@ try { throw "dotnet publish failed with exit code $LASTEXITCODE" } + # Qualcomm NPU (QNN) on ARM64: the app project copies the ONNX Runtime QNN natives over sherpa-onnx's onnxruntime.dll after publish + # (one process can load only one onnxruntime.dll). If that step silently stopped working, the ARM64 installer would ship without the + # NPU and nobody would notice until a Snapdragon PC ran it, so fail the publish instead. x64 must not carry them. + $qnnFiles = @("QnnHtp.dll", "QnnSystem.dll", "onnxruntime_providers_qnn.dll") + if ($RuntimeIdentifier -eq "win-arm64") { + $missingQnn = $qnnFiles | Where-Object { -not (Test-Path (Join-Path $publishDir $_)) } + if ($missingQnn) { + throw "The win-arm64 publish is missing the Qualcomm QNN natives: $($missingQnn -join ', ')." + } + } + elseif (Test-Path (Join-Path $publishDir "QnnHtp.dll")) { + throw "The $RuntimeIdentifier publish contains Qualcomm QNN natives, which belong only in win-arm64." + } + + # Whisper.net ships its natives as content under runtimes\ and runtimes\\ for every OS, and + # publish copies all of them. Keep only the folders for this runtime. + $runtimesDir = Join-Path $publishDir "runtimes" + if (Test-Path $runtimesDir) { + Get-ChildItem $runtimesDir -Directory -Recurse | + Where-Object { $_.Name -match '^(win|linux|linux-musl|osx|macos|maccatalyst|ios|android|browser)(-|$)' -and $_.Name -ne $RuntimeIdentifier } | + Sort-Object { $_.FullName.Length } -Descending | + ForEach-Object { if (Test-Path $_.FullName) { Remove-Item -Recurse -Force $_.FullName } } + Get-ChildItem $runtimesDir -Directory | Where-Object { -not (Get-ChildItem $_.FullName -Recurse -File) } | + ForEach-Object { Remove-Item -Recurse -Force $_.FullName } + } + Write-Host "Published to $publishDir" } finally { diff --git a/src/PrimeDictate.Core/Diagnostics/ActivityFeed.cs b/src/PrimeDictate.Core/Diagnostics/ActivityFeed.cs new file mode 100644 index 0000000..427c9fc --- /dev/null +++ b/src/PrimeDictate.Core/Diagnostics/ActivityFeed.cs @@ -0,0 +1,154 @@ +namespace PrimeDictate.Core.Diagnostics; + +public enum ActivityLevel +{ + Info = 0, + Warning = 1, + Error = 2 +} + +/// One line in the activity feed. Never carries recognized text; callers pass only reasons and counts. +public sealed record ActivityEntry(DateTime TimestampUtc, ActivityLevel Level, string Source, string Message, Guid? SessionId = null, int RepeatCount = 1); + +/// How a dictation session ended, without the words: what the Dictation activity view shows per session. +public enum DictationSessionStatus +{ + Listening = 0, + Processing = 1, + Typed = 2, + NotTyped = 3, + Error = 4, + VoiceCommand = 5, + Discarded = 6 +} + +/// One dictation session: time range, latest status and the app it was aimed at. No text. +public sealed record DictationSessionInfo(Guid Id, DateTime StartedUtc, DateTime EndedUtc, DictationSessionStatus Status, string? AppName); + +/// +/// In-memory, newest-first activity feed fed by : repeated identical lines collapse into one with a +/// count, the global list holds at most 600 entries, and entries and sessions are grouped by dictation session id. It is +/// a diagnostic aid, so by the same rule as the log file nothing recognized is ever put in it. +/// +public sealed class ActivityFeed +{ + public const int MaxGlobalEntries = 600; + public const int MaxSessionEntries = 300; + public const int MaxSessions = 100; + + private readonly object gate = new(); + private readonly List global = []; + private readonly Dictionary> bySession = []; + private readonly List sessions = []; // newest first + + /// Raised after any change, on the thread that made it. + public event Action? Changed; + + public void Add(ActivityLevel level, string source, string message, Guid? sessionId = null, DateTime? timestampUtc = null) + { + var entry = new ActivityEntry(timestampUtc ?? DateTime.UtcNow, level, source, message, sessionId); + lock (this.gate) + { + InsertWithAggregation(this.global, entry, MaxGlobalEntries); + if (sessionId is { } id) + { + if (!this.bySession.TryGetValue(id, out var list)) + { + list = []; + this.bySession[id] = list; + } + + InsertWithAggregation(list, entry, MaxSessionEntries); + } + } + + this.Changed?.Invoke(); + } + + /// Creates or updates a session row. A session never moves back in time: the start stays, the end follows. + public void SetSession(Guid id, DateTime timestampUtc, DictationSessionStatus status, string? appName = null) + { + lock (this.gate) + { + var i = this.sessions.FindIndex(s => s.Id == id); + if (i < 0) + { + this.sessions.Insert(0, new DictationSessionInfo(id, timestampUtc, timestampUtc, status, appName)); + while (this.sessions.Count > MaxSessions) + { + var dropped = this.sessions[^1]; + this.sessions.RemoveAt(this.sessions.Count - 1); + this.bySession.Remove(dropped.Id); + } + } + else + { + var old = this.sessions[i]; + this.sessions[i] = old with + { + EndedUtc = timestampUtc > old.EndedUtc ? timestampUtc : old.EndedUtc, + Status = status, + AppName = string.IsNullOrWhiteSpace(appName) ? old.AppName : appName + }; + } + } + + this.Changed?.Invoke(); + } + + /// Newest first. + public IReadOnlyList Entries() + { + lock (this.gate) + { + return [.. this.global]; + } + } + + /// Newest first. + public IReadOnlyList SessionEntries(Guid id) + { + lock (this.gate) + { + return this.bySession.TryGetValue(id, out var list) ? [.. list] : []; + } + } + + /// Newest first. + public IReadOnlyList Sessions() + { + lock (this.gate) + { + return [.. this.sessions]; + } + } + + public void Clear() + { + lock (this.gate) + { + this.global.Clear(); + this.bySession.Clear(); + this.sessions.Clear(); + } + + this.Changed?.Invoke(); + } + + private static void InsertWithAggregation(List target, ActivityEntry entry, int max) + { + if (target.Count > 0 && target[0].Level == entry.Level && target[0].SessionId == entry.SessionId + && string.Equals(target[0].Source, entry.Source, StringComparison.Ordinal) + && string.Equals(target[0].Message, entry.Message, StringComparison.Ordinal)) + { + target[0] = target[0] with { TimestampUtc = entry.TimestampUtc, RepeatCount = target[0].RepeatCount + 1 }; + return; + } + + target.Insert(0, entry); + if (target.Count > max) + { + target.RemoveRange(max, target.Count - max); + } + } +} diff --git a/src/PrimeDictate.Core/Diagnostics/ActivityText.cs b/src/PrimeDictate.Core/Diagnostics/ActivityText.cs new file mode 100644 index 0000000..7aa0c70 --- /dev/null +++ b/src/PrimeDictate.Core/Diagnostics/ActivityText.cs @@ -0,0 +1,35 @@ +namespace PrimeDictate.Core.Diagnostics; + +/// Plain-text rendering shared by the activity view and its Copy buttons. +public static class ActivityText +{ + public static string Level(ActivityLevel level) => level switch { ActivityLevel.Error => "ERR", ActivityLevel.Warning => "WRN", _ => "INF" }; + + public static string Status(DictationSessionStatus status) => status switch + { + DictationSessionStatus.Listening => "Listening", + DictationSessionStatus.Processing => "Processing", + DictationSessionStatus.Typed => "Typed", + DictationSessionStatus.NotTyped => "Not typed", + DictationSessionStatus.Error => "Error", + DictationSessionStatus.VoiceCommand => "Voice command", + _ => "Discarded" + }; + + public static string Message(ActivityEntry e) => e.RepeatCount > 1 ? $"{e.Message} (x{e.RepeatCount})" : e.Message; + + public static string Line(ActivityEntry e) => + $"{e.TimestampUtc.ToLocalTime():yyyy-MM-dd HH:mm:ss} {Level(e.Level)} [{e.Source}] {Message(e)}"; + + public static string Session(DictationSessionInfo s) + { + var start = s.StartedUtc.ToLocalTime(); + var end = s.EndedUtc.ToLocalTime(); + var range = end - start < TimeSpan.FromSeconds(1) ? $"{start:HH:mm:ss}" : $"{start:HH:mm:ss}-{end:HH:mm:ss}"; + return string.IsNullOrWhiteSpace(s.AppName) ? $"{range} {Status(s.Status)}" : $"{range} {Status(s.Status)} {s.AppName}"; + } + + /// Oldest first, one line each, for pasting into a bug report. + public static string Join(IEnumerable newestFirst) => + string.Join(Environment.NewLine, newestFirst.Reverse().Select(Line)); +} diff --git a/src/PrimeDictate.Core/Diagnostics/AppLog.cs b/src/PrimeDictate.Core/Diagnostics/AppLog.cs index 7983b1d..20030dd 100644 --- a/src/PrimeDictate.Core/Diagnostics/AppLog.cs +++ b/src/PrimeDictate.Core/Diagnostics/AppLog.cs @@ -16,14 +16,50 @@ public static class AppLog /// Where the log goes. Tests point it at a temporary folder. public static string Directory { get; set; } = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "PrimeDictate", "logs"); + /// The in-memory feed behind the Dictation activity view; every call below also lands here. + public static ActivityFeed Feed { get; } = new(); + public static string FilePath => Path.Combine(Directory, "app.log"); + /// + /// Raised (on the thread that logged) whenever an error is logged by or , so the tray can show its + /// "needs attention" icon for a while, as the WPF app did on any error-level log line. Carries no text. + /// + public static event Action? ErrorLogged; + /// Records an exception with where it happened. - public static void Fault(string source, Exception exception) => + public static void Fault(string source, Exception exception, Guid? sessionId = null) + { + Feed.Add(ActivityLevel.Error, source, $"{exception.GetType().FullName}: {Cut(exception.Message, MaxMessage)}", sessionId); Write($"[{source}] {exception.GetType().FullName}: {Cut(exception.Message, MaxMessage)}{Environment.NewLine}{Cut(exception.StackTrace ?? string.Empty, MaxStack)}"); + RaiseErrorLogged(); + } + + /// Records a failure that is not an exception (a listener that gave up, for example). Never pass recognized text. + public static void Error(string source, string message, Guid? sessionId = null) => Event(source, message, ActivityLevel.Error, sessionId); + + private static void RaiseErrorLogged() + { + try + { + ErrorLogged?.Invoke(); + } + catch (Exception ex) when (ex is InvalidOperationException or ObjectDisposedException) + { + // A subscriber that is shutting down must not break the code that logged. + } + } /// Records something the app observed, such as a timeout. Never pass recognized text. - public static void Event(string source, string message) => Write($"[{source}] {Cut(message, MaxMessage)}"); + public static void Event(string source, string message, ActivityLevel level = ActivityLevel.Info, Guid? sessionId = null) + { + Feed.Add(level, source, Cut(message, MaxMessage), sessionId); + Write($"[{source}] {Cut(message, MaxMessage)}"); + if (level == ActivityLevel.Error) + { + RaiseErrorLogged(); + } + } private static string Cut(string text, int max) => text.Length <= max ? text : text[..max] + "…"; diff --git a/src/PrimeDictate.Core/Dictation/CompactRipple.cs b/src/PrimeDictate.Core/Dictation/CompactRipple.cs new file mode 100644 index 0000000..54e786c --- /dev/null +++ b/src/PrimeDictate.Core/Dictation/CompactRipple.cs @@ -0,0 +1,83 @@ +namespace PrimeDictate.Core.Dictation; + +/// +/// The compact overlay's microphone animation, ported from the WPF overlay: three rings that grow outward from the middle of +/// the microphone and fade, brighter while there is voice, plus a glow inside the microphone that follows the level. No UI types, +/// so the numbers are testable and the shell only draws them. Call as audio arrives and at about 30 Hz. +/// +public sealed class CompactRipple +{ + public const int RingCount = 3; + public const double SecondsPerCycle = 2.85; + + private double targetRms; + private double smoothedRms; + private double pulseEnvelope; + private double surfaceEnvelope; + + /// 0 to 1, how far the first ring is through its cycle. + public double Phase { get; private set; } + + /// 0.15 (silent) to 1 (loud): scales how visible the rings are. + public double Intensity { get; private set; } = 0.15; + + /// The glow inside the microphone, 0 to 0.88. + public double GlowOpacity { get; private set; } + + /// Size of the inner glow relative to the microphone, 0.78 to about 1.1. + public double GlowScale { get; private set; } = 0.78; + + public void SetLevel(double rms) => this.targetRms = Math.Max(0, rms); + + public void Reset() + { + this.targetRms = 0; + this.smoothedRms = 0; + this.pulseEnvelope = 0; + this.surfaceEnvelope = 0; + this.Phase = 0; + this.Intensity = 0.15; + this.GlowOpacity = 0; + this.GlowScale = 0.78; + } + + public void Tick(double elapsedSeconds = 0.033) + { + // The target decays so a level that stops arriving fades out by itself; the smoothed value follows it. + this.targetRms *= 0.8; + this.smoothedRms = (this.smoothedRms * 0.7) + (this.targetRms * 0.3); + var voice = Math.Min(1.0, this.smoothedRms * 34.0); + var boost = Math.Min(0.85, this.smoothedRms * 5.0); + + // Fast attack and slow release: the pulse jumps up with speech and lingers instead of breathing in and out. + this.pulseEnvelope = voice > this.pulseEnvelope ? this.pulseEnvelope + ((voice - this.pulseEnvelope) * 0.72) : this.pulseEnvelope * 0.965; + this.surfaceEnvelope = voice > this.surfaceEnvelope ? this.surfaceEnvelope + ((voice - this.surfaceEnvelope) * 0.82) : this.surfaceEnvelope * 0.972; + + this.GlowScale = 0.78 + (this.pulseEnvelope * 0.22) + (boost * 0.12); + this.GlowOpacity = Math.Clamp(0.12 + (this.surfaceEnvelope * 0.62) + (boost * 0.45), 0.0, 0.88); + + this.Phase += elapsedSeconds / SecondsPerCycle; + if (this.Phase >= 1.0) + { + this.Phase -= Math.Floor(this.Phase); + } + + this.Intensity = Math.Clamp(0.15 + (this.surfaceEnvelope * 0.95) + (boost * 0.35), 0.0, 1.0); + } + + /// Scale and opacity of ring (0 to 2); the rings are a third of a cycle apart. + public (double Scale, double Opacity) Ring(int index) + { + var phase = this.Phase + (index / (double)RingCount); + return (Scale(phase), Opacity(phase, this.Intensity)); + } + + /// Starts small in the middle and grows to the edge of the microphone. + public static double Scale(double phase) => 0.32 + (Fraction(phase) * 0.68); + + /// Fades as it grows; a louder voice makes the rings stronger, but never above 0.48. + public static double Opacity(double phase, double voice) => + Math.Clamp(Math.Pow(1.0 - Fraction(phase), 1.85) * (0.08 + (voice * 0.5)), 0.0, 0.48); + + private static double Fraction(double phase) => phase - Math.Floor(phase); +} diff --git a/src/PrimeDictate.Core/Dictation/DefaultMicrophoneFallback.cs b/src/PrimeDictate.Core/Dictation/DefaultMicrophoneFallback.cs index 231d3c0..b694c8d 100644 --- a/src/PrimeDictate.Core/Dictation/DefaultMicrophoneFallback.cs +++ b/src/PrimeDictate.Core/Dictation/DefaultMicrophoneFallback.cs @@ -15,26 +15,29 @@ public sealed class DefaultMicrophoneFallback(IAudioSource inner, Action public ValueTask> ListDevicesAsync(CancellationToken cancellationToken) => inner.ListDevicesAsync(cancellationToken); - public async ValueTask OpenAsync(string? deviceId, CancellationToken cancellationToken) + public ValueTask OpenAsync(string? deviceId, CancellationToken cancellationToken) => + this.OpenAsync(deviceId, MicAccessMode.Shared, cancellationToken); + + public async ValueTask OpenAsync(string? deviceId, MicAccessMode access, CancellationToken cancellationToken) { if (deviceId is null) { - return await inner.OpenAsync(null, cancellationToken).ConfigureAwait(false); + return await inner.OpenAsync(null, access, cancellationToken).ConfigureAwait(false); } try { - var lease = await inner.OpenAsync(deviceId, cancellationToken).ConfigureAwait(false); + var lease = await inner.OpenAsync(deviceId, access, cancellationToken).ConfigureAwait(false); // It is back, so losing it again is worth saying again. Volatile.Write(ref this.reported, 0); return lease; } catch (AudioSourceException ex) when (ex.Kind == AudioSourceErrorKind.DeviceRemoved) { - var lease = await inner.OpenAsync(null, cancellationToken).ConfigureAwait(false); + var lease = await inner.OpenAsync(null, access, cancellationToken).ConfigureAwait(false); if (Interlocked.Exchange(ref this.reported, 1) == 0) { - AppLog.Event("microphone", "The microphone chosen in Settings is not connected; dictation uses the default microphone."); + AppLog.Event("microphone", "The microphone chosen in Settings is not connected; dictation uses the default microphone.", ActivityLevel.Warning); notice("The microphone chosen in Settings is not connected, so the default microphone is used. Pick one in Settings to change that."); } diff --git a/src/PrimeDictate.Core/Dictation/Delivery.cs b/src/PrimeDictate.Core/Dictation/Delivery.cs index 7aa5661..053d81e 100644 --- a/src/PrimeDictate.Core/Dictation/Delivery.cs +++ b/src/PrimeDictate.Core/Dictation/Delivery.cs @@ -14,6 +14,12 @@ public interface IForegroundTarget /// Best-effort return to this window so a late final can still land there. bool TryRestore() => false; + + /// + /// Best-effort insertion straight into the control that had focus, without bringing the window forward + /// (Windows edit controls). False means the caller should restore the window and type instead. + /// + bool TryInjectDirectly(string text) => false; } /// @@ -59,6 +65,7 @@ public sealed record DeliveryResult(DictationDeliveryStatus Status, bool EnterSe /// public static class TranscriptDelivery { + public static DeliveryResult Deliver( string text, IForegroundTarget? target, @@ -81,6 +88,13 @@ public static DeliveryResult Deliver( return new DeliveryResult(DictationDeliveryStatus.SkippedFocusChanged, false, "Focused window changed before transcript typing."); } + // WPF order: without coding-mode Enter, put the text into the original target's focused edit control + // without reactivating it; otherwise (or if that is not possible) restore the window and type. + if (!options.SendEnterAfterCommit && target.TryInjectDirectly(text)) + { + return new DeliveryResult(DictationDeliveryStatus.Injected, false, null); + } + if (!target.TryRestore()) { return new DeliveryResult( diff --git a/src/PrimeDictate.Core/Dictation/DictationController.cs b/src/PrimeDictate.Core/Dictation/DictationController.cs index b67b8f9..94a7593 100644 --- a/src/PrimeDictate.Core/Dictation/DictationController.cs +++ b/src/PrimeDictate.Core/Dictation/DictationController.cs @@ -54,6 +54,7 @@ public sealed class DictationController : IAsyncDisposable private readonly IVoiceCommandProcessor voiceCommands; private readonly TimeProvider time; private readonly ITranscriptRewriter? rewriter; + private readonly IVoiceShellCommandRunner? shellRunner; private readonly SemaphoreSlim gate = new(1, 1); private volatile DictationOptions options = new DictationOptions().Normalized(); private Session? session; @@ -66,7 +67,8 @@ public DictationController( MicrophoneCoordinator? microphone = null, IVoiceCommandProcessor? voiceCommands = null, TimeProvider? time = null, - ITranscriptRewriter? rewriter = null) + ITranscriptRewriter? rewriter = null, + IVoiceShellCommandRunner? shellRunner = null) { this.audioSource = audioSource; this.providerSource = providerSource; @@ -76,6 +78,7 @@ public DictationController( this.voiceCommands = voiceCommands ?? NoVoiceCommands.Instance; this.time = time ?? TimeProvider.System; this.rewriter = rewriter; + this.shellRunner = shellRunner; } public event Action? StateChanged; @@ -96,6 +99,9 @@ public DictationController( public bool IsRecording => Volatile.Read(ref this.session) is not null; + /// How the microphone is open right now (Exclusive or Shared), or null when dictation is not recording. + public MicAccessMode? ActiveMicAccess => Volatile.Read(ref this.session)?.Capture.AccessMode; + public DictationOptions Options { get => this.options; @@ -179,7 +185,11 @@ private async Task StartCoreAsync() micLease = await this.microphone.AcquireAsync(MicrophoneOwner, CancellationToken.None).ConfigureAwait(false); } - capture = await this.audioSource.OpenAsync(opts.InputDeviceId, CancellationToken.None).ConfigureAwait(false); + capture = await this.audioSource.OpenAsync( + opts.InputDeviceId, + opts.ExclusiveMicAccess ? MicAccessMode.Exclusive : MicAccessMode.Shared, + CancellationToken.None).ConfigureAwait(false); + Diagnostics.AppLog.Event("dictation", $"Microphone opened ({capture.AccessMode.ToString().ToLowerInvariant()} access{(opts.ExclusiveMicAccess && capture.AccessMode == MicAccessMode.Shared ? ", exclusive was requested" : string.Empty)})."); } catch (Exception ex) when (ex is MicrophoneBusyException or AudioSourceException) { @@ -188,7 +198,7 @@ private async Task StartCoreAsync() await micLease.DisposeAsync().ConfigureAwait(false); } - Diagnostics.AppLog.Event("dictation", $"Dictation did not start: {ex.Message}"); + Diagnostics.AppLog.Event("dictation", $"Dictation did not start: {ex.Message}", Diagnostics.ActivityLevel.Error); this.Notice?.Invoke(ex.Message); return; } @@ -197,7 +207,8 @@ private async Task StartCoreAsync() var s = new Session(capture, micLease, provider, target, new SpeechActivityTracker(this.time), this.time.GetUtcNow().UtcDateTime); s.Tracker.LevelUpdated += level => this.LevelChanged?.Invoke(level); this.session = s; - s.Reader = Task.Run(() => this.ReadLoopAsync(s)); + Track(s, Diagnostics.DictationSessionStatus.Listening, "Listening started."); + s.Reader =Task.Run(() => this.ReadLoopAsync(s)); s.Preview = Task.Run(() => this.PreviewLoopAsync(s)); this.SetState(DictationState.Listening); } @@ -220,10 +231,12 @@ private async Task StopCoreAsync(string reason, bool commit) { if (!commit || s.DiscardRequested) { + Track(s, Diagnostics.DictationSessionStatus.Discarded, $"Discarded ({reason})."); this.SetState(DictationState.Idle); return; } + Track(s, Diagnostics.DictationSessionStatus.Processing, $"Stopped ({reason}); transcribing."); this.SetState(DictationState.Processing); await this.FinishAsync(s, reason).ConfigureAwait(false); } @@ -414,6 +427,7 @@ private async Task FinishAsync(Session s, string reason) var duration = TimeSpan.FromSeconds(audio.Length / (double)SampleRate); if (audio.Length == 0 || !s.Tracker.HasSpeechEvidence(audio)) { + Track(s, Diagnostics.DictationSessionStatus.Discarded, "No speech was heard; nothing typed."); return; } @@ -421,7 +435,19 @@ private async Task FinishAsync(Session s, string reason) try { var raw = await this.RecognizeAsync(s.Provider, audio, CancellationToken.None).ConfigureAwait(false); - var match = this.voiceCommands.Apply(TranscriptPostProcessor.RemoveTrailingSilenceArtifact(raw, reason.Contains("silence", StringComparison.Ordinal))); + var spoken = TranscriptPostProcessor.RemoveTrailingSilenceArtifact(raw, reason.Contains("silence", StringComparison.Ordinal)); + // Shell commands are matched only here, in the final transcript of a dictation, and only when a runner exists. + var match = this.shellRunner is null ? this.voiceCommands.Apply(spoken) : this.voiceCommands.ApplyFinal(spoken); + if (match.Shell is { } shell && this.shellRunner is not null) + { + // Stop ends the dictation. Continue with nothing left after the phrase also ends it, silently (as the WPF app did): + // the command was the whole utterance, so "No text was recognized" would be wrong. + if (!await this.RunShellCommandAsync(s, shell, this.shellRunner, duration).ConfigureAwait(false) || string.IsNullOrWhiteSpace(match.CleanedText)) + { + return; + } + } + if (match.StopRequested) { return; @@ -436,6 +462,7 @@ private async Task FinishAsync(Session s, string reason) } catch (Exception ex) { + Track(s, Diagnostics.DictationSessionStatus.Error, $"Transcription failed: {ex.GetType().Name}.", Diagnostics.ActivityLevel.Error); this.Notice?.Invoke($"Transcription failed: {ex.Message}"); return; } @@ -447,6 +474,7 @@ private async Task FinishAsync(Session s, string reason) if (transcript.Length == 0) { + Track(s, Diagnostics.DictationSessionStatus.NotTyped, "No text was recognized.", Diagnostics.ActivityLevel.Warning); this.Notice?.Invoke("No text was recognized. Try raising input gain or choosing a different model."); return; } @@ -456,6 +484,12 @@ private async Task FinishAsync(Session s, string reason) string? rewritePrompt = null; if (this.rewriter is not null) { + if (this.rewriter.IsActive) + { + // The overlay shows this status while the model works, as the WPF app did. + this.PartialTranscript?.Invoke(s.Id, RewriteStatusText); + } + try { var rewrite = await this.rewriter.RewriteAsync(transcript, s.Target, CancellationToken.None).ConfigureAwait(false); @@ -482,9 +516,20 @@ private async Task FinishAsync(Session s, string reason) if (result.Status != DictationDeliveryStatus.Injected) { // Why it was not typed (the guard's or the injector's reason); never the transcript itself. - Diagnostics.AppLog.Event("dictation", $"Not typed ({result.Status}): {result.Error}"); + Diagnostics.AppLog.Event("dictation", $"Not typed ({result.Status}): {result.Error}", Diagnostics.ActivityLevel.Warning, s.Id); } + Track( + s, + result.Status switch + { + DictationDeliveryStatus.Injected => Diagnostics.DictationSessionStatus.Typed, + DictationDeliveryStatus.FailedToInject => Diagnostics.DictationSessionStatus.Error, + DictationDeliveryStatus.Discarded => Diagnostics.DictationSessionStatus.Discarded, + _ => Diagnostics.DictationSessionStatus.NotTyped + }, + result.Status == DictationDeliveryStatus.Injected ? $"Typed ({duration.TotalSeconds:0.0} s of audio)." : null); + this.Committed?.Invoke(new DictationCommit( s.Id, this.time.GetUtcNow().UtcDateTime, @@ -500,6 +545,87 @@ private async Task FinishAsync(Session s, string reason) rewritePrompt)); } + /// Updates the session row in the activity feed and, when there is a message, adds it to the session's log. Never text. + private static void Track(Session s, Diagnostics.DictationSessionStatus status, string? message, Diagnostics.ActivityLevel level = Diagnostics.ActivityLevel.Info) + { + var feed = Diagnostics.AppLog.Feed; + feed.SetSession(s.Id, DateTime.UtcNow, status, s.Target?.AppName ?? s.Target?.DisplayName); + if (message is not null) + { + feed.Add(level, "dictation", message, s.Id); + } + } + + /// What the overlay shows while a rewrite runs (the WPF text). + public const string RewriteStatusText = "[AI is processing transcript...]"; + + private static readonly TimeSpan ShellTypeTargetWait = TimeSpan.FromSeconds(2); + private static readonly TimeSpan ShellTypePollInterval = TimeSpan.FromMilliseconds(100); + + /// + /// Runs the user's command for a matched phrase. The command string is the one saved in settings; nothing from the + /// transcript reaches it. Returns true when the rest of the dictation should go on to be typed: only for + /// and only when words remain after the phrase was removed. + /// + private async Task RunShellCommandAsync(Session s, VoiceShellCommandInvocation invocation, IVoiceShellCommandRunner runner, TimeSpan duration) + { + var command = invocation.Command; + var phrase = command.Phrase.Trim(); + var label = $"Voice command: {phrase}"; + DictationDeliveryStatus status; + string? error = null; + try + { + var result = runner.Run(command); + Diagnostics.AppLog.Event("dictation", $"Voice command ran: \"{phrase}\" (pid {result.ProcessId?.ToString() ?? "unknown"}).", sessionId: s.Id); + Track(s, Diagnostics.DictationSessionStatus.VoiceCommand, null); + if (!string.IsNullOrWhiteSpace(invocation.TextToType)) + { + if (!await this.WaitForShellTypeTargetAsync(s.Target).ConfigureAwait(false)) + { + throw new InvalidOperationException("Chained typing skipped because the command did not move focus away from the starting window."); + } + + this.injector.TypeText(invocation.TextToType); + } + + status = DictationDeliveryStatus.CommandExecuted; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + status = DictationDeliveryStatus.CommandFailed; + error = ex.Message; + Diagnostics.AppLog.Event("dictation", $"Voice command failed: \"{phrase}\": {ex.Message}", Diagnostics.ActivityLevel.Error, s.Id); + Track(s, Diagnostics.DictationSessionStatus.Error, null); + this.Notice?.Invoke($"Voice command \"{phrase}\" failed: {ex.Message}"); + } + + this.Committed?.Invoke(new DictationCommit( + s.Id, this.time.GetUtcNow().UtcDateTime, label, status, "Command Prompt", "Command Prompt", "Command Prompt", error, duration, EnterSent: false)); + return status == DictationDeliveryStatus.CommandExecuted && command.CompletionBehavior == VoiceShellCommandCompletionBehavior.Continue; + } + + private async Task WaitForShellTypeTargetAsync(IForegroundTarget? start) + { + if (start is null) + { + await Task.Delay(ShellTypePollInterval, this.time).ConfigureAwait(false); + return true; + } + + var deadline = this.time.GetUtcNow() + ShellTypeTargetWait; + while (this.time.GetUtcNow() < deadline) + { + await Task.Delay(ShellTypePollInterval, this.time).ConfigureAwait(false); + if (!start.IsStillForeground()) + { + return true; + } + } + + return false; + } + private static async Task RecognizeAsync(ITranscriptionProvider provider, ReadOnlyMemory samples, string? language, CancellationToken ct) { var window = provider.Capabilities.MaxWindow ?? TimeSpan.FromSeconds(30); diff --git a/src/PrimeDictate.Core/Dictation/DictationHistoryFilter.cs b/src/PrimeDictate.Core/Dictation/DictationHistoryFilter.cs new file mode 100644 index 0000000..15917cb --- /dev/null +++ b/src/PrimeDictate.Core/Dictation/DictationHistoryFilter.cs @@ -0,0 +1,166 @@ +using System.Text; + +namespace PrimeDictate.Core.Dictation; + +public enum HistoryStatusFilter +{ + All, + Typed, + NotTyped +} + +public enum HistoryTargetKind +{ + All, + Known, + Unknown +} + +/// One choice in the history window's App or Window drop-down. +public sealed record HistoryTargetOption(HistoryTargetKind Kind, string? Value, string DisplayName) +{ + public override string ToString() => this.DisplayName; +} + +/// +/// The history window's filters, as the WPF window had them: search (every word must appear in the final or the original text), delivery +/// status, target app, and target window within that app. Pure functions over the entries, so they are testable without a window. +/// +public static class DictationHistoryFilter +{ + public static readonly HistoryTargetOption AllApps = new(HistoryTargetKind.All, null, "All apps"); + public static readonly HistoryTargetOption AllWindows = new(HistoryTargetKind.All, null, "All windows"); + private static readonly HistoryTargetOption UnknownApp = new(HistoryTargetKind.Unknown, null, "Unknown app"); + private static readonly HistoryTargetOption UnknownWindow = new(HistoryTargetKind.Unknown, null, "Unknown window"); + + public static string AppName(DictationHistoryEntry entry) => Normalize(entry.TargetAppName) ?? string.Empty; + + /// The window title, else the one inside the display name ("Title (0x1234)"), else empty. + public static string WindowTitle(DictationHistoryEntry entry) => + Normalize(entry.TargetWindowTitle) ?? ExtractWindowTitle(entry.TargetDisplayName) ?? string.Empty; + + public static IReadOnlyList Apply( + IEnumerable entries, + string? search, + HistoryStatusFilter status, + HistoryTargetOption app, + HistoryTargetOption window) + { + var terms = (search ?? string.Empty).Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return entries + .Where(e => status switch + { + HistoryStatusFilter.Typed => e.Status == DictationDeliveryStatus.Injected, + HistoryStatusFilter.NotTyped => e.Status != DictationDeliveryStatus.Injected, + _ => true + }) + .Where(e => terms.Length == 0 || terms.All(t => e.Transcript.Contains(t, StringComparison.OrdinalIgnoreCase) + || (e.OriginalTranscript?.Contains(t, StringComparison.OrdinalIgnoreCase) ?? false))) + .Where(e => Matches(AppName(e), app)) + .Where(e => Matches(WindowTitle(e), window)) + .ToList(); + } + + public static IReadOnlyList AppOptions(IReadOnlyCollection entries) + { + var options = new List { AllApps }; + options.AddRange(Known(entries.Select(AppName))); + if (entries.Any(e => AppName(e).Length == 0)) + { + options.Add(UnknownApp); + } + + return options; + } + + /// The windows of the entries that belong to the chosen app. + public static IReadOnlyList WindowOptions(IReadOnlyCollection entries, HistoryTargetOption app) + { + var ofApp = entries.Where(e => Matches(AppName(e), app)).ToList(); + var options = new List { AllWindows }; + options.AddRange(Known(ofApp.Select(WindowTitle))); + if (ofApp.Any(e => WindowTitle(e).Length == 0)) + { + options.Add(UnknownWindow); + } + + return options; + } + + /// The text "Copy details" puts on the clipboard: when and where it went, the result and, if it was rewritten, the original and the prompt. + public static string DetailsText(DictationHistoryEntry e) + { + var sb = new StringBuilder(); + sb.AppendLine($"Timestamp (UTC): {e.TimestampUtc:O}"); + sb.AppendLine($"Thread: {e.SessionId}"); + sb.AppendLine($"Delivery: {DeliveryDisplay(e.Status)}"); + sb.AppendLine($"Target app: {(AppName(e).Length > 0 ? AppName(e) : "Unknown app")}"); + sb.AppendLine($"Target window: {(WindowTitle(e).Length > 0 ? WindowTitle(e) : "Unknown window")}"); + sb.AppendLine($"Audio seconds: {e.AudioSeconds:N1}"); + if (!string.IsNullOrWhiteSpace(e.Error)) + { + sb.AppendLine($"Error: {e.Error}"); + } + + if (!string.IsNullOrWhiteSpace(e.OriginalTranscript)) + { + sb.AppendLine(); + sb.AppendLine("Original Transcript:"); + sb.AppendLine(e.OriginalTranscript); + sb.AppendLine(); + sb.AppendLine("Ollama System Prompt:"); + sb.AppendLine(e.RewriteSystemPrompt); + sb.AppendLine(); + sb.AppendLine("Final Injected Transcript:"); + } + + sb.AppendLine(); + sb.AppendLine(e.Transcript); + return sb.ToString(); + } + + /// The delivery wording of the WPF history window (it never printed the enum name); the two statuses only this app has are worded the same way. + public static string DeliveryDisplay(DictationDeliveryStatus status) => status switch + { + DictationDeliveryStatus.Injected => "Typed into app", + DictationDeliveryStatus.SkippedFocusChanged => "Skipped — focus changed", + DictationDeliveryStatus.SkippedNoFocusGuard => "Skipped — no focus check", + DictationDeliveryStatus.FailedToInject => "Failed to type", + DictationDeliveryStatus.Discarded => "Discarded", + DictationDeliveryStatus.CommandExecuted => "Command ran", + DictationDeliveryStatus.CommandFailed => "Command failed", + _ => status.ToString() + }; + + private static IEnumerable Known(IEnumerable values) => + values.Where(v => v.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(v => v, StringComparer.OrdinalIgnoreCase) + .Select(v => new HistoryTargetOption(HistoryTargetKind.Known, v, v)); + + private static bool Matches(string value, HistoryTargetOption option) => option.Kind switch + { + HistoryTargetKind.Known => string.Equals(value, option.Value, StringComparison.OrdinalIgnoreCase), + HistoryTargetKind.Unknown => value.Length == 0, + _ => true + }; + + private static string? Normalize(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private static string? ExtractWindowTitle(string? targetDisplayName) + { + var text = Normalize(targetDisplayName); + if (text is null) + { + return null; + } + + var handle = text.LastIndexOf(" (0x", StringComparison.OrdinalIgnoreCase); + if (handle > 0 && text.EndsWith(')')) + { + return text[..handle].Trim(); + } + + return text.StartsWith("window 0x", StringComparison.OrdinalIgnoreCase) ? null : text; + } +} diff --git a/src/PrimeDictate.Core/Dictation/DictationOptions.cs b/src/PrimeDictate.Core/Dictation/DictationOptions.cs index 275023b..80a369e 100644 --- a/src/PrimeDictate.Core/Dictation/DictationOptions.cs +++ b/src/PrimeDictate.Core/Dictation/DictationOptions.cs @@ -22,6 +22,12 @@ public sealed record DictationOptions /// public bool TypeWithoutFocusGuard { get; init; } + /// + /// Ask WASAPI for exclusive access to the microphone while dictating (WPF "Request exclusive mic access"). + /// Falls back to shared when the device refuses. Windows only; ignored elsewhere. + /// + public bool ExclusiveMicAccess { get; init; } + /// If focus moved, try to bring the window dictation started in back before typing. public bool ReturnToStartTarget { get; init; } diff --git a/src/PrimeDictate.Core/Dictation/DictationSettings.cs b/src/PrimeDictate.Core/Dictation/DictationSettings.cs index 16dbf9f..8eb6904 100644 --- a/src/PrimeDictate.Core/Dictation/DictationSettings.cs +++ b/src/PrimeDictate.Core/Dictation/DictationSettings.cs @@ -22,12 +22,24 @@ public sealed class DictationSettings public HotkeyDto HistoryHotkey { get; set; } = HotkeyDto.From(HotkeyGesture.DefaultHistory); - /// Model id from the WPF catalog (for example "base.en"). Whisper ONNX only for now. + /// Model id from the WPF catalog (for example "base.en"); read together with . public string? SelectedModelId { get; set; } - [JsonConverter(typeof(JsonStringEnumConverter))] + [JsonConverter(typeof(LegacyBackendConverter))] public LegacyBackend TranscriptionBackend { get; set; } = LegacyBackend.Whisper; + /// + /// A model placed outside the managed folder: a model folder, or for Whisper.net the ggml file. Same name as the WPF setting, so + /// the one the WPF app saved is carried over. When it validates for it is the model used. + /// + public string? ModelPath { get; set; } + + /// + /// Whether first-run setup was finished. The WPF app's flag, honoured on import. Null in a file that predates the flag, which + /// counts as finished (the file only exists because setup or a save happened). + /// + public bool? FirstRunCompleted { get; set; } + public string? SelectedInputDeviceId { get; set; } public double InputGainMultiplier { get; set; } = 1.0; @@ -38,19 +50,50 @@ public sealed class DictationSettings public bool ReturnToStartTargetOnCommit { get; set; } + /// The WPF "Request exclusive mic access while dictating" setting (same name, so it imports as is). Windows only. + public bool ExclusiveMicAccessWhileDictating { get; set; } + public bool PlayAudioCues { get; set; } = true; /// Where the ONNX speech models run: auto, cpu or cuda. Applies at the next start; PRIMEDICTATE_ONNX_DEVICE overrides it. public string OnnxDevice { get; set; } = "auto"; + /// + /// The WPF app's compute setting (Cpu, Gpu or Npu), read from its settings.json. Null when the file had none. + /// Only used for Whisper.net, and only until is chosen in this app. + /// + [JsonConverter(typeof(LegacyComputeInterfaceConverter))] + public LegacyComputeInterface? TranscriptionComputeInterface { get; set; } + + /// Where Whisper.net models run: auto, cpu or gpu. Null follows the WPF value, else Auto. Applies at the next start; PRIMEDICTATE_WHISPERNET_DEVICE overrides it. + public string? WhisperNetDevice { get; set; } + + /// The Whisper.net device this configuration asks for: an explicit choice, else the WPF compute setting, else Auto. + public WhisperNetDevicePreference ResolveWhisperNetDevice() => + this.WhisperNetDevice is { Length: > 0 } chosen + ? WhisperNetDevicePreferences.Parse(chosen) + : this.TranscriptionComputeInterface switch + { + LegacyComputeInterface.Cpu => WhisperNetDevicePreference.Cpu, + LegacyComputeInterface.Gpu => WhisperNetDevicePreference.Gpu, + LegacyComputeInterface.Npu => WhisperNetDevicePreference.Npu, + _ => WhisperNetDevicePreference.Auto + }; + /// Typing speed used for the time-saved figure. Valid range 20 to 120. public int BaselineTypingSpeedWpm { get; set; } = DictationStatsStore.DefaultBaselineWpm; - [JsonConverter(typeof(JsonStringEnumConverter))] + [JsonConverter(typeof(OverlayStyleConverter))] public OverlayStyle OverlayMode { get; set; } = OverlayStyle.CompactMicrophone; public bool IsOverlaySticky { get; set; } + /// + /// Hide the compact microphone while idle. On by default: Justin asked for the overlay to show only while dictating after his first + /// Windows run. Turning it off gives the WPF behavior (the compact microphone stays on screen; see ). + /// + public bool HideOverlayWhenIdle { get; set; } = true; + /// /// Where the user dragged the overlay: the bottom-center point it grows up from, in screen pixels. Null keeps the /// default, a little above the bottom middle of the primary screen. New app only (the WPF overlay did not remember). @@ -59,6 +102,17 @@ public sealed class DictationSettings public int? OverlayAnchorY { get; set; } + /// The automatic update check on launch (at most daily). The tray's "Check for updates..." always works. Same name and meaning as the WPF setting. + public bool CheckForUpdatesAutomatically { get; set; } = true; + + /// What clicking the tray icon does. The WPF default (double click) is kept. + [JsonConverter(typeof(TrayClickBehaviorConverter))] + public TrayClickBehavior TrayClickBehavior { get; set; } = TrayClickBehavior.DoubleClickOpensWorkspace; + + /// Light, dark or the system setting. Dark by default, as the WPF app always was. + [JsonConverter(typeof(AppThemeConverter))] + public AppTheme Theme { get; set; } = AppTheme.Dark; + public List TranscriptReplacements { get; set; } = []; /// Opt-in: listen on the idle microphone for the wake phrase and start dictation. @@ -74,12 +128,18 @@ public sealed class DictationSettings public string VoiceHistoryPhrase { get; set; } = VoiceCommandProcessor.DefaultHistoryPhrase; + /// + /// Phrases that run a command on this computer while dictating. Same shape as the WPF app's list, so imported commands + /// keep their On state (the user chose them there). Commands added in this app start Off. + /// + public List VoiceShellCommands { get; set; } = []; + public VoiceCommandOptions ToVoiceCommandOptions() => new( this.EnableVoiceCommands, this.VoiceDictationPhrase?.Trim() ?? string.Empty, this.VoiceStopPhrase?.Trim() ?? string.Empty, this.VoiceHistoryPhrase?.Trim() ?? string.Empty, - []); + (this.VoiceShellCommands ?? []).Where(c => c is not null).ToList()); public bool EnableOllamaPostProcessing { get; set; } @@ -87,6 +147,7 @@ public sealed class DictationSettings public string OllamaModel { get; set; } = "gemma:2b"; + [JsonConverter(typeof(OllamaModeConverter))] public OllamaMode OllamaMode { get; set; } = OllamaMode.Default; /// New: allow a non-loopback Ollama endpoint. Off by default so speech stays on this computer. @@ -105,7 +166,11 @@ public sealed class DictationSettings AutoCommitSilence = TimeSpan.FromSeconds(this.AutoCommitSilenceSeconds), SendEnterAfterCommit = this.SendEnterAfterCommit, ReturnToStartTarget = this.ReturnToStartTargetOnCommit, + ExclusiveMicAccess = this.ExclusiveMicAccessWhileDictating, TypeWithoutFocusGuard = this.TypeWithoutFocusGuard, + // The WPF app forced English for Whisper (sherpa-onnx and Whisper.net); a multilingual model would otherwise guess the language + // from each short dictation. The transcription workspace keeps its own explicit language choice and does not use these options. + Language = "en", Replacements = this.TranscriptReplacements .Where(r => !string.IsNullOrWhiteSpace(r.Find)) .Select(r => new ReplacementRule(r.Find, r.Replace)) @@ -134,6 +199,34 @@ public enum LegacyBackend QualcommQnn = 4 } +/// The WPF app's TranscriptionComputeInterface values. +public enum LegacyComputeInterface +{ + Cpu = 0, + Gpu = 1, + Npu = 2 +} + +/// Where Whisper.net runs. Npu is the WPF OpenVINO option and only takes effect when the model's OpenVINO files are installed. +public enum WhisperNetDevicePreference +{ + Auto = 0, + Cpu = 1, + Gpu = 2, + Npu = 3 +} + +public static class WhisperNetDevicePreferences +{ + public static WhisperNetDevicePreference Parse(string? text) => text?.Trim().ToLowerInvariant() switch + { + "cpu" => WhisperNetDevicePreference.Cpu, + "gpu" or "cuda" => WhisperNetDevicePreference.Gpu, + "npu" => WhisperNetDevicePreference.Npu, + _ => WhisperNetDevicePreference.Auto + }; +} + public sealed class HotkeyDto { public string KeyCode { get; set; } = ""; @@ -194,16 +287,25 @@ public DictationSettingsLoad Load() var settings = JsonSerializer.Deserialize(File.ReadAllText(file), JsonOptions); if (settings is not null) { + settings.FirstRunCompleted ??= true; + if (imported && settings.EnableOllamaPostProcessing && DictationSettingsValidator.IsRemoteEndpoint(settings.OllamaEndpoint)) + { + // The WPF app posted to any endpoint; this app asks first. Someone who set a remote one up there keeps it working. + settings.OllamaAllowRemoteEndpoint = true; + } + + DictationSettingsValidator.Normalize(settings); return new DictationSettingsLoad(settings, imported, null); } } catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException) { - return new DictationSettingsLoad(new DictationSettings(), false, $"Could not read {file}: {ex.Message}. Using defaults; the file was left untouched."); + // An unreadable file must not look like a finished setup: first run shows again so the user can choose a model and shortcuts. + return new DictationSettingsLoad(new DictationSettings { FirstRunCompleted = false }, false, $"Could not read {file}: {ex.Message}. Using defaults; the file was left untouched."); } } - return new DictationSettingsLoad(new DictationSettings(), false, null); + return new DictationSettingsLoad(new DictationSettings { FirstRunCompleted = false }, false, null); } public void Save(DictationSettings settings) @@ -221,6 +323,8 @@ public static class LegacyBackendExtensions { LegacyBackend.Parakeet => "parakeet-onnx", LegacyBackend.Moonshine => "moonshine-onnx", + LegacyBackend.WhisperNet => "whisper-net", + LegacyBackend.QualcommQnn => "qualcomm-qnn", _ => "whisper-onnx" }; } diff --git a/src/PrimeDictate.Core/Dictation/DictationSettingsValidator.cs b/src/PrimeDictate.Core/Dictation/DictationSettingsValidator.cs new file mode 100644 index 0000000..4dde7e7 --- /dev/null +++ b/src/PrimeDictate.Core/Dictation/DictationSettingsValidator.cs @@ -0,0 +1,281 @@ +using System.Net; + +namespace PrimeDictate.Core.Dictation; + +/// The voice phrases that will be saved: blank ones replaced by the defaults, as the WPF Settings window did. +public sealed record EffectiveVoicePhrases(string Commit, string Stop, string History); + +/// +/// Rules a settings change must satisfy before it is saved, ported from the WPF Settings window (TryBuildSettings) and its +/// startup normalisation (NormalizeShortcutSettings). Pure functions, so the window only shows what they return. +/// +public static class DictationSettingsValidator +{ + /// Null when the three shortcuts are valid and all different. Otherwise a message for the user. + public static string? ValidateHotkeys(HotkeyGesture dictation, HotkeyGesture stop, HotkeyGesture history) + { + foreach (var (gesture, label) in new[] { (dictation, "Start / stop dictation"), (stop, "Emergency stop"), (history, "Open history") }) + { + if (!gesture.IsValid(out var error)) + { + return $"{label}: {error}"; + } + } + + return dictation == stop || dictation == history || stop == history + ? "Each keyboard shortcut must use a different key combination." + : null; + } + + /// + /// Blank phrases take their defaults; with voice commands on, all three blank is an error, and two phrases that say the same + /// words are an error. Null means valid. + /// + public static string? ValidateVoicePhrases(bool voiceCommandsEnabled, string? commit, string? stop, string? history, out EffectiveVoicePhrases effective) + { + var c = commit?.Trim() ?? string.Empty; + var s = stop?.Trim() ?? string.Empty; + var h = history?.Trim() ?? string.Empty; + effective = new EffectiveVoicePhrases( + c.Length == 0 ? VoiceCommandProcessor.DefaultDictationPhrase : c, + s.Length == 0 ? VoiceCommandProcessor.DefaultStopPhrase : s, + h.Length == 0 ? VoiceCommandProcessor.DefaultHistoryPhrase : h); + if (voiceCommandsEnabled && c.Length == 0 && s.Length == 0 && h.Length == 0) + { + return "Voice commands need at least one phrase."; + } + + var owners = new Dictionary(StringComparer.OrdinalIgnoreCase); + return TryReserve(owners, effective.Commit, "Start / stop", out var error) + && TryReserve(owners, effective.Stop, "Emergency stop", out error) + && TryReserve(owners, effective.History, "Open history", out error) + ? null + : error; + } + + /// + /// Checks the phrases of the commands that run programs against the three voice phrases and each other (and the wake phrase, when + /// the wake word is on: saying it would otherwise start a dictation and run a command at once). Incomplete rows are the editor's concern. + /// + public static string? ValidateShellCommands(EffectiveVoicePhrases reserved, IEnumerable commands, bool wakeEnabled, string? wakePhrase) + { + var owners = new Dictionary(StringComparer.OrdinalIgnoreCase); + Reserve(owners, reserved.Commit, "Start / stop phrase"); + Reserve(owners, reserved.Stop, "Stop phrase"); + Reserve(owners, reserved.History, "History phrase"); + if (wakeEnabled) + { + Reserve(owners, WakePhrase.Normalize(wakePhrase), "wake phrase"); + } + + foreach (var command in commands) + { + var phrase = command.Phrase.Trim(); + var key = PhraseKey(phrase); + if (key.Length == 0) + { + return "Computer command phrases need at least one letter or number."; + } + + if (owners.TryGetValue(key, out var existing)) + { + return $"The computer command phrase \"{phrase}\" conflicts with {existing}."; + } + + owners.Add(key, $"computer command \"{phrase}\""); + } + + return null; + } + + /// The wake phrase against the three voice phrases (only when the wake word is on). + public static string? ValidateWakePhrase(bool wakeEnabled, string? wakePhrase, EffectiveVoicePhrases voice) + { + if (!wakeEnabled) + { + return null; + } + + var key = PhraseKey(WakePhrase.Normalize(wakePhrase)); + foreach (var (phrase, label) in new[] { (voice.Commit, "start / stop"), (voice.Stop, "emergency stop"), (voice.History, "history") }) + { + if (key == PhraseKey(phrase)) + { + return $"The wake phrase and the {label} voice phrase must be different."; + } + } + + return null; + } + + /// With rewriting on, the endpoint must be an http(s) address and a model must be named. Null means valid. + public static string? ValidateOllama(bool enabled, string? endpoint, string? model) + { + if (!enabled) + { + return null; + } + + if (!Uri.TryCreate((endpoint ?? string.Empty).Trim(), UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) + { + return "The Ollama endpoint must be an http or https address, for example http://localhost:11434."; + } + + return string.IsNullOrWhiteSpace(model) ? "Name the Ollama model to use, for example gemma:2b." : null; + } + + /// True when the endpoint is not on this computer (it would be refused unless remote endpoints are allowed). + public static bool IsRemoteEndpoint(string? endpoint) => + Uri.TryCreate((endpoint ?? string.Empty).Trim(), UriKind.Absolute, out var uri) + && uri.Scheme is "http" or "https" + && !(uri.IsLoopback || (IPAddress.TryParse(uri.Host, out var ip) && IPAddress.IsLoopback(ip))); + + /// + /// The WPF startup normalisation: an invalid or duplicate shortcut falls back to its default (or an alternative when the default is + /// taken), a blank phrase gets its default, and a stop or history phrase equal to an earlier one is cleared. Returns true when something changed. + /// + public static bool Normalize(DictationSettings settings) + { + var changed = false; + var dictation = Usable(settings.DictationHotkey); + var stop = Usable(settings.StopHotkey); + var history = Usable(settings.HistoryHotkey); + var hotkeysChanged = false; + if (dictation is null) + { + dictation = HotkeyGesture.Default; + hotkeysChanged = true; + } + + if (stop is null || stop == dictation) + { + stop = HotkeyGesture.DefaultStop == dictation ? new HotkeyGesture("VcEnter", true, false, true) : HotkeyGesture.DefaultStop; + hotkeysChanged = true; + } + + if (history is null || history == dictation || history == stop) + { + history = HotkeyGesture.DefaultHistory == dictation || HotkeyGesture.DefaultHistory == stop + ? new HotkeyGesture("VcH", true, false, true) + : HotkeyGesture.DefaultHistory; + hotkeysChanged = true; + } + + if (hotkeysChanged) + { + settings.DictationHotkey = HotkeyDto.From(dictation); + settings.StopHotkey = HotkeyDto.From(stop); + settings.HistoryHotkey = HotkeyDto.From(history); + changed = true; + } + + if (string.IsNullOrWhiteSpace(settings.VoiceDictationPhrase)) + { + settings.VoiceDictationPhrase = VoiceCommandProcessor.DefaultDictationPhrase; + changed = true; + } + + if (string.IsNullOrWhiteSpace(settings.WakeWordPhrase)) + { + settings.WakeWordPhrase = WakePhrase.Default; + changed = true; + } + + if (string.IsNullOrWhiteSpace(settings.VoiceStopPhrase) || Same(settings.VoiceStopPhrase, settings.VoiceDictationPhrase)) + { + settings.VoiceStopPhrase = DefaultUnlessTaken(VoiceCommandProcessor.DefaultStopPhrase, settings.VoiceDictationPhrase); + changed = true; + } + + if (string.IsNullOrWhiteSpace(settings.VoiceHistoryPhrase) || Same(settings.VoiceHistoryPhrase, settings.VoiceDictationPhrase) || Same(settings.VoiceHistoryPhrase, settings.VoiceStopPhrase)) + { + settings.VoiceHistoryPhrase = DefaultUnlessTaken(VoiceCommandProcessor.DefaultHistoryPhrase, settings.VoiceDictationPhrase, settings.VoiceStopPhrase); + changed = true; + } + + if (settings.VoiceShellCommands is null) + { + settings.VoiceShellCommands = []; + changed = true; + } + + if (settings.TranscriptReplacements is null) + { + settings.TranscriptReplacements = []; + changed = true; + } + + return changed; + } + + /// Lower-cased words made of letters and digits, single-spaced: "Thank you!" and "thank you" are the same phrase. + public static string PhraseKey(string? phrase) + { + var words = new List(); + var current = new System.Text.StringBuilder(); + foreach (var character in phrase ?? string.Empty) + { + if (char.IsLetterOrDigit(character)) + { + current.Append(char.ToLowerInvariant(character)); + } + else if (current.Length > 0) + { + words.Add(current.ToString()); + current.Clear(); + } + } + + if (current.Length > 0) + { + words.Add(current.ToString()); + } + + return string.Join(' ', words); + } + + private static HotkeyGesture? Usable(HotkeyDto? dto) + { + if (dto is null) + { + return null; + } + + var gesture = new HotkeyGesture(dto.KeyCode, dto.Ctrl, dto.Shift, dto.Alt); + return gesture.IsValid(out _) ? gesture : null; + } + + private static bool Same(string? left, string? right) => + !string.IsNullOrWhiteSpace(left) && !string.IsNullOrWhiteSpace(right) + && string.Equals(left.Trim(), right.Trim(), StringComparison.OrdinalIgnoreCase); + + private static string DefaultUnlessTaken(string defaultPhrase, params string?[] existing) => + existing.Any(e => Same(defaultPhrase, e)) ? string.Empty : defaultPhrase; + + private static bool TryReserve(Dictionary owners, string phrase, string owner, out string error) + { + var key = PhraseKey(phrase); + if (key.Length > 0 && owners.TryGetValue(key, out var existing)) + { + error = $"Use different phrases for {owner} and {existing} voice commands."; + return false; + } + + if (key.Length > 0) + { + owners.Add(key, owner); + } + + error = string.Empty; + return true; + } + + private static void Reserve(Dictionary owners, string phrase, string owner) + { + var key = PhraseKey(phrase); + if (key.Length > 0) + { + owners[key] = owner; + } + } +} diff --git a/src/PrimeDictate.Core/Dictation/DictationStats.cs b/src/PrimeDictate.Core/Dictation/DictationStats.cs index bdc4206..446e7cf 100644 --- a/src/PrimeDictate.Core/Dictation/DictationStats.cs +++ b/src/PrimeDictate.Core/Dictation/DictationStats.cs @@ -31,7 +31,7 @@ public sealed class DictationStatsState /// Time saved against typing the same words at (20 to 120, else 40), net of speaking time. public TimeSpan TimeSaved(int baselineWpm) { - var baseline = baselineWpm is >= 20 and <= 120 ? baselineWpm : DictationStatsStore.DefaultBaselineWpm; + var baseline = DictationStatsStore.NormalizeBaselineWpm(baselineWpm); var saved = TimeSpan.FromMinutes(this.TotalWords / (double)baseline) - TimeSpan.FromSeconds(this.TotalAudioSeconds); return saved < TimeSpan.Zero ? TimeSpan.Zero : saved; } @@ -70,6 +70,11 @@ public sealed record DictationStatsUpdate(DictationStatsState State, IReadOnlyLi public sealed class DictationStatsStore(string path) { public const int DefaultBaselineWpm = 40; + public const int MinBaselineWpm = 20; + public const int MaxBaselineWpm = 120; + + /// The typing speed to compare against: the value when it is 20 to 120, else the default of 40 (as the WPF app did). + public static int NormalizeBaselineWpm(int wpm) => wpm is >= MinBaselineWpm and <= MaxBaselineWpm ? wpm : DefaultBaselineWpm; private const int MaxDailyBuckets = 370; private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true, Converters = { new JsonStringEnumConverter() } }; diff --git a/src/PrimeDictate.Core/Dictation/OllamaRewriter.cs b/src/PrimeDictate.Core/Dictation/OllamaRewriter.cs index 9ea96aa..7a5aed3 100644 --- a/src/PrimeDictate.Core/Dictation/OllamaRewriter.cs +++ b/src/PrimeDictate.Core/Dictation/OllamaRewriter.cs @@ -6,7 +6,7 @@ namespace PrimeDictate.Core.Dictation; /// Rewrite styles, numbered as in the WPF settings file. -[JsonConverter(typeof(JsonStringEnumConverter))] +[JsonConverter(typeof(OllamaModeConverter))] public enum OllamaMode { Default = 0, @@ -23,6 +23,9 @@ public sealed record RewriteResult(string Text, string? SystemPrompt, bool Rewri /// Optional last step before typing: a local model rewrites the transcript. Any failure keeps the raw text. public interface ITranscriptRewriter { + /// True when a rewrite will be attempted (the controller then shows the "AI is processing" status in the overlay while it runs). + bool IsActive => true; + ValueTask RewriteAsync(string transcript, IForegroundTarget? target, CancellationToken cancellationToken); } @@ -40,6 +43,8 @@ public sealed class OllamaRewriter(Func options, HttpClient? http private static readonly HttpClient Shared = new() { Timeout = TimeSpan.FromSeconds(60) }; private readonly HttpClient client = http ?? Shared; + public bool IsActive => options().Enabled; + public async ValueTask RewriteAsync(string transcript, IForegroundTarget? target, CancellationToken cancellationToken) { var o = options(); diff --git a/src/PrimeDictate.Core/Dictation/OverlayRules.cs b/src/PrimeDictate.Core/Dictation/OverlayRules.cs new file mode 100644 index 0000000..750047b --- /dev/null +++ b/src/PrimeDictate.Core/Dictation/OverlayRules.cs @@ -0,0 +1,130 @@ +namespace PrimeDictate.Core.Dictation; + +/// What the overlay is showing. +public enum OverlayPhase +{ + Ready, + WakeListening, + Listening, + Processing +} + +/// +/// The overlay's and the tray's wording and show/hide rules, ported from the WPF app (TranscriptionOverlayWindow, App.ShouldPersistOverlay, +/// the tray tooltips), kept free of UI types so they can be tested. Nothing here ever receives recognized text except , +/// which only inspects it. +/// +public static class OverlayRules +{ + /// The model family as the WPF header and tooltips name it. + public static string BackendLabel(LegacyBackend backend) => backend switch + { + LegacyBackend.QualcommQnn => "Qualcomm AI Hub Whisper QNN", + LegacyBackend.Moonshine => "Moonshine ONNX", + LegacyBackend.Parakeet => "Parakeet ONNX", + LegacyBackend.WhisperNet => "Whisper.net (GGML)", + _ => "Whisper ONNX" + }; + + /// "Listening [Whisper.net (GGML)]", "Processing [..]", "Wake listening [..]", "Ready [..]". + public static string Header(OverlayPhase phase, string backend) => phase switch + { + OverlayPhase.Listening => $"Listening [{backend}]", + OverlayPhase.Processing => $"Processing [{backend}]", + OverlayPhase.WakeListening => $"Wake listening [{backend}]", + _ => $"Ready [{backend}]" + }; + + /// What the transcript box says when there is no transcript yet. + public static string Placeholder(OverlayPhase phase, string backend) => phase switch + { + OverlayPhase.Listening or OverlayPhase.Processing => $"Listening with {backend}...", + OverlayPhase.WakeListening => "Waiting for wake phrase...", + _ => "Waiting for hotkey..." + }; + + /// The Copy button works only when there is real text: not empty and not one of the placeholders. + public static bool IsCopyable(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + return text != "Waiting for hotkey..." && text != "Waiting for wake phrase..." && text != "Listening..." && !text.StartsWith("Listening with ", StringComparison.Ordinal); + } + + /// "mm:ss", minutes not wrapping at an hour (the WPF timer showed total minutes). + public static string Elapsed(TimeSpan elapsed) => + $"{(int)Math.Max(0, elapsed.TotalMinutes):D2}:{Math.Max(0, elapsed.Seconds):D2}"; + + /// + /// Whether the overlay is on screen. The WPF rule: a compact microphone stays on screen all the time, and so does a pinned overlay + /// (ShouldPersistOverlay); otherwise it shows while dictating, for a notice or the last words. Closing it hides it until + /// the next dictation. is the new opt-out for the compact microphone's always-on rule. + /// + public static bool ShouldShow(bool dismissed, bool active, bool lingering, bool pinned, OverlayStyle configuredStyle, bool hideCompactWhenIdle) + { + if (dismissed) + { + return false; + } + + var staysOnScreen = pinned || (configuredStyle == OverlayStyle.CompactMicrophone && !hideCompactWhenIdle); + return active || lingering || staysOnScreen; + } + + /// + /// The icon's hover text, worded exactly as the WPF tray ("PrimeDictate - Processing [..]", "PrimeDictate - Listening [.., Exclusive]"). + /// Windows shows at most 63 characters, so the wake phrase is dropped when it would not fit. is "Exclusive" + /// or "Shared" while dictating, else null. + /// + public static string TrayTooltip( + OverlayPhase phase, + string backend, + bool needsAttention, + bool wakeFailed, + string? meetingSource, + string wakePhrase, + string? micAccess = null) + { + if (phase == OverlayPhase.Listening) + { + return micAccess is { Length: > 0 } + ? $"PrimeDictate - Listening [{backend}, {micAccess}]" + : $"PrimeDictate - Listening [{backend}]"; + } + + if (phase == OverlayPhase.Processing) + { + return $"PrimeDictate - Processing [{backend}]"; + } + + if (meetingSource is not null) + { + return $"PrimeDictate - Recording meeting ({meetingSource})"; + } + + if (needsAttention || wakeFailed) + { + return wakeFailed ? "PrimeDictate - Wake listening failed" : "PrimeDictate - Needs attention"; + } + + if (phase == OverlayPhase.WakeListening) + { + var phrase = string.IsNullOrWhiteSpace(wakePhrase) ? WakePhrase.Default : wakePhrase.Trim(); + var text = $"PrimeDictate - Wake listening ({phrase})"; + return text.Length <= 63 ? text : "PrimeDictate - Wake listening"; + } + + return $"PrimeDictate - Ready [{backend}]"; + } +} + +/// How long the tray shows its "needs attention" icon after an error, and the rule for it. +public static class TrayAttention +{ + public static readonly TimeSpan Hold = TimeSpan.FromSeconds(10); + + public static bool IsActive(DateTime attentionUntilUtc, DateTime nowUtc, bool wakeFailed) => nowUtc <= attentionUntilUtc || wakeFailed; +} diff --git a/src/PrimeDictate.Core/Dictation/ShellPreferences.cs b/src/PrimeDictate.Core/Dictation/ShellPreferences.cs new file mode 100644 index 0000000..ea42283 --- /dev/null +++ b/src/PrimeDictate.Core/Dictation/ShellPreferences.cs @@ -0,0 +1,149 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace PrimeDictate.Core.Dictation; + +/// What clicking the tray icon does. The first two names are the WPF app's, so its settings.json reads as-is. +public enum TrayClickBehavior +{ + SingleClickOpensWorkspace = 0, + DoubleClickOpensWorkspace = 1, + + /// New: clicking the icon does nothing; the menu's "Open PrimeDictate" is the way in. + ClickDoesNothing = 2 +} + +/// The color scheme of the app's windows. The WPF app was always dark; here the default follows the system. +public enum AppTheme +{ + System = 0, + Light = 1, + Dark = 2 +} + +/// +/// Reads an enum from its name (any case) or number, and falls back to fallback for a value this version does not know, +/// so a hand-edited or older file never stops the whole settings file from loading. maps old spellings. +/// +public class LenientEnumConverter(T fallback) : JsonConverter + where T : struct, Enum +{ + /// Old names that mean a current value (the WPF app's pre-5.x tray names described Settings but opened the workspace). + protected virtual IReadOnlyDictionary Legacy => new Dictionary(); + + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String) + { + var text = reader.GetString() ?? string.Empty; + if (this.Legacy.TryGetValue(text, out var mapped)) + { + return mapped; + } + + return Enum.TryParse(text, ignoreCase: true, out var parsed) && Enum.IsDefined(parsed) ? parsed : fallback; + } + + if (reader.TokenType == JsonTokenType.Number && reader.TryGetInt32(out var number)) + { + var value = (T)Enum.ToObject(typeof(T), number); + return Enum.IsDefined(value) ? value : fallback; + } + + reader.Skip(); + return fallback; + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) => writer.WriteStringValue(value.ToString()); +} + +public sealed class TrayClickBehaviorConverter() : LenientEnumConverter(TrayClickBehavior.DoubleClickOpensWorkspace) +{ + protected override IReadOnlyDictionary Legacy { get; } = new Dictionary + { + ["SingleClickOpensSettings"] = TrayClickBehavior.SingleClickOpensWorkspace, + ["DoubleClickOpensSettings"] = TrayClickBehavior.DoubleClickOpensWorkspace + }; +} + +public sealed class AppThemeConverter() : LenientEnumConverter(AppTheme.Dark); + +/// The enums of a WPF settings.json: a value this version does not know falls back to the default instead of failing the whole import. +public sealed class LegacyBackendConverter() : LenientEnumConverter(LegacyBackend.Whisper); + +public sealed class OverlayStyleConverter() : LenientEnumConverter(OverlayStyle.CompactMicrophone); + +/// An unknown completion behavior reads as Stop, the safe one (nothing more is typed). +public sealed class VoiceShellCompletionConverter() : LenientEnumConverter(VoiceShellCommandCompletionBehavior.Stop); + +public sealed class OllamaModeConverter() : LenientEnumConverter(OllamaMode.Default); + +/// The optional compute setting: an unknown or non-text value reads as "not set" (null), never as a wrong device. +public sealed class LegacyComputeInterfaceConverter : JsonConverter +{ + public override bool HandleNull => true; + + public override LegacyComputeInterface? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + { + return null; + } + + var text = reader.TokenType == JsonTokenType.String ? reader.GetString() : null; + if (reader.TokenType == JsonTokenType.Number && reader.TryGetInt32(out var number)) + { + return Enum.IsDefined(typeof(LegacyComputeInterface), number) ? (LegacyComputeInterface)number : null; + } + + if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) + { + reader.Skip(); + } + + return Enum.TryParse(text, ignoreCase: true, out var parsed) && Enum.IsDefined(parsed) ? parsed : null; + } + + public override void Write(Utf8JsonWriter writer, LegacyComputeInterface? value, JsonSerializerOptions options) + { + if (value is { } v) + { + writer.WriteStringValue(v.ToString()); + } + else + { + writer.WriteNullValue(); + } + } +} + +/// +/// Decides whether a tray click opens the workspace. The tray only reports clicks, so a double click is two clicks close together. +/// A double click is consumed (a third click starts over), so triple-clicking does not open twice. +/// +public sealed class TrayClickDecider(Func behavior, TimeSpan? doubleClickTime = null) +{ + private readonly TimeSpan window = doubleClickTime ?? TimeSpan.FromMilliseconds(500); + private DateTime? lastClickUtc; + + /// True when this click should open the workspace. + public bool OnClick(DateTime nowUtc) + { + switch (behavior()) + { + case TrayClickBehavior.SingleClickOpensWorkspace: + return true; + case TrayClickBehavior.DoubleClickOpensWorkspace: + if (this.lastClickUtc is { } last && nowUtc - last <= this.window) + { + this.lastClickUtc = null; + return true; + } + + this.lastClickUtc = nowUtc; + return false; + default: + return false; + } + } +} diff --git a/src/PrimeDictate.Core/Dictation/VoiceCommandMatcher.cs b/src/PrimeDictate.Core/Dictation/VoiceCommandMatcher.cs index 5554b3c..e0b1a41 100644 --- a/src/PrimeDictate.Core/Dictation/VoiceCommandMatcher.cs +++ b/src/PrimeDictate.Core/Dictation/VoiceCommandMatcher.cs @@ -393,13 +393,17 @@ public enum VoiceShellCommandCompletionBehavior Continue = 1 } -/// A user-defined phrase that runs a command. Matching is ported; running is not (opt-in, dictation only, later). +/// +/// A user-defined phrase that runs a command. The command text comes only from the user's settings, never from what +/// was said. Dictation only; transcription mode has no hook for it. +/// public sealed class VoiceShellCommand { public bool Enabled { get; set; } = true; public string Phrase { get; set; } = ""; + [System.Text.Json.Serialization.JsonConverter(typeof(VoiceShellCompletionConverter))] public VoiceShellCommandCompletionBehavior CompletionBehavior { get; set; } = VoiceShellCommandCompletionBehavior.Stop; public string Command { get; set; } = ""; diff --git a/src/PrimeDictate.Core/Dictation/VoiceCommands.cs b/src/PrimeDictate.Core/Dictation/VoiceCommands.cs index 890384c..ba77a9a 100644 --- a/src/PrimeDictate.Core/Dictation/VoiceCommands.cs +++ b/src/PrimeDictate.Core/Dictation/VoiceCommands.cs @@ -1,6 +1,11 @@ namespace PrimeDictate.Core.Dictation; -public sealed record VoiceCommandResult(string CleanedText, bool CommitRequested, bool StopRequested, bool HistoryRequested) +public sealed record VoiceCommandResult( + string CleanedText, + bool CommitRequested, + bool StopRequested, + bool HistoryRequested, + VoiceShellCommandInvocation? Shell = null) { public static VoiceCommandResult Passthrough(string text) => new(text, false, false, false); } @@ -12,6 +17,12 @@ public sealed record VoiceCommandResult(string CleanedText, bool CommitRequested public interface IVoiceCommandProcessor { VoiceCommandResult Apply(string transcript); + + /// + /// Like for the finished transcript of a dictation, where the user's shell-command phrases are + /// also matched. Live previews never use this, so a preview can never run anything. + /// + VoiceCommandResult ApplyFinal(string transcript) => this.Apply(transcript); } public sealed class NoVoiceCommands : IVoiceCommandProcessor @@ -33,4 +44,10 @@ public VoiceCommandResult Apply(string transcript) var match = VoiceCommandMatcher.Apply(transcript, options()); return new VoiceCommandResult(match.CleanedText, match.CommitRequested, match.StopRequested, match.HistoryRequested); } + + public VoiceCommandResult ApplyFinal(string transcript) + { + var match = VoiceCommandMatcher.Apply(transcript, options(), includeShellCommands: true); + return new VoiceCommandResult(match.CleanedText, match.CommitRequested, match.StopRequested, match.HistoryRequested, match.ShellCommandInvocation); + } } diff --git a/src/PrimeDictate.Core/Dictation/VoiceShellCommandRunner.cs b/src/PrimeDictate.Core/Dictation/VoiceShellCommandRunner.cs new file mode 100644 index 0000000..d3c1b23 --- /dev/null +++ b/src/PrimeDictate.Core/Dictation/VoiceShellCommandRunner.cs @@ -0,0 +1,13 @@ +namespace PrimeDictate.Core.Dictation; + +public readonly record struct VoiceShellCommandResult(int? ProcessId); + +/// +/// Starts a user-configured command. Implementations receive the command exactly as the user saved it in settings; +/// nothing spoken or typed is ever part of it. +/// +public interface IVoiceShellCommandRunner +{ + /// The command is empty or the process did not start. + VoiceShellCommandResult Run(VoiceShellCommand command); +} diff --git a/src/PrimeDictate.Core/Dictation/WakeWord.cs b/src/PrimeDictate.Core/Dictation/WakeWord.cs index 3a30824..304339b 100644 --- a/src/PrimeDictate.Core/Dictation/WakeWord.cs +++ b/src/PrimeDictate.Core/Dictation/WakeWord.cs @@ -136,6 +136,23 @@ public bool IsRunning public event Action? Notice; + /// Raised whenever changes, so the tray can follow it. + public event Action? FailedChanged; + + /// True after the listener gave up (the microphone would not open, no model) until it next starts listening or is disabled. The tray shows it as "Wake listening failed". + public bool HasFailed => Volatile.Read(ref this.failed) != 0; + + private int failed; + private int transcribeFailureReported; + + private void SetFailed(bool value) + { + if (Interlocked.Exchange(ref this.failed, value ? 1 : 0) != (value ? 1 : 0)) + { + this.FailedChanged?.Invoke(); + } + } + public void Configure(bool enabled, string? wakePhrase, string? inputDeviceId, double inputGain) { lock (this.sync) @@ -145,6 +162,26 @@ public void Configure(bool enabled, string? wakePhrase, string? inputDeviceId, d this.deviceId = string.IsNullOrWhiteSpace(inputDeviceId) ? null : inputDeviceId; this.gain = DictationOptions.NormalizeGain(inputGain); } + + if (!enabled) + { + // Turned off: nothing is failing any more. + this.SetFailed(false); + } + + Volatile.Write(ref this.transcribeFailureReported, 0); + } + + /// The listener cannot work (for example no speech model): stops it from running, marks it failed and says why once. + public void Disable(string reason) + { + lock (this.sync) + { + this.enabled = false; + } + + this.SetFailed(true); + this.Notice?.Invoke(reason); } /// Starts listening when enabled and not suspended. Safe to call repeatedly. @@ -246,7 +283,8 @@ private async Task StartLockedAsync() { if (attempt >= OpenRetryDelays.Length) { - Diagnostics.AppLog.Event("wake-word", $"Stopped listening: the microphone did not open after {attempt + 1} tries: {ex.Message}"); + Diagnostics.AppLog.Error("wake-word", $"Stopped listening: the microphone did not open after {attempt + 1} tries: {ex.Message}"); + this.SetFailed(true); this.Notice?.Invoke($"Wake word listening could not open the microphone: {ex.Message}"); return; } @@ -256,6 +294,7 @@ private async Task StartLockedAsync() } } + this.SetFailed(false); var run = new Running(lease); run.Loop = Task.Run(() => this.ListenAsync(run)); lock (this.sync) @@ -364,10 +403,17 @@ private async Task ListenAsync(Running run) } catch (Exception ex) { - this.Notice?.Invoke($"Wake word transcription failed: {ex.Message}"); + // Said once, not on every stretch of speech; reset by the next success or reconfiguration. + if (Interlocked.Exchange(ref this.transcribeFailureReported, 1) == 0) + { + this.Notice?.Invoke($"Wake word transcription failed: {ex.Message}"); + } + continue; } + Volatile.Write(ref this.transcribeFailureReported, 0); + string current; lock (this.sync) { diff --git a/src/PrimeDictate.Core/Pipeline/LiveTranscriptionSession.cs b/src/PrimeDictate.Core/Pipeline/LiveTranscriptionSession.cs index 7a20673..1632786 100644 --- a/src/PrimeDictate.Core/Pipeline/LiveTranscriptionSession.cs +++ b/src/PrimeDictate.Core/Pipeline/LiveTranscriptionSession.cs @@ -249,7 +249,7 @@ public async Task StopAsync(CancellationToken cancellationToken = default) catch (TimeoutException) { this.failure ??= new TimeoutException("The audio capture did not stop in time."); - Diagnostics.AppLog.Event("capture", "The audio capture did not stop within 30 s of Stop."); + Diagnostics.AppLog.Event("capture", "The audio capture did not stop within 30 s of Stop.", Diagnostics.ActivityLevel.Warning); this.Error?.Invoke("The audio capture did not stop in time. What was recorded so far is kept."); } catch (Exception ex) when (ex is not OperationCanceledException) diff --git a/src/PrimeDictate.Core/Providers/AudioContracts.cs b/src/PrimeDictate.Core/Providers/AudioContracts.cs index 989af12..dfc76a5 100644 --- a/src/PrimeDictate.Core/Providers/AudioContracts.cs +++ b/src/PrimeDictate.Core/Providers/AudioContracts.cs @@ -33,6 +33,9 @@ public interface IAudioCaptureLease : IAsyncDisposable string DeviceName { get; } + /// The access the device was actually opened with (an exclusive request can fall back to shared). + MicAccessMode AccessMode => MicAccessMode.Shared; + IAsyncEnumerable ReadFramesAsync(CancellationToken cancellationToken); /// Stops delivery without releasing the device. Frames after resume continue the sequence. @@ -64,6 +67,20 @@ public interface IAudioSource /// Opens a device. Null selects the system default. ValueTask OpenAsync(string? deviceId, CancellationToken cancellationToken); + + /// + /// Opens a device asking for . Exclusive is a request: a source that cannot honour it + /// (not WASAPI, or the device refuses) opens shared, and the lease's says which. + /// + ValueTask OpenAsync(string? deviceId, MicAccessMode access, CancellationToken cancellationToken) => + this.OpenAsync(deviceId, cancellationToken); +} + +/// How a capture device is opened. Exclusive keeps other apps off the microphone while it is open (WASAPI). +public enum MicAccessMode +{ + Shared = 0, + Exclusive = 1 } public sealed record MediaStreamInfo(int Index, string Codec, int SampleRate, int Channels, string? Language, string? Title); diff --git a/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs b/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs index 639ef24..86b00bd 100644 --- a/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs +++ b/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs @@ -10,6 +10,12 @@ namespace PrimeDictate.Core.Settings; /// Transcription-mode defaults. Stored apart from dictation's settings.json, which is /// never rewritten by transcription mode. /// +/// +/// Earlier files also had RequestedBackend, SpeakerDetection, DiarizerModelId, InputDeviceId and CopyImportedMedia. +/// Nothing read them (the run records what it actually used, speaker labels are , imports always +/// keep their media in the session folder, and recording uses the system default microphone), so they were removed; an old file that +/// still has them loads fine. +/// public sealed record TranscriptionPreferences { public const int CurrentSchemaVersion = 1; @@ -18,21 +24,48 @@ public sealed record TranscriptionPreferences public string? AsrModelId { get; init; } - public string RequestedBackend { get; init; } = "cpu"; - public string? Language { get; init; } - public bool SpeakerDetection { get; init; } - - public string? DiarizerModelId { get; init; } - [JsonConverter(typeof(JsonStringEnumConverter))] public AudioRetention AudioRetention { get; init; } = AudioRetention.KeepAudio; - public string? InputDeviceId { get; init; } + /// Meetings: off records only and transcribes after Stop; draft shows a fast model's text while people talk. + public string LiveTextMode { get; init; } = LiveTextModes.Off; + + /// Meetings: after Stop, re-read the recording, label the speakers and mark who talked over whom. + public bool SpeakerLabelsAfterStop { get; init; } = true; + + /// Raise quiet audio before it is transcribed. The saved recording is not changed. + public bool BoostQuietAudio { get; init; } = true; - /// Copy imported files into app storage instead of referencing them in place. - public bool CopyImportedMedia { get; init; } + /// The recording source last used: microphone, system or meeting (both). Null until one was used. + public string? LastSource { get; init; } +} + +/// +/// First-run transcription defaults taken from the dictation settings (the WPF TranscriptionPreferencesSeed): the speech model +/// dictation uses, when there is one, so transcription starts on the model the user already picked. +/// +public static class TranscriptionPreferencesSeed +{ + public static TranscriptionPreferences From(string? dictationModelId, string liveTextMode) => new() + { + AsrModelId = string.IsNullOrWhiteSpace(dictationModelId) ? null : dictationModelId, + LiveTextMode = liveTextMode + }; +} + +public static class LiveTextModes +{ + public const string Off = "off"; + public const string Draft = "draft"; +} + +public static class RecordingSources +{ + public const string Microphone = "microphone"; + public const string System = "system"; + public const string Meeting = "meeting"; } public sealed record PreferencesLoadResult(TranscriptionPreferences Preferences, bool IsReadOnly, string? Warning, string? BackupPath); diff --git a/src/PrimeDictate.Desktop/App.axaml.cs b/src/PrimeDictate.Desktop/App.axaml.cs index 8f80e0e..885eeb7 100644 --- a/src/PrimeDictate.Desktop/App.axaml.cs +++ b/src/PrimeDictate.Desktop/App.axaml.cs @@ -6,8 +6,10 @@ using PrimeDictate.Core.Dictation; using PrimeDictate.Core.Storage; using PrimeDictate.Desktop.Dictation; +using PrimeDictate.Desktop.Updates; using PrimeDictate.Platforms.Speech; using PrimeDictate.Platforms.Startup; +using PrimeDictate.Platforms.Updates; namespace PrimeDictate.Desktop; @@ -17,6 +19,7 @@ public sealed class App : Application internal static SingleInstance? Instance { get; set; } private int exiting; + private UpdateController? updates; public override void Initialize() => AvaloniaXamlLoader.Load(this); @@ -36,15 +39,19 @@ public override void OnFrameworkInitializationCompleted() // ONNX Runtime is one library per process: the GPU build must be in place before any speech model is created. if (!IsSmokeRun(desktop.Args)) { - var saved = new DictationSettingsStore(AppDataPaths.Default).Load().Settings.OnnxDevice; - OnnxRuntimeDevice.Configure(OnnxRuntimeDevice.Effective(saved)); + var savedSettings = new DictationSettingsStore(AppDataPaths.Default).Load().Settings; + // A GPU or NPU choice copied from another PC is judged against this one before the runtime is picked (in memory only). + HardwareNormalization.Normalize(savedSettings, MachineSupport.Current, AppDataPaths.Default.ModelsDirectory); + OnnxRuntimeDevice.Configure(OnnxRuntimeDevice.Effective(savedSettings.OnnxDevice)); + // Whisper.net picks its native build once too, before the first model loads. + WhisperNetRuntime.Configure(WhisperNetRuntime.Effective(savedSettings)); } var window = new MainWindow(); // The window is the default start, so --show and --workspace (the WPF app's flags) are accepted and mean the same. // --background starts in the tray only, where the tray exists (Windows, macOS). - var background = launch.Contains("--background") && !launch.Contains("--show") && !launch.Contains("--workspace") && CanHideToTray; + var background = IsBackgroundLaunch(launch) && !launch.Contains("--show") && !launch.Contains("--workspace") && CanHideToTray; if (!background) { desktop.MainWindow = window; @@ -72,6 +79,36 @@ public override void OnFrameworkInitializationCompleted() base.OnFrameworkInitializationCompleted(); } + /// + /// --background is what the installer's Startup shortcut and the Run value pass. --from-login is what the 6.0.0 shortcut + /// passed; it means the same, so a shortcut left over from 6.0.0 still starts in the tray. + /// + internal static bool IsBackgroundLaunch(string[] args) => args.Contains("--background") || args.Contains("--from-login"); + + /// Windows only: the tray's "Check for updates..." item, the automatic check on launch, and the clean-up of 6.0.0 startup entries. + private void StartUpdates(IClassicDesktopStyleApplicationLifetime desktop, MainWindow window, DictationShell shell) + { + // A run with its own data folder (PRIMEDICTATE_DATA_DIR, used for tests) is an isolated profile: it must not touch the real startup entries or update record. + if (!OperatingSystem.IsWindows() || !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("PRIMEDICTATE_DATA_DIR"))) + { + return; + } + + var launch = new LaunchAtLogin(); + launch.MigrateLegacyEntries(); + var root = AppDataPaths.Default.Root; + this.updates = new UpdateController( + GitHubUpdateService.CurrentApplicationVersion(typeof(App).Assembly), + Path.Combine(root, "updates"), + new UpdateCheckState(Path.Combine(root, "update-check.json"), Path.Combine(root, "settings.json")), + launch, + () => this.ExitAsync(desktop, window, shell), + () => shell.Host.Settings.CheckForUpdatesAutomatically, + () => !shell.Host.IsFirstRun); + shell.UpdateMenu = this.updates; + _ = this.updates.CheckOnLaunchAsync(); + } + private static bool CanHideToTray => OperatingSystem.IsWindows() || OperatingSystem.IsMacOS(); private static void RenderTrayIcons(string directory) @@ -88,7 +125,8 @@ private static void RenderTrayIcons(string directory) private void StartDictation(IClassicDesktopStyleApplicationLifetime desktop, MainWindow window) { - var shell = new DictationShell(desktop, window.Workspace, () => ShowWorkspace(window)); + var shell = new DictationShell(desktop, window, () => ShowWorkspace(window)); + this.StartUpdates(desktop, window, shell); shell.Start(this); shell.ExitRequested += () => _ = this.ExitAsync(desktop, window, shell); desktop.Exit += (_, _) => this.Cleanup(window, shell); @@ -164,6 +202,7 @@ private void Cleanup(MainWindow window, DictationShell shell) shell.DisposeAsync().AsTask().GetAwaiter().GetResult(); // A window that was never shown never raises Closed, so the workspace (and with it the speech worker) is released here. window.Workspace.DisposeAsync().AsTask().Wait(TimeSpan.FromSeconds(10)); + this.updates?.Dispose(); Instance?.Dispose(); } diff --git a/src/PrimeDictate.Desktop/Dictation/CommandsPanel.cs b/src/PrimeDictate.Desktop/Dictation/CommandsPanel.cs new file mode 100644 index 0000000..bcc0b60 --- /dev/null +++ b/src/PrimeDictate.Desktop/Dictation/CommandsPanel.cs @@ -0,0 +1,114 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Layout; +using PrimeDictate.Core.Dictation; +using PrimeDictate.Platforms.Dictation; +using PrimeDictate.Platforms.Input; + +namespace PrimeDictate.Desktop.Dictation; + +/// +/// The keyboard shortcuts and voice phrases that control dictation (start / stop, emergency stop, history), as the WPF "Commands" tab +/// had them. Used by the Settings window and by the last step of first-run setup. Validation happens here, in the form, with the rules in +/// . +/// +public sealed class CommandsPanel : StackPanel +{ + private readonly DictationHost host; + private readonly Action say; + private readonly CheckBox voiceEnabled = FormParts.Check("Listen for command phrases while dictating"); + private readonly TextBox commit = new() { PlaceholderText = VoiceCommandProcessor.DefaultDictationPhrase }; + private readonly TextBox stop = new() { PlaceholderText = VoiceCommandProcessor.DefaultStopPhrase }; + private readonly TextBox history = new() { PlaceholderText = VoiceCommandProcessor.DefaultHistoryPhrase }; + private readonly Dictionary hotkeys = []; + private readonly TextBlock error = FormParts.ErrorLine(); + + /// Where status text goes (the window's status line), for example when hotkeys are unavailable. + public CommandsPanel(DictationHost host, DictationSettings settings, Action say) + { + this.host = host; + this.say = say; + this.Spacing = 10; + this.Children.Add(this.voiceEnabled); + var bindings = settings.ToBindings(); + this.AddCommand(HotkeyAction.ToggleDictation, "Start / stop dictation (the phrase types what you said, then stops)", bindings, this.commit); + this.AddCommand(HotkeyAction.EmergencyStop, "Emergency stop (the phrase stops without typing)", bindings, this.stop); + this.AddCommand(HotkeyAction.ShowHistory, "Open history", bindings, this.history); + this.Children.Add(FormParts.Note("Click Change, then press Ctrl, Shift or Alt plus a key. A blank phrase uses its default.")); + this.Children.Add(this.error); + this.voiceEnabled.IsChecked = settings.EnableVoiceCommands; + this.commit.Text = settings.VoiceDictationPhrase; + this.stop.Text = settings.VoiceStopPhrase; + this.history.Text = settings.VoiceHistoryPhrase; + } + + public bool VoiceCommandsEnabled => this.voiceEnabled.IsChecked == true; + + public HotkeyGesture Gesture(HotkeyAction action) => this.hotkeys[action].Gesture; + + /// Checks the shortcuts and the three phrases, showing the problem under the form. False blocks Save. + public bool TryValidate(out EffectiveVoicePhrases phrases) + { + var phraseProblem = DictationSettingsValidator.ValidateVoicePhrases(this.VoiceCommandsEnabled, this.commit.Text, this.stop.Text, this.history.Text, out phrases); + var problem = DictationSettingsValidator.ValidateHotkeys( + this.Gesture(HotkeyAction.ToggleDictation), this.Gesture(HotkeyAction.EmergencyStop), this.Gesture(HotkeyAction.ShowHistory)) ?? phraseProblem; + FormParts.Show(this.error, problem); + return problem is null; + } + + /// Writes the shortcuts, the voice switch and the (already validated) phrases. + public void Apply(DictationSettings settings, EffectiveVoicePhrases phrases) + { + settings.DictationHotkey = HotkeyDto.From(this.Gesture(HotkeyAction.ToggleDictation)); + settings.StopHotkey = HotkeyDto.From(this.Gesture(HotkeyAction.EmergencyStop)); + settings.HistoryHotkey = HotkeyDto.From(this.Gesture(HotkeyAction.ShowHistory)); + settings.EnableVoiceCommands = this.VoiceCommandsEnabled; + settings.VoiceDictationPhrase = phrases.Commit; + settings.VoiceStopPhrase = phrases.Stop; + settings.VoiceHistoryPhrase = phrases.History; + } + + private void AddCommand(HotkeyAction action, string title, IReadOnlyDictionary bindings, TextBox phrase) + { + var label = new TextBlock { VerticalAlignment = VerticalAlignment.Center, TextWrapping = Avalonia.Media.TextWrapping.Wrap }; + this.hotkeys[action] = (label, bindings[action]); + label.Text = bindings[action].ToString(); + var change = new Button { Content = "Change", Margin = new Thickness(8, 0, 0, 0) }; + change.Click += async (_, _) => + { + if (this.host.HotkeyUnavailableReason is { } reason) + { + this.say(reason); + return; + } + + label.Text = "Press the new shortcut (Esc cancels)..."; + var gesture = await HotkeyCapture.CaptureAsync(TimeSpan.FromSeconds(10), CancellationToken.None); + if (gesture is not null) + { + this.hotkeys[action] = (label, gesture); + } + + label.Text = this.hotkeys[action].Gesture.ToString(); + }; + Grid.SetColumn(change, 1); + phrase.HorizontalAlignment = HorizontalAlignment.Stretch; + this.Children.Add(new Border + { + BorderBrush = new Avalonia.Media.SolidColorBrush(Avalonia.Media.Color.FromArgb(70, 128, 128, 128)), + BorderThickness = new Thickness(1), + CornerRadius = new CornerRadius(4), + Padding = new Thickness(8), + Child = new StackPanel + { + Spacing = 6, + Children = + { + new TextBlock { Text = title, FontWeight = Avalonia.Media.FontWeight.SemiBold, TextWrapping = Avalonia.Media.TextWrapping.Wrap }, + new Grid { ColumnDefinitions = new ColumnDefinitions("*,Auto"), Children = { label, change } }, + FormParts.Row("Voice phrase", phrase) + } + } + }); + } +} diff --git a/src/PrimeDictate.Desktop/Dictation/ConfirmDialog.cs b/src/PrimeDictate.Desktop/Dictation/ConfirmDialog.cs new file mode 100644 index 0000000..183dde0 --- /dev/null +++ b/src/PrimeDictate.Desktop/Dictation/ConfirmDialog.cs @@ -0,0 +1,43 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Layout; +using Avalonia.Media; + +namespace PrimeDictate.Desktop.Dictation; + +/// A small Yes / No question (Avalonia has no message box). The default button is the safe one. +public static class ConfirmDialog +{ + public static async Task AskAsync(Window owner, string title, string message, string yes = "Yes", string no = "No") + { + var result = false; + var dialog = new Window + { + Title = title, + Width = 460, + SizeToContent = SizeToContent.Height, + CanResize = false, + WindowStartupLocation = WindowStartupLocation.CenterOwner + }; + var yesButton = new Button { Content = yes, MinWidth = 90 }; + var noButton = new Button { Content = no, MinWidth = 90, IsDefault = true }; + yesButton.Click += (_, _) => + { + result = true; + dialog.Close(); + }; + noButton.Click += (_, _) => dialog.Close(); + dialog.Content = new StackPanel + { + Margin = new Thickness(20), + Spacing = 16, + Children = + { + new TextBlock { Text = message, TextWrapping = TextWrapping.Wrap }, + new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8, HorizontalAlignment = HorizontalAlignment.Right, Children = { yesButton, noButton } } + } + }; + await dialog.ShowDialog(owner); + return result; + } +} diff --git a/src/PrimeDictate.Desktop/Dictation/CustomModelPanel.cs b/src/PrimeDictate.Desktop/Dictation/CustomModelPanel.cs new file mode 100644 index 0000000..afc5974 --- /dev/null +++ b/src/PrimeDictate.Desktop/Dictation/CustomModelPanel.cs @@ -0,0 +1,158 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Platform.Storage; +using PrimeDictate.Core.Dictation; +using PrimeDictate.Platforms.Speech; + +namespace PrimeDictate.Desktop.Dictation; + +/// +/// The WPF "Resolved model path" box with Browse: shows where the selected model lives and lets the user point at a model that is not +/// in a PrimeDictate models folder (a folder, or for Whisper.net a ggml .bin file). The file rules come from +/// ; this control only asks it. +/// +public sealed class CustomModelPanel : StackPanel +{ + private readonly TextBox path = new() { PlaceholderText = "Model folder, or a Whisper GGML .bin file" }; + private readonly TextBlock status = new() { TextWrapping = Avalonia.Media.TextWrapping.Wrap, Opacity = 0.8 }; + private readonly TextBlock error = FormParts.ErrorLine(); + private bool suppress; + + public CustomModelPanel(string managedFolder) + { + this.Spacing = 6; + var folder = new Button { Content = "Browse folder...", Margin = new Thickness(8, 0, 0, 0) }; + var file = new Button { Content = "Browse file...", Margin = new Thickness(8, 0, 0, 0) }; + folder.Click += async (_, _) => await this.BrowseFolderAsync(); + file.Click += async (_, _) => await this.BrowseFileAsync(); + Grid.SetColumn(folder, 1); + Grid.SetColumn(file, 2); + this.Children.Add(new TextBlock { Text = "Model location (use a downloaded model, or browse to one you already have)", TextWrapping = Avalonia.Media.TextWrapping.Wrap }); + this.Children.Add(new Grid { ColumnDefinitions = new ColumnDefinitions("*,Auto,Auto"), Children = { this.path, folder, file } }); + this.Children.Add(this.status); + this.Children.Add(this.error); + this.Children.Add(FormParts.Note($"Downloaded models are stored in {managedFolder}. Models next to the app, or in a models folder beside where it is started, are found too.")); + this.path.TextChanged += (_, _) => + { + if (!this.suppress) + { + this.Describe(); + } + }; + } + + /// Shows where the selected model lives (blank for none). Called when the model list or its selection changes. + public void ShowSelected(InstalledSpeechModel? selected) + { + this.suppress = true; + this.path.Text = selected?.Directory ?? string.Empty; + this.suppress = false; + FormParts.Show(this.error, null); + this.status.Text = selected is null ? string.Empty : $"Using {selected.DisplayName}."; + } + + /// + /// Decides what to save. A path that differs from the selected model's own is a custom model and must validate (the problem is shown and + /// false returned); then is that model. A blank or unchanged path means the list selection applies. + /// is what goes in ModelPath: the custom path, or null. + /// + public bool TryResolve(InstalledSpeechModel? selected, LegacyBackend? preferred, out string? modelPath, out InstalledSpeechModel? custom) + { + modelPath = null; + custom = null; + FormParts.Show(this.error, null); + var text = this.path.Text?.Trim() ?? string.Empty; + if (text.Length == 0) + { + return true; + } + + if (selected is not null && PathsEqual(text, selected.Directory)) + { + if (selected.IsCustom) + { + custom = selected; + modelPath = selected.Directory; + } + + return true; + } + + if (SpeechModelLocator.TryResolveCustom(text, preferred, out var model, out var problem)) + { + custom = model; + modelPath = model.Directory; + return true; + } + + FormParts.Show(this.error, problem); + return false; + } + + private static bool PathsEqual(string a, string b) + { + try + { + return string.Equals(FormParts.Normalize(a), FormParts.Normalize(b), StringComparison.OrdinalIgnoreCase); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException) + { + return false; + } + } + + private void Describe() + { + var text = this.path.Text?.Trim() ?? string.Empty; + if (text.Length == 0) + { + this.status.Text = string.Empty; + FormParts.Show(this.error, null); + return; + } + + if (SpeechModelLocator.TryResolveCustom(text, null, out var model, out var problem)) + { + FormParts.Show(this.error, null); + this.status.Text = $"Found {model.DisplayName}. It is used when you save."; + } + else + { + this.status.Text = string.Empty; + FormParts.Show(this.error, problem); + } + } + + private async Task BrowseFolderAsync() + { + if (TopLevel.GetTopLevel(this) is not { } top) + { + return; + } + + var picked = await top.StorageProvider.OpenFolderPickerAsync(new FolderPickerOpenOptions { Title = "Select a speech model folder", AllowMultiple = false }); + if (picked.Count > 0 && picked[0].TryGetLocalPath() is { } local) + { + this.path.Text = local; + } + } + + private async Task BrowseFileAsync() + { + if (TopLevel.GetTopLevel(this) is not { } top) + { + return; + } + + var picked = await top.StorageProvider.OpenFilePickerAsync(new FilePickerOpenOptions + { + Title = "Select a Whisper GGML model (.bin)", + AllowMultiple = false, + FileTypeFilter = [new FilePickerFileType("GGML model files") { Patterns = ["*.bin"] }, FilePickerFileTypes.All] + }); + if (picked.Count > 0 && picked[0].TryGetLocalPath() is { } local) + { + this.path.Text = local; + } + } +} diff --git a/src/PrimeDictate.Desktop/Dictation/DictationActivityWindow.cs b/src/PrimeDictate.Desktop/Dictation/DictationActivityWindow.cs new file mode 100644 index 0000000..f2450d3 --- /dev/null +++ b/src/PrimeDictate.Desktop/Dictation/DictationActivityWindow.cs @@ -0,0 +1,133 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Input.Platform; +using Avalonia.Layout; +using Avalonia.Media; +using Avalonia.Threading; +using PrimeDictate.Core.Diagnostics; + +namespace PrimeDictate.Desktop.Dictation; + +/// +/// Dictation activity: dictation sessions (time range, status, app, never the text), the log of the selected session and +/// the app-wide activity log, with Copy buttons for bug reports. Backed by . +/// +public sealed class DictationActivityWindow : Window +{ + private readonly ActivityFeed feed = AppLog.Feed; + private readonly ListBox sessionList = new() { MinWidth = 260 }; + private readonly ListBox sessionLog = new(); + private readonly ListBox globalLog = new(); + private readonly TextBlock sessionHeading = new() { Text = "Select a session", FontWeight = FontWeight.SemiBold }; + private IReadOnlyList sessions = []; + private int refreshQueued; + + public DictationActivityWindow() + { + this.Title = "PrimeDictate: dictation activity"; + this.Width = 980; + this.Height = 640; + this.WindowStartupLocation = WindowStartupLocation.CenterScreen; + + var copyErrors = new Button { Content = "Copy errors" }; + copyErrors.Click += async (_, _) => await this.CopyAsync(this.feed.Entries().Where(e => e.Level == ActivityLevel.Error)); + var copyAll = new Button { Content = "Copy all activity" }; + copyAll.Click += async (_, _) => await this.CopyAsync(this.feed.Entries()); + var clear = new Button { Content = "Clear" }; + clear.Click += (_, _) => this.feed.Clear(); + var buttons = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8, Margin = new Thickness(0, 8, 0, 0), Children = { copyErrors, copyAll, clear } }; + + var left = new DockPanel { Margin = new Thickness(0, 0, 12, 0) }; + var sessionsTitle = new TextBlock { Text = "Dictation sessions", FontWeight = FontWeight.SemiBold, Margin = new Thickness(0, 0, 0, 6) }; + DockPanel.SetDock(sessionsTitle, Dock.Top); + left.Children.Add(sessionsTitle); + left.Children.Add(this.sessionList); + + var perSession = new DockPanel { Height = 240 }; + DockPanel.SetDock(this.sessionHeading, Dock.Top); + perSession.Children.Add(this.sessionHeading); + perSession.Children.Add(this.sessionLog); + DockPanel.SetDock(perSession, Dock.Top); + + var globalTitle = new TextBlock { Text = "All activity (newest first, last 600)", FontWeight = FontWeight.SemiBold, Margin = new Thickness(0, 10, 0, 6) }; + DockPanel.SetDock(globalTitle, Dock.Top); + var right = new DockPanel(); + DockPanel.SetDock(buttons, Dock.Bottom); + right.Children.Add(buttons); + right.Children.Add(perSession); + right.Children.Add(globalTitle); + right.Children.Add(this.globalLog); + + var privacy = new TextBlock { Text = "No recognized text is kept here: only when, what happened and why.", Opacity = 0.6, FontSize = 12, Margin = new Thickness(0, 0, 0, 8) }; + var layout = new DockPanel { Margin = new Thickness(16) }; + DockPanel.SetDock(privacy, Dock.Top); + DockPanel.SetDock(left, Dock.Left); + layout.Children.Add(privacy); + layout.Children.Add(left); + layout.Children.Add(right); + this.Content = layout; + + this.sessionList.SelectionChanged += (_, _) => this.ShowSession(); + this.Opened += (_, _) => this.Refresh(); + this.feed.Changed += this.OnChanged; + this.Closed += (_, _) => this.feed.Changed -= this.OnChanged; + } + + // The feed changes on worker threads, often in bursts; coalesce into one refresh per UI turn. + private void OnChanged() + { + if (Interlocked.Exchange(ref this.refreshQueued, 1) == 1) + { + return; + } + + Dispatcher.UIThread.Post(() => + { + Interlocked.Exchange(ref this.refreshQueued, 0); + this.Refresh(); + }); + } + + private void Refresh() + { + var selected = this.SelectedSession()?.Id; + this.sessions = this.feed.Sessions(); + this.sessionList.ItemsSource = this.sessions.Select(ActivityText.Session).ToList(); + if (selected is { } id) + { + var i = this.sessions.ToList().FindIndex(s => s.Id == id); + if (i >= 0) + { + this.sessionList.SelectedIndex = i; + } + } + + this.globalLog.ItemsSource = this.feed.Entries().Select(ActivityText.Line).ToList(); + this.ShowSession(); + } + + private DictationSessionInfo? SelectedSession() => + this.sessionList.SelectedIndex is >= 0 and var i && i < this.sessions.Count ? this.sessions[i] : null; + + private void ShowSession() + { + if (this.SelectedSession() is not { } session) + { + this.sessionHeading.Text = "Select a session"; + this.sessionLog.ItemsSource = null; + return; + } + + this.sessionHeading.Text = ActivityText.Session(session); + this.sessionLog.ItemsSource = this.feed.SessionEntries(session.Id).Select(ActivityText.Line).ToList(); + } + + private async Task CopyAsync(IEnumerable entries) + { + var text = ActivityText.Join(entries); + if (this.Clipboard is { } clipboard) + { + await clipboard.SetTextAsync(text.Length == 0 ? "(no activity)" : text); + } + } +} diff --git a/src/PrimeDictate.Desktop/Dictation/DictationHistoryWindow.cs b/src/PrimeDictate.Desktop/Dictation/DictationHistoryWindow.cs index cfd80c7..2684174 100644 --- a/src/PrimeDictate.Desktop/Dictation/DictationHistoryWindow.cs +++ b/src/PrimeDictate.Desktop/Dictation/DictationHistoryWindow.cs @@ -7,59 +7,178 @@ namespace PrimeDictate.Desktop.Dictation; -/// Everything dictation typed (or refused to type), searchable, so a lost transcript can be copied back. +/// +/// Everything dictation typed (or refused to type), so a lost transcript can be copied back. Search (every word, in the final or the original +/// text), a delivery filter, an app filter and a window filter, as in the WPF history window; the filters themselves are +/// . Copying is always an explicit click and goes to the clipboard only; nothing is typed from here. +/// public sealed class DictationHistoryWindow : Window { + private static readonly string[] StatusNames = ["All", "Typed into app", "Not typed"]; private readonly DictationHost host; - private readonly TextBox search = new() { PlaceholderText = "Search transcripts, apps and windows" }; + private readonly TextBox search = new() { PlaceholderText = "Search final and original transcript text", MinWidth = 240 }; + private readonly ComboBox statusBox = new() { ItemsSource = StatusNames, SelectedIndex = 0, MinWidth = 150 }; + private readonly ComboBox appBox = new() { MinWidth = 170 }; + private readonly ComboBox windowBox = new() { MinWidth = 190, MaxDropDownHeight = 320 }; + private readonly Button clearFilters = new() { Content = "Clear filters", IsEnabled = false }; private readonly ListBox list = new(); - private readonly TextBox detail = new() { IsReadOnly = true, AcceptsReturn = true, TextWrapping = Avalonia.Media.TextWrapping.Wrap, MinHeight = 120 }; - private readonly TextBlock summary = new() { Opacity = 0.7 }; - private IReadOnlyList entries = []; + private readonly TextBox detail = new() { IsReadOnly = true, AcceptsReturn = true, TextWrapping = Avalonia.Media.TextWrapping.Wrap, MinHeight = 140 }; + private readonly TextBlock summary = new() { Opacity = 0.7, VerticalAlignment = VerticalAlignment.Center }; + private IReadOnlyList all = []; + private IReadOnlyList shown = []; + private IReadOnlyList appOptions = [DictationHistoryFilter.AllApps]; + private IReadOnlyList windowOptions = [DictationHistoryFilter.AllWindows]; + private bool loading; public DictationHistoryWindow(DictationHost host) { this.host = host; this.Title = "PrimeDictate: dictation history"; - this.Width = 760; - this.Height = 560; + this.Width = 860; + this.Height = 640; this.WindowStartupLocation = WindowStartupLocation.CenterScreen; var copy = new Button { Content = "Copy transcript" }; - copy.Click += async (_, _) => await this.CopySelectedAsync(); + copy.Click += async (_, _) => await this.CopyAsync(details: false); + var copyDetails = new Button { Content = "Copy details" }; + copyDetails.Click += async (_, _) => await this.CopyAsync(details: true); var delete = new Button { Content = "Delete" }; delete.Click += (_, _) => this.DeleteSelected(); var clear = new Button { Content = "Clear all..." }; clear.Click += (_, _) => this.ConfirmClear(); - var buttons = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8, Children = { copy, delete, clear } }; + var buttons = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8, Margin = new Thickness(0, 8, 0, 0), Children = { copy, copyDetails, delete, clear } }; + + this.clearFilters.Click += (_, _) => this.ClearFilters(); + var searchRow = new StackPanel + { + Orientation = Orientation.Horizontal, + Spacing = 10, + Children = { new TextBlock { Text = "Search", VerticalAlignment = VerticalAlignment.Center }, this.search, this.summary, this.clearFilters } + }; + var filterRow = new StackPanel + { + Orientation = Orientation.Horizontal, + Spacing = 10, + Margin = new Thickness(0, 8, 0, 8), + Children = + { + new TextBlock { Text = "Status", VerticalAlignment = VerticalAlignment.Center }, this.statusBox, + new TextBlock { Text = "App", VerticalAlignment = VerticalAlignment.Center }, this.appBox, + new TextBlock { Text = "Window", VerticalAlignment = VerticalAlignment.Center }, this.windowBox + } + }; + var top = new StackPanel { Children = { searchRow, filterRow } }; var layout = new DockPanel { Margin = new Thickness(16) }; - DockPanel.SetDock(this.search, Dock.Top); - DockPanel.SetDock(this.summary, Dock.Top); + DockPanel.SetDock(top, Dock.Top); DockPanel.SetDock(buttons, Dock.Bottom); DockPanel.SetDock(this.detail, Dock.Bottom); - layout.Children.Add(this.search); - layout.Children.Add(this.summary); + layout.Children.Add(top); layout.Children.Add(buttons); layout.Children.Add(this.detail); layout.Children.Add(this.list); this.Content = layout; - this.search.TextChanged += (_, _) => this.Reload(); + this.search.TextChanged += (_, _) => this.ApplyFilters(); + this.statusBox.SelectionChanged += (_, _) => this.ApplyFilters(); + this.appBox.SelectionChanged += (_, _) => + { + if (!this.loading) + { + this.RebuildWindowOptions(); + this.ApplyFilters(); + } + }; + this.windowBox.SelectionChanged += (_, _) => this.ApplyFilters(); this.list.SelectionChanged += (_, _) => this.ShowDetail(); - this.Opened += (_, _) => this.Reload(); + this.Opened += (_, _) => + { + this.Reload(); + this.search.Focus(); + }; host.HistoryChanged += this.OnHistoryChanged; this.Closed += (_, _) => host.HistoryChanged -= this.OnHistoryChanged; } + private HistoryStatusFilter Status => (HistoryStatusFilter)Math.Max(0, this.statusBox.SelectedIndex); + + private HistoryTargetOption App => this.appBox.SelectedIndex is >= 0 and var i && i < this.appOptions.Count ? this.appOptions[i] : DictationHistoryFilter.AllApps; + + private HistoryTargetOption WindowChoice => this.windowBox.SelectedIndex is >= 0 and var i && i < this.windowOptions.Count ? this.windowOptions[i] : DictationHistoryFilter.AllWindows; + + private bool HasActiveFilters => + !string.IsNullOrWhiteSpace(this.search.Text) || this.Status != HistoryStatusFilter.All + || this.App.Kind != HistoryTargetKind.All || this.WindowChoice.Kind != HistoryTargetKind.All; + private void OnHistoryChanged(DictationHistoryEntry entry) => Avalonia.Threading.Dispatcher.UIThread.Post(this.Reload); + /// Reads the whole log (it keeps at most 1,000 entries) and rebuilds the App and Window lists from it. private void Reload() { - this.entries = this.host.History.List(this.search.Text); - this.list.ItemsSource = this.entries.Select(Line).ToList(); - this.summary.Text = $"{this.entries.Count:N0} entries"; - this.detail.Text = string.Empty; + this.all = this.host.History.List(limit: DictationHistoryStore.MaxEntries); + this.loading = true; + try + { + var app = this.App; + this.appOptions = DictationHistoryFilter.AppOptions(this.all); + this.appBox.ItemsSource = this.appOptions.Select(o => o.DisplayName).ToList(); + var appIndex = this.appOptions.ToList().FindIndex(o => o.Kind == app.Kind && o.Value == app.Value); + this.appBox.SelectedIndex = Math.Max(0, appIndex); + this.RebuildWindowOptions(); + } + finally + { + this.loading = false; + } + + this.ApplyFilters(); + } + + private void RebuildWindowOptions() + { + var previous = this.WindowChoice; + this.windowOptions = DictationHistoryFilter.WindowOptions(this.all, this.App); + this.windowBox.ItemsSource = this.windowOptions.Select(o => o.DisplayName).ToList(); + var index = this.windowOptions.ToList().FindIndex(o => o.Kind == previous.Kind && o.Value == previous.Value); + this.windowBox.SelectedIndex = Math.Max(0, index); + } + + private void ApplyFilters() + { + if (this.loading) + { + return; + } + + var previous = this.Selected?.Id; + this.shown = DictationHistoryFilter.Apply(this.all, this.search.Text, this.Status, this.App, this.WindowChoice); + this.list.ItemsSource = this.shown.Select(Line).ToList(); + this.summary.Text = $"{this.shown.Count:N0} of {this.all.Count:N0} entries"; + this.clearFilters.IsEnabled = this.HasActiveFilters; + var keep = previous is { } id ? this.shown.ToList().FindIndex(e => e.Id == id) : -1; + this.list.SelectedIndex = keep >= 0 ? keep : (this.shown.Count > 0 ? 0 : -1); + if (this.shown.Count == 0) + { + this.detail.Text = this.all.Count == 0 ? "Nothing has been dictated yet." : "No entries match the filters."; + } + } + + private void ClearFilters() + { + this.loading = true; + try + { + this.search.Text = string.Empty; + this.statusBox.SelectedIndex = 0; + this.appBox.SelectedIndex = 0; + this.RebuildWindowOptions(); + } + finally + { + this.loading = false; + } + + this.ApplyFilters(); } private static string Line(DictationHistoryEntry e) @@ -85,34 +204,28 @@ private static string Line(DictationHistoryEntry e) _ => "discarded" }; - private DictationHistoryEntry? Selected => this.list.SelectedIndex is >= 0 and var i && i < this.entries.Count ? this.entries[i] : null; + private DictationHistoryEntry? Selected => this.list.SelectedIndex is >= 0 and var i && i < this.shown.Count ? this.shown[i] : null; + /// The delivery, target, audio length, error, original and rewrite prompt, then the final text: everything the WPF detail pane showed. private void ShowDetail() { - if (this.Selected is not { } e) - { - return; - } - - var text = e.Transcript; - if (e.OriginalTranscript is not null) + if (this.Selected is { } e) { - text += $"\n\n--- as spoken ---\n{e.OriginalTranscript}"; + this.detail.Text = DictationHistoryFilter.DetailsText(e); } + } - if (e.Error is not null) + private async Task CopyAsync(bool details) + { + if (this.Selected is not { } e) { - text += $"\n\n{e.Error}"; + this.summary.Text = "Select a transcript first, then copy."; + return; } - this.detail.Text = text; - } - - private async Task CopySelectedAsync() - { - if (this.Selected is { } e && this.Clipboard is { } clipboard) + if (this.Clipboard is { } clipboard) { - await clipboard.SetTextAsync(e.Transcript); + await clipboard.SetTextAsync(details ? DictationHistoryFilter.DetailsText(e) : e.Transcript); } } diff --git a/src/PrimeDictate.Desktop/Dictation/DictationOnboardingWindow.cs b/src/PrimeDictate.Desktop/Dictation/DictationOnboardingWindow.cs index 8c38345..debe69f 100644 --- a/src/PrimeDictate.Desktop/Dictation/DictationOnboardingWindow.cs +++ b/src/PrimeDictate.Desktop/Dictation/DictationOnboardingWindow.cs @@ -4,72 +4,274 @@ using Avalonia.Media; using PrimeDictate.Core.Dictation; using PrimeDictate.Platforms.Dictation; +using PrimeDictate.Platforms.Speech; namespace PrimeDictate.Desktop.Dictation; /// -/// First-run check. It says plainly what will work on this computer (microphone, model, hotkeys, typing) and lets the -/// user pick a model. Saving writes dictation-settings.json, which is also what stops it from showing again. +/// First-run setup, the same three steps as the WPF wizard: Welcome (what works here, how commits behave), Model (pick, download or point at +/// one), Commands (shortcuts and voice phrases). Next and Back move between them; Finish validates everything with the same rules as the +/// Settings window and saves. Closing early asks "Finish setup later?", and setup shows again at the next start until it is finished. /// public sealed class DictationOnboardingWindow : Window { + private static readonly string[] StepTitles = ["Welcome", "Choose your model", "Configure commands"]; + private readonly DictationHost host; private readonly ComboBox modelBox = new() { MinWidth = 280 }; private readonly StackPanel checks = new() { Spacing = 8 }; + private readonly NumericUpDown silence = new() { Minimum = 0, Maximum = 30, Increment = 1, FormatString = "0", Width = 120 }; + private readonly CheckBox sendEnter = FormParts.Check("Send Enter after committing the transcript (coding mode)"); + private readonly CheckBox audioCues = FormParts.Check("Play start and stop tones"); + private readonly CheckBox autoUpdate = FormParts.Check("Check GitHub Releases for software updates automatically"); + private readonly ModelDownloadPanel downloads; + private readonly CustomModelPanel customModel; + private readonly CommandsPanel commands; + private readonly Control[] steps; + private readonly ScrollViewer body = new() { HorizontalScrollBarVisibility = Avalonia.Controls.Primitives.ScrollBarVisibility.Disabled }; + private readonly TextBlock stepHeader = new() { FontWeight = FontWeight.SemiBold, Opacity = 0.8 }; + private readonly TextBlock status = new() { TextWrapping = TextWrapping.Wrap, VerticalAlignment = VerticalAlignment.Center, Margin = new Thickness(0, 0, 12, 0) }; + private readonly TextBlock modelError = FormParts.ErrorLine(); + private readonly Button back = new() { Content = "Back" }; + private readonly Button next = new() { Content = "Next", IsDefault = true }; + private List shown = []; + private int step; + private bool allowClose; + private bool asking; public DictationOnboardingWindow(DictationHost host, Action openSettings) { this.host = host; this.Title = "Welcome to PrimeDictate"; - this.Width = 560; - this.SizeToContent = SizeToContent.Height; + this.Width = 600; + this.Height = 680; + this.MaxHeight = 900; this.WindowStartupLocation = WindowStartupLocation.CenterScreen; - var done = new Button { Content = "Start using PrimeDictate", HorizontalAlignment = HorizontalAlignment.Right }; - done.Click += (_, _) => this.Finish(); - var more = new Button { Content = "More settings..." }; - more.Click += (_, _) => + this.downloads = new ModelDownloadPanel(host, id => this.Populate(id)); + this.customModel = new CustomModelPanel(host.ModelsFolder); + this.commands = new CommandsPanel(host, host.Settings, message => this.status.Text = message); + this.modelBox.SelectionChanged += (_, _) => this.ShowSelectedModelPath(); + this.silence.Value = host.Settings.AutoCommitSilenceSeconds; + this.sendEnter.IsChecked = host.Settings.SendEnterAfterCommit; + this.audioCues.IsChecked = host.Settings.PlayAudioCues; + this.autoUpdate.IsChecked = host.Settings.CheckForUpdatesAutomatically; + this.steps = [this.BuildWelcome(), this.BuildModel(), this.BuildCommands(openSettings)]; + + this.back.Click += (_, _) => this.Go(this.step - 1); + this.next.Click += (_, _) => this.OnNext(); + var buttons = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8, Children = { this.back, this.next } }; + var footer = new DockPanel { Margin = new Thickness(24, 10, 24, 16) }; + DockPanel.SetDock(buttons, Dock.Right); + footer.Children.Add(buttons); + footer.Children.Add(this.status); + var header = new StackPanel + { + Margin = new Thickness(24, 20, 24, 4), + Children = { new TextBlock { Text = "Set up PrimeDictate", FontSize = 20, FontWeight = FontWeight.SemiBold }, this.stepHeader } + }; + var root = new DockPanel(); + DockPanel.SetDock(header, Dock.Top); + DockPanel.SetDock(footer, Dock.Bottom); + root.Children.Add(header); + root.Children.Add(footer); + root.Children.Add(this.body); + this.Content = root; + this.Go(0); + this.Opened += (_, _) => this.Populate(); + this.Closing += (_, e) => { - this.Finish(); - openSettings(); + // Only the user closing the window asks; the app shutting down or the OS signing out must not be blocked. + if (this.allowClose || e.CloseReason is not (WindowCloseReason.WindowClosing or WindowCloseReason.Undefined)) + { + return; + } + + e.Cancel = true; + _ = this.ConfirmCloseAsync(); }; + } - var hotkey = host.Settings.ToBindings()[HotkeyAction.ToggleDictation]; - this.Content = new StackPanel + private Control BuildWelcome() => new StackPanel + { + Margin = new Thickness(24, 8, 24, 8), + Spacing = 14, + Children = { - Margin = new Thickness(24), - Spacing = 14, + new TextBlock + { + Text = "PrimeDictate is tuned for dictating into editors, chat apps, documentation tools, terminals and browser text fields. You pick a local speech model, set your shortcut, and the app types the final transcript into the app you are in. Everything runs on this computer.", + TextWrapping = TextWrapping.Wrap + }, + new TextBlock { Text = "Here is what works right now.", FontWeight = FontWeight.SemiBold }, + this.checks, + new TextBlock { Text = "Quick setup checklist", FontWeight = FontWeight.SemiBold, Margin = new Thickness(0, 8, 0, 0) }, + Step("1. Pick a local speech model", "Choose Whisper, Parakeet, Moonshine or Whisper.net; download a model, or browse to one you already have."), + Step("2. Set your dictation shortcut", "PrimeDictate listens globally, so you can start and stop dictation without changing apps."), + Step("3. Tune how commits behave", "Choose silence auto-commit timing, the optional coding-mode Enter, and the start and stop tones."), + new TextBlock { Text = "General dictation behavior", FontWeight = FontWeight.SemiBold, Margin = new Thickness(0, 8, 0, 0) }, + FormParts.Row("Commit after this many seconds of silence (0 = shortcut only)", this.silence), + this.sendEnter, + this.audioCues, + this.autoUpdate, + FormParts.Note("Emergency stop shortcuts and stop phrases discard the active capture without typing text. Use silence auto-commit or the start / stop shortcut when you want to commit."), + FormParts.Note("You can revisit any of these settings from the tray icon later.") + } + }; + + private Control BuildModel() => new StackPanel + { + Margin = new Thickness(24, 8, 24, 8), + Spacing = 10, + Children = + { + new TextBlock { Text = "Choose a speech model", FontWeight = FontWeight.SemiBold }, + FormParts.Note("PrimeDictate supports several local speech engines. Pick a model that fits your computer, then download it or browse to an existing install."), + new TextBlock { Text = "Installed models" }, + this.modelBox, + new TextBlock { Text = "Download a model", FontWeight = FontWeight.SemiBold, Margin = new Thickness(0, 8, 0, 0) }, + this.downloads, + this.customModel, + this.modelError + } + }; + + private Control BuildCommands(Action openSettings) + { + var more = new Button { Content = "More settings...", HorizontalAlignment = HorizontalAlignment.Left }; + more.Click += (_, _) => + { + if (this.TryFinish()) + { + this.Close(); + openSettings(); + } + }; + return new StackPanel + { + Margin = new Thickness(24, 8, 24, 8), + Spacing = 10, Children = { - new TextBlock { Text = "Everything runs on this computer. Here is what works right now.", TextWrapping = TextWrapping.Wrap, FontSize = 16 }, - this.checks, - new TextBlock { Text = "Speech model" }, - this.modelBox, - new ModelDownloadPanel(host, this.Populate), - new TextBlock { Text = $"Press {hotkey} in any app to start dictating, and again to type what you said.", TextWrapping = TextWrapping.Wrap, Opacity = 0.8 }, - new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8, HorizontalAlignment = HorizontalAlignment.Right, Children = { more, done } } + new TextBlock { Text = "Keyboard and voice commands", FontWeight = FontWeight.SemiBold }, + FormParts.Note("These work anywhere. Press the start / stop shortcut to begin dictating, and again to type what you said."), + this.commands, + more } }; - this.Opened += (_, _) => this.Populate(); } - private void Populate() + private static Control Step(string title, string detail) => new Border + { + BorderBrush = new SolidColorBrush(Color.FromArgb(70, 128, 128, 128)), + BorderThickness = new Thickness(1), + CornerRadius = new CornerRadius(6), + Padding = new Thickness(10), + Child = new StackPanel + { + Children = + { + new TextBlock { Text = title, FontWeight = FontWeight.SemiBold }, + new TextBlock { Text = detail, TextWrapping = TextWrapping.Wrap, Opacity = 0.8 } + } + } + }; + + private void Go(int index) + { + this.step = Math.Clamp(index, 0, this.steps.Length - 1); + this.body.Content = this.steps[this.step]; + this.stepHeader.Text = $"Step {this.step + 1} of {this.steps.Length}: {StepTitles[this.step]}"; + this.back.IsEnabled = this.step > 0; + this.next.Content = this.step == this.steps.Length - 1 ? "Finish" : "Next"; + this.status.Text = string.Empty; + } + + private void OnNext() + { + switch (this.step) + { + case 0: + this.Go(1); + // The WPF wizard started the first download as you moved on; do the same when nothing is installed yet. + if (this.host.InstalledModels().Count == 0) + { + this.downloads.StartDownload(id => this.Populate(id)); + } + + break; + case 1: + if (this.TryValidateModel(out _, out _, out _)) + { + this.Go(2); + } + + break; + default: + if (this.TryFinish()) + { + this.Close(); + } + + break; + } + } + + private void Populate(string? select = null) { this.checks.Children.Clear(); this.Add(this.host.UnavailableReason is null, "Microphone", this.host.UnavailableReason ?? "A capture device is available."); var models = this.host.InstalledModels(); this.Add(models.Count > 0, "Speech model", models.Count > 0 ? $"{models.Count} installed." - : $"None installed yet. Download one below (Whisper Tiny English is the fastest to start with), or copy a model folder into {this.host.ModelsFolder}."); + : $"None installed yet. The next step downloads one (Whisper Tiny English is the fastest to start with), or you can copy a model folder into {this.host.ModelsFolder}."); this.Add(this.host.HotkeyUnavailableReason is null, "Global hotkey", this.host.HotkeyUnavailableReason ?? "Available."); this.Add(this.host.FocusGuardAvailable, "Typing into the right window", this.host.FocusGuardAvailable ? "PrimeDictate checks that the window you started in is still in front before typing." : this.host.FocusGuardNotice ?? "Not available here; PrimeDictate will not type unless you allow it in settings."); + var keep = this.modelBox.SelectedIndex is >= 0 and var i && i < this.shown.Count ? this.shown[i] : null; + this.shown = models.Select(m => m.ModelId).ToList(); this.modelBox.ItemsSource = models.Select(m => m.DisplayName).ToList(); - var wanted = this.host.Settings.ResolveModelId(); - this.modelBox.SelectedIndex = models.Count == 0 ? -1 : Math.Max(0, models.ToList().FindIndex(m => m.ModelId == wanted)); + var wanted = select ?? keep ?? this.host.WantedModelId(); + this.modelBox.SelectedIndex = models.Count == 0 ? -1 : Math.Max(0, this.shown.IndexOf(wanted ?? string.Empty)); this.modelBox.IsEnabled = models.Count > 0; + this.ShowSelectedModelPath(); + } + + private InstalledSpeechModel? SelectedModel() + { + var models = this.host.InstalledModels(); + return this.modelBox.SelectedIndex is >= 0 and var i && i < models.Count ? models[i] : null; + } + + private void ShowSelectedModelPath() => this.customModel.ShowSelected(this.SelectedModel()); + + /// A model is needed to finish: an installed one that is selected, or a valid custom path. Same rules as Settings. + private bool TryValidateModel(out string? modelPath, out InstalledSpeechModel? custom, out InstalledSpeechModel? selected) + { + modelPath = null; + custom = null; + selected = this.SelectedModel(); + FormParts.Show(this.modelError, null); + if (this.downloads.IsBusy) + { + FormParts.Show(this.modelError, "Wait for the current model download to finish or cancel it before continuing."); + return false; + } + + if (!this.customModel.TryResolve(selected, selected?.Backend ?? this.host.Settings.TranscriptionBackend, out modelPath, out custom)) + { + return false; + } + + if (custom is null && selected is null) + { + FormParts.Show(this.modelError, "Choose a model to download, or browse to an existing model folder, before continuing."); + return false; + } + + return true; } private void Add(bool ok, string title, string detail) => @@ -85,23 +287,76 @@ private void Add(bool ok, string title, string detail) => Children = { new TextBlock { Text = title, FontWeight = FontWeight.SemiBold }, - new TextBlock { Text = detail, TextWrapping = TextWrapping.Wrap, MaxWidth = 460, Opacity = 0.8 } + new TextBlock { Text = detail, TextWrapping = TextWrapping.Wrap, MaxWidth = 480, Opacity = 0.8 } } } } }); - private void Finish() + /// Validates every step and saves. Problems are shown on the step that has them. + private bool TryFinish() { + if (!this.TryValidateModel(out var modelPath, out var custom, out var selected)) + { + this.Go(1); + return false; + } + + if (!this.commands.TryValidate(out var phrases)) + { + this.Go(2); + this.status.Text = "Not finished: fix the red message above."; + return false; + } + var settings = this.host.Settings; - var models = this.host.InstalledModels(); - if (this.modelBox.SelectedIndex is >= 0 and var i && i < models.Count) + if (custom is not null) { - settings.SelectedModelId = models[i].Id; - settings.TranscriptionBackend = models[i].Backend; + settings.ModelPath = modelPath; + settings.SelectedModelId = custom.Id; + settings.TranscriptionBackend = custom.Backend; + } + else if (selected is not null) + { + settings.ModelPath = null; + settings.SelectedModelId = selected.Id; + settings.TranscriptionBackend = selected.Backend; } + settings.AutoCommitSilenceSeconds = (int)(this.silence.Value ?? 3); + settings.SendEnterAfterCommit = this.sendEnter.IsChecked == true; + settings.PlayAudioCues = this.audioCues.IsChecked == true; + settings.CheckForUpdatesAutomatically = this.autoUpdate.IsChecked != false; + this.commands.Apply(settings, phrases); + settings.FirstRunCompleted = true; this.host.ApplySettings(settings); - this.Close(); + this.allowClose = true; + return true; + } + + private async Task ConfirmCloseAsync() + { + if (this.asking) + { + return; + } + + this.asking = true; + try + { + var later = await ConfirmDialog.AskAsync( + this, + "Finish setup later?", + "Setup is not finished yet. PrimeDictate will keep running in the tray, and this setup window will open again the next time you start the app.\n\nClose setup for now?"); + if (later) + { + this.allowClose = true; + this.Close(); + } + } + finally + { + this.asking = false; + } } } diff --git a/src/PrimeDictate.Desktop/Dictation/DictationOverlayWindow.cs b/src/PrimeDictate.Desktop/Dictation/DictationOverlayWindow.cs index 731ca48..f8ea688 100644 --- a/src/PrimeDictate.Desktop/Dictation/DictationOverlayWindow.cs +++ b/src/PrimeDictate.Desktop/Dictation/DictationOverlayWindow.cs @@ -3,6 +3,7 @@ using Avalonia.Controls.Primitives; using Avalonia.Controls.Shapes; using Avalonia.Input; +using Avalonia.Input.Platform; using Avalonia.Layout; using Avalonia.Media; using Avalonia.Threading; @@ -11,38 +12,72 @@ namespace PrimeDictate.Desktop.Dictation; /// -/// Live dictation preview: a small always-on-top window that never takes focus, so it cannot pull the caret away from -/// the app being dictated into. It shows only while dictating (and briefly for a notice or the final words), unless -/// pinned; the tray icon is there the rest of the time. Compact mode is a microphone pill with the level; full mode adds -/// a text box with the latest words. Both keep one size while words arrive, so nothing jumps. It can be dragged anywhere -/// (the spot is remembered) and closed until the next dictation. Live text is shown here and never typed. +/// Live dictation preview: a small always-on-top window that never takes focus, so it cannot pull the caret away from the app being +/// dictated into. It has the WPF overlay's controls: a pin (keep it on screen), a copy button (copies the last transcript to the clipboard, +/// only when clicked; this is not text injection), a settings button, collapse and expand between the compact microphone and the full panel, close, +/// the elapsed time and a "Local only" badge. Compact mode is the microphone with the ripple animation; full mode adds the particle visualizer and a +/// text box with the latest words. Like the WPF overlay, the compact microphone stays on screen (see ). +/// Both modes keep one size while words arrive, so nothing jumps. It can be dragged anywhere (the spot is remembered). Live text is shown here and never typed. /// public sealed class DictationOverlayWindow : Window { - private const double CompactWidth = 300; + private const double CompactWidth = 350; private const double FullWidth = 460; + private const double MinFullWidth = 360; + private const double TextHeight = 80; private const int MaxShownCharacters = 900; private static readonly TimeSpan NoticeTime = TimeSpan.FromSeconds(6); private static readonly TimeSpan FinalTextTime = TimeSpan.FromSeconds(1.5); private readonly VisualizerControl visualizer = new() { HorizontalAlignment = HorizontalAlignment.Center }; - private readonly TextBlock status = new() { FontSize = 12, Opacity = 0.7, VerticalAlignment = VerticalAlignment.Center }; + private readonly MicRippleControl mic = new() { VerticalAlignment = VerticalAlignment.Center }; + private readonly TextBlock status = new() { FontSize = 13, FontWeight = FontWeight.SemiBold, VerticalAlignment = VerticalAlignment.Center, TextTrimming = TextTrimming.CharacterEllipsis }; + private readonly TextBlock compactStatus = new() { FontSize = 13, FontWeight = FontWeight.SemiBold, VerticalAlignment = VerticalAlignment.Center, TextTrimming = TextTrimming.CharacterEllipsis }; + private readonly TextBlock title = new() { Text = "Live transcript", FontSize = 15, FontWeight = FontWeight.SemiBold, VerticalAlignment = VerticalAlignment.Center }; private readonly TextBlock text = new() { FontSize = 15, LineHeight = 20, TextWrapping = TextWrapping.Wrap }; + private readonly TextBlock timerText = new() { Text = "00:00", FontFamily = new FontFamily("Consolas, Menlo, monospace"), FontSize = 14, VerticalAlignment = VerticalAlignment.Center, HorizontalAlignment = HorizontalAlignment.Center }; private readonly ScrollViewer textView; private readonly Ellipse dot = new() { Width = 10, Height = 10, Fill = Brushes.Gray, VerticalAlignment = VerticalAlignment.Center }; private readonly Border panel; + private readonly Control fullHeader; + private readonly Control compactHeader; + private readonly Control footer; + private readonly Control stateRow; + private readonly ToggleButton pinFull; + private readonly ToggleButton pinCompact; + private readonly Button copyFull; + private readonly Button copyCompact; + private readonly Button collapse; private readonly DispatcherTimer hideTimer; + private readonly DispatcherTimer clockTimer; private readonly DispatcherTimer anchorSaveTimer; private OverlayStyle style = OverlayStyle.CompactMicrophone; + private bool? expandedOverride; private bool sticky; + private bool hideCompactWhenIdle; private bool dismissed; private bool placing; private bool dragging; private bool moved; + private bool clickCandidate; + private PixelPoint pressedAt; private bool showingNotice; + private bool wakeListening; + private string backend = "Whisper ONNX"; + private string transcript = string.Empty; + private string lastTranscript = string.Empty; + private string notice = string.Empty; + private DateTime listeningSince; private DictationState state; private DateTime hideAtUtc = DateTime.MinValue; private PixelPoint? anchor; + private double? userWidth; + private double? userTextHeight; + private bool gripDragging; + private Point gripStart; + private double gripStartWidth; + private double gripStartHeight; + private readonly Border grip; public DictationOverlayWindow() { @@ -56,43 +91,108 @@ public DictationOverlayWindow() this.Background = Brushes.Transparent; this.TransparencyLevelHint = [WindowTransparencyLevel.Transparent]; this.status.Foreground = Brushes.White; + this.title.Foreground = Brushes.White; + this.compactStatus.Foreground = Brushes.White; this.text.Foreground = Brushes.White; + this.timerText.Foreground = Brushes.White; this.textView = new ScrollViewer { Content = this.text, - Height = 80, // four whole lines, so none is cut in half + Height = TextHeight, // four whole lines, so none is cut in half HorizontalScrollBarVisibility = ScrollBarVisibility.Disabled, VerticalScrollBarVisibility = ScrollBarVisibility.Hidden }; - // Not focusable, so pressing it never moves keyboard focus away from the app being dictated into. - var close = new Button + // None of the buttons can take keyboard focus, so pressing one never moves it away from the app being dictated into. + this.pinFull = PinButton(); + this.pinCompact = PinButton(); + this.copyFull = CopyButton(); + this.copyCompact = CopyButton("📋"); + this.collapse = IconButton("⤡", "Back to the compact microphone"); + var minimize = IconButton("—", "Minimize"); + var expand = IconButton("⤢", "Show the full panel"); + var settings = IconButton("⚙", "Settings"); + var settingsCompact = IconButton("⚙", "Settings"); + var close = IconButton("✕", "Hide until the next dictation"); + var closeCompact = IconButton("✕", "Hide until the next dictation"); + foreach (var pin in new[] { this.pinFull, this.pinCompact }) { - Content = "✕", - Focusable = false, - FontSize = 11, - Padding = new Thickness(6, 0), - MinHeight = 0, + pin.Click += (_, _) => this.SetPinned(pin.IsChecked == true, raise: true); + } + + foreach (var copy in new[] { this.copyFull, this.copyCompact }) + { + copy.Click += async (_, _) => await this.CopyLastTranscriptAsync(); + } + + this.collapse.Click += (_, _) => this.Collapse(); + minimize.Click += (_, _) => this.MinimizeClicked(); + expand.Click += (_, _) => this.Expand(); + settings.Click += (_, _) => this.SettingsRequested?.Invoke(); + settingsCompact.Click += (_, _) => this.SettingsRequested?.Invoke(); + close.Click += (_, _) => this.CloseClicked(); + closeCompact.Click += (_, _) => this.CloseClicked(); + + var compactButtons = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 0, Children = { this.copyCompact, this.pinCompact, settingsCompact, expand, closeCompact } }; + this.compactHeader = new DockPanel { LastChildFill = true }; + DockPanel.SetDock(compactButtons, Dock.Right); + ((Panel)this.compactHeader).Children.Add(compactButtons); + ((Panel)this.compactHeader).Children.Add(this.mic); + ((Panel)this.compactHeader).Children.Add(this.compactStatus); + this.mic.Margin = new Thickness(0, 0, 10, 0); + DockPanel.SetDock(this.mic, Dock.Left); + + var fullButtons = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 0, Children = { this.pinFull, settings, this.collapse, minimize, close } }; + this.fullHeader = new DockPanel { LastChildFill = true }; + DockPanel.SetDock(fullButtons, Dock.Right); + ((Panel)this.fullHeader).Children.Add(fullButtons); + ((Panel)this.fullHeader).Children.Add(new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8, Children = { new TextBlock { Text = "🎙", FontSize = 17, VerticalAlignment = VerticalAlignment.Center }, this.title } }); + + this.stateRow = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8, Children = { this.dot, this.status } }; + + var badge = new Border + { + BorderBrush = new SolidColorBrush(Color.FromArgb(0x60, 255, 255, 255)), + BorderThickness = new Thickness(1), + CornerRadius = new CornerRadius(4), + Padding = new Thickness(8, 3), + Child = new TextBlock { Text = "Local only", FontSize = 12, Foreground = Brushes.White, Opacity = 0.8 } + }; + ToolTip.SetTip(badge, "Transcript preview stays on this device. Audio is not uploaded."); + ToolTip.SetTip(this.timerText, "Elapsed dictation time"); + var footerGrid = new Grid { ColumnDefinitions = new ColumnDefinitions("Auto,*,Auto") }; + Grid.SetColumn(this.timerText, 1); + Grid.SetColumn(this.copyFull, 2); + footerGrid.Children.Add(badge); + footerGrid.Children.Add(this.timerText); + footerGrid.Children.Add(this.copyFull); + this.footer = footerGrid; + + this.grip = new Border + { + Width = 16, + Height = 16, + HorizontalAlignment = HorizontalAlignment.Right, Background = Brushes.Transparent, - Foreground = Brushes.White, - Opacity = 0.7, - Cursor = new Cursor(StandardCursorType.Hand), - VerticalAlignment = VerticalAlignment.Center + Cursor = new Cursor(StandardCursorType.BottomRightCorner), + Margin = new Thickness(0, -10, 0, -8), + Child = new TextBlock { Text = "◢", FontSize = 11, Foreground = Brushes.White, Opacity = 0.5, HorizontalAlignment = HorizontalAlignment.Right, VerticalAlignment = VerticalAlignment.Bottom } }; - ToolTip.SetTip(close, "Hide until the next dictation"); - close.Click += (_, _) => this.Dismiss(); - var header = new DockPanel { LastChildFill = true }; - DockPanel.SetDock(close, Dock.Right); - header.Children.Add(close); - header.Children.Add(new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8, Children = { this.dot, this.status } }); + ToolTip.SetTip(this.grip, "Drag to resize"); + this.grip.PointerPressed += this.OnGripPressed; + this.grip.PointerMoved += this.OnGripMoved; + this.grip.PointerReleased += this.OnGripReleased; + this.grip.PointerCaptureLost += (_, _) => this.EndGripDrag(); this.panel = new Border { - Background = new SolidColorBrush(Color.FromArgb(0xE6, 0x1E, 0x1E, 0x24)), + Background = new SolidColorBrush(Color.FromArgb(0xE6, 0x0D, 0x11, 0x17)), + BorderBrush = new SolidColorBrush(Color.FromRgb(0x33, 0x33, 0x33)), + BorderThickness = new Thickness(1), CornerRadius = new CornerRadius(14), - Padding = new Thickness(14, 8, 8, 10), + Padding = new Thickness(14, 10, 8, 12), Cursor = new Cursor(StandardCursorType.SizeAll), - Child = new StackPanel { Spacing = 6, Children = { header, this.textView, this.visualizer } } + Child = new StackPanel { Spacing = 8, Children = { this.compactHeader, this.fullHeader, this.visualizer, this.stateRow, this.textView, this.footer, this.grip } } }; this.Content = this.panel; @@ -101,15 +201,19 @@ public DictationOverlayWindow() { if (this.hideAtUtc != DateTime.MinValue && DateTime.UtcNow >= this.hideAtUtc) { - // The notice or the final words have been shown long enough; a pinned overlay goes back to an empty "Ready". + // The notice or the final words have been shown long enough; what stays on screen goes back to its idle wording. this.hideAtUtc = DateTime.MinValue; this.showingNotice = false; - this.text.Text = string.Empty; + this.notice = string.Empty; + this.transcript = string.Empty; this.Refresh(); } }; this.hideTimer.Start(); + this.clockTimer = new DispatcherTimer { Interval = TimeSpan.FromSeconds(1) }; + this.clockTimer.Tick += (_, _) => this.timerText.Text = OverlayRules.Elapsed(DateTime.UtcNow - this.listeningSince); + // Only a move the user started counts: Windows runs the drag inside BeginMoveDrag, other systems after it returns. // Either way the drag is over once the overlay has been still for a moment, and then the new spot is saved. this.anchorSaveTimer = new DispatcherTimer { Interval = TimeSpan.FromMilliseconds(600) }; @@ -129,15 +233,33 @@ public DictationOverlayWindow() if (e.GetCurrentPoint(this).Properties.IsLeftButtonPressed) { this.dragging = true; + this.clickCandidate = true; + this.pressedAt = this.Position; this.BeginMoveDrag(e); this.anchorSaveTimer.Stop(); this.anchorSaveTimer.Start(); + // On Windows the drag runs inside BeginMoveDrag, so a press that never moved the window ends right here: that is a click. + if (OperatingSystem.IsWindows() && this.clickCandidate && this.Position == this.pressedAt) + { + this.clickCandidate = false; + this.OnPanelClicked(); + } + } + }; + this.PointerReleased += (_, _) => + { + // Elsewhere the window manager reports the release when nothing was dragged. + if (!OperatingSystem.IsWindows() && this.clickCandidate && this.Position == this.pressedAt) + { + this.clickCandidate = false; + this.OnPanelClicked(); } }; this.PositionChanged += (_, _) => { if (this.dragging && !this.placing) { + this.clickCandidate = false; this.anchor = this.CurrentAnchor(); this.moved = true; this.anchorSaveTimer.Stop(); @@ -158,10 +280,27 @@ public DictationOverlayWindow() /// The user moved the overlay: the bottom-center point it now grows from, in screen pixels. public event Action? AnchorMoved; - public void Configure(OverlayStyle style, bool sticky, int? anchorX, int? anchorY) + /// The pin was toggled; the app saves it as IsOverlaySticky. + public event Action? PinChanged; + + /// The gear was clicked. + public event Action? SettingsRequested; + + /// The effective look: the configured one, unless the user expanded or collapsed it for now. + private OverlayStyle EffectiveStyle => this.expandedOverride switch + { + true => OverlayStyle.FullPanel, + false => OverlayStyle.CompactMicrophone, + _ => this.style + }; + + public void Configure(OverlayStyle style, bool sticky, bool hideCompactWhenIdle, int? anchorX, int? anchorY) { this.style = style; - this.sticky = sticky; + this.expandedOverride = null; + this.ResetUserSize(); + this.hideCompactWhenIdle = hideCompactWhenIdle; + this.SetPinned(sticky, raise: false); this.anchor = anchorX is { } x && anchorY is { } y ? new PixelPoint(x, y) : null; this.Refresh(); if (this.IsVisible) @@ -170,14 +309,38 @@ public void Configure(OverlayStyle style, bool sticky, int? anchorX, int? anchor } } + /// The model family shown in the header and tooltip, such as "Whisper.net (GGML)". + public void SetBackendLabel(string label) + { + this.backend = label; + this.Refresh(); + } + + /// True while the wake word is listening on the idle microphone ("Wake listening", yellow). + public void SetWakeListening(bool listening) + { + this.wakeListening = listening; + this.Refresh(); + } + public void SetState(DictationState state) { + var before = this.state; this.state = state; if (state == DictationState.Listening) { + this.RestoreIfMinimized(); this.dismissed = false; this.showingNotice = false; - this.text.Text = string.Empty; + this.notice = string.Empty; + this.transcript = string.Empty; + this.lastTranscript = string.Empty; + this.hideAtUtc = DateTime.MinValue; + if (before != DictationState.Listening) + { + this.listeningSince = DateTime.UtcNow; + this.timerText.Text = "00:00"; + } } else if (state == DictationState.Idle) { @@ -186,68 +349,318 @@ public void SetState(DictationState state) this.hideAtUtc = this.hideAtUtc > finalWords ? this.hideAtUtc : finalWords; } + this.clockTimer.IsEnabled = state == DictationState.Listening; + if (state == DictationState.Idle) + { + this.timerText.Text = "00:00"; + } + this.Refresh(); } - public void SetTranscript(string transcript) + public void SetTranscript(string value) { - if (this.showingNotice) + if (!string.IsNullOrWhiteSpace(value)) { - return; + this.lastTranscript = value.Trim(); } - this.ShowText(transcript.Length > MaxShownCharacters ? "…" + transcript[^MaxShownCharacters..].TrimStart() : transcript); + this.transcript = value.Length > MaxShownCharacters ? "…" + value[^MaxShownCharacters..].TrimStart() : value; this.Refresh(); } - public void SetLevel(double rms) => this.visualizer.SetLevel(rms); + /// Remembers the final transcript so the Copy button can still copy it after dictation ended. Kept in memory only. + public void SetCommitted(string value) + { + if (!string.IsNullOrWhiteSpace(value)) + { + this.lastTranscript = value.Trim(); + this.Refresh(); + } + } + + public void SetLevel(double rms) + { + this.visualizer.SetLevel(rms); + this.mic.SetLevel(rms); + } public void SetNotice(string message) { // A notice is worth seeing even after the overlay was closed (for example why nothing was typed). + this.RestoreIfMinimized(); this.dismissed = false; this.showingNotice = true; - this.ShowText(message); + this.notice = message; this.hideAtUtc = DateTime.UtcNow + NoticeTime; this.Refresh(); } - private void Dismiss() + private static Button IconButton(string glyph, string tip) { + var button = new Button + { + Content = glyph, + Focusable = false, + FontSize = 13, + Padding = new Thickness(7, 2), + MinHeight = 0, + Background = Brushes.Transparent, + Foreground = Brushes.White, + Opacity = 0.75, + Cursor = new Cursor(StandardCursorType.Hand), + VerticalAlignment = VerticalAlignment.Center + }; + ToolTip.SetTip(button, tip); + return button; + } + + private static ToggleButton PinButton() + { + var pin = new ToggleButton + { + Content = "📌", + Focusable = false, + FontSize = 13, + Padding = new Thickness(7, 2), + MinHeight = 0, + Background = Brushes.Transparent, + Foreground = Brushes.White, + Opacity = 0.75, + Cursor = new Cursor(StandardCursorType.Hand), + VerticalAlignment = VerticalAlignment.Center + }; + ToolTip.SetTip(pin, "Keep open on screen"); + return pin; + } + + private static Button CopyButton(string? glyph = null) + { + var copy = IconButton(glyph ?? "📋 Copy", "Copy the last transcript"); + copy.IsEnabled = false; + copy.BorderBrush = new SolidColorBrush(Color.FromArgb(0x60, 255, 255, 255)); + copy.BorderThickness = glyph is null ? new Thickness(1) : new Thickness(0); + return copy; + } + + private void SetPinned(bool pinned, bool raise) + { + this.sticky = pinned; + this.pinFull.IsChecked = pinned; + this.pinCompact.IsChecked = pinned; + var opacity = pinned ? 1.0 : 0.75; + this.pinFull.Opacity = opacity; + this.pinCompact.Opacity = opacity; + if (raise) + { + this.PinChanged?.Invoke(pinned); + } + + this.Refresh(); + } + + private void Expand() + { + this.expandedOverride = true; + this.Refresh(); + } + + private void Collapse() + { + this.ResetUserSize(); + this.expandedOverride = false; + this.Refresh(); + } + + /// + /// The WPF overlay's Minimize: a panel opened from the compact microphone goes back to it, otherwise the window is minimized. A minimized + /// overlay comes back with the next dictation or notice. + /// + private void MinimizeClicked() + { + if (this.expandedOverride == true && this.style == OverlayStyle.CompactMicrophone && !this.sticky) + { + this.Collapse(); + return; + } + + this.WindowState = WindowState.Minimized; + } + + private void RestoreIfMinimized() + { + if (this.WindowState == WindowState.Minimized) + { + this.WindowState = WindowState.Normal; + } + } + + private void ResetUserSize() + { + this.userWidth = null; + this.userTextHeight = null; + } + + /// The resize grip drags in screen coordinates, so the overlay (which sizes itself to its content) never has to take focus or switch to a native resize. + private void OnGripPressed(object? sender, PointerPressedEventArgs e) + { + if (!e.GetCurrentPoint(this.grip).Properties.IsLeftButtonPressed) + { + return; + } + + e.Pointer.Capture(this.grip); + this.gripDragging = true; + this.dragging = true; // keeps the bottom-center anchor from repositioning the window while it grows + this.gripStart = this.ScreenPoint(e); + this.gripStartWidth = this.panel.Bounds.Width; + this.gripStartHeight = this.textView.Bounds.Height; + e.Handled = true; + } + + private void OnGripMoved(object? sender, PointerEventArgs e) + { + if (!this.gripDragging) + { + return; + } + + var now = this.ScreenPoint(e); + this.userWidth = Math.Max(MinFullWidth, this.gripStartWidth + (now.X - this.gripStart.X)); + this.userTextHeight = Math.Max(TextHeight, this.gripStartHeight + (now.Y - this.gripStart.Y)); + this.Refresh(); + e.Handled = true; + } + + private void OnGripReleased(object? sender, PointerReleasedEventArgs e) + { + e.Pointer.Capture(null); + this.EndGripDrag(); + e.Handled = true; + } + + private void EndGripDrag() + { + if (!this.gripDragging) + { + return; + } + + this.gripDragging = false; + this.dragging = false; + // The new size grew from the top-left corner; the spot it now occupies is the one to grow from next time (not saved: the size itself is not remembered, as in WPF). + this.anchor = this.CurrentAnchor(); + } + + private Point ScreenPoint(PointerEventArgs e) + { + var p = this.PointToScreen(e.GetPosition(this)); + var scale = this.DesktopScaling; + return new Point(p.X / scale, p.Y / scale); + } + + /// A click (not a drag) on the compact microphone opens the full panel, as in the WPF overlay. + private void OnPanelClicked() + { + if (this.EffectiveStyle == OverlayStyle.CompactMicrophone) + { + this.Expand(); + } + } + + /// Like the WPF overlay, closing a panel that was opened from the compact microphone goes back to the microphone; otherwise it hides until the next dictation. + private void CloseClicked() + { + if (this.expandedOverride == true && this.style == OverlayStyle.CompactMicrophone && !this.sticky) + { + this.Collapse(); + return; + } + this.dismissed = true; this.showingNotice = false; this.Refresh(); } - private void ShowText(string value) + private async Task CopyLastTranscriptAsync() { - this.text.Text = value; - Dispatcher.UIThread.Post(() => this.textView.ScrollToEnd(), DispatcherPriority.Background); + if (!OverlayRules.IsCopyable(this.lastTranscript) || this.Clipboard is not { } clipboard) + { + return; + } + + try + { + await clipboard.SetTextAsync(this.lastTranscript); + } + catch (Exception ex) when (ex is InvalidOperationException or System.IO.IOException) + { + // The clipboard can be held by another program for a moment; the user can click again. + } } + private OverlayPhase Phase => this.state switch + { + DictationState.Listening => OverlayPhase.Listening, + DictationState.Processing => OverlayPhase.Processing, + _ => this.wakeListening ? OverlayPhase.WakeListening : OverlayPhase.Ready + }; + private void Refresh() { - var active = this.state != DictationState.Idle; - this.dot.Fill = this.state switch + var phase = this.Phase; + var color = phase switch { - DictationState.Listening => Brushes.IndianRed, - DictationState.Processing => Brushes.Orange, - _ => Brushes.SeaGreen + OverlayPhase.Listening => Color.FromRgb(220, 53, 69), + OverlayPhase.Processing => Color.FromRgb(255, 165, 0), + OverlayPhase.WakeListening => Color.FromRgb(255, 214, 10), + _ => Color.FromRgb(32, 164, 112) }; - this.status.Text = this.state switch + this.dot.Fill = new SolidColorBrush(color); + this.mic.SetStateColor(color); + var effective = this.EffectiveStyle; + var full = effective == OverlayStyle.FullPanel; + var header = OverlayRules.Header(phase, this.backend); + this.status.Text = header; + // The compact microphone has room for one short word; the model family is in the tooltip. + this.compactStatus.Text = phase switch { - DictationState.Listening => "Listening", - DictationState.Processing => "Transcribing", + OverlayPhase.Listening => "Listening", + OverlayPhase.Processing => "Transcribing", + OverlayPhase.WakeListening => "Wake listening", _ => "Ready" }; + ToolTip.SetTip(this.panel, header); - var lingering = this.hideAtUtc != DateTime.MinValue && DateTime.UtcNow < this.hideAtUtc; - var full = this.style == OverlayStyle.FullPanel; - this.panel.Width = full ? FullWidth : CompactWidth; + var shown = this.showingNotice + ? this.notice + : this.transcript.Length > 0 ? this.transcript : OverlayRules.Placeholder(phase, this.backend); + this.text.Text = shown; + Dispatcher.UIThread.Post(() => this.textView.ScrollToEnd(), DispatcherPriority.Background); + var copyable = OverlayRules.IsCopyable(this.lastTranscript); + foreach (var copy in new[] { this.copyFull, this.copyCompact }) + { + copy.IsEnabled = copyable; + ToolTip.SetTip(copy, copyable ? "Copy the last transcript" : "Nothing to copy yet: wait for a transcript"); + } + + this.panel.Width = full ? this.userWidth ?? FullWidth : CompactWidth; + this.textView.Height = full ? this.userTextHeight ?? TextHeight : TextHeight; + this.grip.IsVisible = full; + this.panel.CornerRadius = new CornerRadius(full ? 12 : 30); + this.fullHeader.IsVisible = full; + this.compactHeader.IsVisible = !full; + this.stateRow.IsVisible = full; + this.visualizer.IsVisible = full; + this.footer.IsVisible = full; // Full mode keeps its text box while shown, empty or not, so the size never changes; compact mode shows text only for a notice. this.textView.IsVisible = full || this.showingNotice; - this.visualizer.SetRunning(this.state == DictationState.Listening); - var visible = !this.dismissed && (active || lingering || this.sticky); + + this.visualizer.SetRunning(full && phase == OverlayPhase.Listening); + this.mic.SetRunning(!full && phase == OverlayPhase.Listening); + + var lingering = this.hideAtUtc != DateTime.MinValue && DateTime.UtcNow < this.hideAtUtc; + var visible = OverlayRules.ShouldShow(this.dismissed, this.state != DictationState.Idle, lingering, this.sticky, this.style, this.hideCompactWhenIdle); if (visible && !this.IsVisible) { // Placed before it appears (measured first), so it never flashes in a corner. diff --git a/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs b/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs index 0db3a99..bf539a1 100644 --- a/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs +++ b/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs @@ -6,6 +6,8 @@ using Avalonia.Threading; using PrimeDictate.Core.Dictation; using PrimeDictate.Core.Providers; +using PrimeDictate.Core.Settings; +using PrimeDictate.Platforms.Audio; using PrimeDictate.Platforms.Dictation; using PrimeDictate.Platforms.Speech; using PrimeDictate.Platforms.Startup; @@ -18,6 +20,7 @@ public sealed class DictationSettingsWindow : Window { private readonly DictationHost host; private readonly IAudioSource? audio; + private readonly TranscriptionPrefsService prefs; private List shownModels = []; private readonly ComboBox modelBox = new() { MinWidth = 260 }; private readonly ComboBox micBox = new() { MinWidth = 260 }; @@ -25,57 +28,108 @@ public sealed class DictationSettingsWindow : Window private readonly NumericUpDown silence = new() { Minimum = 0, Maximum = 30, Increment = 1, FormatString = "0", Width = 120 }; private readonly CheckBox audioCues = Check("Play start and stop sounds"); private readonly LaunchAtLogin launch = new(); + private readonly IReadOnlyList whisperNetChoices = WhisperNetDeviceChoices.For(MachineSupport.Current); + private readonly ComboBox whisperNetDeviceBox = new() { MinWidth = 260 }; private readonly ComboBox deviceBox = new() { ItemsSource = new[] { "Auto (CUDA if ready, else CPU)", "CPU", "CUDA" }, MinWidth = 260 }; private readonly CheckBox launchAtLogin = Check("Start PrimeDictate when I sign in (tray only)"); + private readonly CheckBox launchEveryone = Check("Start it for everyone on this PC, not just me (asks for administrator permission)"); + private readonly CheckBox autoUpdate = Check("Check for updates automatically (at most once a day; installing always asks first)"); + private readonly ComboBox trayClickBox = new() { ItemsSource = TrayClickNames, MinWidth = 260 }; + private readonly ComboBox themeBox = new() { ItemsSource = new[] { "Follow the system", "Light", "Dark" }, MinWidth = 260 }; + private readonly NumericUpDown baseline = new() { Minimum = DictationStatsStore.MinBaselineWpm, Maximum = DictationStatsStore.MaxBaselineWpm, Increment = 5, FormatString = "0", Width = 120 }; + private readonly CheckBox hideOverlayIdle = Check("Hide the compact microphone when I am not dictating"); private readonly CheckBox sendEnter = Check("Coding mode: press Enter after typing"); private readonly CheckBox returnToStart = Check("If focus moved, return to the window I started in"); - private readonly CheckBox typeWithoutGuard = Check("Type even when the app cannot check which window is in front"); + private readonly CheckBox exclusiveMic = Check("Request exclusive microphone access while dictating (Windows; shared if the device refuses)"); + private readonly CheckBox typeWithoutGuard =Check("Type even when the app cannot check which window is in front"); private readonly CheckBox wakeEnabled = Check("Wake word: start dictation when I say the phrase (listens on the idle microphone, audio stays in memory)"); private readonly TextBox wakePhrase = new() { Width = 260 }; - private readonly CheckBox voiceCommands = Check("Voice commands while dictating"); - private readonly TextBox commitPhrase = new() { Width = 260 }; - private readonly TextBox stopPhrase = new() { Width = 260 }; - private readonly TextBox historyPhrase = new() { Width = 260 }; private readonly ComboBox overlayBox = new() { ItemsSource = new[] { "Compact microphone", "Full panel" } }; private readonly CheckBox sticky = Check("Keep the overlay on screen when not dictating"); private bool resetOverlayPosition; - private readonly TextBox replacements = new() { AcceptsReturn = true, MinHeight = 90, MaxHeight = 220, PlaceholderText = "spoken phrase => replacement (one per line)" }; - private readonly Dictionary hotkeys = []; + // Parity block (WPF features): shortcuts and phrases, Ollama, replacements grid, custom model path, inline errors. + private readonly CommandsPanel commands; + private readonly OllamaPanel ollama; + private readonly ReplacementsEditor replacements; + private readonly CustomModelPanel customModel; + private readonly TextBlock modelError = FormParts.ErrorLine(); + private readonly TextBlock wakeError = FormParts.ErrorLine(); + private readonly TextBlock shellError = FormParts.ErrorLine(); private readonly TextBlock status = new() { TextWrapping = Avalonia.Media.TextWrapping.Wrap, Opacity = 0.75, VerticalAlignment = VerticalAlignment.Center, Margin = new Thickness(0, 0, 12, 0) }; + private readonly ComboBox liveTextBox = new() + { + ItemsSource = new[] { "Off: record only, transcribe after Stop", "Fast draft while people talk, then final pass" }, + MinWidth = 260 + }; + private readonly ComboBox sourceBox = new() { ItemsSource = new[] { "Microphone", "System audio (speakers)", "Microphone + system audio" }, MinWidth = 260 }; + private readonly CheckBox speakerLabels = Check("Meetings: speaker labels after Stop (needs Nemotron and its diarizer)"); + private readonly CheckBox boostAudio = Check("Boost quiet audio for recognition (the saved recording is not changed)"); + private readonly StackPanel shellRows = new() { Spacing = 8 }; + private readonly List shellEditors = []; private DictationSettings working; private IReadOnlyList devices = []; + // The saved microphone when it is not in the list right now (unplugged), kept so Save does not turn it into "System default". + private string? missingMicId; + // Fields that are also changed outside this window (the overlay's pin, the stats window's typing speed): Save writes them + // only when the user changed them here, so it never reverts a change made while the window was open. + private bool loadedSticky; + private int loadedBaseline; + + private static readonly string[] TrayClickNames = ["Double-click the tray icon opens PrimeDictate", "Single click opens PrimeDictate", "Clicking the icon does nothing (use its menu)"]; + private static readonly TrayClickBehavior[] TrayClickValues = [TrayClickBehavior.DoubleClickOpensWorkspace, TrayClickBehavior.SingleClickOpensWorkspace, TrayClickBehavior.ClickDoesNothing]; - public DictationSettingsWindow(DictationHost host, IAudioSource? audio) + private sealed record ShellRow(Control Root, CheckBox On, TextBox Phrase, ComboBox After, TextBox Command); + + public DictationSettingsWindow(DictationHost host, IAudioSource? audio, TranscriptionPrefsService prefs) { this.host = host; this.audio = audio; + this.prefs = prefs; this.working = host.Settings; - this.Title = "PrimeDictate: dictation settings"; + this.commands = new CommandsPanel(host, this.working, message => this.status.Text = message); + this.ollama = new OllamaPanel(this.working); + this.replacements = new ReplacementsEditor(this.working.TranscriptReplacements); + this.customModel = new CustomModelPanel(host.ModelsFolder); + this.modelBox.SelectionChanged += (_, _) => this.ShowSelectedModelPath(); + this.Title = "PrimeDictate: Settings"; this.Width = 560; this.SizeToContent = SizeToContent.Height; this.WindowStartupLocation = WindowStartupLocation.CenterScreen; this.FitToScreen(this.Screens.Primary); var panel = new StackPanel { Margin = new Thickness(20, 20, 20, 10), Spacing = 10 }; + panel.Children.Add(Heading("Dictation")); panel.Children.Add(Row("Model", this.modelBox)); panel.Children.Add(new TextBlock { Text = "Download another model", FontWeight = Avalonia.Media.FontWeight.SemiBold }); - panel.Children.Add(new ModelDownloadPanel(host, this.RefreshModels)); + panel.Children.Add(new ModelDownloadPanel(host, id => this.RefreshModels(id))); + panel.Children.Add(this.customModel); + panel.Children.Add(this.modelError); panel.Children.Add(Row("Speech model device (applies after restart)", this.deviceBox)); panel.Children.Add(new TextBlock { Text = OnnxRuntimeDevice.Summary, Opacity = 0.7, TextWrapping = Avalonia.Media.TextWrapping.Wrap }); + panel.Children.Add(Row("Whisper.net (GGML) model device (applies after restart)", this.whisperNetDeviceBox)); + panel.Children.Add(new TextBlock { Text = AcceleratorSummary(), Opacity = 0.7, TextWrapping = Avalonia.Media.TextWrapping.Wrap }); panel.Children.Add(Row("Microphone", this.micBox)); panel.Children.Add(Row("Input gain", this.gain)); panel.Children.Add(Row("Auto-commit after silence (seconds, 0 = hotkey only)", this.silence)); - foreach (var (action, name) in new[] { (HotkeyAction.ToggleDictation, "Start / stop dictation"), (HotkeyAction.EmergencyStop, "Emergency stop (discard)"), (HotkeyAction.ShowHistory, "Show history") }) - { - panel.Children.Add(this.HotkeyRow(action, name)); - } + panel.Children.Add(Heading("Keyboard and voice commands")); + panel.Children.Add(this.commands); panel.Children.Add(this.audioCues); panel.Children.Add(this.launchAtLogin); + if (this.launch.SupportsAllUsers) + { + panel.Children.Add(this.launchEveryone); + } + + panel.Children.Add(this.autoUpdate); + panel.Children.Add(Row("Clicking the tray icon", this.trayClickBox)); + panel.Children.Add(Row("Color scheme", this.themeBox)); + panel.Children.Add(Row("Compare time saved against typing at (WPM, 20 to 120)", this.baseline)); panel.Children.Add(this.sendEnter); if (OperatingSystem.IsWindows()) { panel.Children.Add(this.returnToStart); + panel.Children.Add(this.exclusiveMic); } if (!host.FocusGuardAvailable) @@ -85,10 +139,7 @@ public DictationSettingsWindow(DictationHost host, IAudioSource? audio) panel.Children.Add(this.wakeEnabled); panel.Children.Add(Row("Wake phrase", this.wakePhrase)); - panel.Children.Add(this.voiceCommands); - panel.Children.Add(Row("Commit phrase (types what you said, then stops)", this.commitPhrase)); - panel.Children.Add(Row("Discard phrase (stops without typing)", this.stopPhrase)); - panel.Children.Add(Row("History phrase", this.historyPhrase)); + panel.Children.Add(this.wakeError); panel.Children.Add(Row("Overlay", this.overlayBox)); panel.Children.Add(new TextBlock { @@ -97,6 +148,7 @@ public DictationSettingsWindow(DictationHost host, IAudioSource? audio) Opacity = 0.7 }); panel.Children.Add(this.sticky); + panel.Children.Add(this.hideOverlayIdle); var resetOverlay = new Button { Content = "Move the overlay back to the bottom of the screen" }; resetOverlay.Click += (_, _) => { @@ -104,15 +156,59 @@ public DictationSettingsWindow(DictationHost host, IAudioSource? audio) this.status.Text = "The overlay goes back to the bottom of the screen when you save."; }; panel.Children.Add(resetOverlay); - panel.Children.Add(new TextBlock { Text = "Replacements" }); + panel.Children.Add(Heading("Replacements")); panel.Children.Add(this.replacements); + panel.Children.Add(Heading("Ollama post-processing")); + panel.Children.Add(this.ollama); + + panel.Children.Add(Heading("Transcription & meetings")); + panel.Children.Add(new TextBlock + { + Text = "These apply to the recording and transcription side of PrimeDictate (the main window). The same options are in its left column; a change in either place shows in both.", + TextWrapping = Avalonia.Media.TextWrapping.Wrap, + Opacity = 0.7 + }); + panel.Children.Add(Row("Recording source (what Record meeting captures)", this.sourceBox)); + panel.Children.Add(Row("Live text during a meeting", this.liveTextBox)); + panel.Children.Add(this.speakerLabels); + panel.Children.Add(this.boostAudio); + + panel.Children.Add(Heading("Commands that run programs (dictation only)")); + panel.Children.Add(new Border + { + Background = new Avalonia.Media.SolidColorBrush(Avalonia.Media.Color.FromArgb(40, 255, 160, 0)), + BorderBrush = new Avalonia.Media.SolidColorBrush(Avalonia.Media.Color.FromRgb(230, 140, 0)), + BorderThickness = new Thickness(1), + CornerRadius = new CornerRadius(4), + Padding = new Thickness(10), + Child = new TextBlock + { + Text = "Warning: when you say one of these phrases while dictating, PrimeDictate runs the command on this computer with your account's permissions. " + + "Only add commands you understand and trust. The command is exactly what you type here; nothing you say is added to it. " + + "They never run from transcription or meetings. New commands start Off. Commands imported from the older PrimeDictate app keep the On/Off setting they had there.", + TextWrapping = Avalonia.Media.TextWrapping.Wrap + } + }); + panel.Children.Add(this.shellRows); + var addShell = new Button { Content = "Add command", HorizontalAlignment = HorizontalAlignment.Left }; + addShell.Click += (_, _) => this.AddShellRow(new VoiceShellCommand { Enabled = false }); + panel.Children.Add(addShell); + panel.Children.Add(this.shellError); // Save and the status line stay in view under the form, which scrolls when it is taller than the screen. - var save = new Button { Content = "Save", VerticalAlignment = VerticalAlignment.Center }; + var save = new Button { Content = "Save", VerticalAlignment = VerticalAlignment.Center, IsDefault = true }; save.Click += (_, _) => this.Save(); + var cancel = new Button { Content = "Cancel", VerticalAlignment = VerticalAlignment.Center, Margin = new Thickness(0, 0, 8, 0) }; + cancel.Click += (_, _) => this.Close(); + var historyButton = new Button { Content = "History", VerticalAlignment = VerticalAlignment.Center, Margin = new Thickness(0, 0, 12, 0) }; + historyButton.Click += (_, _) => this.host.RequestHistory(); var footer = new DockPanel { Margin = new Thickness(20, 10, 20, 16) }; DockPanel.SetDock(save, Dock.Right); + DockPanel.SetDock(cancel, Dock.Right); + DockPanel.SetDock(historyButton, Dock.Left); footer.Children.Add(save); + footer.Children.Add(cancel); + footer.Children.Add(historyButton); footer.Children.Add(this.status); var root = new DockPanel(); DockPanel.SetDock(footer, Dock.Bottom); @@ -124,6 +220,107 @@ public DictationSettingsWindow(DictationHost host, IAudioSource? audio) this.FitToScreen(this.Screens.ScreenFromWindow(this)); await this.LoadAsync(); }; + // A change made in the main window's left column shows here while this window is open. + void OnPrefsChanged(object? origin) + { + if (!ReferenceEquals(origin, this)) + { + this.LoadTranscriptionFields(); + } + } + + prefs.Changed += OnPrefsChanged; + this.Closed += (_, _) => prefs.Changed -= OnPrefsChanged; + } + + private static Control Heading(string text) => new TextBlock + { + Text = text, + FontSize = 16, + FontWeight = Avalonia.Media.FontWeight.SemiBold, + Margin = new Thickness(0, 14, 0, 0) + }; + + private void LoadTranscriptionFields() + { + var p = this.prefs.Current; + this.sourceBox.SelectedIndex = p.LastSource switch { RecordingSources.System => 1, RecordingSources.Meeting => 2, _ => 0 }; + this.liveTextBox.SelectedIndex = p.LiveTextMode == LiveTextModes.Draft ? 1 : 0; + this.speakerLabels.IsChecked = p.SpeakerLabelsAfterStop; + this.boostAudio.IsChecked = p.BoostQuietAudio; + } + + private void AddShellRow(VoiceShellCommand command) + { + var on = new CheckBox { Content = "On", IsChecked = command.Enabled, VerticalAlignment = VerticalAlignment.Center }; + var phrase = new TextBox { Text = command.Phrase, PlaceholderText = "When I say", VerticalAlignment = VerticalAlignment.Center }; + var after = new ComboBox + { + ItemsSource = new[] { "Then stop", "Then keep typing" }, + SelectedIndex = command.CompletionBehavior == VoiceShellCommandCompletionBehavior.Continue ? 1 : 0, + VerticalAlignment = VerticalAlignment.Center + }; + ToolTip.SetTip(after, "Then stop: nothing else from that dictation is typed. Then keep typing: the words after the phrase are typed as usual."); + var cmd = new TextBox { Text = command.Command, PlaceholderText = "Command to run (as you would type it in a terminal)" }; + var remove = new Button { Content = "Remove", VerticalAlignment = VerticalAlignment.Center }; + Grid.SetColumn(phrase, 1); + Grid.SetColumn(after, 2); + Grid.SetColumn(remove, 3); + var top = new Grid { ColumnDefinitions = new ColumnDefinitions("Auto,*,Auto,Auto"), Children = { on, phrase, after, remove } }; + top.ColumnDefinitions[1].MinWidth = 120; + foreach (var child in top.Children.Skip(1)) + { + ((Control)child).Margin = new Thickness(8, 0, 0, 0); + } + + var root = new Border + { + BorderBrush = new Avalonia.Media.SolidColorBrush(Avalonia.Media.Color.FromArgb(70, 128, 128, 128)), + BorderThickness = new Thickness(1), + CornerRadius = new CornerRadius(4), + Padding = new Thickness(8), + Child = new StackPanel { Spacing = 6, Children = { top, cmd } } + }; + var row = new ShellRow(root, on, phrase, after, cmd); + remove.Click += (_, _) => + { + this.shellEditors.Remove(row); + this.shellRows.Children.Remove(root); + }; + this.shellEditors.Add(row); + this.shellRows.Children.Add(root); + } + + /// Rows that are fully empty are dropped; a row with only a phrase or only a command is an error the user must fix. + private bool TryBuildShellCommands(out List commands) + { + commands = []; + FormParts.Show(this.shellError, null); + foreach (var row in this.shellEditors) + { + var phrase = row.Phrase.Text?.Trim() ?? string.Empty; + var command = row.Command.Text?.Trim() ?? string.Empty; + if (phrase.Length == 0 && command.Length == 0) + { + continue; + } + + if (phrase.Length == 0 || command.Length == 0) + { + FormParts.Show(this.shellError, "Each voice command needs both a phrase to say and a command to run."); + return false; + } + + commands.Add(new VoiceShellCommand + { + Enabled = row.On.IsChecked == true, + Phrase = phrase, + Command = command, + CompletionBehavior = row.After.SelectedIndex == 1 ? VoiceShellCommandCompletionBehavior.Continue : VoiceShellCommandCompletionBehavior.Stop + }); + } + + return true; } private static CheckBox Check(string text) => new() { Content = new TextBlock { Text = text, TextWrapping = Avalonia.Media.TextWrapping.Wrap } }; @@ -174,47 +371,32 @@ private void FitToScreen(Screen? screen) DispatcherPriority.Background); } - private Control HotkeyRow(HotkeyAction action, string name) + /// What GPU and NPU runtimes this PC has (the WPF app's runtime summaries). The Qualcomm sentence is shown on Windows ARM64 only, where those models are offered. + private static string AcceleratorSummary() { - var label = new TextBlock { VerticalAlignment = VerticalAlignment.Center, MinWidth = 140 }; - this.hotkeys[action] = (label, this.working.ToBindings()[action]); - label.Text = this.hotkeys[action].Gesture.ToString(); - var change = new Button { Content = "Change" }; - change.Click += async (_, _) => - { - if (this.host.HotkeyUnavailableReason is { } reason) - { - this.status.Text = reason; - return; - } - - label.Text = "Press the new shortcut (Esc cancels)..."; - var gesture = await HotkeyCapture.CaptureAsync(TimeSpan.FromSeconds(10), CancellationToken.None); - if (gesture is not null) - { - this.hotkeys[action] = (label, gesture); - } - - label.Text = this.hotkeys[action].Gesture.ToString(); - }; - // Columns, not a horizontal stack, so "Press the new shortcut..." wraps inside the window instead of running past it. - var title = new TextBlock { Text = name, TextWrapping = Avalonia.Media.TextWrapping.Wrap, VerticalAlignment = VerticalAlignment.Center }; - label.TextWrapping = Avalonia.Media.TextWrapping.Wrap; - label.MinWidth = 0; - label.Margin = new Thickness(10, 0); - Grid.SetColumn(label, 1); - Grid.SetColumn(change, 2); - return new Grid { ColumnDefinitions = new ColumnDefinitions("190,*,Auto"), Children = { title, label, change } }; + var support = MachineSupport.Current; + var summary = support.WhisperNetRuntimeSummary(System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture, OperatingSystem.IsWindows()); + return OperatingSystem.IsWindows() && System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture == System.Runtime.InteropServices.Architecture.Arm64 + ? $"{summary} {PrimeDictate.Platforms.Speech.Qualcomm.QnnRuntimeSupport.RuntimeSummary(support.Qnn)}" + : summary; } - private void RefreshModels() + /// Lists the installed models. A model that was just downloaded () becomes the selected one, as in the WPF Settings window. + private void RefreshModels(string? select = null) { var models = this.host.InstalledModels(); - var current = this.modelBox.SelectedIndex is >= 0 and var ci && ci < this.shownModels.Count ? this.shownModels[ci] : this.working.ResolveModelId(); + var current = select ?? (this.modelBox.SelectedIndex is >= 0 and var ci && ci < this.shownModels.Count ? this.shownModels[ci] : this.host.WantedModelId()); this.shownModels = models.Select(m => m.ModelId).ToList(); this.modelBox.ItemsSource = models.Select(m => m.DisplayName).ToList(); this.modelBox.SelectedIndex = Math.Max(0, this.shownModels.IndexOf(current ?? string.Empty)); this.status.Text = models.Count == 0 ? "No speech model is installed. Download one below." : string.Empty; + this.ShowSelectedModelPath(); + } + + private void ShowSelectedModelPath() + { + var models = this.host.InstalledModels(); + this.customModel.ShowSelected(this.modelBox.SelectedIndex is >= 0 and var i && i < models.Count ? models[i] : null); } private async Task LoadAsync() @@ -226,7 +408,21 @@ private async Task LoadAsync() var names = new List { "System default" }; names.AddRange(this.devices.Select(d => d.Name)); this.micBox.ItemsSource = names; - var index = this.devices.ToList().FindIndex(d => d.Id == this.working.SelectedInputDeviceId); + // The WPF app saved a Windows endpoint id; the list is by device name, so look the id up first. + var saved = this.working.SelectedInputDeviceId; + var wanted = saved is not null && OperatingSystem.IsWindows() && WindowsAudioEndpoints.IsEndpointId(saved) + ? WindowsAudioEndpoints.FriendlyName(saved) ?? saved + : saved; + var index = this.devices.ToList().FindIndex(d => d.Id == wanted); + this.missingMicId = null; + if (index < 0 && saved is not null) + { + this.missingMicId = saved; + names.Add($"{wanted} (not connected)"); + this.micBox.ItemsSource = names; + index = this.devices.Count; + } + this.micBox.SelectedIndex = index < 0 ? 0 : index + 1; } @@ -234,19 +430,35 @@ private async Task LoadAsync() this.silence.Value = this.working.AutoCommitSilenceSeconds; this.audioCues.IsChecked = this.working.PlayAudioCues; this.deviceBox.SelectedIndex = (int)OnnxRuntimeDevice.ParsePreference(this.working.OnnxDevice); - this.launchAtLogin.IsChecked = this.launch.IsEnabled; + // Only the devices this PC can run are listed; a saved choice it cannot run (copied from another PC) shows as Auto. + this.whisperNetDeviceBox.ItemsSource = this.whisperNetChoices.Select(c => c.Label).ToList(); + this.whisperNetDeviceBox.SelectedIndex = WhisperNetDeviceChoices.IndexOf(this.whisperNetChoices, this.working.ResolveWhisperNetDevice()); + var scope = this.launch.Scope; + this.launchAtLogin.IsChecked = scope != LoginScope.Off; + this.launchEveryone.IsChecked = scope == LoginScope.AllUsers; + this.autoUpdate.IsChecked = this.working.CheckForUpdatesAutomatically; + this.trayClickBox.SelectedIndex = Math.Max(0, Array.IndexOf(TrayClickValues, this.working.TrayClickBehavior)); + this.themeBox.SelectedIndex = (int)this.working.Theme; + this.loadedBaseline = DictationStatsStore.NormalizeBaselineWpm(this.working.BaselineTypingSpeedWpm); + this.baseline.Value = this.loadedBaseline; + this.hideOverlayIdle.IsChecked = this.working.HideOverlayWhenIdle; this.sendEnter.IsChecked = this.working.SendEnterAfterCommit; this.returnToStart.IsChecked = this.working.ReturnToStartTargetOnCommit; + this.exclusiveMic.IsChecked = this.working.ExclusiveMicAccessWhileDictating; this.typeWithoutGuard.IsChecked = this.working.TypeWithoutFocusGuard; this.wakeEnabled.IsChecked = this.working.EnableWakeWord; this.wakePhrase.Text = this.working.WakeWordPhrase; - this.voiceCommands.IsChecked = this.working.EnableVoiceCommands; - this.commitPhrase.Text = this.working.VoiceDictationPhrase; - this.stopPhrase.Text = this.working.VoiceStopPhrase; - this.historyPhrase.Text = this.working.VoiceHistoryPhrase; this.overlayBox.SelectedIndex = (int)this.working.OverlayMode; - this.sticky.IsChecked = this.working.IsOverlaySticky; - this.replacements.Text = string.Join('\n', this.working.TranscriptReplacements.Select(r => $"{r.Find} => {r.Replace}")); + this.loadedSticky = this.working.IsOverlaySticky; + this.sticky.IsChecked = this.loadedSticky; + this.LoadTranscriptionFields(); + this.shellRows.Children.Clear(); + this.shellEditors.Clear(); + foreach (var command in this.working.VoiceShellCommands ?? []) + { + this.AddShellRow(command); + } + var notes = new List(); if (this.host.FocusGuardNotice is { } guardNotice) { @@ -264,58 +476,130 @@ private async Task LoadAsync() } } - private void ApplyLaunchAtLogin() + /// Returns false (and shows the problem in the window) when the change failed; the caller then keeps the window open and does not save, as the WPF app did. + private bool ApplyLaunchAtLogin() { - var wanted = this.launchAtLogin.IsChecked == true; - if (wanted != this.launch.IsEnabled && this.launch.Apply(wanted) is { } problem) + var wanted = this.launchAtLogin.IsChecked != true ? LoginScope.Off + : this.launch.SupportsAllUsers && this.launchEveryone.IsChecked == true ? LoginScope.AllUsers + : LoginScope.CurrentUser; + if (wanted != this.launch.Scope && this.launch.Apply(wanted) is { } problem) { this.status.Text = problem; - this.launchAtLogin.IsChecked = this.launch.IsEnabled; + var actual = this.launch.Scope; + this.launchAtLogin.IsChecked = actual != LoginScope.Off; + this.launchEveryone.IsChecked = actual == LoginScope.AllUsers; + this.status.Text = $"Not saved: {problem}"; + return false; } + + return true; } private void Save() { + // Validate everything first and show each problem next to its field; nothing is applied until all of it is valid. + FormParts.Show(this.wakeError, null); + var ok = this.TryBuildShellCommands(out var shellCommands); + ok &= this.commands.TryValidate(out var phrases); + var wakeProblem = DictationSettingsValidator.ValidateWakePhrase(this.wakeEnabled.IsChecked == true, this.wakePhrase.Text, phrases); + FormParts.Show(this.wakeError, wakeProblem); + ok &= wakeProblem is null; + if (ok) + { + var shellProblem = DictationSettingsValidator.ValidateShellCommands(phrases, shellCommands, this.wakeEnabled.IsChecked == true, this.wakePhrase.Text); + FormParts.Show(this.shellError, shellProblem); + ok &= shellProblem is null; + } + + ok &= this.ollama.TryValidate(); var models = this.host.InstalledModels(); + var selected = this.modelBox.SelectedIndex is >= 0 and var mi && mi < models.Count ? models[mi] : null; + var modelOk = this.customModel.TryResolve(selected, selected?.Backend ?? this.working.TranscriptionBackend, out var modelPath, out var customModel); + ok &= modelOk; + if (!ok) + { + this.status.Text = "Not saved: fix the red messages above."; + return; + } + + if (!this.ApplyLaunchAtLogin()) + { + return; + } + var s = this.working; - if (this.modelBox.SelectedIndex is >= 0 and var mi && mi < models.Count) + if (customModel is not null) + { + s.ModelPath = modelPath; + s.SelectedModelId = customModel.Id; + s.TranscriptionBackend = customModel.Backend; + } + else { - s.SelectedModelId = models[mi].Id; - s.TranscriptionBackend = models[mi].Backend; + s.ModelPath = null; + if (selected is { IsCustom: false }) + { + s.SelectedModelId = selected.Id; + s.TranscriptionBackend = selected.Backend; + } } - s.SelectedInputDeviceId = this.micBox.SelectedIndex is > 0 and var di && di - 1 < this.devices.Count ? this.devices[di - 1].Id : null; + s.FirstRunCompleted = true; + this.commands.Apply(s, phrases); + this.ollama.Apply(s); + s.TranscriptReplacements = this.replacements.Build(); + // Null (the Windows default) only when the user picked "System default"; an unplugged saved microphone stays selected. + s.SelectedInputDeviceId = this.micBox.SelectedIndex is > 0 and var di + ? (di - 1 < this.devices.Count ? this.devices[di - 1].Id : this.missingMicId) + : (this.audio is null ? s.SelectedInputDeviceId : null); s.InputGainMultiplier = this.gain.Value; s.AutoCommitSilenceSeconds = (int)(this.silence.Value ?? 3); s.PlayAudioCues = this.audioCues.IsChecked == true; s.OnnxDevice = ((OnnxDevicePreference)Math.Max(0, this.deviceBox.SelectedIndex)).ToString().ToLowerInvariant(); - this.ApplyLaunchAtLogin(); + // Only write a choice when the user made one, so an untouched WPF value this PC does not offer is not silently turned into Auto. + if (WhisperNetDeviceChoices.ValueToSave(this.whisperNetChoices, this.whisperNetDeviceBox.SelectedIndex, s) is { } device) + { + s.WhisperNetDevice = device; + } + s.SendEnterAfterCommit = this.sendEnter.IsChecked == true; s.ReturnToStartTargetOnCommit = this.returnToStart.IsChecked == true; + s.ExclusiveMicAccessWhileDictating = this.exclusiveMic.IsChecked == true; s.TypeWithoutFocusGuard = this.typeWithoutGuard.IsChecked == true; s.EnableWakeWord = this.wakeEnabled.IsChecked == true; s.WakeWordPhrase = WakePhrase.Normalize(this.wakePhrase.Text); - s.EnableVoiceCommands = this.voiceCommands.IsChecked == true; - s.VoiceDictationPhrase = this.commitPhrase.Text?.Trim() ?? string.Empty; - s.VoiceStopPhrase = this.stopPhrase.Text?.Trim() ?? string.Empty; - s.VoiceHistoryPhrase = this.historyPhrase.Text?.Trim() ?? string.Empty; s.OverlayMode = (OverlayStyle)Math.Max(0, this.overlayBox.SelectedIndex); - s.IsOverlaySticky = this.sticky.IsChecked == true; + if ((this.sticky.IsChecked == true) != this.loadedSticky) + { + s.IsOverlaySticky = this.sticky.IsChecked == true; + } + + s.HideOverlayWhenIdle = this.hideOverlayIdle.IsChecked == true; + s.CheckForUpdatesAutomatically = this.autoUpdate.IsChecked == true; + s.TrayClickBehavior = TrayClickValues[Math.Clamp(this.trayClickBox.SelectedIndex, 0, TrayClickValues.Length - 1)]; + s.Theme = (AppTheme)Math.Clamp(this.themeBox.SelectedIndex, 0, 2); + var baselineShown = DictationStatsStore.NormalizeBaselineWpm((int)Math.Round(this.baseline.Value ?? DictationStatsStore.DefaultBaselineWpm)); + if (baselineShown != this.loadedBaseline) + { + s.BaselineTypingSpeedWpm = baselineShown; + } if (this.resetOverlayPosition) { s.OverlayAnchorX = null; s.OverlayAnchorY = null; } - s.DictationHotkey = HotkeyDto.From(this.hotkeys[HotkeyAction.ToggleDictation].Gesture); - s.StopHotkey = HotkeyDto.From(this.hotkeys[HotkeyAction.EmergencyStop].Gesture); - s.HistoryHotkey = HotkeyDto.From(this.hotkeys[HotkeyAction.ShowHistory].Gesture); - s.TranscriptReplacements = (this.replacements.Text ?? string.Empty) - .Split('\n', StringSplitOptions.RemoveEmptyEntries) - .Select(line => line.Split("=>", 2)) - .Where(parts => parts.Length == 2 && !string.IsNullOrWhiteSpace(parts[0])) - .Select(parts => new ReplacementDto { Find = parts[0].Trim(), Replace = parts[1].Trim() }) - .ToList(); + s.VoiceShellCommands = shellCommands; this.host.ApplySettings(s); + var source = this.sourceBox.SelectedIndex switch { 1 => RecordingSources.System, 2 => RecordingSources.Meeting, _ => RecordingSources.Microphone }; + this.prefs.Update( + p => p with + { + LastSource = source, + LiveTextMode = this.liveTextBox.SelectedIndex == 1 ? LiveTextModes.Draft : LiveTextModes.Off, + SpeakerLabelsAfterStop = this.speakerLabels.IsChecked == true, + BoostQuietAudio = this.boostAudio.IsChecked == true + }, + this); this.Close(); } } diff --git a/src/PrimeDictate.Desktop/Dictation/DictationShell.cs b/src/PrimeDictate.Desktop/Dictation/DictationShell.cs index 36aff18..193f3a7 100644 --- a/src/PrimeDictate.Desktop/Dictation/DictationShell.cs +++ b/src/PrimeDictate.Desktop/Dictation/DictationShell.cs @@ -4,8 +4,11 @@ using Avalonia.Media; using Avalonia.Media.Imaging; using Avalonia.Platform; +using Avalonia.Styling; using Avalonia.Threading; +using PrimeDictate.Core.Diagnostics; using PrimeDictate.Core.Dictation; +using PrimeDictate.Core.Providers; using PrimeDictate.Platforms; using PrimeDictate.Platforms.Dictation; using PrimeDictate.Platforms.Speech; @@ -20,25 +23,31 @@ namespace PrimeDictate.Desktop.Dictation; public sealed class DictationShell : IAsyncDisposable { private readonly DictationHost host; + private readonly MainWindow window; private readonly TranscriptionWorkspaceService workspace; private readonly IClassicDesktopStyleApplicationLifetime lifetime; private readonly Action showWorkspace; private readonly DictationOverlayWindow overlay = new(); private readonly TrayIcon tray = new(); private readonly NativeMenuItem toggleItem = new("Start dictation"); + private readonly NativeMenuItem recordItem = new("Record meeting"); + private string modelName = string.Empty; private readonly IAudioCuePlayer cues = new ProcessAudioCuePlayer(); private DictationState lastState = DictationState.Idle; private DateTime errorUntilUtc; + private readonly TrayClickDecider clickDecider; private DictationSettingsWindow? settingsWindow; private DictationHistoryWindow? historyWindow; private DictationStatsWindow? statsWindow; - public DictationShell(IClassicDesktopStyleApplicationLifetime lifetime, TranscriptionWorkspaceService workspace, Action showWorkspace, DictationHost? host = null) + public DictationShell(IClassicDesktopStyleApplicationLifetime lifetime, MainWindow window, Action showWorkspace, DictationHost? host = null) { this.lifetime = lifetime; - this.workspace = workspace; + this.window = window; + this.workspace = window.Workspace; this.showWorkspace = showWorkspace; this.host = host ?? new DictationHost(workspace.Paths, workspace.Microphone, workspace.MicrophoneSource, new SharpHookHotkeySource()); + this.clickDecider = new TrayClickDecider(() => this.host.Settings.TrayClickBehavior); } public DictationHost Host => this.host; @@ -47,7 +56,24 @@ public void Start(Application app) { this.ConfigureOverlay(); this.overlay.AnchorMoved += (x, y) => this.host.RememberOverlayAnchor(x, y); + this.overlay.PinChanged += pinned => this.host.RememberOverlayPinned(pinned); + this.overlay.SettingsRequested += this.ShowSettings; + ApplyTheme(this.host.Settings.Theme); + // Any error-level log line turns the tray icon to "needs attention" for a while, as in the WPF app. + AppLog.ErrorLogged += this.OnErrorLogged; + if (this.host.Wake is { } wake) + { + // The failed flag is read live from the listener; this only refreshes the tray and overlay when it flips either way. + wake.FailedChanged += () => Dispatcher.UIThread.Post(this.RefreshStatusSurfaces); + } + this.BuildTray(app); + this.window.DictationHistoryRequested += this.ShowHistory; + this.window.DictationStatsRequested += this.ShowStats; + this.window.SettingsRequested += this.ShowSettings; + this.window.DictationActivityRequested += this.ShowActivity; + this.window.RecordingChanged += this.OnMeetingRecordingChanged; + this.RefreshHeader(refreshModel: true); this.host.Notice += message => Dispatcher.UIThread.Post(() => this.OnNotice(message)); this.host.HistoryRequested += () => Dispatcher.UIThread.Post(this.ShowHistory); if (this.host.Controller is { } controller) @@ -83,6 +109,9 @@ public void Start(Application app) this.overlay.SetState(DictationState.Idle); } + /// The update command for the tray's "Check for updates..." item. Set before Start; null leaves the item out. + public PrimeDictate.Desktop.Updates.IUpdateMenu? UpdateMenu { get; set; } + /// Raised when the user picks Exit in the tray menu. The app decides how to leave (see App.ExitAsync). public event Action? ExitRequested; @@ -98,7 +127,10 @@ public async Task PrepareExitAsync() public ValueTask DisposeAsync() { + AppLog.ErrorLogged -= this.OnErrorLogged; this.tray.IsVisible = false; + this.tray.Icon = null; + TrayIconRenderer.DisposeCached(); this.overlay.Close(); return this.host.DisposeAsync(); } @@ -112,30 +144,108 @@ private void BuildTray(Application app) await Task.Run(controller.ToggleAsync); } }; - var workspaceItem = new NativeMenuItem("Open workspace"); + // Start or stop a live recording in the workspace through the same code the Record and Stop buttons use, so either + // one can end what the other started. + this.recordItem.Click += async (_, _) => + { + this.showWorkspace(); + if (this.window.IsRecording) + { + await this.window.StopMeetingRecordingAsync(); + } + else + { + await this.window.StartMeetingRecordingAsync(); + } + }; + var workspaceItem = new NativeMenuItem("Open PrimeDictate"); workspaceItem.Click += (_, _) => this.showWorkspace(); - var historyItem = new NativeMenuItem("Dictation history..."); + var historyItem = new NativeMenuItem("Dictation history…"); historyItem.Click += (_, _) => this.ShowHistory(); - var statsItem = new NativeMenuItem("Dictation stats..."); + var statsItem = new NativeMenuItem("Stats…"); statsItem.Click += (_, _) => this.ShowStats(); - var settingsItem = new NativeMenuItem("Dictation settings..."); + var activityItem = new NativeMenuItem("Dictation activity…"); + activityItem.Click += (_, _) => this.ShowActivity(); + var settingsItem = new NativeMenuItem("Settings…"); settingsItem.Click += (_, _) => this.ShowSettings(); var quitItem = new NativeMenuItem("Exit PrimeDictate"); quitItem.Click += (_, _) => this.ExitRequested?.Invoke(); - this.tray.Menu = [this.toggleItem, workspaceItem, historyItem, statsItem, settingsItem, new NativeMenuItemSeparator(), quitItem]; - this.tray.Clicked += (_, _) => this.showWorkspace(); + this.tray.Menu = [this.toggleItem, this.recordItem, workspaceItem, historyItem, statsItem, activityItem, settingsItem]; + if (this.UpdateMenu is { } updates) + { + var updateItem = new NativeMenuItem(updates.MenuText); + updateItem.Click += async (_, _) => await updates.CheckNowAsync(); + updates.MenuChanged += () => + { + updateItem.Header = updates.MenuText; + updateItem.IsEnabled = updates.MenuEnabled; + }; + this.tray.Menu.Items.Add(updateItem); + } + + this.tray.Menu.Items.Add(new NativeMenuItemSeparator()); + this.tray.Menu.Items.Add(quitItem); + // The tray reports single clicks only; a double click is two close together (see TrayClickDecider). "Open PrimeDictate" in the menu always works. + this.tray.Clicked += (_, _) => + { + if (this.clickDecider.OnClick(DateTime.UtcNow)) + { + this.showWorkspace(); + } + }; this.tray.Icon = TrayIconRenderer.Create(this.CurrentTrayState()); - this.tray.ToolTipText = "PrimeDictate: ready"; + this.tray.ToolTipText = "PrimeDictate - Ready"; TrayIcon.SetIcons(app, [this.tray]); } + /// The WPF order: recording, then processing, then "needs attention", then the wake word, then ready. private TrayVisualState CurrentTrayState() => - this.errorUntilUtc > DateTime.UtcNow ? TrayVisualState.Error - : this.lastState == DictationState.Listening ? TrayVisualState.Recording + this.lastState == DictationState.Listening || this.window.IsRecording ? TrayVisualState.Recording : this.lastState == DictationState.Processing ? TrayVisualState.Processing - : this.host.Wake?.IsRunning == true ? TrayVisualState.AlwaysListening + : TrayAttention.IsActive(this.errorUntilUtc, DateTime.UtcNow, this.WakeFailed) ? TrayVisualState.Error + : this.WakeListening ? TrayVisualState.AlwaysListening : TrayVisualState.Ready; + /// Wake word on and nothing else going on, as in the WPF app (it shows even in the gap while the listener restarts). + private bool WakeListening => this.host.Settings.EnableWakeWord && this.lastState == DictationState.Idle && !this.WakeFailed; + + /// Wake word on and the listener gave up. Derived live, so it clears when the listener recovers or the wake word is turned off. + private bool WakeFailed => this.host.Settings.EnableWakeWord && this.host.Wake?.HasFailed == true; + + private void OnErrorLogged() => Dispatcher.UIThread.Post(this.HoldAttention); + + private void HoldAttention() + { + this.errorUntilUtc = DateTime.UtcNow + TrayAttention.Hold; + this.RefreshTrayIcon(); + this.RefreshTooltip(); + DispatcherTimer.RunOnce(() => + { + this.RefreshTrayIcon(); + this.RefreshTooltip(); + }, TrayAttention.Hold + TimeSpan.FromSeconds(0.5)); + } + + private void RefreshStatusSurfaces() + { + this.overlay.SetWakeListening(this.WakeListening); + this.RefreshTrayIcon(); + this.RefreshTooltip(); + } + + private static void ApplyTheme(AppTheme theme) + { + if (Application.Current is { } app) + { + app.RequestedThemeVariant = theme switch + { + AppTheme.Light => ThemeVariant.Light, + AppTheme.Dark => ThemeVariant.Dark, + _ => ThemeVariant.Default + }; + } + } + private void RefreshTrayIcon() => this.tray.Icon = TrayIconRenderer.Create(this.CurrentTrayState()); private void OnState(DictationState state) @@ -146,35 +256,97 @@ private void OnState(DictationState state) { this.cues.Play(DictationAudioCue.Start); } - else if (this.lastState == DictationState.Listening) + else if (state == DictationState.Processing && this.lastState == DictationState.Listening) { + // As in the WPF app the stop cue marks the start of processing; a discard, emergency stop or device error plays none. this.cues.Play(DictationAudioCue.Stop); } } + // As in WPF: Windows Mouse Sonar pulse when recording starts and when processing starts. + if (state != this.lastState && state is DictationState.Listening or DictationState.Processing) + { + WindowsMousePointerIndicator.PulseSoon(message => AppLog.Event("dictation", message)); + } + this.lastState = state; this.overlay.SetState(state); - this.RefreshTrayIcon(); - this.tray.ToolTipText = state switch + this.RefreshStatusSurfaces(); + this.toggleItem.Header = state == DictationState.Listening ? "Stop and type" : "Start dictation"; + this.RefreshHeader(refreshModel: false); + } + + private void OnMeetingRecordingChanged() + { + this.recordItem.Header = this.window.IsRecording ? "Stop meeting recording" : "Record meeting"; + this.RefreshStatusSurfaces(); + } + + /// Dictation states win over a meeting recording in the tooltip, since dictation is the short, active thing (). + private void RefreshTooltip() => + this.tray.ToolTipText = OverlayRules.TrayTooltip( + this.lastState switch + { + DictationState.Listening => OverlayPhase.Listening, + DictationState.Processing => OverlayPhase.Processing, + _ => this.WakeListening ? OverlayPhase.WakeListening : OverlayPhase.Ready + }, + this.BackendLabel, + TrayAttention.IsActive(this.errorUntilUtc, DateTime.UtcNow, this.WakeFailed), + this.WakeFailed, + this.window.IsRecording ? this.window.RecordingSourceLabel : null, + this.host.Settings.WakeWordPhrase, + this.MicAccessLabel); + + /// While dictating on Windows, which microphone access was granted (as the WPF tooltip showed). + private string? MicAccessLabel => this.lastState != DictationState.Listening ? null + : this.host.Controller?.ActiveMicAccess switch { - DictationState.Listening => "PrimeDictate: listening", - DictationState.Processing => "PrimeDictate: transcribing", - _ => "PrimeDictate: ready" + MicAccessMode.Exclusive => "Exclusive", + MicAccessMode.Shared when OperatingSystem.IsWindows() => "Shared", + _ => null }; - this.toggleItem.Header = state == DictationState.Listening ? "Stop and type" : "Start dictation"; + + private string BackendLabel => OverlayRules.BackendLabel(this.host.Settings.TranscriptionBackend); + + /// The main window's header line: dictation state, the model dictation uses and the hotkey. + private void RefreshHeader(bool refreshModel) + { + if (this.host.Controller is null) + { + this.window.SetDictationStatus("Dictation unavailable: " + (this.host.UnavailableReason ?? "no microphone")); + return; + } + + if (refreshModel) + { + var wanted = this.host.WantedModelId(); + var installed = this.host.InstalledModels(); + this.modelName = SpeechModelLocator.Resolve(installed, this.host.Settings)?.DisplayName + ?? (installed.Count > 0 ? installed[0].DisplayName : "no model installed"); + } + + var state = this.lastState switch + { + DictationState.Listening => "Listening", + DictationState.Processing => "Transcribing", + _ => "Ready" + }; + var hotkey = this.host.Settings.ToBindings()[HotkeyAction.ToggleDictation]; + this.window.SetDictationStatus($"Dictation: {state} · {this.modelName} · {hotkey}"); } private void OnCommitted(DictationCommit commit) { + // So the overlay's Copy button can still copy the last transcript once dictation is over (kept in memory, never logged). + this.overlay.SetCommitted(commit.Transcript); switch (commit.Status) { case DictationDeliveryStatus.SkippedFocusChanged: case DictationDeliveryStatus.SkippedNoFocusGuard: case DictationDeliveryStatus.FailedToInject: this.overlay.SetNotice($"Not typed: {commit.Error}\n\"{commit.Transcript}\""); - this.errorUntilUtc = DateTime.UtcNow + TimeSpan.FromSeconds(8); - this.RefreshTrayIcon(); - DispatcherTimer.RunOnce(this.RefreshTrayIcon, TimeSpan.FromSeconds(8.5)); + this.HoldAttention(); break; } } @@ -193,6 +365,20 @@ private void ShowHistory() this.historyWindow.Show(); } + private DictationActivityWindow? activityWindow; + + private void ShowActivity() + { + if (this.activityWindow is { IsVisible: true }) + { + this.activityWindow.Activate(); + return; + } + + this.activityWindow = new DictationActivityWindow(); + this.activityWindow.Show(); + } + private void ShowStats() { if (this.statsWindow is { IsVisible: true }) @@ -213,14 +399,22 @@ private void ShowSettings() return; } - this.settingsWindow = new DictationSettingsWindow(this.host, this.workspace.MicrophoneSource); - this.settingsWindow.Closed += (_, _) => this.ConfigureOverlay(); + this.settingsWindow = new DictationSettingsWindow(this.host, this.workspace.MicrophoneSource, this.window.Prefs); + this.settingsWindow.Closed += (_, _) => + { + this.ConfigureOverlay(); + ApplyTheme(this.host.Settings.Theme); + this.RefreshHeader(refreshModel: true); + this.RefreshStatusSurfaces(); + }; this.settingsWindow.Show(); } private void ConfigureOverlay() { var settings = this.host.Settings; - this.overlay.Configure(settings.OverlayMode, settings.IsOverlaySticky, settings.OverlayAnchorX, settings.OverlayAnchorY); + this.overlay.SetBackendLabel(this.BackendLabel); + this.overlay.Configure(settings.OverlayMode, settings.IsOverlaySticky, settings.HideOverlayWhenIdle, settings.OverlayAnchorX, settings.OverlayAnchorY); + this.overlay.SetWakeListening(this.WakeListening); } } diff --git a/src/PrimeDictate.Desktop/Dictation/DictationStatsWindow.cs b/src/PrimeDictate.Desktop/Dictation/DictationStatsWindow.cs index b353348..54297c1 100644 --- a/src/PrimeDictate.Desktop/Dictation/DictationStatsWindow.cs +++ b/src/PrimeDictate.Desktop/Dictation/DictationStatsWindow.cs @@ -13,16 +13,66 @@ public sealed class DictationStatsWindow : Window { private static readonly Color[] Palette = [Color.Parse("#38BDF8"), Color.Parse("#A78BFA"), Color.Parse("#34D399"), Color.Parse("#F59E0B")]; + private readonly DictationHost host; + private readonly StackPanel content = new() { Spacing = 16 }; + private readonly NumericUpDown baselineBox = new() + { + Minimum = DictationStatsStore.MinBaselineWpm, + Maximum = DictationStatsStore.MaxBaselineWpm, + Increment = 5, + FormatString = "0", + Width = 120 + }; + + private bool loading; + public DictationStatsWindow(DictationHost host) { + this.host = host; this.Title = "PrimeDictate: dictation stats"; this.Width = 620; this.SizeToContent = SizeToContent.Height; this.WindowStartupLocation = WindowStartupLocation.CenterScreen; + this.content.Margin = new Thickness(24); + this.Content = new ScrollViewer { Content = this.content, HorizontalScrollBarVisibility = Avalonia.Controls.Primitives.ScrollBarVisibility.Disabled }; + this.MaxHeight = 760; + this.baselineBox.ValueChanged += (_, _) => this.OnBaselineChanged(); + this.Render(); + } + + /// + /// The "compare against typing at N WPM" setting, as on the WPF Impact tab: whole numbers from 20 to 120, saved at once, and the + /// time-saved figure follows. A value outside the range (an empty box) is not saved. + /// + private void OnBaselineChanged() + { + if (this.loading || this.baselineBox.Value is not { } value) + { + return; + } + + var wpm = (int)Math.Round(value); + if (wpm is < DictationStatsStore.MinBaselineWpm or > DictationStatsStore.MaxBaselineWpm || wpm == this.host.Settings.BaselineTypingSpeedWpm) + { + return; + } - var state = host.Stats(); - var baseline = host.Settings.BaselineTypingSpeedWpm; - var panel = new StackPanel { Margin = new Thickness(24), Spacing = 16 }; + this.host.Settings.BaselineTypingSpeedWpm = wpm; + this.host.ApplySettings(this.host.Settings); + this.Render(); + } + + private void Render() + { + var state = this.host.Stats(); + var baseline = DictationStatsStore.NormalizeBaselineWpm(this.host.Settings.BaselineTypingSpeedWpm); + this.loading = true; + this.baselineBox.Value = baseline; + this.loading = false; + var panel = this.content; + panel.Children.Clear(); + panel.Children.Add(new TextBlock { Text = "Dictation impact", FontSize = 18, FontWeight = FontWeight.SemiBold }); + panel.Children.Add(new TextBlock { Text = "Local stats are counted from successful transcript commits and stay on this PC. Only text that was typed into an app counts.", Opacity = 0.7, TextWrapping = TextWrapping.Wrap }); panel.Children.Add(new StackPanel { Orientation = Orientation.Horizontal, @@ -30,33 +80,54 @@ public DictationStatsWindow(DictationHost host) Children = { Figure("Words typed", state.TotalWords.ToString("N0", CultureInfo.InvariantCulture)), - Figure("Time saved", FormatDuration(state.TimeSaved(baseline))), - Figure("Speaking speed", $"{state.AverageWordsPerMinute:N0} WPM"), - Figure("Dictations", state.InjectedSessions.ToString("N0", CultureInfo.InvariantCulture)) + Figure("Net time saved", FormatDuration(state.TimeSaved(baseline))), + Figure("Average speech pace", $"{state.AverageWordsPerMinute:N0} WPM"), + Figure("Successful sessions", state.InjectedSessions.ToString("N0", CultureInfo.InvariantCulture)) + } + }); + panel.Children.Add(new StackPanel + { + Orientation = Orientation.Horizontal, + Spacing = 8, + Children = + { + new TextBlock { Text = "Compare against typing at", VerticalAlignment = VerticalAlignment.Center }, + this.baselineBox, + new TextBlock { Text = "WPM (20 to 120)", VerticalAlignment = VerticalAlignment.Center } } }); - panel.Children.Add(new TextBlock { Text = $"Time saved compares with typing at {(baseline is >= 20 and <= 120 ? baseline : DictationStatsStore.DefaultBaselineWpm)} WPM. Only text that was typed into an app counts.", Opacity = 0.7, TextWrapping = TextWrapping.Wrap }); panel.Children.Add(new TextBlock { Text = "Last 14 days", FontWeight = FontWeight.SemiBold }); panel.Children.Add(this.Bars(state)); - panel.Children.Add(new TextBlock { Text = "Milestones", FontWeight = FontWeight.SemiBold }); + panel.Children.Add(new TextBlock { Text = "Achievements", FontWeight = FontWeight.SemiBold }); foreach (var a in DictationStatsStore.Achievements) { var unlocked = state.UnlockedAchievementIds.Contains(a.Id) || state.TotalWords >= a.WordThreshold; var remaining = Math.Max(0, a.WordThreshold - state.TotalWords); - panel.Children.Add(new StackPanel + var accent = unlocked ? Color.Parse("#34D399") : Color.Parse("#64748B"); + panel.Children.Add(new Border { - Orientation = Orientation.Horizontal, - Spacing = 10, - Children = + BorderBrush = new SolidColorBrush(accent), + BorderThickness = new Thickness(1), + CornerRadius = new CornerRadius(8), + Padding = new Thickness(12), + Child = new DockPanel { - new TextBlock { Text = unlocked ? "✓" : "○", Foreground = unlocked ? Brushes.SeaGreen : Brushes.Gray, Width = 16 }, - new TextBlock { Text = a.Title, FontWeight = FontWeight.SemiBold, MinWidth = 190 }, - new TextBlock { Text = unlocked ? a.Message : $"{Math.Min(100, state.TotalWords * 100.0 / a.WordThreshold):N0}% - {remaining:N0} words to go", Opacity = 0.8, TextWrapping = TextWrapping.Wrap, MaxWidth = 340 } + LastChildFill = true, + Children = + { + new TextBlock { Text = unlocked ? "Unlocked" : "Locked", Foreground = new SolidColorBrush(accent), FontWeight = FontWeight.SemiBold, [DockPanel.DockProperty] = Dock.Right, Margin = new Thickness(14, 0, 0, 0) }, + new StackPanel + { + Children = + { + new TextBlock { Text = a.Title, FontWeight = FontWeight.SemiBold }, + new TextBlock { Text = unlocked ? a.Message : $"{Math.Min(100, state.TotalWords * 100.0 / a.WordThreshold):N0}% complete - {remaining:N0} words to go.", Opacity = 0.8, TextWrapping = TextWrapping.Wrap } + } + } + } } }); } - - this.Content = panel; } private static StackPanel Figure(string label, string value) => new() @@ -68,7 +139,7 @@ private Control Bars(DictationStatsState state) { var days = state.LastDays(DateTime.Today); var max = Math.Max(1, days.Max(d => d.Words)); - var row = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 6, Height = 150 }; + var row = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 6, Height = 166 }; for (var i = 0; i < days.Count; i++) { var (day, words) = days[i]; @@ -79,6 +150,7 @@ private Control Bars(DictationStatsState state) Width = 36, Children = { + new TextBlock { Text = words.ToString(CultureInfo.InvariantCulture), FontSize = 10, HorizontalAlignment = HorizontalAlignment.Center, Opacity = 0.7 }, new Border { Height = height, Background = new SolidColorBrush(Palette[i % Palette.Length]), CornerRadius = new CornerRadius(3) }, new TextBlock { Text = day.ToString("M/d", CultureInfo.InvariantCulture), FontSize = 10, HorizontalAlignment = HorizontalAlignment.Center, Opacity = 0.7 } } diff --git a/src/PrimeDictate.Desktop/Dictation/FormParts.cs b/src/PrimeDictate.Desktop/Dictation/FormParts.cs new file mode 100644 index 0000000..5a6ec97 --- /dev/null +++ b/src/PrimeDictate.Desktop/Dictation/FormParts.cs @@ -0,0 +1,35 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Layout; +using Avalonia.Media; + +namespace PrimeDictate.Desktop.Dictation; + +/// Small building blocks shared by the settings controls and the first-run wizard. +internal static class FormParts +{ + private static readonly IBrush ErrorBrush = new SolidColorBrush(Color.FromRgb(224, 82, 82)); + + public static CheckBox Check(string text) => new() { Content = new TextBlock { Text = text, TextWrapping = TextWrapping.Wrap } }; + + public static Control Row(string label, Control input) + { + // Under its label, not centered in the window. + input.HorizontalAlignment = HorizontalAlignment.Left; + return new StackPanel { Spacing = 4, Children = { new TextBlock { Text = label, TextWrapping = TextWrapping.Wrap }, input } }; + } + + public static TextBlock Note(string text) => new() { Text = text, TextWrapping = TextWrapping.Wrap, Opacity = 0.7 }; + + /// A red line that stays hidden until gives it a message. + public static TextBlock ErrorLine() => new() { Foreground = ErrorBrush, TextWrapping = TextWrapping.Wrap, IsVisible = false, FontWeight = FontWeight.SemiBold }; + + public static void Show(TextBlock line, string? message) + { + line.Text = message ?? string.Empty; + line.IsVisible = !string.IsNullOrEmpty(message); + } + + public static string Normalize(string? path) => + string.IsNullOrWhiteSpace(path) ? string.Empty : System.IO.Path.GetFullPath(path.Trim().Trim('"')).TrimEnd(System.IO.Path.DirectorySeparatorChar, System.IO.Path.AltDirectorySeparatorChar); +} diff --git a/src/PrimeDictate.Desktop/Dictation/MicRippleControl.cs b/src/PrimeDictate.Desktop/Dictation/MicRippleControl.cs new file mode 100644 index 0000000..1320bb1 --- /dev/null +++ b/src/PrimeDictate.Desktop/Dictation/MicRippleControl.cs @@ -0,0 +1,116 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Media; +using Avalonia.Threading; +using PrimeDictate.Core.Dictation; + +namespace PrimeDictate.Desktop.Dictation; + +/// +/// The compact overlay's microphone: a dark disc with the microphone glyph in the state color, a white glow inside that follows the +/// voice level, and three rings that grow outward and fade (the WPF compact-mode ripple, driven by ). +/// The animation timer only runs while it is told the microphone is live; a still frame is drawn otherwise. +/// +public sealed class MicRippleControl : Control +{ + private const double Size = 46; + private static readonly Geometry Body = Geometry.Parse("M12,3 C9.8,3 8,4.8 8,7 L8,12 C8,14.2 9.8,16 12,16 C14.2,16 16,14.2 16,12 L16,7 C16,4.8 14.2,3 12,3 Z"); + private static readonly Geometry Cradle = Geometry.Parse("M6,11 L6,12 C6,15.31 8.69,18 12,18 C15.31,18 18,15.31 18,12 L18,11"); + private static readonly Geometry Stem = Geometry.Parse("M12,18 L12,21 M9,21 L15,21"); + private static readonly IBrush Face = new SolidColorBrush(Color.FromRgb(0x1A, 0x1F, 0x26)); + private static readonly byte[] RingAlpha = [140, 115, 90]; + + private readonly CompactRipple ripple = new(); + private readonly DispatcherTimer timer = new() { Interval = TimeSpan.FromMilliseconds(33) }; + private Color state = Color.FromRgb(32, 164, 112); + + public MicRippleControl() + { + this.Width = Size; + this.Height = Size; + this.IsHitTestVisible = false; + this.timer.Tick += (_, _) => + { + this.ripple.Tick(); + this.InvalidateVisual(); + }; + } + + public void SetLevel(double rms) => this.ripple.SetLevel(rms); + + /// Ready green, wake yellow, listening red, processing green, as in the WPF overlay. + public void SetStateColor(Color color) + { + this.state = color; + this.InvalidateVisual(); + } + + /// Animates while the microphone is live; stopping resets the rings so an idle microphone sits still. + public void SetRunning(bool running) + { + if (running) + { + this.timer.Start(); + return; + } + + this.timer.Stop(); + this.ripple.Reset(); + this.InvalidateVisual(); + } + + public override void Render(DrawingContext context) + { + var center = new Point(Size / 2, Size / 2); + var radius = Size / 2; + var ink = new SolidColorBrush(this.state); + context.DrawEllipse(Face, null, center, radius, radius); + + // The glow is lit from inside, so it is clipped to the disc. + using (context.PushClip(new Rect(0, 0, Size, Size))) + { + var glow = this.ripple.GlowScale * radius; + var brush = new RadialGradientBrush + { + Center = new RelativePoint(0.5, 0.42, RelativeUnit.Relative), + GradientOrigin = new RelativePoint(0.5, 0.42, RelativeUnit.Relative), + RadiusX = RelativeScalar.Parse("38%"), + RadiusY = RelativeScalar.Parse("38%"), + GradientStops = + { + new GradientStop(Color.FromArgb(0xF2, 255, 255, 255), 0), + new GradientStop(Color.FromArgb(0x66, 255, 255, 255), 0.35), + new GradientStop(Color.FromArgb(0x00, 255, 255, 255), 1) + } + }; + using (context.PushOpacity(this.ripple.GlowOpacity)) + { + context.DrawEllipse(brush, null, center, glow, glow); + } + + for (var i = 0; i < CompactRipple.RingCount; i++) + { + var (scale, opacity) = this.ripple.Ring(i); + var ring = new Pen(new SolidColorBrush(Color.FromArgb(RingAlpha[i], this.state.R, this.state.G, this.state.B)), 1.5); + using (context.PushOpacity(opacity)) + { + context.DrawEllipse(null, ring, center, radius * scale * 0.92, radius * scale * 0.92); + } + } + } + + context.DrawEllipse(null, new Pen(ink, 2), center, radius - 1, radius - 1); + + // The glyph is drawn on a 24x24 grid, scaled to 28 px (as in the WPF overlay) and nudged up a little. + const double glyph = 28; + var scaleFactor = glyph / 24.0; + var offset = new Vector((Size - glyph) / 2, ((Size - glyph) / 2) - 1); + using (context.PushTransform(Matrix.CreateScale(scaleFactor, scaleFactor) * Matrix.CreateTranslation(offset.X, offset.Y))) + { + context.DrawGeometry(ink, null, Body); + var line = new Pen(ink, 1.8, lineCap: PenLineCap.Round, lineJoin: PenLineJoin.Round); + context.DrawGeometry(null, line, Cradle); + context.DrawGeometry(null, line, Stem); + } + } +} diff --git a/src/PrimeDictate.Desktop/Dictation/ModelDownloadPanel.cs b/src/PrimeDictate.Desktop/Dictation/ModelDownloadPanel.cs index e6d65ce..11ecebc 100644 --- a/src/PrimeDictate.Desktop/Dictation/ModelDownloadPanel.cs +++ b/src/PrimeDictate.Desktop/Dictation/ModelDownloadPanel.cs @@ -6,7 +6,7 @@ namespace PrimeDictate.Desktop.Dictation; -/// Pick a catalog model, download it with progress, cancel it. Calls once a model is ready. +/// Pick a catalog model, download it with progress, cancel it. Calls with the model's id once a model is ready. public sealed class ModelDownloadPanel : StackPanel { private readonly DictationHost host; @@ -15,16 +15,26 @@ public sealed class ModelDownloadPanel : StackPanel private readonly Button cancel = new() { Content = "Cancel", IsVisible = false, Margin = new Avalonia.Thickness(8, 0, 0, 0) }; private readonly ProgressBar bar = new() { Minimum = 0, Maximum = 1, IsVisible = false }; private readonly TextBlock text = new() { TextWrapping = Avalonia.Media.TextWrapping.Wrap, Opacity = 0.8 }; + private readonly IReadOnlyList options; private CancellationTokenSource? running; - public ModelDownloadPanel(DictationHost host, Action installed) + private static string FamilyName(PrimeDictate.Core.Dictation.LegacyBackend backend) => backend switch + { + PrimeDictate.Core.Dictation.LegacyBackend.WhisperNet => "Whisper.net", + PrimeDictate.Core.Dictation.LegacyBackend.QualcommQnn => "Qualcomm NPU", + _ => backend.ToString() + }; + + public ModelDownloadPanel(DictationHost host, Action installed) { this.host = host; this.Spacing = 6; - this.choice.ItemsSource = SpeechModelCatalog.Options - .Select(o => $"{o.Backend}: {o.DisplayName} ({SpeechModelCatalog.FormatSize(o.ApproximateBytes)}){(o.Recommended ? " - recommended" : string.Empty)}") + // Qualcomm NPU models are offered only on a Windows ARM64 build that has the QNN runtime, as in the WPF app. + this.options = SpeechModelCatalog.AvailableOptions(MachineSupport.Current); + this.choice.ItemsSource = this.options + .Select(o => $"{FamilyName(o.Backend)}: {o.DisplayName} ({SpeechModelCatalog.FormatSize(o.ApproximateBytes)}){(o.Recommended ? " - recommended" : string.Empty)}") .ToList(); - this.choice.SelectedIndex = 0; + this.choice.SelectedIndex = PreselectedIndex(this.options, host.Settings); this.start.Click += async (_, _) => await this.RunAsync(installed); this.cancel.Click += (_, _) => this.running?.Cancel(); // The list takes what is left of the row (the model names are long), so the buttons stay inside the window. @@ -35,12 +45,40 @@ public ModelDownloadPanel(DictationHost host, Action installed) this.Children.Add(this.text); } - private async Task RunAsync(Action installed) + /// + /// The model shown first, as the WPF Settings window chose it: the one the settings already name, else the recommended model of that + /// family (Whisper by default), else the first. For a new install that is Whisper Tiny English. + /// + internal static int PreselectedIndex(IReadOnlyList options, PrimeDictate.Core.Dictation.DictationSettings settings) + { + var wanted = settings.ResolveModelId(); + var index = wanted is null ? -1 : options.ToList().FindIndex(o => o.ModelId == wanted); + if (index < 0) + { + index = options.ToList().FindIndex(o => o.Backend == settings.TranscriptionBackend && o.Recommended); + } + + return Math.Max(0, index); + } + + /// True while a download is running. + public bool IsBusy => this.running is not null; + + /// Starts downloading the model currently shown in the list (first-run setup does this when no model is installed, as the WPF wizard did). + public void StartDownload(Action installed) + { + if (!this.IsBusy) + { + _ = this.RunAsync(installed); + } + } + + private async Task RunAsync(Action installed) { - var option = SpeechModelCatalog.Options[Math.Max(0, this.choice.SelectedIndex)]; + var option = this.options[Math.Max(0, this.choice.SelectedIndex)]; this.running = new CancellationTokenSource(); this.SetBusy(true); - this.text.Text = $"Downloading {option.DisplayName} from GitHub..."; + this.text.Text = $"Downloading {option.DisplayName}..."; var progress = new Progress(p => Dispatcher.UIThread.Post(() => { this.bar.IsIndeterminate = p.Fraction is null; @@ -49,6 +87,7 @@ private async Task RunAsync(Action installed) { "download" => $"Downloading {option.DisplayName}: {p.BytesDownloaded / (1024 * 1024):N0} MB{(p.TotalBytes is { } t ? $" of {t / (1024 * 1024):N0} MB" : string.Empty)}", "extract" => "Unpacking...", + "tokenizer" => "Adding the Whisper tokenizer...", _ => "Ready." }; })); @@ -56,7 +95,7 @@ private async Task RunAsync(Action installed) { await this.host.DownloadModelAsync(option, progress, this.running.Token); this.text.Text = $"{option.DisplayName} is installed."; - installed(); + installed(option.ModelId); } catch (OperationCanceledException) { diff --git a/src/PrimeDictate.Desktop/Dictation/OllamaPanel.cs b/src/PrimeDictate.Desktop/Dictation/OllamaPanel.cs new file mode 100644 index 0000000..16c3d73 --- /dev/null +++ b/src/PrimeDictate.Desktop/Dictation/OllamaPanel.cs @@ -0,0 +1,70 @@ +using Avalonia; +using Avalonia.Controls; +using PrimeDictate.Core.Dictation; + +namespace PrimeDictate.Desktop.Dictation; + +/// Optional rewriting of the final transcript with a local Ollama model (the WPF "Ollama post-processing" block). +public sealed class OllamaPanel : StackPanel +{ + private static readonly (OllamaMode Mode, string Label)[] Modes = + [ + (OllamaMode.Default, "Default (Grammar & Spelling)"), + (OllamaMode.Prompt, "Prompt (AI Prompt Engineer)"), + (OllamaMode.Bug, "Bug Report (QA Engineer)"), + (OllamaMode.Update, "Update (Changelog/Status)"), + (OllamaMode.Communication, "Communication (Email/Message)"), + (OllamaMode.Blog, "Blog (Draft/Post)"), + (OllamaMode.VibeCoding, "Vibe Coding (Agent Instructions)") + ]; + + private readonly CheckBox enabled = FormParts.Check("Enable local Ollama formatting"); + private readonly TextBox endpoint = new() { Width = 300 }; + private readonly TextBox model = new() { Width = 200 }; + private readonly ComboBox mode = new() { ItemsSource = Modes.Select(m => m.Label).ToList(), Width = 300 }; + private readonly CheckBox allowRemote = FormParts.Check("Allow an Ollama server on another computer (your dictated words are sent there)"); + private readonly TextBlock remoteNote = FormParts.Note("This endpoint is not on this computer. Until you allow remote endpoints, your words are typed as spoken and not sent."); + private readonly TextBlock error = FormParts.ErrorLine(); + + public OllamaPanel(DictationSettings settings) + { + this.Spacing = 8; + this.Children.Add(FormParts.Note("After you stop, the final text can be rewritten by a model running in Ollama before it is typed. If Ollama is off or fails, the text is typed as spoken.")); + this.Children.Add(this.enabled); + this.Children.Add(FormParts.Row("Endpoint", this.endpoint)); + this.Children.Add(FormParts.Row("Model", this.model)); + this.Children.Add(FormParts.Row("Mode", this.mode)); + this.Children.Add(this.allowRemote); + this.Children.Add(this.remoteNote); + this.Children.Add(this.error); + this.enabled.IsChecked = settings.EnableOllamaPostProcessing; + this.endpoint.Text = settings.OllamaEndpoint; + this.model.Text = settings.OllamaModel; + this.mode.SelectedIndex = Math.Max(0, Array.FindIndex(Modes, m => m.Mode == settings.OllamaMode)); + this.allowRemote.IsChecked = settings.OllamaAllowRemoteEndpoint; + this.endpoint.TextChanged += (_, _) => this.UpdateNote(); + this.allowRemote.IsCheckedChanged += (_, _) => this.UpdateNote(); + this.UpdateNote(); + } + + /// With rewriting on, the endpoint and model must be usable. The problem is shown under the form. + public bool TryValidate() + { + var problem = DictationSettingsValidator.ValidateOllama(this.enabled.IsChecked == true, this.endpoint.Text, this.model.Text); + FormParts.Show(this.error, problem); + return problem is null; + } + + public void Apply(DictationSettings settings) + { + settings.EnableOllamaPostProcessing = this.enabled.IsChecked == true; + // Left blank while off, the defaults come back rather than an empty address being saved. + settings.OllamaEndpoint = string.IsNullOrWhiteSpace(this.endpoint.Text) ? OllamaOptions.Disabled.Endpoint : this.endpoint.Text.Trim(); + settings.OllamaModel = string.IsNullOrWhiteSpace(this.model.Text) ? OllamaOptions.Disabled.Model : this.model.Text.Trim(); + settings.OllamaMode = Modes[Math.Clamp(this.mode.SelectedIndex, 0, Modes.Length - 1)].Mode; + settings.OllamaAllowRemoteEndpoint = this.allowRemote.IsChecked == true; + } + + private void UpdateNote() => + this.remoteNote.IsVisible = DictationSettingsValidator.IsRemoteEndpoint(this.endpoint.Text) && this.allowRemote.IsChecked != true; +} diff --git a/src/PrimeDictate.Desktop/Dictation/ReplacementsEditor.cs b/src/PrimeDictate.Desktop/Dictation/ReplacementsEditor.cs new file mode 100644 index 0000000..8c5a685 --- /dev/null +++ b/src/PrimeDictate.Desktop/Dictation/ReplacementsEditor.cs @@ -0,0 +1,57 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Layout; +using PrimeDictate.Core.Dictation; + +namespace PrimeDictate.Desktop.Dictation; + +/// The WPF "Replacements" grid: a Find column, a "Replace with" column, Add rule and Remove per row. +public sealed class ReplacementsEditor : StackPanel +{ + private readonly StackPanel rows = new() { Spacing = 6 }; + private readonly List<(Control Root, TextBox Find, TextBox Replace)> editors = []; + + public ReplacementsEditor(IEnumerable rules) + { + this.Spacing = 8; + this.Children.Add(FormParts.Note("After transcription, replace phrases in the final text before optional AI polish and typing. Matching is case-insensitive; longer phrases are applied first.")); + var header = new Grid { ColumnDefinitions = new ColumnDefinitions("*,*,Auto") }; + header.Children.Add(new TextBlock { Text = "Find", FontWeight = Avalonia.Media.FontWeight.SemiBold }); + var replaceHeader = new TextBlock { Text = "Replace with", FontWeight = Avalonia.Media.FontWeight.SemiBold, Margin = new Thickness(8, 0, 0, 0) }; + Grid.SetColumn(replaceHeader, 1); + header.Children.Add(replaceHeader); + this.Children.Add(header); + this.Children.Add(this.rows); + var add = new Button { Content = "Add rule", HorizontalAlignment = HorizontalAlignment.Left }; + add.Click += (_, _) => this.AddRow(new ReplacementDto()); + this.Children.Add(add); + foreach (var rule in rules) + { + this.AddRow(rule); + } + } + + /// The rules to save: rows with nothing to find are dropped, the rest trimmed (as the WPF Save did). + public List Build() => this.editors + .Where(e => !string.IsNullOrWhiteSpace(e.Find.Text)) + .Select(e => new ReplacementDto { Find = e.Find.Text!.Trim(), Replace = e.Replace.Text?.Trim() ?? string.Empty }) + .ToList(); + + private void AddRow(ReplacementDto rule) + { + var find = new TextBox { Text = rule.Find, PlaceholderText = "spoken phrase" }; + var replace = new TextBox { Text = rule.Replace, PlaceholderText = "replacement", Margin = new Thickness(8, 0, 0, 0) }; + var remove = new Button { Content = "Remove", Margin = new Thickness(8, 0, 0, 0) }; + Grid.SetColumn(replace, 1); + Grid.SetColumn(remove, 2); + var root = new Grid { ColumnDefinitions = new ColumnDefinitions("*,*,Auto"), Children = { find, replace, remove } }; + var row = (Control: (Control)root, Find: find, Replace: replace); + remove.Click += (_, _) => + { + this.editors.Remove(row); + this.rows.Children.Remove(root); + }; + this.editors.Add(row); + this.rows.Children.Add(root); + } +} diff --git a/src/PrimeDictate.Desktop/Dictation/TrayIconRenderer.cs b/src/PrimeDictate.Desktop/Dictation/TrayIconRenderer.cs index 1324840..949fe78 100644 --- a/src/PrimeDictate.Desktop/Dictation/TrayIconRenderer.cs +++ b/src/PrimeDictate.Desktop/Dictation/TrayIconRenderer.cs @@ -22,7 +22,31 @@ public static class TrayIconRenderer { private const int Size = 64; - public static WindowIcon Create(TrayVisualState state) => new(Render(state)); + private static readonly Dictionary Cache = []; + + /// One icon per state, drawn the first time it is needed and reused after, as the WPF app keeps one icon per state. UI thread only. + public static WindowIcon Create(TrayVisualState state) + { + if (!Cache.TryGetValue(state, out var entry)) + { + var bitmap = Render(state); + entry = (new WindowIcon(bitmap), bitmap); + Cache[state] = entry; + } + + return entry.Icon; + } + + /// Releases the cached icons when the tray goes away. + public static void DisposeCached() + { + foreach (var (_, bitmap) in Cache.Values) + { + bitmap.Dispose(); + } + + Cache.Clear(); + } public static RenderTargetBitmap Render(TrayVisualState state) { diff --git a/src/PrimeDictate.Desktop/MainWindow.axaml b/src/PrimeDictate.Desktop/MainWindow.axaml index bbbd52e..70f67df 100644 --- a/src/PrimeDictate.Desktop/MainWindow.axaml +++ b/src/PrimeDictate.Desktop/MainWindow.axaml @@ -5,14 +5,30 @@ Title="PrimeDictate" Width="1100" Height="720" MinWidth="720" MinHeight="480"> - - - - - + + +