From 5c59eb8da56a4d90ded9f2b9babdda9f75b89f77 Mon Sep 17 00:00:00 2001 From: CakeRepository Date: Wed, 30 Sep 2026 15:14:08 -0500 Subject: [PATCH 01/15] Run voice shell commands in dictation (opt-in list, never in transcription) Port the WPF runner: the command comes only from settings, is matched only in the final transcript of a dictation, and Stop/Continue decide whether the rest is typed. Imported WPF commands keep their Enabled state; tests use a fake runner. Co-Authored-By: Claude Opus 5.5 --- .../Dictation/DictationController.cs | 82 ++++++++++- .../Dictation/DictationSettings.cs | 8 +- .../Dictation/VoiceCommandMatcher.cs | 5 +- .../Dictation/VoiceCommands.cs | 19 ++- .../Dictation/VoiceShellCommandRunner.cs | 13 ++ .../Dictation/DictationHost.cs | 3 +- .../ProcessVoiceShellCommandRunner.cs | 44 ++++++ .../DictationLoopTests.cs | 129 ++++++++++++++++++ .../DictationSettingsTests.cs | 28 ++++ 9 files changed, 325 insertions(+), 6 deletions(-) create mode 100644 src/PrimeDictate.Core/Dictation/VoiceShellCommandRunner.cs create mode 100644 src/PrimeDictate.Platforms/Dictation/ProcessVoiceShellCommandRunner.cs diff --git a/src/PrimeDictate.Core/Dictation/DictationController.cs b/src/PrimeDictate.Core/Dictation/DictationController.cs index b67b8f9..10fda7e 100644 --- a/src/PrimeDictate.Core/Dictation/DictationController.cs +++ b/src/PrimeDictate.Core/Dictation/DictationController.cs @@ -54,6 +54,7 @@ public sealed class DictationController : IAsyncDisposable private readonly IVoiceCommandProcessor voiceCommands; private readonly TimeProvider time; private readonly ITranscriptRewriter? rewriter; + private readonly IVoiceShellCommandRunner? shellRunner; private readonly SemaphoreSlim gate = new(1, 1); private volatile DictationOptions options = new DictationOptions().Normalized(); private Session? session; @@ -66,7 +67,8 @@ public DictationController( MicrophoneCoordinator? microphone = null, IVoiceCommandProcessor? voiceCommands = null, TimeProvider? time = null, - ITranscriptRewriter? rewriter = null) + ITranscriptRewriter? rewriter = null, + IVoiceShellCommandRunner? shellRunner = null) { this.audioSource = audioSource; this.providerSource = providerSource; @@ -76,6 +78,7 @@ public DictationController( this.voiceCommands = voiceCommands ?? NoVoiceCommands.Instance; this.time = time ?? TimeProvider.System; this.rewriter = rewriter; + this.shellRunner = shellRunner; } public event Action? StateChanged; @@ -421,7 +424,17 @@ private async Task FinishAsync(Session s, string reason) try { var raw = await this.RecognizeAsync(s.Provider, audio, CancellationToken.None).ConfigureAwait(false); - var match = this.voiceCommands.Apply(TranscriptPostProcessor.RemoveTrailingSilenceArtifact(raw, reason.Contains("silence", StringComparison.Ordinal))); + var spoken = TranscriptPostProcessor.RemoveTrailingSilenceArtifact(raw, reason.Contains("silence", StringComparison.Ordinal)); + // Shell commands are matched only here, in the final transcript of a dictation, and only when a runner exists. + var match = this.shellRunner is null ? this.voiceCommands.Apply(spoken) : this.voiceCommands.ApplyFinal(spoken); + if (match.Shell is { } shell && this.shellRunner is not null) + { + if (!await this.RunShellCommandAsync(s, shell, this.shellRunner, duration).ConfigureAwait(false)) + { + return; + } + } + if (match.StopRequested) { return; @@ -500,6 +513,71 @@ private async Task FinishAsync(Session s, string reason) rewritePrompt)); } + private static readonly TimeSpan ShellTypeTargetWait = TimeSpan.FromSeconds(2); + private static readonly TimeSpan ShellTypePollInterval = TimeSpan.FromMilliseconds(100); + + /// + /// Runs the user's command for a matched phrase. The command string is the one saved in settings; nothing from the + /// transcript reaches it. Returns true when the rest of the dictation should go on to be typed: only for + /// and only when words remain after the phrase was removed. + /// + private async Task RunShellCommandAsync(Session s, VoiceShellCommandInvocation invocation, IVoiceShellCommandRunner runner, TimeSpan duration) + { + var command = invocation.Command; + var phrase = command.Phrase.Trim(); + var label = $"Voice command: {phrase}"; + DictationDeliveryStatus status; + string? error = null; + try + { + var result = runner.Run(command); + Diagnostics.AppLog.Event("dictation", $"Voice command ran: \"{phrase}\" (pid {result.ProcessId?.ToString() ?? "unknown"})."); + if (!string.IsNullOrWhiteSpace(invocation.TextToType)) + { + if (!await this.WaitForShellTypeTargetAsync(s.Target).ConfigureAwait(false)) + { + throw new InvalidOperationException("Chained typing skipped because the command did not move focus away from the starting window."); + } + + this.injector.TypeText(invocation.TextToType); + } + + status = DictationDeliveryStatus.CommandExecuted; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + status = DictationDeliveryStatus.CommandFailed; + error = ex.Message; + Diagnostics.AppLog.Event("dictation", $"Voice command failed: \"{phrase}\": {ex.Message}"); + this.Notice?.Invoke($"Voice command \"{phrase}\" failed: {ex.Message}"); + } + + this.Committed?.Invoke(new DictationCommit( + s.Id, this.time.GetUtcNow().UtcDateTime, label, status, "Command", null, null, error, duration, EnterSent: false)); + return status == DictationDeliveryStatus.CommandExecuted && command.CompletionBehavior == VoiceShellCommandCompletionBehavior.Continue; + } + + private async Task WaitForShellTypeTargetAsync(IForegroundTarget? start) + { + if (start is null) + { + await Task.Delay(ShellTypePollInterval, this.time).ConfigureAwait(false); + return true; + } + + var deadline = this.time.GetUtcNow() + ShellTypeTargetWait; + while (this.time.GetUtcNow() < deadline) + { + await Task.Delay(ShellTypePollInterval, this.time).ConfigureAwait(false); + if (!start.IsStillForeground()) + { + return true; + } + } + + return false; + } + private static async Task RecognizeAsync(ITranscriptionProvider provider, ReadOnlyMemory samples, string? language, CancellationToken ct) { var window = provider.Capabilities.MaxWindow ?? TimeSpan.FromSeconds(30); diff --git a/src/PrimeDictate.Core/Dictation/DictationSettings.cs b/src/PrimeDictate.Core/Dictation/DictationSettings.cs index 16dbf9f..d109d6d 100644 --- a/src/PrimeDictate.Core/Dictation/DictationSettings.cs +++ b/src/PrimeDictate.Core/Dictation/DictationSettings.cs @@ -74,12 +74,18 @@ public sealed class DictationSettings public string VoiceHistoryPhrase { get; set; } = VoiceCommandProcessor.DefaultHistoryPhrase; + /// + /// Phrases that run a command on this computer while dictating. Same shape as the WPF app's list, so imported commands + /// keep their On state (the user chose them there). Commands added in this app start Off. + /// + public List VoiceShellCommands { get; set; } = []; + public VoiceCommandOptions ToVoiceCommandOptions() => new( this.EnableVoiceCommands, this.VoiceDictationPhrase?.Trim() ?? string.Empty, this.VoiceStopPhrase?.Trim() ?? string.Empty, this.VoiceHistoryPhrase?.Trim() ?? string.Empty, - []); + (this.VoiceShellCommands ?? []).Where(c => c is not null).ToList()); public bool EnableOllamaPostProcessing { get; set; } diff --git a/src/PrimeDictate.Core/Dictation/VoiceCommandMatcher.cs b/src/PrimeDictate.Core/Dictation/VoiceCommandMatcher.cs index 5554b3c..5d5f837 100644 --- a/src/PrimeDictate.Core/Dictation/VoiceCommandMatcher.cs +++ b/src/PrimeDictate.Core/Dictation/VoiceCommandMatcher.cs @@ -393,7 +393,10 @@ public enum VoiceShellCommandCompletionBehavior Continue = 1 } -/// A user-defined phrase that runs a command. Matching is ported; running is not (opt-in, dictation only, later). +/// +/// A user-defined phrase that runs a command. The command text comes only from the user's settings, never from what +/// was said. Dictation only; transcription mode has no hook for it. +/// public sealed class VoiceShellCommand { public bool Enabled { get; set; } = true; diff --git a/src/PrimeDictate.Core/Dictation/VoiceCommands.cs b/src/PrimeDictate.Core/Dictation/VoiceCommands.cs index 890384c..ba77a9a 100644 --- a/src/PrimeDictate.Core/Dictation/VoiceCommands.cs +++ b/src/PrimeDictate.Core/Dictation/VoiceCommands.cs @@ -1,6 +1,11 @@ namespace PrimeDictate.Core.Dictation; -public sealed record VoiceCommandResult(string CleanedText, bool CommitRequested, bool StopRequested, bool HistoryRequested) +public sealed record VoiceCommandResult( + string CleanedText, + bool CommitRequested, + bool StopRequested, + bool HistoryRequested, + VoiceShellCommandInvocation? Shell = null) { public static VoiceCommandResult Passthrough(string text) => new(text, false, false, false); } @@ -12,6 +17,12 @@ public sealed record VoiceCommandResult(string CleanedText, bool CommitRequested public interface IVoiceCommandProcessor { VoiceCommandResult Apply(string transcript); + + /// + /// Like for the finished transcript of a dictation, where the user's shell-command phrases are + /// also matched. Live previews never use this, so a preview can never run anything. + /// + VoiceCommandResult ApplyFinal(string transcript) => this.Apply(transcript); } public sealed class NoVoiceCommands : IVoiceCommandProcessor @@ -33,4 +44,10 @@ public VoiceCommandResult Apply(string transcript) var match = VoiceCommandMatcher.Apply(transcript, options()); return new VoiceCommandResult(match.CleanedText, match.CommitRequested, match.StopRequested, match.HistoryRequested); } + + public VoiceCommandResult ApplyFinal(string transcript) + { + var match = VoiceCommandMatcher.Apply(transcript, options(), includeShellCommands: true); + return new VoiceCommandResult(match.CleanedText, match.CommitRequested, match.StopRequested, match.HistoryRequested, match.ShellCommandInvocation); + } } diff --git a/src/PrimeDictate.Core/Dictation/VoiceShellCommandRunner.cs b/src/PrimeDictate.Core/Dictation/VoiceShellCommandRunner.cs new file mode 100644 index 0000000..d3c1b23 --- /dev/null +++ b/src/PrimeDictate.Core/Dictation/VoiceShellCommandRunner.cs @@ -0,0 +1,13 @@ +namespace PrimeDictate.Core.Dictation; + +public readonly record struct VoiceShellCommandResult(int? ProcessId); + +/// +/// Starts a user-configured command. Implementations receive the command exactly as the user saved it in settings; +/// nothing spoken or typed is ever part of it. +/// +public interface IVoiceShellCommandRunner +{ + /// The command is empty or the process did not start. + VoiceShellCommandResult Run(VoiceShellCommand command); +} diff --git a/src/PrimeDictate.Platforms/Dictation/DictationHost.cs b/src/PrimeDictate.Platforms/Dictation/DictationHost.cs index 61fe0dc..e007aee 100644 --- a/src/PrimeDictate.Platforms/Dictation/DictationHost.cs +++ b/src/PrimeDictate.Platforms/Dictation/DictationHost.cs @@ -59,7 +59,8 @@ public DictationHost( injector ?? new SharpHookTextInjector(), microphone, voiceCommands ?? new VoiceCommandProcessor(() => this.Settings.ToVoiceCommandOptions()), - rewriter: new OllamaRewriter(() => this.Settings.ToOllamaOptions(), report: message => this.Notice?.Invoke(message))); + rewriter: new OllamaRewriter(() => this.Settings.ToOllamaOptions(), report: message => this.Notice?.Invoke(message)), + shellRunner: new ProcessVoiceShellCommandRunner()); this.Controller.Notice += message => this.Notice?.Invoke(message); this.Controller.HistoryRequested += () => this.HistoryRequested?.Invoke(); this.Controller.Committed += this.OnCommitted; diff --git a/src/PrimeDictate.Platforms/Dictation/ProcessVoiceShellCommandRunner.cs b/src/PrimeDictate.Platforms/Dictation/ProcessVoiceShellCommandRunner.cs new file mode 100644 index 0000000..d2df186 --- /dev/null +++ b/src/PrimeDictate.Platforms/Dictation/ProcessVoiceShellCommandRunner.cs @@ -0,0 +1,44 @@ +using System.Diagnostics; +using PrimeDictate.Core.Dictation; + +namespace PrimeDictate.Platforms.Dictation; + +/// +/// Runs a voice shell command through the platform's command interpreter (cmd.exe /d /c on Windows, /bin/sh -c +/// elsewhere) with no window, from the user's profile folder. The command is passed as one argument, exactly as saved. +/// +public sealed class ProcessVoiceShellCommandRunner : IVoiceShellCommandRunner +{ + public VoiceShellCommandResult Run(VoiceShellCommand command) + { + ArgumentNullException.ThrowIfNull(command); + var text = command.Command.Trim(); + if (text.Length == 0) + { + throw new InvalidOperationException("Voice command has no command configured."); + } + + var startInfo = new ProcessStartInfo + { + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + }; + if (OperatingSystem.IsWindows()) + { + var interpreter = Environment.GetEnvironmentVariable("ComSpec"); + startInfo.FileName = string.IsNullOrWhiteSpace(interpreter) ? "cmd.exe" : interpreter; + startInfo.ArgumentList.Add("/d"); + startInfo.ArgumentList.Add("/c"); + } + else + { + startInfo.FileName = "/bin/sh"; + startInfo.ArgumentList.Add("-c"); + } + + startInfo.ArgumentList.Add(text); + using var process = Process.Start(startInfo) ?? throw new InvalidOperationException("The system did not start the command process."); + return new VoiceShellCommandResult(process.Id); + } +} diff --git a/tests/PrimeDictate.Core.Tests/DictationLoopTests.cs b/tests/PrimeDictate.Core.Tests/DictationLoopTests.cs index 701cfff..d061882 100644 --- a/tests/PrimeDictate.Core.Tests/DictationLoopTests.cs +++ b/tests/PrimeDictate.Core.Tests/DictationLoopTests.cs @@ -299,6 +299,135 @@ private sealed class ManualTime : TimeProvider public override DateTimeOffset GetUtcNow() => DateTimeOffset.UtcNow; } + private static readonly VoiceCommandOptions ShellOptionsBase = new(true, "thank you", "potato farmer", "show me the money", []); + + private static async Task<(List Commits, FakeInjector Injector, FakeShellRunner Runner)> DictateAsync( + string transcript, + VoiceShellCommand[] commands, + FakeShellRunner? runner = null, + bool withRunner = true) + { + runner ??= new FakeShellRunner(); + var source = new FakeSource(); + var inj = new FakeInjector(); + var commits = new List(); + var processor = new VoiceCommandProcessor(() => ShellOptionsBase with { ShellCommands = commands }); + await using var controller = new DictationController( + source, () => new FakeProvider(transcript), new FakeGuard(), inj, voiceCommands: processor, shellRunner: withRunner ? runner : null); + controller.Options = new DictationOptions { AutoCommitSilence = TimeSpan.Zero }; + controller.Committed += commits.Add; + await controller.ToggleAsync(); + source.Lease!.Push(Tone(1.0)); + await WaitFor(() => controller.IsRecording); + await controller.ToggleAsync(); + return (commits, inj, runner); + } + + [Fact] + public async Task Shell_command_runs_the_saved_command_and_stop_types_nothing() + { + var saved = new VoiceShellCommand { Phrase = "open notes", Command = "notepad.exe C:\\todo.txt", Enabled = true }; + var (commits, inj, runner) = await DictateAsync("please rm -rf everything open notes", [saved]); + + // The runner receives the command exactly as saved, never words from the transcript. + Assert.Same(saved, Assert.Single(runner.Ran)); + Assert.Equal("notepad.exe C:\\todo.txt", runner.Ran[0].Command); + Assert.Empty(inj.Typed); + var commit = Assert.Single(commits); + Assert.Equal(DictationDeliveryStatus.CommandExecuted, commit.Status); + Assert.Equal("Voice command: open notes", commit.Transcript); + } + + [Fact] + public async Task Shell_command_with_continue_types_the_words_that_remain() + { + var saved = new VoiceShellCommand { Phrase = "open notes", Command = "notepad", CompletionBehavior = VoiceShellCommandCompletionBehavior.Continue }; + var (commits, inj, runner) = await DictateAsync("open notes remember the milk", [saved]); + + Assert.Single(runner.Ran); + Assert.Equal(["remember the milk"], inj.Typed); + Assert.Equal([DictationDeliveryStatus.CommandExecuted, DictationDeliveryStatus.Injected], commits.Select(c => c.Status).ToArray()); + } + + [Fact] + public async Task Shell_command_with_continue_and_nothing_left_types_nothing() + { + var saved = new VoiceShellCommand { Phrase = "open notes", Command = "notepad", CompletionBehavior = VoiceShellCommandCompletionBehavior.Continue }; + var (_, inj, runner) = await DictateAsync("open notes", [saved]); + + Assert.Single(runner.Ran); + Assert.Empty(inj.Typed); + } + + [Fact] + public async Task Disabled_or_unmatched_shell_commands_never_run_and_the_text_is_typed() + { + var off = new VoiceShellCommand { Phrase = "open notes", Command = "notepad", Enabled = false }; + var (_, inj, runner) = await DictateAsync("open notes today", [off]); + Assert.Empty(runner.Ran); + Assert.Equal(["open notes today"], inj.Typed); + } + + [Fact] + public async Task Shell_command_failure_is_reported_and_nothing_is_typed() + { + var saved = new VoiceShellCommand { Phrase = "open notes", Command = "notepad", CompletionBehavior = VoiceShellCommandCompletionBehavior.Continue }; + var (commits, inj, _) = await DictateAsync("open notes hello", [saved], new FakeShellRunner { Fail = true }); + + Assert.Empty(inj.Typed); + var commit = Assert.Single(commits); + Assert.Equal(DictationDeliveryStatus.CommandFailed, commit.Status); + Assert.Equal("cannot start", commit.Error); + } + + [Fact] + public async Task Without_a_runner_the_phrase_is_just_dictated_text() + { + var saved = new VoiceShellCommand { Phrase = "open notes", Command = "notepad" }; + var (_, inj, runner) = await DictateAsync("open notes", [saved], withRunner: false); + Assert.Empty(runner.Ran); + Assert.Equal(["open notes"], inj.Typed); + } + + [Fact] + public async Task A_live_preview_never_runs_a_shell_command() + { + var saved = new VoiceShellCommand { Phrase = "open notes", Command = "notepad" }; + var runner = new FakeShellRunner(); + var source = new FakeSource(); + var partials = new List(); + var processor = new VoiceCommandProcessor(() => ShellOptionsBase with { ShellCommands = [saved] }); + await using var controller = new DictationController( + source, () => new FakeProvider("open notes"), new FakeGuard(), new FakeInjector(), voiceCommands: processor, shellRunner: runner); + controller.Options = new DictationOptions { AutoCommitSilence = TimeSpan.Zero }; + controller.PartialTranscript += (_, t) => partials.Add(t); + await controller.ToggleAsync(); + source.Lease!.Push(Tone(1.0)); + await WaitFor(() => partials.Count > 0); + + Assert.Empty(runner.Ran); + await controller.DiscardAsync(); + Assert.Empty(runner.Ran); + } + + private sealed class FakeShellRunner : IVoiceShellCommandRunner + { + public List Ran { get; } = []; + + public bool Fail { get; init; } + + public VoiceShellCommandResult Run(VoiceShellCommand command) + { + if (this.Fail) + { + throw new InvalidOperationException("cannot start"); + } + + this.Ran.Add(command); + return new VoiceShellCommandResult(42); + } + } + private sealed class StopWordCommands : IVoiceCommandProcessor { public VoiceCommandResult Apply(string transcript) => diff --git a/tests/PrimeDictate.Core.Tests/DictationSettingsTests.cs b/tests/PrimeDictate.Core.Tests/DictationSettingsTests.cs index 1be7523..934a4b1 100644 --- a/tests/PrimeDictate.Core.Tests/DictationSettingsTests.cs +++ b/tests/PrimeDictate.Core.Tests/DictationSettingsTests.cs @@ -53,6 +53,34 @@ public void Wpf_settings_are_imported_read_only_and_never_rewritten() Assert.False(store.Load().ImportedFromWpf); } + [Fact] + public void Wpf_voice_shell_commands_are_imported_with_their_enabled_state_and_new_ones_round_trip() + { + var paths = new AppDataPaths(this.root); + File.WriteAllText(Path.Combine(this.root, "settings.json"), """ + { + "VoiceShellCommands": [ + { "Enabled": true, "Phrase": "open notes", "CompletionBehavior": "Continue", "Command": "notepad" }, + { "Enabled": false, "Phrase": "lock screen", "CompletionBehavior": "Stop", "Command": "rundll32 user32.dll,LockWorkStation" } + ] + } + """); + var store = new DictationSettingsStore(paths); + var load = store.Load(); + var commands = load.Settings.VoiceShellCommands; + Assert.Equal(2, commands.Count); + Assert.True(commands[0].Enabled); + Assert.Equal(VoiceShellCommandCompletionBehavior.Continue, commands[0].CompletionBehavior); + Assert.False(commands[1].Enabled); + Assert.Equal(2, load.Settings.ToVoiceCommandOptions().ShellCommands.Count); + + store.Save(load.Settings); + var again = store.Load().Settings.VoiceShellCommands; + Assert.Equal(["open notes", "lock screen"], again.Select(c => c.Phrase).ToArray()); + Assert.Equal([true, false], again.Select(c => c.Enabled).ToArray()); + Assert.Empty(new DictationSettings().VoiceShellCommands); + } + [Fact] public void Missing_or_broken_files_fall_back_to_defaults_without_touching_them() { From 44f134a7afecd1c8e9722584572990b1c8536e55 Mon Sep 17 00:00:00 2001 From: CakeRepository Date: Wed, 30 Sep 2026 15:18:27 -0500 Subject: [PATCH 02/15] Whisper.net (ggml) models in the new app for dictation and the workspace Adds the Whisper.net packages, a ggml catalog/locator/downloader in Platforms/Speech sharing the WPF install folder, a ref-counted shared model cache, WhisperNetProvider, compute-device handling (WPF TranscriptionComputeInterface plus a new Auto/CPU/GPU setting), settings migration for TranscriptionBackend=WhisperNet, workspace picker default, tests and the status doc. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/stage-7-dictation-status.md | 5 +- .../Dictation/DictationSettings.cs | 51 +++ src/PrimeDictate.Desktop/App.axaml.cs | 6 +- .../Dictation/DictationSettingsWindow.cs | 17 + .../Dictation/ModelDownloadPanel.cs | 4 +- src/PrimeDictate.Desktop/MainWindow.axaml.cs | 2 +- .../Dictation/DictationHost.cs | 9 +- .../PrimeDictate.Platforms.csproj | 6 + .../Speech/ModelDownloader.cs | 50 +++ .../Speech/SharedResourceCache.cs | 97 +++++ .../Speech/SpeechModels.cs | 65 +++- .../Speech/WhisperNetProvider.cs | 251 ++++++++++++ .../Speech/WhisperNetRuntime.cs | 79 ++++ .../TranscriptionWorkspaceService.cs | 42 +- .../ModelDownloaderTests.cs | 3 +- .../WhisperNetRealRunTests.cs | 89 +++++ .../WhisperNetTests.cs | 366 ++++++++++++++++++ 17 files changed, 1122 insertions(+), 20 deletions(-) create mode 100644 src/PrimeDictate.Platforms/Speech/SharedResourceCache.cs create mode 100644 src/PrimeDictate.Platforms/Speech/WhisperNetProvider.cs create mode 100644 src/PrimeDictate.Platforms/Speech/WhisperNetRuntime.cs create mode 100644 tests/PrimeDictate.Core.Tests/WhisperNetRealRunTests.cs create mode 100644 tests/PrimeDictate.Core.Tests/WhisperNetTests.cs diff --git a/docs/architecture/stage-7-dictation-status.md b/docs/architecture/stage-7-dictation-status.md index 5d2b181..f052ec5 100644 --- a/docs/architecture/stage-7-dictation-status.md +++ b/docs/architecture/stage-7-dictation-status.md @@ -2,7 +2,7 @@ Branch `claude/dictation-parity-4k0tl9`, following `stage-7-parity-plan.md`. "Ran" means executed on Linux in a container; "Compiled only" means it builds but has not been run on that OS. -**Status (2026-09-30):** merged into `main` (PR #3). Release 6.0.0 ships the WPF app; this app is not in the installers yet. Hotkey and wake-word dictation, the overlay and the settings window were first run on Windows the same day (see the Windows notes below). Before it can replace the WPF app it still needs Whisper.net models (the WPF backend Justin uses), an updater and its own installer. +**Status (2026-09-30):** merged into `main` (PR #3). Release 6.0.0 ships the WPF app; this app is not in the installers yet. Hotkey and wake-word dictation, the overlay and the settings window were first run on Windows the same day (see the Windows notes below). Whisper.net models (the WPF backend Justin uses) are now in this app (see the Whisper.net row). Before it can replace the WPF app it still needs an updater and its own installer. ## Done @@ -31,7 +31,8 @@ Branch `claude/dictation-parity-4k0tl9`, following `stage-7-parity-plan.md`. "Ra | Audio cues (synthesized WAV; Windows `PlaySound`, macOS `afplay`, Linux `paplay`/`aplay`) | `AudioCues.cs`, `ProcessAudioCuePlayer.cs` | WAV generation ran; playback not heard on any OS | | Tray icon: the WPF "voice wire" icon redrawn in Avalonia, five states | `TrayIconRenderer.cs` | Rendered under Xvfb and viewed (`--render-tray-icons `); not seen in a real tray | | `--show` / `--workspace` (accepted; the window is the default start) and new `--background` (tray only, Windows and macOS) | `App.axaml.cs` | Not run on Windows | -| Parakeet TDT v2/v3 and Moonshine v1/v2 (CPU) as dictation models, with download (same ids, folders, archives and file rules as the WPF catalogs) | `Platforms/Speech/SpeechModels.cs`, `SherpaWhisperProvider.cs` (Parakeet and Moonshine providers), `DictationHost.cs` | Ran, here on Linux with the real models: downloaded through the app's downloader, validated, and transcribed the sherpa test clips correctly (Moonshine tiny v2, Moonshine base v1, Parakeet v3), plus unit tests for discovery and settings. Not run: Windows, live dictation with these models, GPU. Moonshine QNN/NPU and Whisper.net are not ported. The wake listener still prefers Whisper tiny/base | +| Parakeet TDT v2/v3 and Moonshine v1/v2 (CPU) as dictation models, with download (same ids, folders, archives and file rules as the WPF catalogs) | `Platforms/Speech/SpeechModels.cs`, `SherpaWhisperProvider.cs` (Parakeet and Moonshine providers), `DictationHost.cs` | Ran, here on Linux with the real models: downloaded through the app's downloader, validated, and transcribed the sherpa test clips correctly (Moonshine tiny v2, Moonshine base v1, Parakeet v3), plus unit tests for discovery and settings. Not run: Windows, live dictation with these models, GPU. Moonshine QNN/NPU is not ported (Whisper.net is, next row). The wake listener still prefers Whisper tiny/base | +| Whisper.net (ggml) models for dictation and the transcription workspace: same ids (large-v3-turbo, large-v3, base.en, tiny.en), file names, `models/whisper.net/` folder, Hugging Face URLs and sizes as the WPF catalog, so an installed file is shared. Single-file download (temp file, size check, then move into place) in Settings and first run. `WhisperNetProvider` shares one loaded model per file (ref-counted), so dictation and the workspace load large-v3-turbo on the GPU once; one inference at a time per loaded model. English-only models are forced to "en"; other models use "en" unless a language or "auto" is given, as in WPF. Device: the WPF `TranscriptionComputeInterface` (Cpu/Gpu/Npu) is read from `settings.json`, plus a new Auto/CPU/GPU choice in Settings (`WhisperNetDevice`, applies after restart; `PRIMEDICTATE_WHISPERNET_DEVICE` overrides). Library order as WPF (CPU: Cpu, CpuNoAvx; GPU: Cuda, Vulkan, Cpu, CpuNoAvx; Auto and NPU: OpenVINO first only when the model has its OpenVINO files, else CPU), applied once before the first model loads; the loaded library is logged and a notice is shown when the GPU was wanted but not loaded. A WPF file with `TranscriptionBackend=WhisperNet` now selects `whisper-net:` (before it looked for a Whisper ONNX model of that name). The workspace picker starts on the dictation model, else large-v3-turbo. Wake word still prefers small sherpa Whisper models | Ran, on Windows 11 with an RTX 5070 and the real `ggml-large-v3-turbo.bin`: a 6.2 s synthetic clip through the provider loaded the **Cuda** runtime, 2.5 s for the first call including model load, 0.24 s warm; a second provider on the same file reused the loaded model (one load, two leases). CPU for comparison: 16 s per call. Unit tests: catalog, locator, download, id mapping, settings migration (WPF file byte-identical), library order, cache ref-counting. Publish: `win-x64` and `win-arm64` self-contained both build; the win-x64 output has `runtimes/{cuda,vulkan,openvino}/win-x64` and `runtimes/win-x64` whisper natives (Whisper.net also copies its linux and macOS natives; 344 MB of output in total). Not run: Vulkan, OpenVINO/NPU, the download through the app UI, dictation by hotkey with this model, the new settings control looked at | | ONNX models on CUDA (Auto/CPU/CUDA setting, ONNX Runtime GPU pack, checked fallback with a note, fp32 Whisper, fp16 Parakeet v2) | `Platforms/Speech/OnnxRuntimeDevice.cs`, `docs/architecture/onnx-cuda.md`, `scripts/Install-OnnxGpuRuntime.ps1` | Ran: unit tests with a fake GPU probe. Not run: any real GPU, the GPU DLLs, Windows | | Clean exit (tray Exit, `--quit`, one running instance, recording saved and dictation dropped on exit) | `Platforms/Startup/SingleInstance.cs`, `App.axaml.cs`, `Program.cs` | Ran: unit tests; on Linux under Xvfb `--quit` shut the app down, a second launch exited without a second copy, `--quit` with nothing running exits 0. Not run: on Windows, with a real recording | | Whisper model download (same 7 models, GitHub release URLs and install folders as WPF, so installs are shared; staged unpack, validated before it is moved into place; progress and cancel in Settings and first run) | `Platforms/Speech/WhisperModelDownloader.cs`, `ModelDownloadPanel.cs` | Ran: unit tests with a fake HTTP server and real `tar` (good archive, incomplete archive, HTTP error, broken archive, already installed). Confirmed from this container that the real tiny.en archive is reachable, 118,071,777 bytes as in the catalog, and holds both full and int8 files (the locator prefers int8). Not run: a full real download through the app, the progress UI, `tar` on Windows | diff --git a/src/PrimeDictate.Core/Dictation/DictationSettings.cs b/src/PrimeDictate.Core/Dictation/DictationSettings.cs index 16dbf9f..8aae668 100644 --- a/src/PrimeDictate.Core/Dictation/DictationSettings.cs +++ b/src/PrimeDictate.Core/Dictation/DictationSettings.cs @@ -43,6 +43,28 @@ public sealed class DictationSettings /// Where the ONNX speech models run: auto, cpu or cuda. Applies at the next start; PRIMEDICTATE_ONNX_DEVICE overrides it. public string OnnxDevice { get; set; } = "auto"; + /// + /// The WPF app's compute setting (Cpu, Gpu or Npu), read from its settings.json. Null when the file had none. + /// Only used for Whisper.net, and only until is chosen in this app. + /// + [JsonConverter(typeof(JsonStringEnumConverter))] + public LegacyComputeInterface? TranscriptionComputeInterface { get; set; } + + /// Where Whisper.net models run: auto, cpu or gpu. Null follows the WPF value, else Auto. Applies at the next start; PRIMEDICTATE_WHISPERNET_DEVICE overrides it. + public string? WhisperNetDevice { get; set; } + + /// The Whisper.net device this configuration asks for: an explicit choice, else the WPF compute setting, else Auto. + public WhisperNetDevicePreference ResolveWhisperNetDevice() => + this.WhisperNetDevice is { Length: > 0 } chosen + ? WhisperNetDevicePreferences.Parse(chosen) + : this.TranscriptionComputeInterface switch + { + LegacyComputeInterface.Cpu => WhisperNetDevicePreference.Cpu, + LegacyComputeInterface.Gpu => WhisperNetDevicePreference.Gpu, + LegacyComputeInterface.Npu => WhisperNetDevicePreference.Npu, + _ => WhisperNetDevicePreference.Auto + }; + /// Typing speed used for the time-saved figure. Valid range 20 to 120. public int BaselineTypingSpeedWpm { get; set; } = DictationStatsStore.DefaultBaselineWpm; @@ -134,6 +156,34 @@ public enum LegacyBackend QualcommQnn = 4 } +/// The WPF app's TranscriptionComputeInterface values. +public enum LegacyComputeInterface +{ + Cpu = 0, + Gpu = 1, + Npu = 2 +} + +/// Where Whisper.net runs. Npu is the WPF OpenVINO option and only takes effect when the model's OpenVINO files are installed. +public enum WhisperNetDevicePreference +{ + Auto = 0, + Cpu = 1, + Gpu = 2, + Npu = 3 +} + +public static class WhisperNetDevicePreferences +{ + public static WhisperNetDevicePreference Parse(string? text) => text?.Trim().ToLowerInvariant() switch + { + "cpu" => WhisperNetDevicePreference.Cpu, + "gpu" or "cuda" => WhisperNetDevicePreference.Gpu, + "npu" => WhisperNetDevicePreference.Npu, + _ => WhisperNetDevicePreference.Auto + }; +} + public sealed class HotkeyDto { public string KeyCode { get; set; } = ""; @@ -221,6 +271,7 @@ public static class LegacyBackendExtensions { LegacyBackend.Parakeet => "parakeet-onnx", LegacyBackend.Moonshine => "moonshine-onnx", + LegacyBackend.WhisperNet => "whisper-net", _ => "whisper-onnx" }; } diff --git a/src/PrimeDictate.Desktop/App.axaml.cs b/src/PrimeDictate.Desktop/App.axaml.cs index 8f80e0e..24aa58c 100644 --- a/src/PrimeDictate.Desktop/App.axaml.cs +++ b/src/PrimeDictate.Desktop/App.axaml.cs @@ -36,8 +36,10 @@ public override void OnFrameworkInitializationCompleted() // ONNX Runtime is one library per process: the GPU build must be in place before any speech model is created. if (!IsSmokeRun(desktop.Args)) { - var saved = new DictationSettingsStore(AppDataPaths.Default).Load().Settings.OnnxDevice; - OnnxRuntimeDevice.Configure(OnnxRuntimeDevice.Effective(saved)); + var savedSettings = new DictationSettingsStore(AppDataPaths.Default).Load().Settings; + OnnxRuntimeDevice.Configure(OnnxRuntimeDevice.Effective(savedSettings.OnnxDevice)); + // Whisper.net picks its native build once too, before the first model loads. + WhisperNetRuntime.Configure(WhisperNetRuntime.Effective(savedSettings)); } var window = new MainWindow(); diff --git a/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs b/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs index 0db3a99..6aa6ee4 100644 --- a/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs +++ b/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs @@ -25,6 +25,7 @@ public sealed class DictationSettingsWindow : Window private readonly NumericUpDown silence = new() { Minimum = 0, Maximum = 30, Increment = 1, FormatString = "0", Width = 120 }; private readonly CheckBox audioCues = Check("Play start and stop sounds"); private readonly LaunchAtLogin launch = new(); + private readonly ComboBox whisperNetDeviceBox = new() { ItemsSource = new[] { "Auto (CPU, or the NPU when its files are installed)", "CPU", "GPU (CUDA or Vulkan, CPU if it fails)" }, MinWidth = 260 }; private readonly ComboBox deviceBox = new() { ItemsSource = new[] { "Auto (CUDA if ready, else CPU)", "CPU", "CUDA" }, MinWidth = 260 }; private readonly CheckBox launchAtLogin = Check("Start PrimeDictate when I sign in (tray only)"); private readonly CheckBox sendEnter = Check("Coding mode: press Enter after typing"); @@ -62,6 +63,7 @@ public DictationSettingsWindow(DictationHost host, IAudioSource? audio) panel.Children.Add(new ModelDownloadPanel(host, this.RefreshModels)); panel.Children.Add(Row("Speech model device (applies after restart)", this.deviceBox)); panel.Children.Add(new TextBlock { Text = OnnxRuntimeDevice.Summary, Opacity = 0.7, TextWrapping = Avalonia.Media.TextWrapping.Wrap }); + panel.Children.Add(Row("Whisper.net (GGML) model device (applies after restart)", this.whisperNetDeviceBox)); panel.Children.Add(Row("Microphone", this.micBox)); panel.Children.Add(Row("Input gain", this.gain)); panel.Children.Add(Row("Auto-commit after silence (seconds, 0 = hotkey only)", this.silence)); @@ -234,6 +236,13 @@ private async Task LoadAsync() this.silence.Value = this.working.AutoCommitSilenceSeconds; this.audioCues.IsChecked = this.working.PlayAudioCues; this.deviceBox.SelectedIndex = (int)OnnxRuntimeDevice.ParsePreference(this.working.OnnxDevice); + // Npu (a WPF-only value) is shown as Auto, which still uses the NPU when the model has its OpenVINO files. + this.whisperNetDeviceBox.SelectedIndex = this.working.ResolveWhisperNetDevice() switch + { + WhisperNetDevicePreference.Cpu => 1, + WhisperNetDevicePreference.Gpu => 2, + _ => 0 + }; this.launchAtLogin.IsChecked = this.launch.IsEnabled; this.sendEnter.IsChecked = this.working.SendEnterAfterCommit; this.returnToStart.IsChecked = this.working.ReturnToStartTargetOnCommit; @@ -289,6 +298,14 @@ private void Save() s.AutoCommitSilenceSeconds = (int)(this.silence.Value ?? 3); s.PlayAudioCues = this.audioCues.IsChecked == true; s.OnnxDevice = ((OnnxDevicePreference)Math.Max(0, this.deviceBox.SelectedIndex)).ToString().ToLowerInvariant(); + // Only write a choice when the user made one, so an untouched WPF "Npu" is not silently turned into Auto. + var shownDevice = this.whisperNetDeviceBox.SelectedIndex switch { 1 => "cpu", 2 => "gpu", _ => "auto" }; + var keepsNpu = s.WhisperNetDevice is null && s.ResolveWhisperNetDevice() == WhisperNetDevicePreference.Npu && shownDevice == "auto"; + if (!keepsNpu) + { + s.WhisperNetDevice = shownDevice; + } + this.ApplyLaunchAtLogin(); s.SendEnterAfterCommit = this.sendEnter.IsChecked == true; s.ReturnToStartTargetOnCommit = this.returnToStart.IsChecked == true; diff --git a/src/PrimeDictate.Desktop/Dictation/ModelDownloadPanel.cs b/src/PrimeDictate.Desktop/Dictation/ModelDownloadPanel.cs index e6d65ce..53171c2 100644 --- a/src/PrimeDictate.Desktop/Dictation/ModelDownloadPanel.cs +++ b/src/PrimeDictate.Desktop/Dictation/ModelDownloadPanel.cs @@ -22,7 +22,7 @@ public ModelDownloadPanel(DictationHost host, Action installed) this.host = host; this.Spacing = 6; this.choice.ItemsSource = SpeechModelCatalog.Options - .Select(o => $"{o.Backend}: {o.DisplayName} ({SpeechModelCatalog.FormatSize(o.ApproximateBytes)}){(o.Recommended ? " - recommended" : string.Empty)}") + .Select(o => $"{(o.Backend == PrimeDictate.Core.Dictation.LegacyBackend.WhisperNet ? "Whisper.net" : o.Backend.ToString())}: {o.DisplayName} ({SpeechModelCatalog.FormatSize(o.ApproximateBytes)}){(o.Recommended ? " - recommended" : string.Empty)}") .ToList(); this.choice.SelectedIndex = 0; this.start.Click += async (_, _) => await this.RunAsync(installed); @@ -40,7 +40,7 @@ private async Task RunAsync(Action installed) var option = SpeechModelCatalog.Options[Math.Max(0, this.choice.SelectedIndex)]; this.running = new CancellationTokenSource(); this.SetBusy(true); - this.text.Text = $"Downloading {option.DisplayName} from GitHub..."; + this.text.Text = $"Downloading {option.DisplayName}..."; var progress = new Progress(p => Dispatcher.UIThread.Post(() => { this.bar.IsIndeterminate = p.Fraction is null; diff --git a/src/PrimeDictate.Desktop/MainWindow.axaml.cs b/src/PrimeDictate.Desktop/MainWindow.axaml.cs index 5f135ed..ea31719 100644 --- a/src/PrimeDictate.Desktop/MainWindow.axaml.cs +++ b/src/PrimeDictate.Desktop/MainWindow.axaml.cs @@ -186,7 +186,7 @@ private async Task InitializeWorkspaceAsync() await this.workspace.InitializeAsync(CancellationToken.None); this.models = this.workspace.AvailableModels(); this.ModelBox.ItemsSource = this.models.Select(m => m.DisplayName).ToList(); - this.ModelBox.SelectedIndex = this.models.Count > 0 ? 0 : -1; + this.ModelBox.SelectedIndex = TranscriptionWorkspaceService.DefaultModelIndex(this.models, this.workspace.DictationModelId()); await this.ReloadSessionsAsync(); var notes = new List(); if (this.models.Count == 0) diff --git a/src/PrimeDictate.Platforms/Dictation/DictationHost.cs b/src/PrimeDictate.Platforms/Dictation/DictationHost.cs index 61fe0dc..a04bae3 100644 --- a/src/PrimeDictate.Platforms/Dictation/DictationHost.cs +++ b/src/PrimeDictate.Platforms/Dictation/DictationHost.cs @@ -20,8 +20,8 @@ public sealed class DictationHost : IAsyncDisposable private readonly IHotkeySource? hotkeys; private readonly IForegroundTargetGuard guard; private readonly object providerSync = new(); - private SherpaOfflineProvider? provider; - private SherpaOfflineProvider? wakeProvider; + private ITranscriptionProvider? provider; + private ITranscriptionProvider? wakeProvider; private string? wakeProviderId; private string? providerId; private string? lastModelNotice; @@ -212,7 +212,7 @@ public async ValueTask DisposeAsync() await this.Wake.DisposeAsync().ConfigureAwait(false); } - SherpaOfflineProvider?[] old; + ITranscriptionProvider?[] old; lock (this.providerSync) { old = [this.provider, this.wakeProvider]; @@ -342,8 +342,9 @@ private void OnCommitted(DictationCommit commit) } } - private static SherpaOfflineProvider CreateProvider(InstalledSpeechModel model) => model.Backend switch + private static ITranscriptionProvider CreateProvider(InstalledSpeechModel model) => model.Backend switch { + LegacyBackend.WhisperNet => new WhisperNetProvider(model), LegacyBackend.Parakeet => new SherpaParakeetProvider(model), LegacyBackend.Moonshine => new SherpaMoonshineProvider(model), _ => new SherpaWhisperProvider(WhisperOnnxModelLocator.TryResolve(model.Directory, out var whisper) diff --git a/src/PrimeDictate.Platforms/PrimeDictate.Platforms.csproj b/src/PrimeDictate.Platforms/PrimeDictate.Platforms.csproj index c3d2157..7f8f9ea 100644 --- a/src/PrimeDictate.Platforms/PrimeDictate.Platforms.csproj +++ b/src/PrimeDictate.Platforms/PrimeDictate.Platforms.csproj @@ -15,6 +15,12 @@ + + + + + + diff --git a/src/PrimeDictate.Platforms/Speech/ModelDownloader.cs b/src/PrimeDictate.Platforms/Speech/ModelDownloader.cs index 8e1ce4a..2c00b5c 100644 --- a/src/PrimeDictate.Platforms/Speech/ModelDownloader.cs +++ b/src/PrimeDictate.Platforms/Speech/ModelDownloader.cs @@ -36,6 +36,11 @@ public async Task DownloadAsync( return destination; } + if (option.IsSingleFile) + { + return await this.DownloadFileAsync(option, destination, progress, cancellationToken).ConfigureAwait(false); + } + var archive = Path.Combine(whisperRoot, option.ArchiveFileName + ".download"); var scratch = Path.Combine(whisperRoot, option.InstallDirectoryName + ".extract"); try @@ -94,6 +99,51 @@ public async Task DownloadAsync( } } + /// A single-file model (Whisper.net ggml): written to {file}.download, checked, then moved into place. + private async Task DownloadFileAsync(ModelDownloadOption option, string destination, IProgress? progress, CancellationToken cancellationToken) + { + var temp = destination + ".download"; + try + { + DeleteQuietly(temp); + long downloaded = 0; + long? total = null; + using (var response = await this.client.GetAsync(this.UriFor?.Invoke(option) ?? option.DownloadUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false)) + { + response.EnsureSuccessStatusCode(); + total = response.Content.Headers.ContentLength; + await using var source = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + await using var file = new FileStream(temp, FileMode.Create, FileAccess.Write, FileShare.None, 128 * 1024, useAsync: true); + var buffer = new byte[128 * 1024]; + int read; + while ((read = await source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + await file.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false); + downloaded += read; + progress?.Report(new ModelDownloadProgress("download", downloaded, total)); + } + } + + if (total is > 0 && downloaded != total) + { + throw new IOException($"The download ended early ({downloaded:N0} of {total:N0} bytes)."); + } + + if (!SpeechModelLocator.IsValidGgml(option, temp)) + { + throw new InvalidOperationException($"The downloaded file is too small to be the {option.DisplayName} model ({downloaded:N0} bytes)."); + } + + File.Move(temp, destination, overwrite: true); + progress?.Report(new ModelDownloadProgress("ready", downloaded, downloaded)); + return destination; + } + finally + { + DeleteQuietly(temp); + } + } + private static void DeleteQuietly(string path) { try diff --git a/src/PrimeDictate.Platforms/Speech/SharedResourceCache.cs b/src/PrimeDictate.Platforms/Speech/SharedResourceCache.cs new file mode 100644 index 0000000..eaff11f --- /dev/null +++ b/src/PrimeDictate.Platforms/Speech/SharedResourceCache.cs @@ -0,0 +1,97 @@ +namespace PrimeDictate.Platforms.Speech; + +/// +/// Shares one expensive resource (a loaded speech model) between several users. The first for a key creates +/// it, later ones get the same instance, and it is disposed when the last lease is released. Creation runs under the cache lock, so +/// two users asking for the same model at once load it once. +/// +public sealed class SharedResourceCache where T : class, IDisposable +{ + private readonly object gate = new(); + private readonly Dictionary entries = new(StringComparer.OrdinalIgnoreCase); + + /// How many resources are currently loaded. + public int Count + { + get + { + lock (this.gate) + { + return this.entries.Count; + } + } + } + + /// How many leases are open on (0 when it is not loaded). + public int LeaseCount(string key) + { + lock (this.gate) + { + return this.entries.TryGetValue(key, out var entry) ? entry.Leases : 0; + } + } + + public Lease Acquire(string key, Func create) + { + lock (this.gate) + { + if (!this.entries.TryGetValue(key, out var entry)) + { + // A failed create leaves nothing behind, so the next caller tries again. + entry = new Entry(create()); + this.entries[key] = entry; + } + + entry.Leases++; + return new Lease(this, key, entry.Value); + } + } + + private void Release(string key) + { + T? dispose = null; + lock (this.gate) + { + if (this.entries.TryGetValue(key, out var entry) && --entry.Leases <= 0) + { + this.entries.Remove(key); + dispose = entry.Value; + } + } + + // Outside the lock: unloading a large model can take a moment and must not stall other users. + dispose?.Dispose(); + } + + private sealed class Entry(T value) + { + public T Value { get; } = value; + + public int Leases { get; set; } + } + + /// One user's hold on a shared resource. Disposing it more than once is harmless. + public sealed class Lease : IDisposable + { + private readonly SharedResourceCache owner; + private readonly string key; + private int released; + + internal Lease(SharedResourceCache owner, string key, T value) + { + this.owner = owner; + this.key = key; + this.Value = value; + } + + public T Value { get; } + + public void Dispose() + { + if (Interlocked.Exchange(ref this.released, 1) == 0) + { + this.owner.Release(this.key); + } + } + } +} diff --git a/src/PrimeDictate.Platforms/Speech/SpeechModels.cs b/src/PrimeDictate.Platforms/Speech/SpeechModels.cs index 358e176..54ce95b 100644 --- a/src/PrimeDictate.Platforms/Speech/SpeechModels.cs +++ b/src/PrimeDictate.Platforms/Speech/SpeechModels.cs @@ -3,6 +3,7 @@ namespace PrimeDictate.Platforms.Speech; /// An installed speech model of any family, as dictation sees it. +/// is the model folder, except for Whisper.net where it is the path of the ggml .bin file. public sealed record InstalledSpeechModel(LegacyBackend Backend, string Id, string DisplayName, string Directory, bool IsEnglishOnly) { /// Stable id such as whisper-onnx:base.en or parakeet-onnx:parakeet-tdt-0.6b-v3. @@ -19,13 +20,19 @@ public sealed record ModelDownloadOption( string Description, long ApproximateBytes, string InstallDirectoryName, - bool Recommended = false) + bool Recommended = false, + string? FileName = null) { + /// True for a model that is one file (Whisper.net ggml .bin) rather than an archive that unpacks to a folder. + public bool IsSingleFile => this.FileName is not null; + public string SubFolder => SpeechModelLocator.SubFolder(this.Backend); public string ArchiveFileName => $"{this.InstallDirectoryName}.tar.bz2"; - public Uri DownloadUri => new($"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/{this.ArchiveFileName}"); + public Uri DownloadUri => this.FileName is { } file + ? new($"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/{file}") + : new($"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/{this.ArchiveFileName}"); public string ModelId => $"{SpeechModelLocator.Prefix(this.Backend)}:{this.Id}"; } @@ -45,10 +52,21 @@ public static class SpeechModelCatalog new(LegacyBackend.Parakeet, "parakeet-tdt-0.6b-v2", "Parakeet TDT 0.6B v2", "The earlier English Parakeet release.", 690L * 1024 * 1024, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8"), new(LegacyBackend.Parakeet, "parakeet-tdt-0.6b-v2-fp16", "Parakeet TDT 0.6B v2 (fp16, for the GPU)", "English Parakeet in half precision. Made for CUDA; large and slower on the CPU.", 1_120_982_957, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-fp16"), new(LegacyBackend.Moonshine, "moonshine-tiny-v2-en", "Moonshine Tiny v2 (English)", "Very small and fast English model.", 83_886_080, "sherpa-onnx-moonshine-tiny-en-quantized-2026-02-27", true), - new(LegacyBackend.Moonshine, "moonshine-base-en", "Moonshine Base (English) v1", "The original Moonshine model.", 250_807_309, "sherpa-onnx-moonshine-base-en-int8") + new(LegacyBackend.Moonshine, "moonshine-base-en", "Moonshine Base (English) v1", "The original Moonshine model.", 250_807_309, "sherpa-onnx-moonshine-base-en-int8"), + GgmlWhisper("large-v3-turbo", "Large V3 Turbo (GGML, Whisper.net)", "The fastest Whisper V3 option. Runs on the GPU (CUDA or Vulkan) when available.", 1_618_426_976), + GgmlWhisper("large-v3", "Large V3 (GGML, Whisper.net)", "The highest-accuracy Whisper V3 model. Runs on the GPU when available.", 4_277_163_902, recommended: true), + GgmlWhisper("base.en", "Base English (GGML, Whisper.net)", "Standard English GGML model.", 147_964_352), + GgmlWhisper("tiny.en", "Tiny English (GGML, Whisper.net)", "Very small English GGML model.", 77_720_256) ]; - public static string FormatSize(long bytes) => $"{bytes / (1024d * 1024d):N0} MB"; + public static string FormatSize(long bytes) => bytes >= 1024L * 1024 * 1024 ? $"{bytes / (1024d * 1024d * 1024d):N1} GB" : $"{bytes / (1024d * 1024d):N0} MB"; + + /// + /// A Whisper.net ggml model: one ggml-{id}.bin under models/whisper.net/, downloaded from the whisper.cpp repository on Hugging Face. + /// Same ids, file names and sizes as the WPF catalog, so both apps share the installed file. + /// + private static ModelDownloadOption GgmlWhisper(string id, string name, string description, long bytes, bool recommended = false) => + new(LegacyBackend.WhisperNet, id, name, description, bytes, $"ggml-{id}.bin", recommended, FileName: $"ggml-{id}.bin"); private static ModelDownloadOption Whisper(string id, string name, string description, long bytes, bool recommended = false) => new(LegacyBackend.Whisper, id, name, description, bytes, $"sherpa-onnx-whisper-{id}", recommended); @@ -63,11 +81,24 @@ public static class SpeechModelLocator { LegacyBackend.Parakeet => "parakeet", LegacyBackend.Moonshine => "moonshine", + LegacyBackend.WhisperNet => "whisper.net", _ => "whisper" }; + /// + /// A ggml file is valid when it exists and is at least half the catalog size. The sizes are approximate (the published file + /// can differ by a few MB), so this only rejects an empty or cut-off file, not a slightly different build. + /// + public static bool IsValidGgml(ModelDownloadOption option, string path) + { + var info = new FileInfo(path); + return info.Exists && info.Length > 0 && info.Length >= option.ApproximateBytes / 2; + } + + /// For a directory-based model, the folder; for a single-file model, the file path. public static bool IsValid(ModelDownloadOption option, string directory) => option.Backend switch { + LegacyBackend.WhisperNet => IsValidGgml(option, directory), LegacyBackend.Parakeet => IsParakeet(directory), LegacyBackend.Moonshine => ResolveMoonshine(directory) is not null, _ => WhisperOnnxModelLocator.TryResolve(directory, out _) @@ -123,19 +154,41 @@ public static IReadOnlyList Discover(string modelsRoot) foreach (var option in SpeechModelCatalog.Options.Where(o => o.Backend != LegacyBackend.Whisper)) { - var dir = Path.Combine(modelsRoot, option.SubFolder, option.InstallDirectoryName); + var dir = InstallPath(modelsRoot, option); if (IsValid(option, dir)) { - found.Add(new InstalledSpeechModel(option.Backend, option.Id, option.DisplayName, Path.GetFullPath(dir), option.Id is "parakeet-tdt-0.6b-v2" or "parakeet-tdt-0.6b-v2-fp16" || option.Id.EndsWith("-en", StringComparison.Ordinal))); + var english = option.Backend == LegacyBackend.WhisperNet + ? option.Id.EndsWith(".en", StringComparison.Ordinal) + : option.Id is "parakeet-tdt-0.6b-v2" or "parakeet-tdt-0.6b-v2-fp16" || option.Id.EndsWith("-en", StringComparison.Ordinal); + found.Add(new InstalledSpeechModel(option.Backend, option.Id, option.DisplayName, Path.GetFullPath(dir), english)); } } return found; } + /// Where the model lives: its folder, or for a Whisper.net model the .bin file itself. public static string InstallPath(string modelsRoot, ModelDownloadOption option) => Path.Combine(modelsRoot, option.SubFolder, option.InstallDirectoryName); + /// The catalog option for a Whisper.net model id such as large-v3-turbo, or null. + public static ModelDownloadOption? FindWhisperNet(string? id) => + SpeechModelCatalog.Options.FirstOrDefault(o => o.Backend == LegacyBackend.WhisperNet && string.Equals(o.Id, id, StringComparison.OrdinalIgnoreCase)); + + /// OpenVINO encoder files that sit next to a ggml model (the WPF app downloads these for the Intel NPU). Null when either is missing. + public static string? WhisperNetOpenVinoEncoder(string modelPath) + { + var directory = Path.GetDirectoryName(modelPath); + if (string.IsNullOrWhiteSpace(directory)) + { + return null; + } + + var stem = Path.GetFileNameWithoutExtension(modelPath); + var xml = Path.Combine(directory, $"{stem}-encoder-openvino.xml"); + return File.Exists(xml) && File.Exists(Path.Combine(directory, $"{stem}-encoder-openvino.bin")) ? xml : null; + } + private static string? Find(string directory, params string[] names) => names.Select(n => Path.Combine(directory, n)).FirstOrDefault(File.Exists); } diff --git a/src/PrimeDictate.Platforms/Speech/WhisperNetProvider.cs b/src/PrimeDictate.Platforms/Speech/WhisperNetProvider.cs new file mode 100644 index 0000000..9f16013 --- /dev/null +++ b/src/PrimeDictate.Platforms/Speech/WhisperNetProvider.cs @@ -0,0 +1,251 @@ +using System.Diagnostics; +using PrimeDictate.Core.Diagnostics; +using PrimeDictate.Core.Dictation; +using PrimeDictate.Core.Providers; +using PrimeDictate.Core.Transcripts; +using Whisper.net; +using Whisper.net.LibraryLoader; + +namespace PrimeDictate.Platforms.Speech; + +/// +/// A loaded ggml model shared by every that uses the same file. Dictation and the transcription workspace +/// each create their own processor from it, but the weights (and the GPU memory) are loaded once. serializes the +/// processors, because whisper.cpp runs one inference at a time per context. +/// +public sealed class WhisperNetModel : IDisposable +{ + private readonly WhisperFactory factory; + + private WhisperNetModel(WhisperFactory factory, string? openVinoEncoder, string openVinoDevice) + { + this.factory = factory; + this.OpenVinoEncoder = openVinoEncoder; + this.OpenVinoDevice = openVinoDevice; + } + + public SemaphoreSlim Gate { get; } = new(1, 1); + + public string? OpenVinoEncoder { get; } + + public string OpenVinoDevice { get; } + + internal static WhisperNetModel Load(string modelPath) + { + var preference = WhisperNetRuntime.Preference; + var openVino = preference is WhisperNetDevicePreference.Auto or WhisperNetDevicePreference.Npu + ? SpeechModelLocator.WhisperNetOpenVinoEncoder(modelPath) + : null; + WhisperNetRuntime.ApplyOnce(openVino is not null); + var clock = Stopwatch.StartNew(); + var factory = WhisperFactory.FromPath(modelPath); + clock.Stop(); + var loaded = RuntimeOptions.LoadedLibrary; + AppLog.Event("whisper.net", $"Loaded {Path.GetFileName(modelPath)} in {clock.ElapsedMilliseconds} ms; requested {preference}, native runtime {loaded?.ToString() ?? "unknown"}."); + if (WhisperNetRuntime.GpuWantedButNotLoaded(preference, loaded)) + { + var message = $"GPU was requested for Whisper.net, but it loaded {loaded?.ToString() ?? "no GPU runtime"} instead. Check the NVIDIA driver (CUDA) or Vulkan support if GPU speed is expected."; + AppLog.Event("whisper.net", message); + WhisperNetRuntime.Notice?.Invoke(message); + } + + return new WhisperNetModel(factory, openVino, WhisperNetRuntime.OpenVinoDevice(preference)); + } + + public WhisperProcessor CreateProcessor(string? language) + { + var builder = this.factory.CreateBuilder(); + builder = language is null ? builder.WithLanguageDetection() : builder.WithLanguage(language); + if (this.OpenVinoEncoder is { } encoder && RuntimeOptions.LoadedLibrary == RuntimeLibrary.OpenVino) + { + var cache = Path.Combine(Path.GetDirectoryName(encoder) ?? ".", ".openvino-cache", this.OpenVinoDevice.ToLowerInvariant()); + Directory.CreateDirectory(cache); + builder = builder.WithOpenVinoEncoder(encoder, this.OpenVinoDevice, cache); + } + + return builder.Build(); + } + + public void Dispose() + { + this.factory.Dispose(); + this.Gate.Dispose(); + } +} + +/// +/// Whisper through Whisper.net (whisper.cpp, ggml models), on the GPU when the runtime allows. Loads on first use. Segments carry the +/// model's own timestamps. A processor is bound to one language, so it is rebuilt if the language changes. +/// +public sealed class WhisperNetProvider : ITranscriptionProvider +{ + private static readonly SharedResourceCache Models = new(); + private readonly SemaphoreSlim gate = new(1, 1); + private readonly InstalledSpeechModel model; + private readonly WhisperNetDevicePreference preference = WhisperNetRuntime.Preference; + private readonly Func.Lease> acquire; + private SharedResourceCache.Lease? lease; + private WhisperProcessor? processor; + private string? processorLanguage; + private bool disposed; + + public WhisperNetProvider(InstalledSpeechModel model) + { + this.model = model; + this.acquire = path => Models.Acquire(path, () => WhisperNetModel.Load(path)); + this.ModelId = model.ModelId; + this.Capabilities = new TranscriptionProviderCapabilities( + SupportsFiles: true, + LiveMode: LiveRecognitionMode.BufferedWindows, + Timing: TimingCapabilities.SegmentTimestamps, + CombinedDiarization: false, + MaxSpeakers: null, + MaxWindow: TimeSpan.FromSeconds(30), + Languages: model.IsEnglishOnly ? ["en"] : ["auto"], + RequiredSampleRate: 16_000); + } + + /// The loaded-model cache shared by all providers in the process. Exposed so the app can check that nothing stays loaded after exit. + public static SharedResourceCache SharedModels => Models; + + public string ModelId { get; } + + public TranscriptionProviderCapabilities Capabilities { get; } + + /// What ran, read after the first recognition. A GPU request that ended on the CPU says so. + public EffectiveRuntime Runtime + { + get + { + var loaded = RuntimeOptions.LoadedLibrary; + var effective = loaded?.ToString().ToLowerInvariant() ?? "whisper.net"; + var wantedGpu = this.preference == WhisperNetDevicePreference.Gpu; + var fellBack = loaded is not null && WhisperNetRuntime.GpuWantedButNotLoaded(this.preference, loaded); + var requested = wantedGpu && !fellBack ? effective : wantedGpu ? "cuda" : this.preference.ToString().ToLowerInvariant(); + return new EffectiveRuntime("whisper.net", "1.9.0", requested, effective, fellBack ? "The GPU runtime could not be loaded." : null); + } + } + + public ValueTask StartStreamingAsync(string? language, bool diarize, CancellationToken cancellationToken) => + throw new NotSupportedException("This model runs as Buffered Live; native streaming is not available."); + + /// + /// English-only models always get "en". Otherwise an explicit language wins ("en-US" becomes "en"), "auto" asks Whisper to detect it, + /// and no language means "en", as in the WPF app. + /// + public static string? ResolveLanguage(string? language, bool englishOnly) + { + if (englishOnly || string.IsNullOrWhiteSpace(language)) + { + return "en"; + } + + return string.Equals(language.Trim(), "auto", StringComparison.OrdinalIgnoreCase) + ? null + : language.Split('-')[0].Trim().ToLowerInvariant(); + } + + public async ValueTask> RecognizeWindowAsync(ReadOnlyMemory samples, string? language, CancellationToken cancellationToken) + { + if (samples.Length == 0) + { + return []; + } + + var resolved = ResolveLanguage(language, this.model.IsEnglishOnly); + var copy = samples.ToArray(); + await this.gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + ObjectDisposedException.ThrowIf(this.disposed, this); + return await Task.Run(() => this.RunAsync(copy, resolved, cancellationToken), cancellationToken).ConfigureAwait(false); + } + finally + { + this.gate.Release(); + } + } + + private async Task> RunAsync(float[] samples, string? language, CancellationToken cancellationToken) + { + this.lease ??= this.acquire(this.model.Directory); + var shared = this.lease.Value; + // One inference at a time per loaded model, across every provider that shares it. + await shared.Gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (this.processor is null || this.processorLanguage != language) + { + if (this.processor is not null) + { + await this.processor.DisposeAsync().ConfigureAwait(false); + this.processor = null; + } + + this.processor = shared.CreateProcessor(language); + this.processorLanguage = language; + } + + var segments = new List(); + await foreach (var segment in this.processor.ProcessAsync(samples, cancellationToken).ConfigureAwait(false)) + { + var text = segment.Text.Trim(); + // Whisper prints this marker for silence; it is not speech. + if (text.Length == 0 || string.Equals(text, "[BLANK_AUDIO]", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + // Whisper.net reports token probabilities, not a calibrated confidence score. + segments.Add(new RecognizedSegment(segment.Start, segment.End, text, null, null, null, TimingProvenance.Model)); + } + + return segments; + } + finally + { + shared.Gate.Release(); + } + } + + public async ValueTask DisposeAsync() + { + await this.gate.WaitAsync().ConfigureAwait(false); + try + { + if (this.disposed) + { + return; + } + + this.disposed = true; + if (this.processor is not null) + { + // The processor shares the native context, so it is released under the model's gate. + var shared = this.lease?.Value; + if (shared is not null) + { + await shared.Gate.WaitAsync().ConfigureAwait(false); + } + + try + { + await this.processor.DisposeAsync().ConfigureAwait(false); + } + finally + { + shared?.Gate.Release(); + } + + this.processor = null; + } + + this.lease?.Dispose(); + this.lease = null; + } + finally + { + this.gate.Release(); + } + } +} diff --git a/src/PrimeDictate.Platforms/Speech/WhisperNetRuntime.cs b/src/PrimeDictate.Platforms/Speech/WhisperNetRuntime.cs new file mode 100644 index 0000000..d036273 --- /dev/null +++ b/src/PrimeDictate.Platforms/Speech/WhisperNetRuntime.cs @@ -0,0 +1,79 @@ +using PrimeDictate.Core.Dictation; +using Whisper.net.LibraryLoader; + +namespace PrimeDictate.Platforms.Speech; + +/// +/// Chooses the native whisper.cpp build Whisper.net loads. is process-global and the native library cannot be +/// swapped once loaded, so the order is applied once, just before the first model is created; a changed setting applies at the next start. +/// Same orders as the WPF app: CPU -> Cpu, CpuNoAvx; GPU -> Cuda, Vulkan, Cpu, CpuNoAvx; Auto and NPU use OpenVINO first only when the +/// model has its OpenVINO encoder files, else the CPU. +/// +public static class WhisperNetRuntime +{ + public const string EnvironmentVariable = "PRIMEDICTATE_WHISPERNET_DEVICE"; + + private static readonly object Gate = new(); + private static bool applied; + + public static WhisperNetDevicePreference Preference { get; private set; } = WhisperNetDevicePreference.Auto; + + /// Told about a GPU that was wanted but not loaded. Never carries recognized text. + public static Action? Notice { get; set; } + + /// The environment variable wins over the saved setting, as for the ONNX device. + public static WhisperNetDevicePreference Effective(DictationSettings settings) => + Environment.GetEnvironmentVariable(EnvironmentVariable) is { Length: > 0 } fromEnvironment + ? WhisperNetDevicePreferences.Parse(fromEnvironment) + : settings.ResolveWhisperNetDevice(); + + /// Call once at startup before any Whisper.net model is created. Ignored once a model has been loaded. + public static void Configure(WhisperNetDevicePreference preference, Action? notice = null) + { + lock (Gate) + { + if (applied) + { + return; + } + + Preference = preference; + Notice = notice; + } + } + + public static IReadOnlyList LibraryOrder(WhisperNetDevicePreference preference, bool hasOpenVinoEncoder) => preference switch + { + WhisperNetDevicePreference.Cpu => [RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx], + WhisperNetDevicePreference.Gpu => [RuntimeLibrary.Cuda, RuntimeLibrary.Vulkan, RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx], + _ when hasOpenVinoEncoder => [RuntimeLibrary.OpenVino, RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx], + _ => [RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx] + }; + + /// The OpenVINO device name for the encoder: the NPU when asked for, else the CPU. + public static string OpenVinoDevice(WhisperNetDevicePreference preference) => + preference == WhisperNetDevicePreference.Npu ? "NPU" : "CPU"; + + /// The library Whisper.net actually loaded, or null before the first model. + public static string? LoadedLibraryName => RuntimeOptions.LoadedLibrary?.ToString(); + + /// True when the GPU was asked for and a GPU build is not what loaded. + public static bool GpuWantedButNotLoaded(WhisperNetDevicePreference preference, RuntimeLibrary? loaded) => + preference == WhisperNetDevicePreference.Gpu && loaded is not (RuntimeLibrary.Cuda or RuntimeLibrary.Vulkan); + + /// Sets the library order the first time a model is created. Later calls keep the first order (the native library is already loaded). + internal static void ApplyOnce(bool hasOpenVinoEncoder) + { + lock (Gate) + { + if (applied) + { + return; + } + + applied = true; + RuntimeOptions.LoadedLibrary = null; + RuntimeOptions.RuntimeLibraryOrder = [.. LibraryOrder(Preference, hasOpenVinoEncoder)]; + } + } +} diff --git a/src/PrimeDictate.Platforms/TranscriptionWorkspaceService.cs b/src/PrimeDictate.Platforms/TranscriptionWorkspaceService.cs index 8a8c587..a998060 100644 --- a/src/PrimeDictate.Platforms/TranscriptionWorkspaceService.cs +++ b/src/PrimeDictate.Platforms/TranscriptionWorkspaceService.cs @@ -181,7 +181,7 @@ public IReadOnlyList AvailableModels() choices.Add(new SpeechModelChoice(model.ModelId, model.DisplayName, model.IsEnglishOnly ? "en" : null, false, null)); } - foreach (var model in SpeechModelLocator.Discover(this.paths.ModelsDirectory).Where(m => m.Backend is LegacyBackend.Parakeet or LegacyBackend.Moonshine)) + foreach (var model in SpeechModelLocator.Discover(this.paths.ModelsDirectory).Where(m => m.Backend is LegacyBackend.Parakeet or LegacyBackend.Moonshine or LegacyBackend.WhisperNet)) { this.onnxModels[model.ModelId] = model; choices.Add(new SpeechModelChoice(model.ModelId, model.DisplayName, model.IsEnglishOnly ? "en" : null, false, null)); @@ -210,6 +210,39 @@ public IReadOnlyList AvailableModels() /// Any other choice is used as it is. Null when nothing but Nemotron is installed. /// /// "No live text": the meeting is only recorded, and the final pass transcribes it after Stop. + /// + /// The model the picker starts on: the one dictation is set to (so a user upgrading from the WPF app gets the same model), else + /// Whisper.net Large V3 Turbo when it is installed, else the first in the list. Returns -1 for an empty list. + /// + public static int DefaultModelIndex(IReadOnlyList available, string? dictationModelId) + { + if (available.Count == 0) + { + return -1; + } + + foreach (var wanted in new[] { dictationModelId, "whisper-net:large-v3-turbo" }) + { + if (wanted is null) + { + continue; + } + + for (var i = 0; i < available.Count; i++) + { + if (string.Equals(available[i].ModelId, wanted, StringComparison.Ordinal)) + { + return i; + } + } + } + + return 0; + } + + /// The model id dictation is set to (from the WPF file or this app's settings), or null. + public string? DictationModelId() => new DictationSettingsStore(this.paths).Load().Settings.ResolveModelId(); + public static SpeechModelChoice RecordOnly { get; } = new(RecordOnlyProvider.Id, "Record only", null, false, null); public static SpeechModelChoice? LiveDraftModel(SpeechModelChoice selected, IReadOnlyList available) @@ -317,7 +350,12 @@ private async Task ProviderAsync(SpeechModelChoice model } else if (this.onnxModels.TryGetValue(model.ModelId, out var onnx)) { - provider = onnx.Backend == LegacyBackend.Parakeet ? new SherpaParakeetProvider(onnx) : new SherpaMoonshineProvider(onnx); + provider = onnx.Backend switch + { + LegacyBackend.WhisperNet => new WhisperNetProvider(onnx), + LegacyBackend.Parakeet => new SherpaParakeetProvider(onnx), + _ => new SherpaMoonshineProvider(onnx) + }; } else if (this.nemotron.FirstOrDefault(n => $"nemotron:{n.Asr.Id}" == model.ModelId) is { } setup) { diff --git a/tests/PrimeDictate.Core.Tests/ModelDownloaderTests.cs b/tests/PrimeDictate.Core.Tests/ModelDownloaderTests.cs index a42c1ea..5ca94c0 100644 --- a/tests/PrimeDictate.Core.Tests/ModelDownloaderTests.cs +++ b/tests/PrimeDictate.Core.Tests/ModelDownloaderTests.cs @@ -96,7 +96,8 @@ public void Catalog_ids_match_the_wpf_folders_so_installs_are_shared() { Assert.Equal("sherpa-onnx-whisper-base.en", SpeechModelCatalog.Options.First(o => o.Id == "base.en").InstallDirectoryName); Assert.EndsWith("/asr-models/sherpa-onnx-whisper-base.en.tar.bz2", Tiny.DownloadUri.AbsoluteUri.Replace("tiny.en", "base.en")); - Assert.Equal(12, SpeechModelCatalog.Options.Count); + Assert.Equal(12, SpeechModelCatalog.Options.Count(o => o.Backend != PrimeDictate.Core.Dictation.LegacyBackend.WhisperNet)); + Assert.Equal(4, SpeechModelCatalog.Options.Count(o => o.Backend == PrimeDictate.Core.Dictation.LegacyBackend.WhisperNet)); Assert.Equal("parakeet-onnx:parakeet-tdt-0.6b-v3", SpeechModelCatalog.Options.First(o => o.Backend == PrimeDictate.Core.Dictation.LegacyBackend.Parakeet).ModelId); } } diff --git a/tests/PrimeDictate.Core.Tests/WhisperNetRealRunTests.cs b/tests/PrimeDictate.Core.Tests/WhisperNetRealRunTests.cs new file mode 100644 index 0000000..810e716 --- /dev/null +++ b/tests/PrimeDictate.Core.Tests/WhisperNetRealRunTests.cs @@ -0,0 +1,89 @@ +using System.Diagnostics; +using PrimeDictate.Core.Dictation; +using PrimeDictate.Platforms.Speech; +using Xunit.Abstractions; + +namespace PrimeDictate.Core.Tests; + +/// +/// Runs a real ggml model through . Does nothing unless PRIMEDICTATE_WHISPERNET_MODEL (a .bin path) +/// and PRIMEDICTATE_WHISPERNET_WAV (16 kHz 16-bit mono WAV) are set, so the default test run never needs a model. Optional +/// PRIMEDICTATE_WHISPERNET_DEVICE is cpu, gpu or auto (default gpu). Reports timings and the runtime, never the recognized text. +/// +public sealed class WhisperNetRealRunTests(ITestOutputHelper output) +{ + [Fact] + public async Task Transcribes_a_wav_with_a_real_model_when_asked_to() + { + var modelPath = Environment.GetEnvironmentVariable("PRIMEDICTATE_WHISPERNET_MODEL"); + var wavPath = Environment.GetEnvironmentVariable("PRIMEDICTATE_WHISPERNET_WAV"); + if (string.IsNullOrWhiteSpace(modelPath) || string.IsNullOrWhiteSpace(wavPath)) + { + return; + } + + var device = WhisperNetDevicePreferences.Parse(Environment.GetEnvironmentVariable(WhisperNetRuntime.EnvironmentVariable) ?? "gpu"); + var notices = new List(); + WhisperNetRuntime.Configure(device, notices.Add); + + var option = SpeechModelLocator.FindWhisperNet(Path.GetFileNameWithoutExtension(modelPath).Replace("ggml-", string.Empty)) + ?? throw new InvalidOperationException("The model file name is not a catalog model."); + var model = new InstalledSpeechModel(LegacyBackend.WhisperNet, option.Id, option.DisplayName, Path.GetFullPath(modelPath), option.Id.EndsWith(".en", StringComparison.Ordinal)); + var samples = ReadPcm16Mono16k(wavPath); + output.WriteLine($"Audio: {samples.Length / 16000d:N1} s; requested device: {device}"); + + await using var provider = new WhisperNetProvider(model); + var first = Stopwatch.StartNew(); + var segments = await provider.RecognizeWindowAsync(samples, "en", CancellationToken.None); + first.Stop(); + output.WriteLine($"First call (includes model load): {first.ElapsedMilliseconds} ms, {segments.Count} segment(s), {segments.Sum(s => s.Text.Length)} characters"); + output.WriteLine($"Runtime: {provider.Runtime.EffectiveBackend} (requested {provider.Runtime.RequestedBackend}); fallback: {provider.Runtime.FallbackReason ?? "none"}"); + + // A second provider on the same file must reuse the loaded model, not load it again. + await using var second = new WhisperNetProvider(model); + Assert.Equal(1, WhisperNetProvider.SharedModels.LeaseCount(model.Directory)); + var warm = Stopwatch.StartNew(); + var again = await second.RecognizeWindowAsync(samples, "en", CancellationToken.None); + warm.Stop(); + Assert.Equal(1, WhisperNetProvider.SharedModels.Count); + Assert.Equal(2, WhisperNetProvider.SharedModels.LeaseCount(model.Directory)); + output.WriteLine($"Second provider, same model (shared load): {warm.ElapsedMilliseconds} ms, {again.Count} segment(s)"); + + var timed = Stopwatch.StartNew(); + _ = await provider.RecognizeWindowAsync(samples, "en", CancellationToken.None); + timed.Stop(); + output.WriteLine($"Warm transcribe: {timed.ElapsedMilliseconds} ms"); + foreach (var notice in notices) + { + output.WriteLine("Notice: " + notice); + } + + Assert.NotEmpty(segments); + } + + private static float[] ReadPcm16Mono16k(string path) + { + var bytes = File.ReadAllBytes(path); + var offset = 12; + while (offset + 8 <= bytes.Length) + { + var id = System.Text.Encoding.ASCII.GetString(bytes, offset, 4); + var size = BitConverter.ToInt32(bytes, offset + 4); + if (id == "data") + { + var count = Math.Min(size, bytes.Length - offset - 8) / 2; + var samples = new float[count]; + for (var i = 0; i < count; i++) + { + samples[i] = BitConverter.ToInt16(bytes, offset + 8 + (i * 2)) / 32768f; + } + + return samples; + } + + offset += 8 + size + (size & 1); + } + + throw new InvalidDataException("No data chunk in the WAV."); + } +} diff --git a/tests/PrimeDictate.Core.Tests/WhisperNetTests.cs b/tests/PrimeDictate.Core.Tests/WhisperNetTests.cs new file mode 100644 index 0000000..cc47a3f --- /dev/null +++ b/tests/PrimeDictate.Core.Tests/WhisperNetTests.cs @@ -0,0 +1,366 @@ +using System.Net; +using PrimeDictate.Core.Dictation; +using PrimeDictate.Core.Storage; +using PrimeDictate.Platforms; +using PrimeDictate.Platforms.Speech; +using Whisper.net.LibraryLoader; + +namespace PrimeDictate.Core.Tests; + +public sealed class WhisperNetTests : IDisposable +{ + private readonly string root = Path.Combine(Path.GetTempPath(), "pd-wn-" + Guid.NewGuid().ToString("N")); + + public WhisperNetTests() => Directory.CreateDirectory(this.root); + + public void Dispose() => Directory.Delete(this.root, recursive: true); + + private static ModelDownloadOption Option(string id) => SpeechModelCatalog.Options.Single(o => o.Backend == LegacyBackend.WhisperNet && o.Id == id); + + private string Install(string id, long? size = null) + { + var option = Option(id); + var path = SpeechModelLocator.InstallPath(this.root, option); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + using var file = File.Create(path); + file.SetLength(size ?? option.ApproximateBytes); + return path; + } + + // --- catalog and locator --- + + [Fact] + public void Catalog_matches_the_wpf_ids_files_folder_and_urls() + { + var wn = SpeechModelCatalog.Options.Where(o => o.Backend == LegacyBackend.WhisperNet).ToList(); + Assert.Equal(["large-v3-turbo", "large-v3", "base.en", "tiny.en"], wn.Select(o => o.Id)); + Assert.All(wn, o => + { + Assert.Equal($"ggml-{o.Id}.bin", o.FileName); + Assert.True(o.IsSingleFile); + Assert.Equal("whisper.net", o.SubFolder); + Assert.Equal(new Uri($"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-{o.Id}.bin"), o.DownloadUri); + Assert.Equal($"whisper-net:{o.Id}", o.ModelId); + }); + Assert.Equal(1_618_426_976, Option("large-v3-turbo").ApproximateBytes); + Assert.Equal(4_277_163_902, Option("large-v3").ApproximateBytes); + Assert.Equal(147_964_352, Option("base.en").ApproximateBytes); + Assert.Equal(77_720_256, Option("tiny.en").ApproximateBytes); + } + + [Fact] + public void Archive_models_are_not_single_files() + { + Assert.All(SpeechModelCatalog.Options.Where(o => o.Backend != LegacyBackend.WhisperNet), o => Assert.False(o.IsSingleFile)); + } + + [Fact] + public void The_locator_finds_an_installed_model_at_the_shared_wpf_path() + { + var path = this.Install("large-v3-turbo", size: 1_624_555_275); + Assert.Equal(Path.Combine(this.root, "whisper.net", "ggml-large-v3-turbo.bin"), path); + + var found = Assert.Single(SpeechModelLocator.Discover(this.root), m => m.Backend == LegacyBackend.WhisperNet); + Assert.Equal("whisper-net:large-v3-turbo", found.ModelId); + Assert.Equal(Path.GetFullPath(path), found.Directory); + Assert.False(found.IsEnglishOnly); + } + + [Fact] + public void English_only_ggml_models_are_marked() + { + this.Install("base.en"); + Assert.True(Assert.Single(SpeechModelLocator.Discover(this.root)).IsEnglishOnly); + } + + [Fact] + public void An_empty_or_cut_off_file_is_not_an_installed_model() + { + this.Install("large-v3-turbo", size: 0); + this.Install("base.en", size: 1024); + Assert.Empty(SpeechModelLocator.Discover(this.root)); + } + + [Fact] + public void FindWhisperNet_is_case_insensitive_and_null_for_unknown_ids() + { + Assert.Equal("large-v3-turbo", SpeechModelLocator.FindWhisperNet("Large-V3-Turbo")?.Id); + Assert.Null(SpeechModelLocator.FindWhisperNet("medium")); + Assert.Null(SpeechModelLocator.FindWhisperNet(null)); + } + + [Fact] + public void OpenVino_encoder_needs_both_sidecar_files() + { + var model = this.Install("large-v3", size: 10); + Assert.Null(SpeechModelLocator.WhisperNetOpenVinoEncoder(model)); + var xml = Path.Combine(this.root, "whisper.net", "ggml-large-v3-encoder-openvino.xml"); + File.WriteAllText(xml, "x"); + Assert.Null(SpeechModelLocator.WhisperNetOpenVinoEncoder(model)); + File.WriteAllText(Path.Combine(this.root, "whisper.net", "ggml-large-v3-encoder-openvino.bin"), "x"); + Assert.Equal(xml, SpeechModelLocator.WhisperNetOpenVinoEncoder(model)); + } + + // --- download --- + + private sealed class Handler(byte[] body, HttpStatusCode status = HttpStatusCode.OK, long? declaredLength = null) : HttpMessageHandler + { + public List Requests { get; } = []; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + this.Requests.Add(request.RequestUri!); + var content = new ByteArrayContent(body); + if (declaredLength is { } length) + { + content.Headers.ContentLength = length; + } + + return Task.FromResult(new HttpResponseMessage(status) { Content = content }); + } + } + + // A tiny catalog entry so the test does not need a real 77 MB body: same rules, small size. + private static readonly ModelDownloadOption Small = Option("tiny.en") with { ApproximateBytes = 1000 }; + + [Fact] + public async Task A_model_file_is_written_to_a_temp_file_then_moved_into_place() + { + var handler = new Handler(new byte[1500]); + var stages = new List(); + var path = await new ModelDownloader(new HttpClient(handler)).DownloadAsync(Small, this.root, new Progress(p => stages.Add(p.Stage))); + + Assert.Equal(Path.Combine(this.root, "whisper.net", "ggml-tiny.en.bin"), path); + Assert.Equal(1500, new FileInfo(path).Length); + Assert.Equal(Small.DownloadUri, Assert.Single(handler.Requests)); + Assert.DoesNotContain(Directory.GetFiles(Path.Combine(this.root, "whisper.net")), f => f.EndsWith(".download")); + + // Already installed: nothing is fetched again. + var again = new Handler([]); + await new ModelDownloader(new HttpClient(again)).DownloadAsync(Small, this.root); + Assert.Empty(again.Requests); + } + + [Fact] + public async Task A_download_that_is_too_small_installs_nothing() + { + await Assert.ThrowsAsync(() => + new ModelDownloader(new HttpClient(new Handler(new byte[10]))).DownloadAsync(Small, this.root)); + Assert.Empty(Directory.GetFiles(Path.Combine(this.root, "whisper.net"))); + } + + [Fact] + public async Task A_cut_connection_installs_nothing() + { + await Assert.ThrowsAsync(() => + new ModelDownloader(new HttpClient(new Handler(new byte[1500], declaredLength: 4000))).DownloadAsync(Small, this.root)); + Assert.Empty(Directory.GetFiles(Path.Combine(this.root, "whisper.net"))); + } + + [Fact] + public async Task An_http_error_installs_nothing() + { + await Assert.ThrowsAsync(() => + new ModelDownloader(new HttpClient(new Handler([], HttpStatusCode.NotFound))).DownloadAsync(Small, this.root)); + Assert.Empty(Directory.GetFiles(Path.Combine(this.root, "whisper.net"))); + } + + // --- id mapping and settings migration --- + + [Fact] + public void A_whisper_net_selection_maps_to_the_whisper_net_model_id() + { + var settings = new DictationSettings { TranscriptionBackend = LegacyBackend.WhisperNet, SelectedModelId = "large-v3-turbo" }; + Assert.Equal("whisper-net:large-v3-turbo", settings.ResolveModelId()); + Assert.Equal("whisper-onnx:base.en", new DictationSettings { SelectedModelId = "base.en" }.ResolveModelId()); + } + + private const string WpfWhisperNetJson = """ + { + "FirstRunCompleted": true, + "TranscriptionBackend": "WhisperNet", + "SelectedModelId": "large-v3-turbo", + "TranscriptionComputeInterface": "Gpu" + } + """; + + [Fact] + public void The_wpf_whisper_net_setup_is_imported_without_touching_the_wpf_file() + { + var store = new DictationSettingsStore(new AppDataPaths(this.root)); + File.WriteAllText(store.WpfSettingsPath, WpfWhisperNetJson); + + var settings = store.Load().Settings; + Assert.Equal("whisper-net:large-v3-turbo", settings.ResolveModelId()); + Assert.Equal(LegacyComputeInterface.Gpu, settings.TranscriptionComputeInterface); + Assert.Equal(WhisperNetDevicePreference.Gpu, settings.ResolveWhisperNetDevice()); + + store.Save(settings); + Assert.Equal(WpfWhisperNetJson, File.ReadAllText(store.WpfSettingsPath)); + Assert.Equal(WhisperNetDevicePreference.Gpu, store.Load().Settings.ResolveWhisperNetDevice()); + } + + [Theory] + [InlineData("\"Cpu\"", WhisperNetDevicePreference.Cpu)] + [InlineData("\"Gpu\"", WhisperNetDevicePreference.Gpu)] + [InlineData("\"Npu\"", WhisperNetDevicePreference.Npu)] + [InlineData("1", WhisperNetDevicePreference.Gpu)] + public void The_wpf_compute_interface_maps_to_a_device(string json, WhisperNetDevicePreference expected) + { + var store = new DictationSettingsStore(new AppDataPaths(this.root)); + File.WriteAllText(store.WpfSettingsPath, $$"""{ "TranscriptionComputeInterface": {{json}} }"""); + Assert.Equal(expected, store.Load().Settings.ResolveWhisperNetDevice()); + } + + [Fact] + public void An_explicit_choice_in_this_app_beats_the_wpf_value_and_no_value_means_auto() + { + Assert.Equal(WhisperNetDevicePreference.Auto, new DictationSettings().ResolveWhisperNetDevice()); + var settings = new DictationSettings { TranscriptionComputeInterface = LegacyComputeInterface.Gpu, WhisperNetDevice = "cpu" }; + Assert.Equal(WhisperNetDevicePreference.Cpu, settings.ResolveWhisperNetDevice()); + Assert.Equal(WhisperNetDevicePreference.Gpu, WhisperNetDevicePreferences.Parse("CUDA")); + Assert.Equal(WhisperNetDevicePreference.Auto, WhisperNetDevicePreferences.Parse("nonsense")); + } + + // --- runtime order and language --- + + [Fact] + public void Runtime_library_order_matches_the_wpf_app() + { + Assert.Equal([RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx], WhisperNetRuntime.LibraryOrder(WhisperNetDevicePreference.Cpu, false)); + Assert.Equal([RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx], WhisperNetRuntime.LibraryOrder(WhisperNetDevicePreference.Cpu, true)); + Assert.Equal( + [RuntimeLibrary.Cuda, RuntimeLibrary.Vulkan, RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx], + WhisperNetRuntime.LibraryOrder(WhisperNetDevicePreference.Gpu, false)); + Assert.Equal([RuntimeLibrary.OpenVino, RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx], WhisperNetRuntime.LibraryOrder(WhisperNetDevicePreference.Auto, true)); + Assert.Equal([RuntimeLibrary.OpenVino, RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx], WhisperNetRuntime.LibraryOrder(WhisperNetDevicePreference.Npu, true)); + Assert.Equal([RuntimeLibrary.Cpu, RuntimeLibrary.CpuNoAvx], WhisperNetRuntime.LibraryOrder(WhisperNetDevicePreference.Auto, false)); + } + + [Fact] + public void A_gpu_request_that_loaded_the_cpu_is_reported() + { + Assert.True(WhisperNetRuntime.GpuWantedButNotLoaded(WhisperNetDevicePreference.Gpu, RuntimeLibrary.Cpu)); + Assert.True(WhisperNetRuntime.GpuWantedButNotLoaded(WhisperNetDevicePreference.Gpu, null)); + Assert.False(WhisperNetRuntime.GpuWantedButNotLoaded(WhisperNetDevicePreference.Gpu, RuntimeLibrary.Cuda)); + Assert.False(WhisperNetRuntime.GpuWantedButNotLoaded(WhisperNetDevicePreference.Gpu, RuntimeLibrary.Vulkan)); + Assert.False(WhisperNetRuntime.GpuWantedButNotLoaded(WhisperNetDevicePreference.Cpu, RuntimeLibrary.Cpu)); + } + + [Theory] + [InlineData(null, false, "en")] + [InlineData("en-US", false, "en")] + [InlineData("de-DE", false, "de")] + [InlineData("auto", false, null)] + [InlineData("de", true, "en")] + [InlineData("auto", true, "en")] + public void Language_is_resolved_like_the_wpf_app_with_auto_detection_on_request(string? input, bool englishOnly, string? expected) => + Assert.Equal(expected, WhisperNetProvider.ResolveLanguage(input, englishOnly)); + + // --- workspace picker default --- + + [Fact] + public void The_workspace_picker_starts_on_the_dictation_model_then_turbo_then_the_first() + { + SpeechModelChoice Choice(string id) => new(id, id, null, false, null); + var all = new[] { Choice("whisper-onnx:tiny.en"), Choice("whisper-net:base.en"), Choice("whisper-net:large-v3-turbo") }; + Assert.Equal(1, TranscriptionWorkspaceService.DefaultModelIndex(all, "whisper-net:base.en")); + Assert.Equal(2, TranscriptionWorkspaceService.DefaultModelIndex(all, null)); + Assert.Equal(2, TranscriptionWorkspaceService.DefaultModelIndex(all, "whisper-onnx:missing")); + Assert.Equal(0, TranscriptionWorkspaceService.DefaultModelIndex(all.Take(2).ToArray(), null)); + Assert.Equal(-1, TranscriptionWorkspaceService.DefaultModelIndex([], null)); + } + + // --- shared factory cache --- + + private sealed class FakeModel : IDisposable + { + public int Disposed { get; private set; } + + public void Dispose() => this.Disposed++; + } + + [Fact] + public void Two_users_of_one_model_share_a_single_load_and_the_last_release_disposes_it() + { + var cache = new SharedResourceCache(); + var created = 0; + FakeModel Create() { created++; return new FakeModel(); } + + var a = cache.Acquire("m", Create); + var b = cache.Acquire("M", Create); // paths compare case-insensitively on Windows + Assert.Same(a.Value, b.Value); + Assert.Equal(1, created); + Assert.Equal(2, cache.LeaseCount("m")); + + a.Dispose(); + Assert.Equal(0, a.Value.Disposed); + Assert.Equal(1, cache.LeaseCount("m")); + + b.Dispose(); + b.Dispose(); // releasing twice must not over-release + Assert.Equal(1, b.Value.Disposed); + Assert.Equal(0, cache.Count); + } + + [Fact] + public void A_double_dispose_of_one_lease_does_not_unload_a_model_another_user_holds() + { + var cache = new SharedResourceCache(); + var a = cache.Acquire("m", () => new FakeModel()); + var b = cache.Acquire("m", () => new FakeModel()); + a.Dispose(); + a.Dispose(); + Assert.Equal(0, b.Value.Disposed); + Assert.Equal(1, cache.LeaseCount("m")); + } + + [Fact] + public void Different_models_load_separately_and_a_released_model_loads_again_on_next_use() + { + var cache = new SharedResourceCache(); + var turbo = cache.Acquire("turbo", () => new FakeModel()); + var tiny = cache.Acquire("tiny", () => new FakeModel()); + Assert.NotSame(turbo.Value, tiny.Value); + Assert.Equal(2, cache.Count); + + var first = turbo.Value; + turbo.Dispose(); + Assert.Equal(1, first.Disposed); + using var again = cache.Acquire("turbo", () => new FakeModel()); + Assert.NotSame(first, again.Value); + tiny.Dispose(); + } + + [Fact] + public void A_failed_load_leaves_nothing_behind_and_can_be_retried() + { + var cache = new SharedResourceCache(); + Assert.Throws(() => cache.Acquire("m", () => throw new InvalidOperationException("no gpu"))); + Assert.Equal(0, cache.Count); + using var lease = cache.Acquire("m", () => new FakeModel()); + Assert.Equal(1, cache.LeaseCount("m")); + } + + [Fact] + public async Task Concurrent_users_load_the_model_once() + { + var cache = new SharedResourceCache(); + var created = 0; + var leases = await Task.WhenAll(Enumerable.Range(0, 16).Select(_ => Task.Run(() => cache.Acquire("m", () => + { + Interlocked.Increment(ref created); + Thread.Sleep(20); + return new FakeModel(); + })))); + Assert.Equal(1, created); + Assert.Equal(16, cache.LeaseCount("m")); + var model = leases[0].Value; + foreach (var lease in leases) + { + lease.Dispose(); + } + + Assert.Equal(1, model.Disposed); + } +} From 36a4db36e1bec33cf1f35f37df050c9aee6ef111 Mon Sep 17 00:00:00 2001 From: CakeRepository Date: Wed, 30 Sep 2026 15:19:01 -0500 Subject: [PATCH 03/15] One app: tray front door, header status, unified Settings Tray gets Record meeting (shared path with the Record button), Open PrimeDictate and Settings; the window header shows dictation state, model and hotkey with history, stats and settings buttons; transcription options move into TranscriptionPreferences and a Transcription and meetings section in Settings; transcription model defaults to the dictation model. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/stage-7-dictation-status.md | 12 +- .../Settings/TranscriptionPreferences.cs | 25 ++ src/PrimeDictate.Desktop/App.axaml.cs | 2 +- .../Dictation/DictationSettingsWindow.cs | 177 +++++++++- .../Dictation/DictationShell.cs | 92 +++++- src/PrimeDictate.Desktop/MainWindow.axaml | 26 +- src/PrimeDictate.Desktop/MainWindow.axaml.cs | 302 ++++++++++++++---- .../TranscriptionPrefsService.cs | 79 +++++ tests/PrimeDictate.Core.Tests/StorageTests.cs | 21 ++ 9 files changed, 652 insertions(+), 84 deletions(-) create mode 100644 src/PrimeDictate.Desktop/TranscriptionPrefsService.cs diff --git a/docs/architecture/stage-7-dictation-status.md b/docs/architecture/stage-7-dictation-status.md index 5d2b181..a36332d 100644 --- a/docs/architecture/stage-7-dictation-status.md +++ b/docs/architecture/stage-7-dictation-status.md @@ -58,9 +58,19 @@ Final-only typing, no clipboard, no live retyping into the target, Enter only af ## Not done yet -- Voice shell commands: matching is ported, running is not. Off until an explicit opt-in with a visible warning exists. Never in transcription mode. +- (Voice shell commands are now done, see the table.) - Updater, installers. - Wake word uses a Tiny/Base Whisper model when installed, else the dictation model (as WPF does). Moonshine/Whisper.net wake models are not ported. - The WPF app's focused-edit-control insertion (`WindowsFocusedTextControl`) and direct injection into the original target. - Compact-mode ripple animation and the copy/pin buttons of the WPF overlay, macOS `NSPanel`, Linux X11 hints. - First-run onboarding, launch at login, updater, packaging. + +## One app: tray, window and settings (6.1.0 work) + +| Piece | Where | Verified | +|---|---|---| +| Voice shell commands run in dictation only: matched in the final transcript (never in the live preview, never in transcription), the command string comes only from settings, Stop/Continue as in WPF (Continue types the words that remain), history row "Voice command: phrase". Imported WPF commands keep their On state; commands added here start Off; the Settings editor (On / Say / After / Command) shows a warning | `DictationController.cs`, `VoiceShellCommandRunner.cs` (interface), `ProcessVoiceShellCommandRunner.cs` (cmd.exe /d /c, /bin/sh -c), `DictationSettingsWindow.cs` | Ran: unit tests with a fake runner. Real process start and the editor not exercised by hand | +| Tray menu: Start dictation, Record meeting (flips to Stop meeting recording; same code as the Record button, so either can stop the other; opens the window), Open PrimeDictate, Dictation history, Stats, Settings, Exit. Tray icon and tooltip show recording during a meeting | `DictationShell.cs`, `MainWindow.axaml.cs` | Compiled and started on Windows; menu clicks and a real recording not exercised | +| Main window header: dictation state, model, hotkey, buttons for history, stats, settings | `MainWindow.axaml` | Started on Windows and looked at | +| Transcription model defaults to the dictation model when none was chosen for transcription | `MainWindow.axaml.cs` | Not exercised (no model in the test data dir) | +| One Settings window with a "Transcription & meetings" section (source, live text, speaker labels, boost), kept in sync with the main window; stored in `transcription-settings.json`, the old `meeting-live-text.txt` is carried over once | `TranscriptionPrefsService.cs`, `TranscriptionPreferences.cs` | Defaults and load: unit tests. Sync not exercised by hand | diff --git a/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs b/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs index 639ef24..bb467f9 100644 --- a/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs +++ b/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs @@ -33,6 +33,31 @@ public sealed record TranscriptionPreferences /// Copy imported files into app storage instead of referencing them in place. public bool CopyImportedMedia { get; init; } + + /// Meetings: off records only and transcribes after Stop; draft shows a fast model's text while people talk. + public string LiveTextMode { get; init; } = LiveTextModes.Off; + + /// Meetings: after Stop, re-read the recording, label the speakers and mark who talked over whom. + public bool SpeakerLabelsAfterStop { get; init; } = true; + + /// Raise quiet audio before it is transcribed. The saved recording is not changed. + public bool BoostQuietAudio { get; init; } = true; + + /// The recording source last used: microphone, system or meeting (both). Null until one was used. + public string? LastSource { get; init; } +} + +public static class LiveTextModes +{ + public const string Off = "off"; + public const string Draft = "draft"; +} + +public static class RecordingSources +{ + public const string Microphone = "microphone"; + public const string System = "system"; + public const string Meeting = "meeting"; } public sealed record PreferencesLoadResult(TranscriptionPreferences Preferences, bool IsReadOnly, string? Warning, string? BackupPath); diff --git a/src/PrimeDictate.Desktop/App.axaml.cs b/src/PrimeDictate.Desktop/App.axaml.cs index 8f80e0e..5bf9396 100644 --- a/src/PrimeDictate.Desktop/App.axaml.cs +++ b/src/PrimeDictate.Desktop/App.axaml.cs @@ -88,7 +88,7 @@ private static void RenderTrayIcons(string directory) private void StartDictation(IClassicDesktopStyleApplicationLifetime desktop, MainWindow window) { - var shell = new DictationShell(desktop, window.Workspace, () => ShowWorkspace(window)); + var shell = new DictationShell(desktop, window,() => ShowWorkspace(window)); shell.Start(this); shell.ExitRequested += () => _ = this.ExitAsync(desktop, window, shell); desktop.Exit += (_, _) => this.Cleanup(window, shell); diff --git a/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs b/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs index 0db3a99..120bb34 100644 --- a/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs +++ b/src/PrimeDictate.Desktop/Dictation/DictationSettingsWindow.cs @@ -6,6 +6,7 @@ using Avalonia.Threading; using PrimeDictate.Core.Dictation; using PrimeDictate.Core.Providers; +using PrimeDictate.Core.Settings; using PrimeDictate.Platforms.Dictation; using PrimeDictate.Platforms.Speech; using PrimeDictate.Platforms.Startup; @@ -18,6 +19,7 @@ public sealed class DictationSettingsWindow : Window { private readonly DictationHost host; private readonly IAudioSource? audio; + private readonly TranscriptionPrefsService prefs; private List shownModels = []; private readonly ComboBox modelBox = new() { MinWidth = 260 }; private readonly ComboBox micBox = new() { MinWidth = 260 }; @@ -42,21 +44,35 @@ public sealed class DictationSettingsWindow : Window private readonly TextBox replacements = new() { AcceptsReturn = true, MinHeight = 90, MaxHeight = 220, PlaceholderText = "spoken phrase => replacement (one per line)" }; private readonly Dictionary hotkeys = []; private readonly TextBlock status = new() { TextWrapping = Avalonia.Media.TextWrapping.Wrap, Opacity = 0.75, VerticalAlignment = VerticalAlignment.Center, Margin = new Thickness(0, 0, 12, 0) }; + private readonly ComboBox liveTextBox = new() + { + ItemsSource = new[] { "Off: record only, transcribe after Stop", "Fast draft while people talk, then final pass" }, + MinWidth = 260 + }; + private readonly ComboBox sourceBox = new() { ItemsSource = new[] { "Microphone", "System audio (speakers)", "Microphone + system audio" }, MinWidth = 260 }; + private readonly CheckBox speakerLabels = Check("Meetings: speaker labels after Stop (needs Nemotron and its diarizer)"); + private readonly CheckBox boostAudio = Check("Boost quiet audio for recognition (the saved recording is not changed)"); + private readonly StackPanel shellRows = new() { Spacing = 8 }; + private readonly List shellEditors = []; private DictationSettings working; private IReadOnlyList devices = []; - public DictationSettingsWindow(DictationHost host, IAudioSource? audio) + private sealed record ShellRow(Control Root, CheckBox On, TextBox Phrase, ComboBox After, TextBox Command); + + public DictationSettingsWindow(DictationHost host, IAudioSource? audio, TranscriptionPrefsService prefs) { this.host = host; this.audio = audio; + this.prefs = prefs; this.working = host.Settings; - this.Title = "PrimeDictate: dictation settings"; + this.Title = "PrimeDictate: Settings"; this.Width = 560; this.SizeToContent = SizeToContent.Height; this.WindowStartupLocation = WindowStartupLocation.CenterScreen; this.FitToScreen(this.Screens.Primary); var panel = new StackPanel { Margin = new Thickness(20, 20, 20, 10), Spacing = 10 }; + panel.Children.Add(Heading("Dictation")); panel.Children.Add(Row("Model", this.modelBox)); panel.Children.Add(new TextBlock { Text = "Download another model", FontWeight = Avalonia.Media.FontWeight.SemiBold }); panel.Children.Add(new ModelDownloadPanel(host, this.RefreshModels)); @@ -107,6 +123,39 @@ public DictationSettingsWindow(DictationHost host, IAudioSource? audio) panel.Children.Add(new TextBlock { Text = "Replacements" }); panel.Children.Add(this.replacements); + panel.Children.Add(Heading("Transcription & meetings")); + panel.Children.Add(new TextBlock + { + Text = "These apply to the recording and transcription side of PrimeDictate (the main window). The same options are in its left column; a change in either place shows in both.", + TextWrapping = Avalonia.Media.TextWrapping.Wrap, + Opacity = 0.7 + }); + panel.Children.Add(Row("Recording source (what Record meeting captures)", this.sourceBox)); + panel.Children.Add(Row("Live text during a meeting", this.liveTextBox)); + panel.Children.Add(this.speakerLabels); + panel.Children.Add(this.boostAudio); + + panel.Children.Add(Heading("Commands that run programs (dictation only)")); + panel.Children.Add(new Border + { + Background = new Avalonia.Media.SolidColorBrush(Avalonia.Media.Color.FromArgb(40, 255, 160, 0)), + BorderBrush = new Avalonia.Media.SolidColorBrush(Avalonia.Media.Color.FromRgb(230, 140, 0)), + BorderThickness = new Thickness(1), + CornerRadius = new CornerRadius(4), + Padding = new Thickness(10), + Child = new TextBlock + { + Text = "Warning: when you say one of these phrases while dictating, PrimeDictate runs the command on this computer with your account's permissions. " + + "Only add commands you understand and trust. The command is exactly what you type here; nothing you say is added to it. " + + "They never run from transcription or meetings. New commands start Off. Commands imported from the older PrimeDictate app keep the On/Off setting they had there.", + TextWrapping = Avalonia.Media.TextWrapping.Wrap + } + }); + panel.Children.Add(this.shellRows); + var addShell = new Button { Content = "Add command", HorizontalAlignment = HorizontalAlignment.Left }; + addShell.Click += (_, _) => this.AddShellRow(new VoiceShellCommand { Enabled = false }); + panel.Children.Add(addShell); + // Save and the status line stay in view under the form, which scrolls when it is taller than the screen. var save = new Button { Content = "Save", VerticalAlignment = VerticalAlignment.Center }; save.Click += (_, _) => this.Save(); @@ -124,6 +173,106 @@ public DictationSettingsWindow(DictationHost host, IAudioSource? audio) this.FitToScreen(this.Screens.ScreenFromWindow(this)); await this.LoadAsync(); }; + // A change made in the main window's left column shows here while this window is open. + void OnPrefsChanged(object? origin) + { + if (!ReferenceEquals(origin, this)) + { + this.LoadTranscriptionFields(); + } + } + + prefs.Changed += OnPrefsChanged; + this.Closed += (_, _) => prefs.Changed -= OnPrefsChanged; + } + + private static Control Heading(string text) => new TextBlock + { + Text = text, + FontSize = 16, + FontWeight = Avalonia.Media.FontWeight.SemiBold, + Margin = new Thickness(0, 14, 0, 0) + }; + + private void LoadTranscriptionFields() + { + var p = this.prefs.Current; + this.sourceBox.SelectedIndex = p.LastSource switch { RecordingSources.System => 1, RecordingSources.Meeting => 2, _ => 0 }; + this.liveTextBox.SelectedIndex = p.LiveTextMode == LiveTextModes.Draft ? 1 : 0; + this.speakerLabels.IsChecked = p.SpeakerLabelsAfterStop; + this.boostAudio.IsChecked = p.BoostQuietAudio; + } + + private void AddShellRow(VoiceShellCommand command) + { + var on = new CheckBox { Content = "On", IsChecked = command.Enabled, VerticalAlignment = VerticalAlignment.Center }; + var phrase = new TextBox { Text = command.Phrase, PlaceholderText = "When I say", VerticalAlignment = VerticalAlignment.Center }; + var after = new ComboBox + { + ItemsSource = new[] { "Then stop", "Then keep typing" }, + SelectedIndex = command.CompletionBehavior == VoiceShellCommandCompletionBehavior.Continue ? 1 : 0, + VerticalAlignment = VerticalAlignment.Center + }; + ToolTip.SetTip(after, "Then stop: nothing else from that dictation is typed. Then keep typing: the words after the phrase are typed as usual."); + var cmd = new TextBox { Text = command.Command, PlaceholderText = "Command to run (as you would type it in a terminal)" }; + var remove = new Button { Content = "Remove", VerticalAlignment = VerticalAlignment.Center }; + Grid.SetColumn(phrase, 1); + Grid.SetColumn(after, 2); + Grid.SetColumn(remove, 3); + var top = new Grid { ColumnDefinitions = new ColumnDefinitions("Auto,*,Auto,Auto"), Children = { on, phrase, after, remove } }; + top.ColumnDefinitions[1].MinWidth = 120; + foreach (var child in top.Children.Skip(1)) + { + ((Control)child).Margin = new Thickness(8, 0, 0, 0); + } + + var root = new Border + { + BorderBrush = new Avalonia.Media.SolidColorBrush(Avalonia.Media.Color.FromArgb(70, 128, 128, 128)), + BorderThickness = new Thickness(1), + CornerRadius = new CornerRadius(4), + Padding = new Thickness(8), + Child = new StackPanel { Spacing = 6, Children = { top, cmd } } + }; + var row = new ShellRow(root, on, phrase, after, cmd); + remove.Click += (_, _) => + { + this.shellEditors.Remove(row); + this.shellRows.Children.Remove(root); + }; + this.shellEditors.Add(row); + this.shellRows.Children.Add(root); + } + + /// Rows that are fully empty are dropped; a row with only a phrase or only a command is an error the user must fix. + private bool TryBuildShellCommands(out List commands) + { + commands = []; + foreach (var row in this.shellEditors) + { + var phrase = row.Phrase.Text?.Trim() ?? string.Empty; + var command = row.Command.Text?.Trim() ?? string.Empty; + if (phrase.Length == 0 && command.Length == 0) + { + continue; + } + + if (phrase.Length == 0 || command.Length == 0) + { + this.status.Text = "Each voice command needs both a phrase to say and a command to run."; + return false; + } + + commands.Add(new VoiceShellCommand + { + Enabled = row.On.IsChecked == true, + Phrase = phrase, + Command = command, + CompletionBehavior = row.After.SelectedIndex == 1 ? VoiceShellCommandCompletionBehavior.Continue : VoiceShellCommandCompletionBehavior.Stop + }); + } + + return true; } private static CheckBox Check(string text) => new() { Content = new TextBlock { Text = text, TextWrapping = Avalonia.Media.TextWrapping.Wrap } }; @@ -247,6 +396,14 @@ private async Task LoadAsync() this.overlayBox.SelectedIndex = (int)this.working.OverlayMode; this.sticky.IsChecked = this.working.IsOverlaySticky; this.replacements.Text = string.Join('\n', this.working.TranscriptReplacements.Select(r => $"{r.Find} => {r.Replace}")); + this.LoadTranscriptionFields(); + this.shellRows.Children.Clear(); + this.shellEditors.Clear(); + foreach (var command in this.working.VoiceShellCommands ?? []) + { + this.AddShellRow(command); + } + var notes = new List(); if (this.host.FocusGuardNotice is { } guardNotice) { @@ -276,6 +433,11 @@ private void ApplyLaunchAtLogin() private void Save() { + if (!this.TryBuildShellCommands(out var shellCommands)) + { + return; + } + var models = this.host.InstalledModels(); var s = this.working; if (this.modelBox.SelectedIndex is >= 0 and var mi && mi < models.Count) @@ -315,7 +477,18 @@ private void Save() .Where(parts => parts.Length == 2 && !string.IsNullOrWhiteSpace(parts[0])) .Select(parts => new ReplacementDto { Find = parts[0].Trim(), Replace = parts[1].Trim() }) .ToList(); + s.VoiceShellCommands = shellCommands; this.host.ApplySettings(s); + var source = this.sourceBox.SelectedIndex switch { 1 => RecordingSources.System, 2 => RecordingSources.Meeting, _ => RecordingSources.Microphone }; + this.prefs.Update( + p => p with + { + LastSource = source, + LiveTextMode = this.liveTextBox.SelectedIndex == 1 ? LiveTextModes.Draft : LiveTextModes.Off, + SpeakerLabelsAfterStop = this.speakerLabels.IsChecked == true, + BoostQuietAudio = this.boostAudio.IsChecked == true + }, + this); this.Close(); } } diff --git a/src/PrimeDictate.Desktop/Dictation/DictationShell.cs b/src/PrimeDictate.Desktop/Dictation/DictationShell.cs index 36aff18..7039580 100644 --- a/src/PrimeDictate.Desktop/Dictation/DictationShell.cs +++ b/src/PrimeDictate.Desktop/Dictation/DictationShell.cs @@ -20,12 +20,15 @@ namespace PrimeDictate.Desktop.Dictation; public sealed class DictationShell : IAsyncDisposable { private readonly DictationHost host; + private readonly MainWindow window; private readonly TranscriptionWorkspaceService workspace; private readonly IClassicDesktopStyleApplicationLifetime lifetime; private readonly Action showWorkspace; private readonly DictationOverlayWindow overlay = new(); private readonly TrayIcon tray = new(); private readonly NativeMenuItem toggleItem = new("Start dictation"); + private readonly NativeMenuItem recordItem = new("Record meeting"); + private string modelName = string.Empty; private readonly IAudioCuePlayer cues = new ProcessAudioCuePlayer(); private DictationState lastState = DictationState.Idle; private DateTime errorUntilUtc; @@ -33,10 +36,11 @@ public sealed class DictationShell : IAsyncDisposable private DictationHistoryWindow? historyWindow; private DictationStatsWindow? statsWindow; - public DictationShell(IClassicDesktopStyleApplicationLifetime lifetime, TranscriptionWorkspaceService workspace, Action showWorkspace, DictationHost? host = null) + public DictationShell(IClassicDesktopStyleApplicationLifetime lifetime, MainWindow window, Action showWorkspace, DictationHost? host = null) { this.lifetime = lifetime; - this.workspace = workspace; + this.window = window; + this.workspace = window.Workspace; this.showWorkspace = showWorkspace; this.host = host ?? new DictationHost(workspace.Paths, workspace.Microphone, workspace.MicrophoneSource, new SharpHookHotkeySource()); } @@ -48,6 +52,11 @@ public void Start(Application app) this.ConfigureOverlay(); this.overlay.AnchorMoved += (x, y) => this.host.RememberOverlayAnchor(x, y); this.BuildTray(app); + this.window.DictationHistoryRequested += this.ShowHistory; + this.window.DictationStatsRequested += this.ShowStats; + this.window.SettingsRequested += this.ShowSettings; + this.window.RecordingChanged += this.OnMeetingRecordingChanged; + this.RefreshHeader(refreshModel: true); this.host.Notice += message => Dispatcher.UIThread.Post(() => this.OnNotice(message)); this.host.HistoryRequested += () => Dispatcher.UIThread.Post(this.ShowHistory); if (this.host.Controller is { } controller) @@ -112,17 +121,32 @@ private void BuildTray(Application app) await Task.Run(controller.ToggleAsync); } }; - var workspaceItem = new NativeMenuItem("Open workspace"); + // Start or stop a live recording in the workspace through the same code the Record and Stop buttons use, so either + // one can end what the other started. + this.recordItem.Click += async (_, _) => + { + this.showWorkspace(); + if (this.window.IsRecording) + { + await this.window.StopMeetingRecordingAsync(); + } + else + { + await this.window.StartMeetingRecordingAsync(); + } + }; + var workspaceItem = new NativeMenuItem("Open PrimeDictate"); workspaceItem.Click += (_, _) => this.showWorkspace(); - var historyItem = new NativeMenuItem("Dictation history..."); + var historyItem = new NativeMenuItem("Dictation history…"); historyItem.Click += (_, _) => this.ShowHistory(); - var statsItem = new NativeMenuItem("Dictation stats..."); + var statsItem = new NativeMenuItem("Stats…"); statsItem.Click += (_, _) => this.ShowStats(); - var settingsItem = new NativeMenuItem("Dictation settings..."); + var settingsItem = new NativeMenuItem("Settings…"); settingsItem.Click += (_, _) => this.ShowSettings(); var quitItem = new NativeMenuItem("Exit PrimeDictate"); quitItem.Click += (_, _) => this.ExitRequested?.Invoke(); - this.tray.Menu = [this.toggleItem, workspaceItem, historyItem, statsItem, settingsItem, new NativeMenuItemSeparator(), quitItem]; + // The updater's "Check for updates…" item goes just before Exit. + this.tray.Menu = [this.toggleItem, this.recordItem, workspaceItem, historyItem, statsItem, settingsItem, new NativeMenuItemSeparator(), quitItem]; this.tray.Clicked += (_, _) => this.showWorkspace(); this.tray.Icon = TrayIconRenderer.Create(this.CurrentTrayState()); this.tray.ToolTipText = "PrimeDictate: ready"; @@ -131,7 +155,7 @@ private void BuildTray(Application app) private TrayVisualState CurrentTrayState() => this.errorUntilUtc > DateTime.UtcNow ? TrayVisualState.Error - : this.lastState == DictationState.Listening ? TrayVisualState.Recording + : this.lastState == DictationState.Listening || this.window.IsRecording ? TrayVisualState.Recording : this.lastState == DictationState.Processing ? TrayVisualState.Processing : this.host.Wake?.IsRunning == true ? TrayVisualState.AlwaysListening : TrayVisualState.Ready; @@ -155,13 +179,53 @@ private void OnState(DictationState state) this.lastState = state; this.overlay.SetState(state); this.RefreshTrayIcon(); - this.tray.ToolTipText = state switch + this.RefreshTooltip(); + this.toggleItem.Header = state == DictationState.Listening ? "Stop and type" : "Start dictation"; + this.RefreshHeader(refreshModel: false); + } + + private void OnMeetingRecordingChanged() + { + this.recordItem.Header = this.window.IsRecording ? "Stop meeting recording" : "Record meeting"; + this.RefreshTrayIcon(); + this.RefreshTooltip(); + } + + /// Dictation states win over a meeting recording in the tooltip, since dictation is the short, active thing. + private void RefreshTooltip() => + this.tray.ToolTipText = this.lastState switch { DictationState.Listening => "PrimeDictate: listening", DictationState.Processing => "PrimeDictate: transcribing", + _ when this.window.IsRecording => $"PrimeDictate: recording meeting ({this.window.RecordingSourceLabel})", _ => "PrimeDictate: ready" }; - this.toggleItem.Header = state == DictationState.Listening ? "Stop and type" : "Start dictation"; + + /// The main window's header line: dictation state, the model dictation uses and the hotkey. + private void RefreshHeader(bool refreshModel) + { + if (this.host.Controller is null) + { + this.window.SetDictationStatus("Dictation unavailable: " + (this.host.UnavailableReason ?? "no microphone")); + return; + } + + if (refreshModel) + { + var wanted = this.host.Settings.ResolveModelId(); + var installed = this.host.InstalledModels(); + this.modelName = installed.FirstOrDefault(m => m.ModelId == wanted)?.DisplayName + ?? (installed.Count > 0 ? installed[0].DisplayName : "no model installed"); + } + + var state = this.lastState switch + { + DictationState.Listening => "Listening", + DictationState.Processing => "Transcribing", + _ => "Ready" + }; + var hotkey = this.host.Settings.ToBindings()[HotkeyAction.ToggleDictation]; + this.window.SetDictationStatus($"Dictation: {state} · {this.modelName} · {hotkey}"); } private void OnCommitted(DictationCommit commit) @@ -213,8 +277,12 @@ private void ShowSettings() return; } - this.settingsWindow = new DictationSettingsWindow(this.host, this.workspace.MicrophoneSource); - this.settingsWindow.Closed += (_, _) => this.ConfigureOverlay(); + this.settingsWindow = new DictationSettingsWindow(this.host, this.workspace.MicrophoneSource, this.window.Prefs); + this.settingsWindow.Closed += (_, _) => + { + this.ConfigureOverlay(); + this.RefreshHeader(refreshModel: true); + }; this.settingsWindow.Show(); } diff --git a/src/PrimeDictate.Desktop/MainWindow.axaml b/src/PrimeDictate.Desktop/MainWindow.axaml index bbbd52e..e3f4f47 100644 --- a/src/PrimeDictate.Desktop/MainWindow.axaml +++ b/src/PrimeDictate.Desktop/MainWindow.axaml @@ -5,14 +5,28 @@ Title="PrimeDictate" Width="1100" Height="720" MinWidth="720" MinHeight="480"> - - - - - + + +