From cbec0d9608be6613a9493f6cbc75d2f0a8b02509 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 18:28:50 +0000 Subject: [PATCH 01/77] Retarget the Windows app to .NET 10 Behavior-preserving upgrade from net8.0-windows to net10.0-windows. Pins the SDK in global.json and points CI setup-dotnet at it. No code, dependency, installer identity, or settings format changes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GUzvd3VYM6n8xTXtBdDt5f --- .github/workflows/build.yml | 2 +- PrimeDictate.csproj | 2 +- README.md | 2 +- docs/architecture/stage-1-baseline.md | 57 +++++++++++++++++++++++++++ global.json | 7 ++++ installer/README.md | 2 +- 6 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 docs/architecture/stage-1-baseline.md create mode 100644 global.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 735b6f0..b991935 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 with: - dotnet-version: "8.0.x" + global-json-file: global.json - name: Compute version metadata id: version shell: pwsh diff --git a/PrimeDictate.csproj b/PrimeDictate.csproj index dbb0fa1..2934f97 100644 --- a/PrimeDictate.csproj +++ b/PrimeDictate.csproj @@ -3,7 +3,7 @@ 1.24.4 WinExe - net8.0-windows + net10.0-windows enable enable true diff --git a/README.md b/README.md index 94930d9..abd5ac5 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ Pro tips: ## Requirements - Windows is the primary supported platform. -- [.NET 8 SDK](https://dotnet.microsoft.com/download/dotnet/8.0) is required to build from source. +- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) is required to build from source (pinned in `global.json`). - Local transcription uses local model files only. - sherpa-onnx backends require extracted ONNX model folders and tokens. - Whisper.net uses GGML `.bin` files and optional hardware-specific sidecars. diff --git a/docs/architecture/stage-1-baseline.md b/docs/architecture/stage-1-baseline.md new file mode 100644 index 0000000..5b5ad92 --- /dev/null +++ b/docs/architecture/stage-1-baseline.md @@ -0,0 +1,57 @@ +# Stage 1: baseline audit and .NET 10 upgrade + +Scope: behavior-preserving retarget of the existing Windows app from `net8.0-windows` to +`net10.0-windows`. No application code, dependency versions, installer identity, or settings +format changed. + +## Baseline + +- Audited `main` at `87e0aaba0837206a8e30407f37922bc3b2208d81` (the brief's planning snapshot). + `main` had not moved when this stage started; `origin` had no other branches or open PRs. +- Verified in source (not by running the app): + - Backend IDs are the numeric enum `TranscriptionBackendKind` (`Whisper = 0`, `Parakeet = 1`, + `Moonshine = 2`, `WhisperNet = 3`, `QualcommQnn = 4`) and are persisted in `settings.json`. + - `ITranscriptionEngine.TranscribeAsync` returns `ValueTask`; structured segments are + discarded. + - The live preview loop re-transcribes a rolling snapshot capped at 12 s + (`LivePreviewMaxAudio` in `Program.cs`). + - `DictationController` routes final text through voice command matching and can run configured + shell commands (`VoiceShellCommandRunner`), which is why transcription mode needs its own + controller. + - The WiX payload harvests the publish folder with a wildcard (`installer/wix/shared/AppPayload.wxs`) + and uses `MajorUpgrade` with the unchanged `UpgradeCode`, so runtime file renames between .NET 8 + and .NET 10 do not require installer edits. +- `AGENTS.md` still describes the pre-5.x layout (no Whisper.net, Qualcomm, wake word, history, or + Ollama). It is being reconciled in a separate thread, so it is intentionally untouched here. + +## Changes + +| File | Change | +|------|--------| +| `PrimeDictate.csproj` | `net8.0-windows` to `net10.0-windows`. | +| `global.json` | New. Pins SDK `10.0.100` with `rollForward: latestFeature` (any stable 10.0.x SDK ≥ 10.0.100). | +| `.github/workflows/build.yml` | `setup-dotnet` reads `global.json` instead of `8.0.x`. | +| `README.md`, `installer/README.md` | Build prerequisite is the .NET 10 SDK. | + +## What was verified, and where + +All checks ran on Linux x64 with the .NET SDK 10.0.112 (Ubuntu package) and +`-p:EnableWindowsTargeting=true`. Windows-only code was compiled, never executed. + +| Check | Result | +|-------|--------| +| `dotnet build -c Release` on `main` (net8.0-windows) | 0 warnings, 0 errors | +| `dotnet build -c Release` after retarget (net10.0-windows) | 0 warnings, 0 errors | +| `dotnet publish -r win-x64` and `-r win-arm64`, self-contained, both TFMs | Succeeded | +| Publish-folder diff, .NET 8 vs .NET 10 | Only framework files differ (new WPF/runtime assemblies, versioned `mscordaccore_*`). All native payloads (`onnxruntime.dll`, `sherpa-onnx*.dll`, QNN `Qnn*.dll`/`libQnnHtp*`, Whisper.net runtimes) are present with identical file lists; `onnxruntime.dll` and `sherpa-onnx-c-api.dll` hashes match for both RIDs, so the ARM64 QNN copy target still wins. | +| Source scan for .NET 9/10 breaking-change hotspots (BinaryFormatter, clipboard `DataObject` payloads, obsoletion warnings) | None found; only `Clipboard.SetText` is used. | + +Not verified (needs a Windows machine or the Windows CI runner): + +- Launching the app, hotkey, WASAPI capture, overlay, text injection, tray, settings migration. +- Any model inference, including QNN on ARM64 hardware. +- WiX MSI build and an upgrade install over 5.1.0 (WiX build runs only on Windows). The PR's CI run + exercises build, publish, and MSI creation on `windows-latest`. + +Windows behavior tests are owned by the "Add a first test project" thread; they should target +`net10.0-windows` once this lands. diff --git a/global.json b/global.json new file mode 100644 index 0000000..d46d21e --- /dev/null +++ b/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "10.0.100", + "rollForward": "latestFeature", + "allowPrerelease": false + } +} diff --git a/installer/README.md b/installer/README.md index 00f5671..f5e84ab 100644 --- a/installer/README.md +++ b/installer/README.md @@ -22,7 +22,7 @@ After install, users open PrimeDictate and choose a model in first-run setup or ## Prerequisites (maintainer) -- [.NET 8 SDK](https://dotnet.microsoft.com/download/dotnet/8.0) +- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) (version pinned in `global.json`) ## Build From c45e83ee030dfddf82be8c51a6c6af3ac230258e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 18:38:23 +0000 Subject: [PATCH 02/77] docs: note package assemblies that switch to net10 builds Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GUzvd3VYM6n8xTXtBdDt5f --- docs/architecture/stage-1-baseline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/stage-1-baseline.md b/docs/architecture/stage-1-baseline.md index 5b5ad92..29982c5 100644 --- a/docs/architecture/stage-1-baseline.md +++ b/docs/architecture/stage-1-baseline.md @@ -43,7 +43,7 @@ All checks ran on Linux x64 with the .NET SDK 10.0.112 (Ubuntu package) and | `dotnet build -c Release` on `main` (net8.0-windows) | 0 warnings, 0 errors | | `dotnet build -c Release` after retarget (net10.0-windows) | 0 warnings, 0 errors | | `dotnet publish -r win-x64` and `-r win-arm64`, self-contained, both TFMs | Succeeded | -| Publish-folder diff, .NET 8 vs .NET 10 | Only framework files differ (new WPF/runtime assemblies, versioned `mscordaccore_*`). All native payloads (`onnxruntime.dll`, `sherpa-onnx*.dll`, QNN `Qnn*.dll`/`libQnnHtp*`, Whisper.net runtimes) are present with identical file lists; `onnxruntime.dll` and `sherpa-onnx-c-api.dll` hashes match for both RIDs, so the ARM64 QNN copy target still wins. | +| Publish-folder diff, .NET 8 vs .NET 10 | Same file list apart from framework files (new WPF/runtime assemblies, versioned `mscordaccore_*`, localized WPF resources). All native payloads (`onnxruntime.dll`, `sherpa-onnx-c-api.dll`, QNN `Qnn*.dll`/`libQnnHtp*`, Whisper.net native runtimes) are present; `onnxruntime.dll` and `sherpa-onnx-c-api.dll` hashes match for both RIDs, so the ARM64 QNN copy target still wins. Three managed package assemblies change content at the same package version because their packages ship a `net10.0` build that NuGet now selects instead of the `net8.0` one: `SharpHook.dll` (7.1.1), `Whisper.net.dll` (1.9.0), and `sherpa-onnx.dll` (1.13.0). These are the main runtime risk in this stage and need a smoke test on Windows (hotkey, text entry, Whisper.net and sherpa transcription). | | Source scan for .NET 9/10 breaking-change hotspots (BinaryFormatter, clipboard `DataObject` payloads, obsoletion warnings) | None found; only `Clipboard.SetText` is used. | Not verified (needs a Windows machine or the Windows CI runner): From 868c4f5f4fec30ab4340040ce27abc66b0dfed4e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 18:40:47 +0000 Subject: [PATCH 03/77] Add portable transcription core and wire it into the Windows app PrimeDictate.Core (net10.0) holds platform-neutral audio primitives, the versioned transcript model and events, session state, provider and model-registry contracts, microphone/model coordination, SQLite session storage, and separate transcription preferences with migrations. The Windows app references Core, keeps the dictation string path unchanged, and adds structured timed results (native timestamps for Whisper.net, approximate chunk timing elsewhere). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GUzvd3VYM6n8xTXtBdDt5f --- PrimeDictate.csproj | 6 + PrimeDictate.sln | 49 ++ TranscriptionEngines.cs | 108 +++- TranscriptionPreferencesSeed.cs | 39 ++ docs/architecture/stage-2-foundations.md | 72 +++ .../Audio/AudioConversion.cs | 82 +++ src/PrimeDictate.Core/Audio/AudioFormat.cs | 36 ++ src/PrimeDictate.Core/Audio/AudioFrame.cs | 83 +++ .../Audio/RecordedAudioTimeline.cs | 116 ++++ .../Audio/StreamingResampler.cs | 195 ++++++ .../Coordination/MicrophoneCoordinator.cs | 169 +++++ .../Coordination/ModelLeaseScheduler.cs | 128 ++++ .../Models/ModelContracts.cs | 82 +++ .../PrimeDictate.Core.csproj | 19 + .../Providers/AudioContracts.cs | 90 +++ .../Providers/RecognitionContracts.cs | 98 +++ .../Sessions/ITranscriptionSessionStore.cs | 48 ++ .../Sessions/TranscriptionSessionOptions.cs | 32 + .../TranscriptionSessionStateMachine.cs | 40 ++ .../Settings/TranscriptionPreferences.cs | 146 +++++ src/PrimeDictate.Core/Storage/AppDataPaths.cs | 56 ++ .../SqliteTranscriptionSessionStore.cs | 586 ++++++++++++++++++ .../Transcripts/TranscriptDocumentReducer.cs | 138 +++++ .../Transcripts/TranscriptModel.cs | 169 +++++ .../Transcripts/TranscriptionEvents.cs | 37 ++ tests/PrimeDictate.Core.Tests/AudioTests.cs | 178 ++++++ .../CoordinationTests.cs | 138 +++++ .../PrimeDictate.Core.Tests.csproj | 25 + tests/PrimeDictate.Core.Tests/StorageTests.cs | 283 +++++++++ tests/PrimeDictate.Core.Tests/TestData.cs | 40 ++ .../TranscriptTests.cs | 150 +++++ 31 files changed, 3430 insertions(+), 8 deletions(-) create mode 100644 TranscriptionPreferencesSeed.cs create mode 100644 docs/architecture/stage-2-foundations.md create mode 100644 src/PrimeDictate.Core/Audio/AudioConversion.cs create mode 100644 src/PrimeDictate.Core/Audio/AudioFormat.cs create mode 100644 src/PrimeDictate.Core/Audio/AudioFrame.cs create mode 100644 src/PrimeDictate.Core/Audio/RecordedAudioTimeline.cs create mode 100644 src/PrimeDictate.Core/Audio/StreamingResampler.cs create mode 100644 src/PrimeDictate.Core/Coordination/MicrophoneCoordinator.cs create mode 100644 src/PrimeDictate.Core/Coordination/ModelLeaseScheduler.cs create mode 100644 src/PrimeDictate.Core/Models/ModelContracts.cs create mode 100644 src/PrimeDictate.Core/PrimeDictate.Core.csproj create mode 100644 src/PrimeDictate.Core/Providers/AudioContracts.cs create mode 100644 src/PrimeDictate.Core/Providers/RecognitionContracts.cs create mode 100644 src/PrimeDictate.Core/Sessions/ITranscriptionSessionStore.cs create mode 100644 src/PrimeDictate.Core/Sessions/TranscriptionSessionOptions.cs create mode 100644 src/PrimeDictate.Core/Sessions/TranscriptionSessionStateMachine.cs create mode 100644 src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs create mode 100644 src/PrimeDictate.Core/Storage/AppDataPaths.cs create mode 100644 src/PrimeDictate.Core/Storage/SqliteTranscriptionSessionStore.cs create mode 100644 src/PrimeDictate.Core/Transcripts/TranscriptDocumentReducer.cs create mode 100644 src/PrimeDictate.Core/Transcripts/TranscriptModel.cs create mode 100644 src/PrimeDictate.Core/Transcripts/TranscriptionEvents.cs create mode 100644 tests/PrimeDictate.Core.Tests/AudioTests.cs create mode 100644 tests/PrimeDictate.Core.Tests/CoordinationTests.cs create mode 100644 tests/PrimeDictate.Core.Tests/PrimeDictate.Core.Tests.csproj create mode 100644 tests/PrimeDictate.Core.Tests/StorageTests.cs create mode 100644 tests/PrimeDictate.Core.Tests/TestData.cs create mode 100644 tests/PrimeDictate.Core.Tests/TranscriptTests.cs diff --git a/PrimeDictate.csproj b/PrimeDictate.csproj index 2934f97..6030df7 100644 --- a/PrimeDictate.csproj +++ b/PrimeDictate.csproj @@ -18,6 +18,8 @@ Copyright © FlowDevs.io git installer\wix\assets\PrimeDictate.ico + + $(DefaultItemExcludes);src\**;tests\** @@ -33,6 +35,10 @@ + + + + diff --git a/PrimeDictate.sln b/PrimeDictate.sln index 6762353..0f5f494 100644 --- a/PrimeDictate.sln +++ b/PrimeDictate.sln @@ -1,23 +1,72 @@ + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.5.2.0 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PrimeDictate", "PrimeDictate.csproj", "{6044D142-08F7-4FF1-CE4A-9D712AF04711}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PrimeDictate.Core", "src\PrimeDictate.Core\PrimeDictate.Core.csproj", "{4FB02F11-2FF8-413B-A890-1E2FAE755C92}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05-4346-4AA6-1389-037BE0695223}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PrimeDictate.Core.Tests", "tests\PrimeDictate.Core.Tests\PrimeDictate.Core.Tests.csproj", "{C0F52246-2A96-4B15-BEC6-B141EC8C61E8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Debug|x64.ActiveCfg = Debug|Any CPU + {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Debug|x64.Build.0 = Debug|Any CPU + {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Debug|x86.ActiveCfg = Debug|Any CPU + {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Debug|x86.Build.0 = Debug|Any CPU {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Release|Any CPU.ActiveCfg = Release|Any CPU {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Release|Any CPU.Build.0 = Release|Any CPU + {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Release|x64.ActiveCfg = Release|Any CPU + {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Release|x64.Build.0 = Release|Any CPU + {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Release|x86.ActiveCfg = Release|Any CPU + {6044D142-08F7-4FF1-CE4A-9D712AF04711}.Release|x86.Build.0 = Release|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Debug|x64.ActiveCfg = Debug|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Debug|x64.Build.0 = Debug|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Debug|x86.ActiveCfg = Debug|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Debug|x86.Build.0 = Debug|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Release|Any CPU.Build.0 = Release|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Release|x64.ActiveCfg = Release|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Release|x64.Build.0 = Release|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Release|x86.ActiveCfg = Release|Any CPU + {4FB02F11-2FF8-413B-A890-1E2FAE755C92}.Release|x86.Build.0 = Release|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Debug|x64.ActiveCfg = Debug|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Debug|x64.Build.0 = Debug|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Debug|x86.ActiveCfg = Debug|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Debug|x86.Build.0 = Debug|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Release|Any CPU.Build.0 = Release|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Release|x64.ActiveCfg = Release|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Release|x64.Build.0 = Release|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Release|x86.ActiveCfg = Release|Any CPU + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {4FB02F11-2FF8-413B-A890-1E2FAE755C92} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {C0F52246-2A96-4B15-BEC6-B141EC8C61E8} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9A418AF3-E9F2-40D5-91CA-B3E5B169DADD} EndGlobalSection diff --git a/TranscriptionEngines.cs b/TranscriptionEngines.cs index 3079509..f029675 100644 --- a/TranscriptionEngines.cs +++ b/TranscriptionEngines.cs @@ -1,6 +1,8 @@ using System.IO; using System.Runtime.InteropServices; using SherpaOnnx; +using PrimeDictate.Core.Providers; +using PrimeDictate.Core.Transcripts; using Whisper.net.LibraryLoader; namespace PrimeDictate; @@ -23,6 +25,27 @@ ValueTask TranscribeAsync( CancellationToken cancellationToken); } +/// +/// Engines that can return timed segments. Dictation keeps using the string path in +/// ; transcription mode uses this one. +/// +internal interface IStructuredTranscriptionEngine : ITranscriptionEngine +{ + ValueTask> TranscribeSegmentsAsync( + PcmAudioBuffer audio, + TranscriptionEngineConfiguration configuration, + CancellationToken cancellationToken); +} + +internal static class StructuredTranscription +{ + /// Wraps text from an engine without timing. No word timings or confidence are invented. + public static IReadOnlyList FromPlainText(string? text, TimeSpan windowDuration) => + string.IsNullOrWhiteSpace(text) + ? [] + : [new RecognizedSegment(TimeSpan.Zero, windowDuration, text.Trim(), Words: null, Confidence: null, SpeakerLabel: null, TimingProvenance.ApproximateChunk)]; +} + internal sealed class TranscriptionEngineHost : IAsyncDisposable { private readonly SemaphoreSlim engineGate = new(initialCount: 1, maxCount: 1); @@ -81,6 +104,39 @@ public async ValueTask TranscribeAsync( } } + /// + /// Structured recognition for transcription mode. Engines without native timing return + /// one segment spanning the buffer, marked as approximate chunk timing. + /// + public async ValueTask> TranscribeSegmentsAsync( + PcmAudioBuffer audio, + CancellationToken cancellationToken = default) + { + if (audio.IsEmpty) + { + return []; + } + + var configurationSnapshot = this.GetConfigurationSnapshot(); + await this.engineGate.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + var engine = await this.EnsureEngineAsync(configurationSnapshot).ConfigureAwait(false); + if (engine is IStructuredTranscriptionEngine structured) + { + return await structured.TranscribeSegmentsAsync(audio, configurationSnapshot, cancellationToken).ConfigureAwait(false); + } + + var text = await engine.TranscribeAsync(audio, configurationSnapshot, cancellationToken).ConfigureAwait(false); + return StructuredTranscription.FromPlainText(text, audio.Duration); + } + finally + { + this.engineGate.Release(); + } + } + public async ValueTask DisposeAsync() { await this.engineGate.WaitAsync().ConfigureAwait(false); @@ -421,7 +477,7 @@ protected override void ConfigureRecognizer(OfflineRecognizerConfig config, stri } } -internal sealed class WhisperNetTranscriptionEngine : ITranscriptionEngine +internal sealed class WhisperNetTranscriptionEngine : IStructuredTranscriptionEngine { private readonly SemaphoreSlim syncRoot = new SemaphoreSlim(1, 1); private Whisper.net.WhisperFactory? factory; @@ -438,11 +494,49 @@ public async ValueTask TranscribeAsync( PcmAudioBuffer audio, TranscriptionEngineConfiguration configuration, CancellationToken cancellationToken) + { + var segments = await this.ProcessSegmentsAsync(audio, configuration, cancellationToken).ConfigureAwait(false); + var sb = new System.Text.StringBuilder(); + foreach (var segment in segments) + { + if (!string.IsNullOrWhiteSpace(segment.Text)) + { + sb.Append(segment.Text); + } + } + + return sb.ToString().Trim(); + } + + public async ValueTask> TranscribeSegmentsAsync( + PcmAudioBuffer audio, + TranscriptionEngineConfiguration configuration, + CancellationToken cancellationToken) + { + var segments = await this.ProcessSegmentsAsync(audio, configuration, cancellationToken).ConfigureAwait(false); + return segments + .Where(segment => !string.IsNullOrWhiteSpace(segment.Text)) + .Select(segment => new RecognizedSegment( + segment.Start, + segment.End, + segment.Text.Trim(), + Words: null, + // Whisper.net reports token probabilities, not a calibrated confidence score. + Confidence: null, + SpeakerLabel: null, + TimingProvenance.Model)) + .ToList(); + } + + private async ValueTask> ProcessSegmentsAsync( + PcmAudioBuffer audio, + TranscriptionEngineConfiguration configuration, + CancellationToken cancellationToken) { var sampleCount = TranscriptionAudio.GetPcm16MonoSampleCount(audio, this.Name); if (sampleCount == 0) { - return string.Empty; + return []; } cancellationToken.ThrowIfCancellationRequested(); @@ -456,15 +550,13 @@ public async ValueTask TranscribeAsync( return await Task.Run(async () => { var processorInstance = this.EnsureProcessor(configuration); - var sb = new System.Text.StringBuilder(); + var segments = new List(); await foreach (var segment in processorInstance.ProcessAsync(samples, cancellationToken).ConfigureAwait(false)) { - if (!string.IsNullOrWhiteSpace(segment.Text)) - { - sb.Append(segment.Text); - } + segments.Add(segment); } - return sb.ToString().Trim(); + + return segments; }, cancellationToken).ConfigureAwait(false); } finally diff --git a/TranscriptionPreferencesSeed.cs b/TranscriptionPreferencesSeed.cs new file mode 100644 index 0000000..c54f2c6 --- /dev/null +++ b/TranscriptionPreferencesSeed.cs @@ -0,0 +1,39 @@ +using PrimeDictate.Core.Settings; + +namespace PrimeDictate; + +/// +/// First-run transcription defaults copied from the dictation settings. Dictation's +/// settings.json is only read here, never rewritten. +/// +internal static class TranscriptionPreferencesSeed +{ + public static TranscriptionPreferences FromAppSettings(AppSettings settings) => new() + { + AsrModelId = LegacyModelIds.From(settings.TranscriptionBackend, settings.SelectedModelId), + RequestedBackend = settings.TranscriptionComputeInterface switch + { + TranscriptionComputeInterface.Gpu => "gpu", + TranscriptionComputeInterface.Npu => "npu", + _ => "cpu" + }, + InputDeviceId = settings.SelectedInputDeviceId + }; +} + +/// Stable model IDs for the existing backends, used until the model registry owns them. +internal static class LegacyModelIds +{ + public static string From(TranscriptionBackendKind backend, string? selectedModelId) + { + var family = backend switch + { + TranscriptionBackendKind.Parakeet => "parakeet-onnx", + TranscriptionBackendKind.Moonshine => "moonshine", + TranscriptionBackendKind.WhisperNet => "whisper-net", + TranscriptionBackendKind.QualcommQnn => "qualcomm-qnn", + _ => "whisper-onnx" + }; + return string.IsNullOrWhiteSpace(selectedModelId) ? $"{family}:auto" : $"{family}:{selectedModelId.Trim()}"; + } +} diff --git a/docs/architecture/stage-2-foundations.md b/docs/architecture/stage-2-foundations.md new file mode 100644 index 0000000..a7017bc --- /dev/null +++ b/docs/architecture/stage-2-foundations.md @@ -0,0 +1,72 @@ +# Stage 2: shared foundations + +Adds `src/PrimeDictate.Core` (`net10.0`, no WPF, Windows Forms, WASAPI, or registry +dependencies) and wires the existing Windows app to it without changing dictation behavior. + +## What is in Core + +| Area | Types | Notes | +|------|-------|-------| +| Audio | `AudioFormat`, `AudioFrame`, `AudioConversion`, `StreamingResampler`, `RecordedAudioTimeline` | Frames always own their samples (copied out of callback/pooled memory). Offsets are 64-bit per-channel sample counts, never UI time. The resampler is a windowed-sinc polyphase filter whose output does not depend on chunk boundaries and maps offsets back to the source exactly. The timeline maps the continuous recorded-audio clock to wall-clock capture intervals, so Pause records nothing and Resume continues the offset. | +| Transcript model | `TranscriptDocument`, `TranscriptSegment`, `WordTiming`, `SpeakerAttribution`, `TranscriptSpeaker`, `RecognitionRunInfo`, `MediaMetadata`, `AudioReference` | Schema-versioned. Raw recognizer text and user edits are separate fields. Timing carries provenance (`Model`, `Alignment`, `ApproximateChunk`). Confidence and word timings are nullable. Each rerun gets a new `ResultVersion`; older results and their edits are kept. Speakers are an ID-to-name map. | +| Events | `SessionStarted`, `SegmentUpserted`, `SegmentFinalized`, `SpeakerUpdated`, `ProgressChanged`, `SessionCompleted`, `SessionFailed` | App-owned names. `TranscriptDocumentReducer` applies them idempotently by segment ID and revision; stale or duplicate updates are ignored, and provider updates never overwrite user edits or speaker names. | +| Sessions | `TranscriptionSessionStateMachine`, `TranscriptionSessionOptions`, `ITranscriptionSessionStore` | Options are an immutable snapshot taken at session start. Pause, Finalizing, Interrupted, and rerun-after-Completed are explicit states. | +| Provider contracts | `IAudioSource`/`IAudioCaptureLease`, `IAudioDecoder`, `IAudioPlayback`, `ITranscriptionProvider`, `IStreamingRecognitionSession`, `IDiarizationProvider` | Capabilities distinguish native streaming from buffered windows, report timing support, and report the effective runtime/backend separately from the requested one. | +| Models | `ModelDescriptor`, `ModelStatus`, `ModelAvailability`, `IModelRegistry` | Speech models and speaker-detection models are different `ModelKind`s. Availability separates Installed (files present) from Ready (a probe actually ran). | +| Coordination | `MicrophoneCoordinator`, `ModelLeaseScheduler` | One microphone owner at a time. Acquiring suspends active background consumers (wake word, idle dictation) and releasing resumes exactly those; a busy dictation blocks acquisition instead of being cut off. Model leases serialize non-thread-safe engines per model and grant live work ahead of queued file windows. | +| Persistence | `SqliteTranscriptionSessionStore`, `AppDataPaths`, `TranscriptionPreferencesStore` | See below. | + +## Persistence decisions + +- **SQLite metadata, media on disk.** `%LocalAppData%\PrimeDictate\transcription\sessions.db` + (Windows; `~/Library/Application Support/PrimeDictate` on macOS, `~/.local/share/PrimeDictate` + on Linux). WAL journal, `synchronous=FULL`, schema version in `PRAGMA user_version`. A database + from a newer schema is refused and left untouched. +- **Checkpoints hold final content only.** Provisional segments are never written; after a crash + they are recomputed from retained audio. On startup, sessions left Running/Paused/Finalizing are + marked Interrupted and offered for recovery. +- **Deletion boundaries.** Deleting a session removes its rows, its media directory, and owned audio + that lies under the media root. Referenced originals are never deleted, and an "owned" path + outside the media root is refused rather than deleted. `DeleteOwnedAudioAsync` implements + transcript-only retention. +- **Permissions.** On macOS/Linux the data directories are created `0700` and the database and + settings files `0600`. On Windows they inherit the per-user `LocalAppData` ACL. No encryption at + rest is claimed. +- **Separate transcription settings.** `transcription-settings.json` sits next to dictation's + `settings.json`, which transcription mode only reads (to seed first-run defaults via + `TranscriptionPreferencesSeed`) and never rewrites. Migrations are versioned and idempotent, back + up the file before rewriting it, keep unreadable files as a backup, and load files from a newer + version read-only. +- Dictation history (`TranscriptionHistoryStore`) is unchanged and remains separate from sessions. + +## Changes to the Windows app + +- `PrimeDictate.csproj` references Core and excludes `src/**` and `tests/**` from its default globs + (the project sits at the repository root). +- `IStructuredTranscriptionEngine` and `TranscriptionEngineHost.TranscribeSegmentsAsync` add timed + results beside the unchanged string path. Whisper.net returns its native segment timestamps + (`TimingProvenance.Model`); its token probabilities are not exposed as confidence. Other engines + are wrapped as one `ApproximateChunk` segment. Dictation still calls `TranscribeAsync`, whose + output is built exactly as before. +- `LegacyModelIds` gives the existing backends stable IDs (`whisper-onnx:`, `parakeet-onnx:`, + `moonshine:`, `whisper-net:`, `qualcomm-qnn:`) for the model registry to adopt. + +## Verification + +Run on Linux x64, .NET SDK 10.0.112: + +- `dotnet test tests/PrimeDictate.Core.Tests`: 51 tests pass. They cover resampler length, + chunk-invariance, passband gain, alias rejection, and impulse timing; frame ownership; PCM16 round + trip; pause/resume offsets and wall-clock mapping; reducer idempotency, stale revisions, edit + preservation, speaker renaming, rerun versions, repeated legitimate phrases, and invalid + intervals; state transitions; microphone suspend/restore/rollback; model-lease priority and + cancellation; SQLite round trip (Unicode, words, speakers, media), partials excluded, stale + revisions, interruption marking, deletion boundaries, transcript-only retention, paging, newer + schema refusal; and preferences seeding, migration and backup, newer-version read-only, and + corrupt-file recovery. +- Solution build (`EnableWindowsTargeting`): 0 warnings. Self-contained `win-x64`/`win-arm64` + publish includes `PrimeDictate.Core.dll` and the matching native `e_sqlite3.dll`. + +Not verified: anything executing on Windows, macOS, or on a Linux desktop. The structured Whisper.net +path compiles but has not run. No UI uses Core yet, and neither the model registry implementation +nor capture, decoder, or playback adapters exist yet (stages 3 to 5). diff --git a/src/PrimeDictate.Core/Audio/AudioConversion.cs b/src/PrimeDictate.Core/Audio/AudioConversion.cs new file mode 100644 index 0000000..c8e719b --- /dev/null +++ b/src/PrimeDictate.Core/Audio/AudioConversion.cs @@ -0,0 +1,82 @@ +using System.Buffers.Binary; + +namespace PrimeDictate.Core.Audio; + +/// +/// Sample-format boundaries. Engines that need PCM16 convert here, once, at the point of use. +/// +public static class AudioConversion +{ + public static void Pcm16ToFloat(ReadOnlySpan pcm16, Span destination) + { + var count = pcm16.Length / 2; + if (destination.Length < count) + { + throw new ArgumentException("Destination is too small.", nameof(destination)); + } + + for (var i = 0; i < count; i++) + { + destination[i] = BinaryPrimitives.ReadInt16LittleEndian(pcm16.Slice(i * 2, 2)) / 32768f; + } + } + + public static void FloatToPcm16(ReadOnlySpan samples, Span destination) + { + if (destination.Length < samples.Length * 2) + { + throw new ArgumentException("Destination is too small.", nameof(destination)); + } + + for (var i = 0; i < samples.Length; i++) + { + var scaled = Math.Clamp(samples[i], -1f, 1f) * 32767f; + BinaryPrimitives.WriteInt16LittleEndian(destination.Slice(i * 2, 2), (short)MathF.Round(scaled)); + } + } + + /// + /// Averages channels into mono. Downmixing is explicit so callers can instead pick + /// one channel of a multi-track source. + /// + public static float[] DownmixToMono(ReadOnlySpan interleaved, int channels) + { + ArgumentOutOfRangeException.ThrowIfLessThan(channels, 1); + if (channels == 1) + { + return interleaved.ToArray(); + } + + var frames = interleaved.Length / channels; + var mono = new float[frames]; + for (var f = 0; f < frames; f++) + { + var sum = 0f; + var baseIndex = f * channels; + for (var c = 0; c < channels; c++) + { + sum += interleaved[baseIndex + c]; + } + + mono[f] = sum / channels; + } + + return mono; + } + + /// Extracts one channel from interleaved audio. + public static float[] SelectChannel(ReadOnlySpan interleaved, int channels, int channel) + { + ArgumentOutOfRangeException.ThrowIfLessThan(channels, 1); + ArgumentOutOfRangeException.ThrowIfNegative(channel); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(channel, channels); + var frames = interleaved.Length / channels; + var result = new float[frames]; + for (var f = 0; f < frames; f++) + { + result[f] = interleaved[(f * channels) + channel]; + } + + return result; + } +} diff --git a/src/PrimeDictate.Core/Audio/AudioFormat.cs b/src/PrimeDictate.Core/Audio/AudioFormat.cs new file mode 100644 index 0000000..270b19f --- /dev/null +++ b/src/PrimeDictate.Core/Audio/AudioFormat.cs @@ -0,0 +1,36 @@ +namespace PrimeDictate.Core.Audio; + +public enum AudioSampleFormat +{ + Pcm16 = 0, + Float32 = 1 +} + +/// Describes interleaved audio samples. +public readonly record struct AudioFormat(int SampleRate, int Channels, AudioSampleFormat SampleFormat) +{ + /// Timeline format used by every current speech engine: 16 kHz mono float. + public static AudioFormat SpeechTimeline { get; } = new(16_000, 1, AudioSampleFormat.Float32); + + public int BytesPerSample => this.SampleFormat == AudioSampleFormat.Pcm16 ? 2 : 4; + + public void Validate() + { + if (this.SampleRate <= 0) + { + throw new ArgumentOutOfRangeException(nameof(this.SampleRate), this.SampleRate, "Sample rate must be positive."); + } + + if (this.Channels <= 0) + { + throw new ArgumentOutOfRangeException(nameof(this.Channels), this.Channels, "Channel count must be positive."); + } + } + + /// Converts a per-channel sample offset to exact time. + public TimeSpan ToTime(long sampleOffset) => + TimeSpan.FromTicks(checked(sampleOffset * TimeSpan.TicksPerSecond / this.SampleRate)); + + /// Converts time to the nearest preceding per-channel sample offset. + public long ToSampleOffset(TimeSpan time) => time.Ticks * this.SampleRate / TimeSpan.TicksPerSecond; +} diff --git a/src/PrimeDictate.Core/Audio/AudioFrame.cs b/src/PrimeDictate.Core/Audio/AudioFrame.cs new file mode 100644 index 0000000..cd5f341 --- /dev/null +++ b/src/PrimeDictate.Core/Audio/AudioFrame.cs @@ -0,0 +1,83 @@ +namespace PrimeDictate.Core.Audio; + +/// +/// A block of interleaved audio samples on a source's timeline. +/// +/// +/// A frame always owns its sample array. Capture adapters must copy out of pooled or +/// callback-owned buffers before creating a frame, so consumers can hold frames across +/// channel boundaries without reading memory that was returned to its owner. +/// +public sealed class AudioFrame +{ + private readonly float[] samples; + + private AudioFrame(float[] samples, AudioFormat format, long sequenceNumber, long sampleOffset) + { + this.samples = samples; + this.Format = format; + this.SequenceNumber = sequenceNumber; + this.SampleOffset = sampleOffset; + } + + public AudioFormat Format { get; } + + /// Monotonic per-source counter; a gap means frames were lost upstream. + public long SequenceNumber { get; } + + /// Per-channel sample index of the first sample, counted from the start of the source. + public long SampleOffset { get; } + + /// Interleaved samples in the range [-1, 1]. + public ReadOnlyMemory Samples => this.samples; + + public int SamplesPerChannel => this.samples.Length / this.Format.Channels; + + public long EndSampleOffset => this.SampleOffset + this.SamplesPerChannel; + + public TimeSpan Start => this.Format.ToTime(this.SampleOffset); + + public TimeSpan End => this.Format.ToTime(this.EndSampleOffset); + + /// Creates a frame from a copy of . + public static AudioFrame CopyFrom( + ReadOnlySpan interleaved, + AudioFormat format, + long sequenceNumber, + long sampleOffset) + { + Validate(interleaved.Length, format, sequenceNumber, sampleOffset); + return new AudioFrame(interleaved.ToArray(), format with { SampleFormat = AudioSampleFormat.Float32 }, sequenceNumber, sampleOffset); + } + + /// Creates a frame from little-endian PCM16 bytes, converting to float. + public static AudioFrame FromPcm16( + ReadOnlySpan pcm16, + int sampleRate, + int channels, + long sequenceNumber, + long sampleOffset) + { + if (pcm16.Length % 2 != 0) + { + throw new ArgumentException("PCM16 data must contain whole samples.", nameof(pcm16)); + } + + var floats = new float[pcm16.Length / 2]; + AudioConversion.Pcm16ToFloat(pcm16, floats); + var format = new AudioFormat(sampleRate, channels, AudioSampleFormat.Float32); + Validate(floats.Length, format, sequenceNumber, sampleOffset); + return new AudioFrame(floats, format, sequenceNumber, sampleOffset); + } + + private static void Validate(int sampleCount, AudioFormat format, long sequenceNumber, long sampleOffset) + { + format.Validate(); + ArgumentOutOfRangeException.ThrowIfNegative(sequenceNumber); + ArgumentOutOfRangeException.ThrowIfNegative(sampleOffset); + if (sampleCount % format.Channels != 0) + { + throw new ArgumentException("Interleaved sample count must be a multiple of the channel count."); + } + } +} diff --git a/src/PrimeDictate.Core/Audio/RecordedAudioTimeline.cs b/src/PrimeDictate.Core/Audio/RecordedAudioTimeline.cs new file mode 100644 index 0000000..a7eca41 --- /dev/null +++ b/src/PrimeDictate.Core/Audio/RecordedAudioTimeline.cs @@ -0,0 +1,116 @@ +namespace PrimeDictate.Core.Audio; + +/// A span of continuously captured audio and the wall-clock time it was captured. +public sealed record CaptureInterval( + TimeSpan RecordedStart, + TimeSpan RecordedDuration, + DateTimeOffset WallClockStart, + CaptureIntervalEnd EndReason); + +public enum CaptureIntervalEnd +{ + Open = 0, + Paused = 1, + Stopped = 2, + /// Capture failed or samples were lost; the next interval does not continue this one. + Gap = 3 +} + +/// +/// Maps the continuous recorded-audio timeline (what playback and transcripts use) to the +/// wall-clock intervals when capture actually ran. Pausing records nothing and adds no +/// recorded time. Recorded offsets come from sample counts, never from a UI clock. +/// +public sealed class RecordedAudioTimeline +{ + private readonly List intervals = []; + private readonly AudioFormat format; + private long recordedSamples; + private long openIntervalStartSample = -1; + private DateTimeOffset openIntervalWallClock; + + public RecordedAudioTimeline(AudioFormat format) + { + format.Validate(); + this.format = format; + } + + public bool IsCapturing => this.openIntervalStartSample >= 0; + + public TimeSpan RecordedDuration => this.format.ToTime(this.recordedSamples); + + public long RecordedSamples => this.recordedSamples; + + public IReadOnlyList Intervals => this.intervals; + + public void BeginInterval(DateTimeOffset wallClockStart) + { + if (this.IsCapturing) + { + throw new InvalidOperationException("A capture interval is already open."); + } + + this.openIntervalStartSample = this.recordedSamples; + this.openIntervalWallClock = wallClockStart; + } + + /// Appends captured samples; returns the recorded-timeline offset of the first one. + public long Append(int samplesPerChannel) + { + if (!this.IsCapturing) + { + throw new InvalidOperationException("Audio arrived while no capture interval is open (paused or stopped)."); + } + + ArgumentOutOfRangeException.ThrowIfNegative(samplesPerChannel); + var offset = this.recordedSamples; + this.recordedSamples += samplesPerChannel; + return offset; + } + + public CaptureInterval EndInterval(CaptureIntervalEnd reason) + { + if (!this.IsCapturing) + { + throw new InvalidOperationException("No capture interval is open."); + } + + if (reason == CaptureIntervalEnd.Open) + { + throw new ArgumentOutOfRangeException(nameof(reason)); + } + + var interval = new CaptureInterval( + this.format.ToTime(this.openIntervalStartSample), + this.format.ToTime(this.recordedSamples - this.openIntervalStartSample), + this.openIntervalWallClock, + reason); + this.intervals.Add(interval); + this.openIntervalStartSample = -1; + return interval; + } + + /// Returns the wall-clock time a recorded offset was captured, or null if it is past the end. + public DateTimeOffset? ToWallClock(TimeSpan recordedOffset) + { + foreach (var interval in this.intervals) + { + if (recordedOffset >= interval.RecordedStart && + recordedOffset < interval.RecordedStart + interval.RecordedDuration) + { + return interval.WallClockStart + (recordedOffset - interval.RecordedStart); + } + } + + if (this.IsCapturing) + { + var openStart = this.format.ToTime(this.openIntervalStartSample); + if (recordedOffset >= openStart && recordedOffset < this.RecordedDuration) + { + return this.openIntervalWallClock + (recordedOffset - openStart); + } + } + + return null; + } +} diff --git a/src/PrimeDictate.Core/Audio/StreamingResampler.cs b/src/PrimeDictate.Core/Audio/StreamingResampler.cs new file mode 100644 index 0000000..8c45189 --- /dev/null +++ b/src/PrimeDictate.Core/Audio/StreamingResampler.cs @@ -0,0 +1,195 @@ +namespace PrimeDictate.Core.Audio; + +/// +/// Streaming mono resampler using a windowed-sinc polyphase filter. +/// +/// +/// Output sample n corresponds exactly to input time n * inputRate / outputRate, +/// so offsets on the output timeline can be mapped back to the source without drift. +/// Chunk boundaries do not affect the result: feeding the same input in different chunk +/// sizes produces identical output. Call once at end of stream to drain +/// the filter's look-ahead. +/// +public sealed class StreamingResampler +{ + private const int BaseHalfTaps = 16; + + private readonly int upFactor; + private readonly int downFactor; + private readonly int halfTaps; + private readonly float[][]? phases; + private float[] buffer; + private long bufferStart; + private int bufferCount; + private long nextOutput; + private long totalInput; + private bool flushed; + + public StreamingResampler(int inputRate, int outputRate) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(inputRate, 0); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(outputRate, 0); + this.InputRate = inputRate; + this.OutputRate = outputRate; + var gcd = Gcd(inputRate, outputRate); + this.upFactor = outputRate / gcd; + this.downFactor = inputRate / gcd; + + if (this.upFactor == this.downFactor) + { + this.halfTaps = 0; + this.buffer = []; + return; + } + + var ratio = Math.Min(1.0, (double)this.upFactor / this.downFactor); + this.halfTaps = (int)Math.Ceiling(BaseHalfTaps / ratio); + // Cutoff in cycles per input sample, slightly below Nyquist of the lower rate. + var cutoff = 0.5 * ratio * 0.94; + this.phases = BuildPhases(this.upFactor, this.halfTaps, cutoff); + + // Leading zeros stand in for silence before the first sample. + this.buffer = new float[Math.Max(1024, this.halfTaps * 4)]; + this.bufferCount = this.halfTaps - 1; + this.bufferStart = -(this.halfTaps - 1); + } + + public int InputRate { get; } + + public int OutputRate { get; } + + /// Total output samples produced so far; also the offset of the next output sample. + public long OutputSamplesProduced => this.nextOutput; + + public float[] Process(ReadOnlySpan input) + { + if (this.flushed) + { + throw new InvalidOperationException("The resampler has been flushed."); + } + + this.totalInput += input.Length; + if (this.phases is null) + { + this.nextOutput += input.Length; + return input.ToArray(); + } + + this.Append(input); + return this.Drain(limit: long.MaxValue); + } + + /// Drains the filter at end of stream; output length totals ceil(input * out / in). + public float[] Flush() + { + if (this.flushed) + { + return []; + } + + this.flushed = true; + if (this.phases is null) + { + return []; + } + + var expectedTotal = (long)Math.Ceiling((double)this.totalInput * this.upFactor / this.downFactor); + Span zeros = stackalloc float[this.halfTaps + 1]; + this.Append(zeros); + return this.Drain(expectedTotal); + } + + private float[] Drain(long limit) + { + var output = new List(); + var available = this.bufferStart + this.bufferCount; + while (this.nextOutput < limit) + { + var product = this.nextOutput * this.downFactor; + var center = product / this.upFactor; + if (center + this.halfTaps >= available) + { + break; + } + + var phase = this.phases![(int)(product % this.upFactor)]; + var first = (int)(center - this.halfTaps + 1 - this.bufferStart); + var sum = 0f; + for (var k = 0; k < phase.Length; k++) + { + sum += phase[k] * this.buffer[first + k]; + } + + output.Add(sum); + this.nextOutput++; + } + + this.Trim(); + return output.ToArray(); + } + + private void Append(ReadOnlySpan input) + { + if (this.bufferCount + input.Length > this.buffer.Length) + { + Array.Resize(ref this.buffer, Math.Max(this.buffer.Length * 2, this.bufferCount + input.Length)); + } + + input.CopyTo(this.buffer.AsSpan(this.bufferCount)); + this.bufferCount += input.Length; + } + + private void Trim() + { + var nextCenter = this.nextOutput * this.downFactor / this.upFactor; + var keepFrom = nextCenter - this.halfTaps + 1; + var drop = (int)Math.Clamp(keepFrom - this.bufferStart, 0, this.bufferCount); + if (drop == 0) + { + return; + } + + this.buffer.AsSpan(drop, this.bufferCount - drop).CopyTo(this.buffer); + this.bufferCount -= drop; + this.bufferStart += drop; + } + + private static float[][] BuildPhases(int phaseCount, int halfTaps, double cutoff) + { + var phases = new float[phaseCount][]; + var tapCount = halfTaps * 2; + for (var p = 0; p < phaseCount; p++) + { + var coefficients = new double[tapCount]; + var fraction = (double)p / phaseCount; + var sum = 0.0; + for (var k = 0; k < tapCount; k++) + { + // Distance in input samples between this tap and the exact output position. + var t = (k - halfTaps + 1) - fraction; + var sincArg = 2.0 * cutoff * t; + var sinc = Math.Abs(sincArg) < 1e-12 ? 1.0 : Math.Sin(Math.PI * sincArg) / (Math.PI * sincArg); + var w = (t / halfTaps + 1.0) / 2.0; + var window = Math.Abs(t) >= halfTaps + ? 0.0 + : 0.42 - (0.5 * Math.Cos(2 * Math.PI * w)) + (0.08 * Math.Cos(4 * Math.PI * w)); + coefficients[k] = sinc * window; + sum += coefficients[k]; + } + + phases[p] = coefficients.Select(c => (float)(c / sum)).ToArray(); + } + + return phases; + } + + private static int Gcd(int a, int b) + { + while (b != 0) + { + (a, b) = (b, a % b); + } + + return a; + } +} diff --git a/src/PrimeDictate.Core/Coordination/MicrophoneCoordinator.cs b/src/PrimeDictate.Core/Coordination/MicrophoneCoordinator.cs new file mode 100644 index 0000000..e91ac08 --- /dev/null +++ b/src/PrimeDictate.Core/Coordination/MicrophoneCoordinator.cs @@ -0,0 +1,169 @@ +namespace PrimeDictate.Core.Coordination; + +/// +/// Background microphone users (dictation, wake word) that yield to a transcription session. +/// +public interface IMicrophoneConsumer +{ + string Name { get; } + + /// True while the consumer is mid-utterance and must not be interrupted. + bool IsBusy { get; } + + /// True when the user has this consumer enabled and it currently holds or listens on the microphone. + bool IsActive { get; } + + ValueTask SuspendAsync(string reason, CancellationToken cancellationToken); + + ValueTask ResumeAsync(CancellationToken cancellationToken); +} + +public sealed class MicrophoneBusyException(string message) : InvalidOperationException(message); + +/// +/// Grants one exclusive microphone session at a time. Acquiring suspends active background +/// consumers and releasing restores exactly those consumers, so the user's prior choice +/// (for example, wake word on or off) is preserved. +/// +public sealed class MicrophoneCoordinator +{ + private readonly object sync = new(); + private readonly List consumers = []; + private Lease? current; + + public event EventHandler? Changed; + + public string? CurrentOwner + { + get + { + lock (this.sync) + { + return this.current?.Owner; + } + } + } + + /// Why background consumers are suspended, for display; null when nothing holds the microphone. + public string? SuspensionReason + { + get + { + lock (this.sync) + { + return this.current?.Reason; + } + } + } + + public void Register(IMicrophoneConsumer consumer) + { + ArgumentNullException.ThrowIfNull(consumer); + lock (this.sync) + { + if (!this.consumers.Contains(consumer)) + { + this.consumers.Add(consumer); + } + } + } + + public async ValueTask AcquireAsync(string owner, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(owner); + Lease lease; + IMicrophoneConsumer[] toSuspend; + lock (this.sync) + { + if (this.current is not null) + { + throw new MicrophoneBusyException($"The microphone is in use by {this.current.Owner}."); + } + + var busy = this.consumers.FirstOrDefault(c => c.IsBusy); + if (busy is not null) + { + throw new MicrophoneBusyException($"{busy.Name} is in progress. Finish it before starting {owner}."); + } + + toSuspend = this.consumers.Where(c => c.IsActive).ToArray(); + lease = new Lease(this, owner, $"Paused while {owner} is using the microphone."); + this.current = lease; + } + + var suspended = new List(); + try + { + foreach (var consumer in toSuspend) + { + await consumer.SuspendAsync(lease.Reason, cancellationToken).ConfigureAwait(false); + suspended.Add(consumer); + } + } + catch + { + await ResumeAllAsync(suspended).ConfigureAwait(false); + lock (this.sync) + { + this.current = null; + } + + throw; + } + + lease.Suspended = suspended; + this.Changed?.Invoke(this, EventArgs.Empty); + return lease; + } + + private async ValueTask ReleaseAsync(Lease lease) + { + lock (this.sync) + { + if (!ReferenceEquals(this.current, lease)) + { + return; + } + + this.current = null; + } + + await ResumeAllAsync(lease.Suspended).ConfigureAwait(false); + this.Changed?.Invoke(this, EventArgs.Empty); + } + + private static async ValueTask ResumeAllAsync(IReadOnlyList consumers) + { + List? errors = null; + foreach (var consumer in consumers) + { + try + { + await consumer.ResumeAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) + { + (errors ??= []).Add(ex); + } + } + + if (errors is not null) + { + throw new AggregateException("One or more microphone consumers failed to resume.", errors); + } + } + + private sealed class Lease(MicrophoneCoordinator owner, string name, string reason) : IAsyncDisposable + { + private int disposed; + + public string Owner { get; } = name; + + public string Reason { get; } = reason; + + public IReadOnlyList Suspended { get; set; } = []; + + public ValueTask DisposeAsync() => + Interlocked.Exchange(ref this.disposed, 1) == 0 ? owner.ReleaseAsync(this) : ValueTask.CompletedTask; + } +} diff --git a/src/PrimeDictate.Core/Coordination/ModelLeaseScheduler.cs b/src/PrimeDictate.Core/Coordination/ModelLeaseScheduler.cs new file mode 100644 index 0000000..b11961d --- /dev/null +++ b/src/PrimeDictate.Core/Coordination/ModelLeaseScheduler.cs @@ -0,0 +1,128 @@ +namespace PrimeDictate.Core.Coordination; + +public enum ModelLeasePriority +{ + /// Imported-file jobs. They take one lease per window so live work can interleave. + Background = 0, + + /// Live sessions; always granted ahead of queued background work. + Live = 1 +} + +/// +/// Serializes access to native model instances that are not thread-safe. Each model key has +/// one holder at a time; waiters are granted by priority, then arrival order. +/// +public sealed class ModelLeaseScheduler +{ + private readonly object sync = new(); + private readonly Dictionary keys = new(StringComparer.Ordinal); + private long arrivalCounter; + + public int QueuedCount(string modelKey) + { + lock (this.sync) + { + return this.keys.TryGetValue(modelKey, out var state) ? state.Waiters.Count : 0; + } + } + + public ValueTask AcquireAsync(string modelKey, ModelLeasePriority priority, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(modelKey); + cancellationToken.ThrowIfCancellationRequested(); + Waiter waiter; + lock (this.sync) + { + if (!this.keys.TryGetValue(modelKey, out var state)) + { + state = new KeyState(); + this.keys[modelKey] = state; + } + + if (!state.Held) + { + state.Held = true; + return ValueTask.FromResult(new Lease(this, modelKey)); + } + + waiter = new Waiter(priority, this.arrivalCounter++); + state.Waiters.Add(waiter); + } + + if (cancellationToken.CanBeCanceled) + { + waiter.Registration = cancellationToken.Register(() => this.Cancel(modelKey, waiter, cancellationToken)); + } + + return new ValueTask(waiter.Completion.Task); + } + + private void Cancel(string modelKey, Waiter waiter, CancellationToken cancellationToken) + { + lock (this.sync) + { + if (!this.keys.TryGetValue(modelKey, out var state) || !state.Waiters.Remove(waiter)) + { + return; + } + } + + waiter.Completion.TrySetCanceled(cancellationToken); + } + + private void Release(string modelKey) + { + Waiter? next; + lock (this.sync) + { + var state = this.keys[modelKey]; + next = state.Waiters + .OrderByDescending(w => w.Priority) + .ThenBy(w => w.Arrival) + .FirstOrDefault(); + if (next is null) + { + state.Held = false; + this.keys.Remove(modelKey); + return; + } + + state.Waiters.Remove(next); + } + + next.Registration.Dispose(); + next.Completion.TrySetResult(new Lease(this, modelKey)); + } + + private sealed class KeyState + { + public bool Held { get; set; } + + public List Waiters { get; } = []; + } + + private sealed class Waiter(ModelLeasePriority priority, long arrival) + { + public ModelLeasePriority Priority { get; } = priority; + + public long Arrival { get; } = arrival; + + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public CancellationTokenRegistration Registration { get; set; } + } + + private sealed class Lease(ModelLeaseScheduler scheduler, string modelKey) : IDisposable + { + private int disposed; + + public void Dispose() + { + if (Interlocked.Exchange(ref this.disposed, 1) == 0) + { + scheduler.Release(modelKey); + } + } + } +} diff --git a/src/PrimeDictate.Core/Models/ModelContracts.cs b/src/PrimeDictate.Core/Models/ModelContracts.cs new file mode 100644 index 0000000..3dba272 --- /dev/null +++ b/src/PrimeDictate.Core/Models/ModelContracts.cs @@ -0,0 +1,82 @@ +using System.Runtime.InteropServices; +using PrimeDictate.Core.Providers; + +namespace PrimeDictate.Core.Models; + +public enum ModelKind +{ + /// Produces words. Listed under "Speech model". + SpeechRecognition = 0, + + /// Finds speakers. Listed under "Speaker detection", never as a speech model. + SpeakerDiarization = 1 +} + +public enum ModelAvailability +{ + Missing = 0, + Downloading = 1, + /// Required files are present; not yet proven to load. + Installed = 2, + /// A load probe succeeded on this machine with the reported effective backend. + Ready = 3, + /// Cannot run on this OS, architecture, or runtime. + Unsupported = 4, + FailedValidation = 5 +} + +public sealed record ModelFile(string RelativePath, long? SizeBytes, string? Sha256); + +public sealed record PlatformRequirement(OSPlatform? OperatingSystem, Architecture? ProcessArchitecture, string? Note); + +public sealed record ModelDescriptor( + string Id, + ModelKind Kind, + string Family, + string DisplayName, + string ArtifactFormat, + string? ArtifactRevision, + IReadOnlyList ExpectedFiles, + long? DownloadSizeBytes, + IReadOnlyList Languages, + bool SupportsFiles, + LiveRecognitionMode LiveMode, + TimingCapabilities Timing, + bool SupportsDiarization, + IReadOnlyList SupportedPlatforms, + string? License); + +public sealed record ModelStatus( + ModelDescriptor Descriptor, + ModelAvailability Availability, + string? InstalledPath, + string? Reason, + EffectiveRuntime? ProbedRuntime); + +public interface IModelRegistry +{ + ValueTask> ListAsync(ModelKind kind, CancellationToken cancellationToken); + + ValueTask GetStatusAsync(string modelId, CancellationToken cancellationToken); + + /// + /// Loads the model and runs a short inference to prove it works, reporting the backend that + /// actually ran. A loadable library is not proof of accelerator use. + /// + ValueTask ProbeAsync(string modelId, string requestedBackend, CancellationToken cancellationToken); +} + +public static class PlatformRequirements +{ + public static bool IsSatisfied(IReadOnlyList requirements, OSPlatform os, Architecture architecture) => + requirements.Count == 0 || + requirements.Any(r => + (r.OperatingSystem is null || r.OperatingSystem.Value == os) && + (r.ProcessArchitecture is null || r.ProcessArchitecture.Value == architecture)); + + public static OSPlatform Current() => + OperatingSystem.IsWindows() ? OSPlatform.Windows + : OperatingSystem.IsMacOS() ? OSPlatform.OSX + : OperatingSystem.IsLinux() ? OSPlatform.Linux + : OSPlatform.Create("UNKNOWN"); +} diff --git a/src/PrimeDictate.Core/PrimeDictate.Core.csproj b/src/PrimeDictate.Core/PrimeDictate.Core.csproj new file mode 100644 index 0000000..a1e66c8 --- /dev/null +++ b/src/PrimeDictate.Core/PrimeDictate.Core.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + PrimeDictate.Core + Platform-neutral transcription contracts, session model, and persistence for PrimeDictate. + + + + + + + + + + + diff --git a/src/PrimeDictate.Core/Providers/AudioContracts.cs b/src/PrimeDictate.Core/Providers/AudioContracts.cs new file mode 100644 index 0000000..5ba5a75 --- /dev/null +++ b/src/PrimeDictate.Core/Providers/AudioContracts.cs @@ -0,0 +1,90 @@ +using PrimeDictate.Core.Audio; + +namespace PrimeDictate.Core.Providers; + +public enum AudioSourceErrorKind +{ + PermissionDenied = 0, + DeviceRemoved = 1, + DeviceChanged = 2, + FormatUnsupported = 3, + Unknown = 4 +} + +public sealed class AudioSourceException(AudioSourceErrorKind kind, string message, Exception? inner = null) + : Exception(message, inner) +{ + public AudioSourceErrorKind Kind { get; } = kind; +} + +/// +/// An exclusive lease on a capture device. Disposing the lease stops capture and releases +/// the device. completes when capture stops and throws +/// on device errors. +/// +public interface IAudioCaptureLease : IAsyncDisposable +{ + AudioFormat Format { get; } + + string DeviceId { get; } + + string DeviceName { get; } + + IAsyncEnumerable ReadFramesAsync(CancellationToken cancellationToken); + + /// Stops delivery without releasing the device. Frames after resume continue the sequence. + ValueTask PauseAsync(CancellationToken cancellationToken); + + ValueTask ResumeAsync(CancellationToken cancellationToken); +} + +public sealed record AudioInputDevice(string Id, string Name, bool IsDefault); + +public interface IAudioSource +{ + ValueTask> ListDevicesAsync(CancellationToken cancellationToken); + + /// Opens a device. Null selects the system default. + ValueTask OpenAsync(string? deviceId, CancellationToken cancellationToken); +} + +public sealed record MediaStreamInfo(int Index, string Codec, int SampleRate, int Channels, string? Language, string? Title); + +public sealed record MediaProbeResult( + string ContainerFormat, + TimeSpan? Duration, + IReadOnlyList AudioStreams); + +public sealed class MediaDecodeException(string errorCode, string message, Exception? inner = null) + : Exception(message, inner) +{ + /// Stable code for UI and diagnostics, for example "unsupported-format" or "corrupt". + public string ErrorCode { get; } = errorCode; +} + +/// +/// Decodes local media incrementally. Implementations treat input as untrusted, never build +/// shell commands from paths, and never reach the network. +/// +public interface IAudioDecoder +{ + ValueTask ProbeAsync(string path, CancellationToken cancellationToken); + + /// + /// Streams the selected audio stream as frames in the source's native rate and channel + /// layout. Offsets are counted from the start of the stream. + /// + IAsyncEnumerable DecodeAsync(string path, int streamIndex, CancellationToken cancellationToken); +} + +/// Plays session audio. Positions are on the recorded-audio timeline. +public interface IAudioPlayback : IAsyncDisposable +{ + ValueTask OpenAsync(string path, CancellationToken cancellationToken); + + ValueTask PlayAsync(TimeSpan from, TimeSpan? until, CancellationToken cancellationToken); + + ValueTask PauseAsync(CancellationToken cancellationToken); + + TimeSpan Position { get; } +} diff --git a/src/PrimeDictate.Core/Providers/RecognitionContracts.cs b/src/PrimeDictate.Core/Providers/RecognitionContracts.cs new file mode 100644 index 0000000..7342528 --- /dev/null +++ b/src/PrimeDictate.Core/Providers/RecognitionContracts.cs @@ -0,0 +1,98 @@ +using PrimeDictate.Core.Audio; +using PrimeDictate.Core.Transcripts; + +namespace PrimeDictate.Core.Providers; + +public enum LiveRecognitionMode +{ + /// Provider cannot run live. + None = 0, + + /// Provider keeps streaming state and consumes new audio incrementally. + NativeStreaming = 1, + + /// Provider recognizes bounded windows; shown to users as "Buffered Live". + BufferedWindows = 2 +} + +[Flags] +public enum TimingCapabilities +{ + None = 0, + SegmentTimestamps = 1, + WordTimestamps = 2, + Confidence = 4 +} + +public sealed record TranscriptionProviderCapabilities( + bool SupportsFiles, + LiveRecognitionMode LiveMode, + TimingCapabilities Timing, + bool CombinedDiarization, + int? MaxSpeakers, + TimeSpan? MaxWindow, + IReadOnlyList Languages, + int RequiredSampleRate); + +/// What actually ran, reported after load, not what was requested. +public sealed record EffectiveRuntime(string RuntimeName, string? RuntimeVersion, string RequestedBackend, string EffectiveBackend, string? FallbackReason); + +/// A recognized span in a provider's own window, before session placement. +public sealed record RecognizedSegment( + TimeSpan Start, + TimeSpan End, + string Text, + IReadOnlyList? Words, + double? Confidence, + string? SpeakerLabel, + TimingProvenance Provenance); + +/// Update from a streaming recognizer. is stable across partials. +public sealed record StreamingUpdate(string UtteranceId, RecognizedSegment Segment, bool IsFinal); + +public interface IStreamingRecognitionSession : IAsyncDisposable +{ + /// Feeds audio on the session timeline. Frames must be in the provider's required format. + ValueTask WriteAsync(AudioFrame frame, CancellationToken cancellationToken); + + /// Signals end of input; the update stream completes after the last final result. + ValueTask CompleteAsync(CancellationToken cancellationToken); + + IAsyncEnumerable ReadUpdatesAsync(CancellationToken cancellationToken); +} + +/// +/// A loaded speech model. Instances are not assumed thread-safe; callers obtain one through a +/// model lease from the resource coordinator. +/// +public interface ITranscriptionProvider : IAsyncDisposable +{ + string ModelId { get; } + + TranscriptionProviderCapabilities Capabilities { get; } + + EffectiveRuntime Runtime { get; } + + /// + /// Recognizes one bounded mono window at . + /// Offsets in the result are relative to the window start. + /// + ValueTask> RecognizeWindowAsync(ReadOnlyMemory samples, string? language, CancellationToken cancellationToken); + + /// Starts native streaming. Throws unless LiveMode is NativeStreaming. + ValueTask StartStreamingAsync(string? language, bool diarize, CancellationToken cancellationToken); +} + +public sealed record SpeakerTurn(string SpeakerLabel, TimeSpan Start, TimeSpan End, double? Confidence); + +public sealed record DiarizationCapabilities(int MaxSpeakers, bool SupportsStreaming, int RequiredSampleRate); + +/// Finds anonymous speaker activity on the same timeline as recognition. +public interface IDiarizationProvider : IAsyncDisposable +{ + string ModelId { get; } + + DiarizationCapabilities Capabilities { get; } + + ValueTask> DiarizeAsync(ReadOnlyMemory samples, TimeSpan offset, CancellationToken cancellationToken); +} diff --git a/src/PrimeDictate.Core/Sessions/ITranscriptionSessionStore.cs b/src/PrimeDictate.Core/Sessions/ITranscriptionSessionStore.cs new file mode 100644 index 0000000..c109a79 --- /dev/null +++ b/src/PrimeDictate.Core/Sessions/ITranscriptionSessionStore.cs @@ -0,0 +1,48 @@ +using PrimeDictate.Core.Transcripts; + +namespace PrimeDictate.Core.Sessions; + +public sealed record TranscriptSessionSummary( + Guid SessionId, + string Title, + TranscriptSourceType SourceType, + TranscriptSessionStatus Status, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt, + TimeSpan? Duration, + int SegmentCount); + +public sealed record SessionDeletionResult(bool Existed, IReadOnlyList DeletedFiles, IReadOnlyList KeptExternalFiles); + +/// +/// Versioned local persistence for transcription sessions. Separate from dictation history. +/// +public interface ITranscriptionSessionStore : IAsyncDisposable +{ + ValueTask InitializeAsync(CancellationToken cancellationToken); + + /// + /// Persists metadata, runs, speakers, audio references, and final segments in one + /// transaction. Provisional segments are not persisted; after a crash they are recomputed + /// from retained audio. + /// + ValueTask SaveCheckpointAsync(TranscriptDocument document, CancellationToken cancellationToken); + + ValueTask LoadAsync(Guid sessionId, CancellationToken cancellationToken); + + ValueTask> ListAsync(int skip, int take, CancellationToken cancellationToken); + + /// + /// Call once at startup: sessions left running, paused, or finalizing are marked + /// and returned for recovery. + /// + ValueTask> MarkInterruptedSessionsAsync(CancellationToken cancellationToken); + + /// Deletes owned audio and keeps the transcript (transcript-only retention). + ValueTask> DeleteOwnedAudioAsync(Guid sessionId, CancellationToken cancellationToken); + + /// + /// Deletes the session's records and the audio it owns. Referenced originals are never deleted. + /// + ValueTask DeleteAsync(Guid sessionId, CancellationToken cancellationToken); +} diff --git a/src/PrimeDictate.Core/Sessions/TranscriptionSessionOptions.cs b/src/PrimeDictate.Core/Sessions/TranscriptionSessionOptions.cs new file mode 100644 index 0000000..59fa199 --- /dev/null +++ b/src/PrimeDictate.Core/Sessions/TranscriptionSessionOptions.cs @@ -0,0 +1,32 @@ +namespace PrimeDictate.Core.Sessions; + +public enum AudioRetention +{ + /// Keep session audio for playback and reruns. + KeepAudio = 0, + + /// Delete owned audio once the transcript is complete. Playback and reruns become unavailable. + TranscriptOnly = 1 +} + +public enum DownmixMode +{ + Average = 0, + SingleChannel = 1 +} + +/// +/// Immutable options captured when a session or rerun starts. Settings changes after this +/// point never affect a running session. +/// +public sealed record TranscriptionSessionOptions( + string AsrModelId, + string? AsrModelRevision, + string RequestedBackend, + string? Language, + string? DiarizerModelId, + AudioRetention AudioRetention, + DownmixMode Downmix, + int? SourceChannel, + int? SourceTrack, + string? InputDeviceId); diff --git a/src/PrimeDictate.Core/Sessions/TranscriptionSessionStateMachine.cs b/src/PrimeDictate.Core/Sessions/TranscriptionSessionStateMachine.cs new file mode 100644 index 0000000..77a93ea --- /dev/null +++ b/src/PrimeDictate.Core/Sessions/TranscriptionSessionStateMachine.cs @@ -0,0 +1,40 @@ +using PrimeDictate.Core.Transcripts; + +namespace PrimeDictate.Core.Sessions; + +/// Allowed status transitions for a transcription session. +public static class TranscriptionSessionStateMachine +{ + private static readonly Dictionary Allowed = new() + { + [TranscriptSessionStatus.Created] = [TranscriptSessionStatus.Running, TranscriptSessionStatus.Canceled, TranscriptSessionStatus.Failed], + [TranscriptSessionStatus.Running] = + [ + TranscriptSessionStatus.Paused, TranscriptSessionStatus.Finalizing, TranscriptSessionStatus.Failed, + TranscriptSessionStatus.Canceled, TranscriptSessionStatus.Interrupted + ], + [TranscriptSessionStatus.Paused] = + [ + TranscriptSessionStatus.Running, TranscriptSessionStatus.Finalizing, TranscriptSessionStatus.Failed, + TranscriptSessionStatus.Canceled, TranscriptSessionStatus.Interrupted + ], + [TranscriptSessionStatus.Finalizing] = [TranscriptSessionStatus.Completed, TranscriptSessionStatus.Failed, TranscriptSessionStatus.Interrupted], + // A rerun with another model starts a new result version on a finished session. + [TranscriptSessionStatus.Completed] = [TranscriptSessionStatus.Running], + [TranscriptSessionStatus.Failed] = [TranscriptSessionStatus.Running, TranscriptSessionStatus.Canceled], + [TranscriptSessionStatus.Interrupted] = [TranscriptSessionStatus.Running, TranscriptSessionStatus.Finalizing, TranscriptSessionStatus.Canceled], + [TranscriptSessionStatus.Canceled] = [] + }; + + public static bool CanTransition(TranscriptSessionStatus from, TranscriptSessionStatus to) => + Allowed.TryGetValue(from, out var targets) && targets.Contains(to); + + public static TranscriptSessionStatus Transition(TranscriptSessionStatus from, TranscriptSessionStatus to) => + CanTransition(from, to) + ? to + : throw new InvalidOperationException($"A session cannot move from {from} to {to}."); + + /// Statuses that mean work was in flight when the app last ran. + public static bool NeedsRecovery(TranscriptSessionStatus status) => + status is TranscriptSessionStatus.Running or TranscriptSessionStatus.Paused or TranscriptSessionStatus.Finalizing; +} diff --git a/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs b/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs new file mode 100644 index 0000000..639ef24 --- /dev/null +++ b/src/PrimeDictate.Core/Settings/TranscriptionPreferences.cs @@ -0,0 +1,146 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using PrimeDictate.Core.Sessions; +using PrimeDictate.Core.Storage; + +namespace PrimeDictate.Core.Settings; + +/// +/// Transcription-mode defaults. Stored apart from dictation's settings.json, which is +/// never rewritten by transcription mode. +/// +public sealed record TranscriptionPreferences +{ + public const int CurrentSchemaVersion = 1; + + public int SchemaVersion { get; init; } = CurrentSchemaVersion; + + public string? AsrModelId { get; init; } + + public string RequestedBackend { get; init; } = "cpu"; + + public string? Language { get; init; } + + public bool SpeakerDetection { get; init; } + + public string? DiarizerModelId { get; init; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public AudioRetention AudioRetention { get; init; } = AudioRetention.KeepAudio; + + public string? InputDeviceId { get; init; } + + /// Copy imported files into app storage instead of referencing them in place. + public bool CopyImportedMedia { get; init; } +} + +public sealed record PreferencesLoadResult(TranscriptionPreferences Preferences, bool IsReadOnly, string? Warning, string? BackupPath); + +/// +/// Loads and saves with versioned, idempotent migrations. +/// A backup is written before any migration rewrites the file. A file from a newer app version +/// is left untouched and loaded read-only. +/// +public sealed class TranscriptionPreferencesStore +{ + private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true }; + + private readonly string path; + private readonly IReadOnlyDictionary> migrations; + + public TranscriptionPreferencesStore(string path) + : this(path, new Dictionary>()) + { + } + + /// Keyed by source version; each upgrades by exactly one version. + internal TranscriptionPreferencesStore(string path, IReadOnlyDictionary> migrations) + { + this.path = path; + this.migrations = migrations; + } + + public string FilePath => this.path; + + /// Creates first-run defaults, typically copied from the dictation model choice. + public PreferencesLoadResult Load(Func seed, int targetVersion = TranscriptionPreferences.CurrentSchemaVersion) + { + if (!File.Exists(this.path)) + { + return new PreferencesLoadResult(seed() with { SchemaVersion = targetVersion }, IsReadOnly: false, Warning: null, BackupPath: null); + } + + JsonObject root; + try + { + root = JsonNode.Parse(File.ReadAllText(this.path)) as JsonObject + ?? throw new JsonException("Root is not an object."); + } + catch (JsonException) + { + var corruptBackup = this.Backup("corrupt"); + return new PreferencesLoadResult( + seed() with { SchemaVersion = targetVersion }, + IsReadOnly: false, + Warning: "Transcription settings could not be read and were reset. The unreadable file was kept.", + BackupPath: corruptBackup); + } + + var version = root["SchemaVersion"]?.GetValue() ?? 0; + if (version > targetVersion) + { + var newer = root.Deserialize(JsonOptions) ?? seed(); + return new PreferencesLoadResult( + newer, + IsReadOnly: true, + Warning: $"Transcription settings were saved by a newer PrimeDictate (schema {version}). They will not be changed.", + BackupPath: null); + } + + string? backupPath = null; + if (version < targetVersion) + { + backupPath = this.Backup($"v{version}"); + while (version < targetVersion) + { + if (!this.migrations.TryGetValue(version, out var migrate)) + { + throw new InvalidOperationException($"No migration from transcription settings schema {version}."); + } + + root = migrate(root); + version++; + root["SchemaVersion"] = version; + } + + this.WriteAtomic(root.ToJsonString(JsonOptions)); + } + + var preferences = root.Deserialize(JsonOptions) ?? seed(); + return new PreferencesLoadResult(preferences, IsReadOnly: false, Warning: null, BackupPath: backupPath); + } + + public void Save(TranscriptionPreferences preferences) + { + ArgumentNullException.ThrowIfNull(preferences); + this.WriteAtomic(JsonSerializer.Serialize(preferences, JsonOptions)); + } + + private string Backup(string label) + { + var backup = $"{this.path}.{label}.{DateTime.UtcNow:yyyyMMddHHmmss}.bak"; + File.Copy(this.path, backup, overwrite: true); + return backup; + } + + private void WriteAtomic(string json) + { + var directory = Path.GetDirectoryName(this.path) ?? throw new InvalidOperationException("Settings path has no directory."); + AppDataPaths.EnsurePrivateDirectory(directory); + var temp = this.path + ".tmp"; + File.WriteAllText(temp, json); + AppDataPaths.RestrictFile(temp); + File.Move(temp, this.path, overwrite: true); + } +} diff --git a/src/PrimeDictate.Core/Storage/AppDataPaths.cs b/src/PrimeDictate.Core/Storage/AppDataPaths.cs new file mode 100644 index 0000000..49032cf --- /dev/null +++ b/src/PrimeDictate.Core/Storage/AppDataPaths.cs @@ -0,0 +1,56 @@ +namespace PrimeDictate.Core.Storage; + +/// +/// Per-user application data locations. On Windows this is the same +/// %LocalAppData%\PrimeDictate root existing installs use for settings, history, and +/// models. .NET maps LocalApplicationData to ~/Library/Application Support on macOS and +/// $XDG_DATA_HOME (default ~/.local/share) on Linux. +/// +public sealed class AppDataPaths +{ + public AppDataPaths(string root) + { + ArgumentException.ThrowIfNullOrWhiteSpace(root); + this.Root = Path.GetFullPath(root); + } + + public static AppDataPaths Default { get; } = new(Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData, Environment.SpecialFolderOption.DoNotVerify), + "PrimeDictate")); + + public string Root { get; } + + public string ModelsDirectory => Path.Combine(this.Root, "models"); + + public string TranscriptionDirectory => Path.Combine(this.Root, "transcription"); + + public string SessionDatabasePath => Path.Combine(this.TranscriptionDirectory, "sessions.db"); + + public string SessionMediaDirectory => Path.Combine(this.TranscriptionDirectory, "media"); + + public string SessionTempDirectory => Path.Combine(this.TranscriptionDirectory, "tmp"); + + public string TranscriptionPreferencesPath => Path.Combine(this.Root, "transcription-settings.json"); + + /// Creates a directory readable only by the current user where the OS supports it. + public static void EnsurePrivateDirectory(string path) + { + if (OperatingSystem.IsWindows()) + { + // LocalAppData is already per-user; inherit its ACL. + Directory.CreateDirectory(path); + return; + } + + Directory.CreateDirectory(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + + public static void RestrictFile(string path) + { + if (!OperatingSystem.IsWindows() && File.Exists(path)) + { + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + } +} diff --git a/src/PrimeDictate.Core/Storage/SqliteTranscriptionSessionStore.cs b/src/PrimeDictate.Core/Storage/SqliteTranscriptionSessionStore.cs new file mode 100644 index 0000000..55ab913 --- /dev/null +++ b/src/PrimeDictate.Core/Storage/SqliteTranscriptionSessionStore.cs @@ -0,0 +1,586 @@ +using System.Text.Json; +using Microsoft.Data.Sqlite; +using PrimeDictate.Core.Sessions; +using PrimeDictate.Core.Transcripts; + +namespace PrimeDictate.Core.Storage; + +/// +/// SQLite metadata store. Media stays on disk under the session media directory; the database +/// holds only paths. Transcript text never goes to logs from here. +/// +public sealed class SqliteTranscriptionSessionStore : ITranscriptionSessionStore +{ + public const int CurrentSchemaVersion = 1; + + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.General); + + private readonly string databasePath; + private readonly string mediaRoot; + private readonly SemaphoreSlim gate = new(1, 1); + private SqliteConnection? connection; + + public SqliteTranscriptionSessionStore(string databasePath, string mediaRoot) + { + this.databasePath = Path.GetFullPath(databasePath); + this.mediaRoot = Path.GetFullPath(mediaRoot); + } + + public static SqliteTranscriptionSessionStore Create(AppDataPaths paths) => + new(paths.SessionDatabasePath, paths.SessionMediaDirectory); + + public async ValueTask InitializeAsync(CancellationToken cancellationToken) + { + await this.gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (this.connection is not null) + { + return; + } + + AppDataPaths.EnsurePrivateDirectory(Path.GetDirectoryName(this.databasePath)!); + AppDataPaths.EnsurePrivateDirectory(this.mediaRoot); + var builder = new SqliteConnectionStringBuilder + { + DataSource = this.databasePath, + Mode = SqliteOpenMode.ReadWriteCreate, + Pooling = false + }; + var conn = new SqliteConnection(builder.ToString()); + await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + AppDataPaths.RestrictFile(this.databasePath); + await ExecuteAsync(conn, null, "PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; PRAGMA synchronous=FULL;", cancellationToken).ConfigureAwait(false); + + var version = await ScalarLongAsync(conn, "PRAGMA user_version;", cancellationToken).ConfigureAwait(false); + if (version > CurrentSchemaVersion) + { + await conn.DisposeAsync().ConfigureAwait(false); + throw new InvalidOperationException( + $"The transcription database was created by a newer PrimeDictate (schema {version}). It was not modified."); + } + + if (version < 1) + { + await using var tx = conn.BeginTransaction(); + await ExecuteAsync(conn, tx, SchemaV1, cancellationToken).ConfigureAwait(false); + await ExecuteAsync(conn, tx, "PRAGMA user_version = 1;", cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + + this.connection = conn; + } + finally + { + this.gate.Release(); + } + } + + public async ValueTask SaveCheckpointAsync(TranscriptDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + await this.gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var conn = this.RequireConnection(); + await using var tx = conn.BeginTransaction(); + + await using (var cmd = conn.CreateCommand()) + { + cmd.Transaction = tx; + cmd.CommandText = """ + INSERT INTO sessions (id, schema_version, title, source_type, status, created_at, updated_at, + duration_ticks, language, active_result_version, media_json) + VALUES ($id, $schema, $title, $source, $status, $created, $updated, $duration, $language, $active, $media) + ON CONFLICT(id) DO UPDATE SET + schema_version = excluded.schema_version, title = excluded.title, status = excluded.status, + updated_at = excluded.updated_at, duration_ticks = excluded.duration_ticks, + language = excluded.language, active_result_version = excluded.active_result_version, + media_json = excluded.media_json; + """; + cmd.Parameters.AddWithValue("$id", document.SessionId.ToString("D")); + cmd.Parameters.AddWithValue("$schema", document.SchemaVersion); + cmd.Parameters.AddWithValue("$title", document.Title); + cmd.Parameters.AddWithValue("$source", (int)document.SourceType); + cmd.Parameters.AddWithValue("$status", (int)document.Status); + cmd.Parameters.AddWithValue("$created", document.CreatedAt.ToString("O")); + cmd.Parameters.AddWithValue("$updated", document.UpdatedAt.ToString("O")); + cmd.Parameters.AddWithValue("$duration", (object?)document.Duration?.Ticks ?? DBNull.Value); + cmd.Parameters.AddWithValue("$language", (object?)document.Language ?? DBNull.Value); + cmd.Parameters.AddWithValue("$active", document.ActiveResultVersion); + cmd.Parameters.AddWithValue("$media", document.Media is null ? DBNull.Value : JsonSerializer.Serialize(document.Media, JsonOptions)); + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + foreach (var run in document.Runs) + { + await using var cmd = conn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = """ + INSERT INTO runs (session_id, result_version, run_json) VALUES ($id, $version, $json) + ON CONFLICT(session_id, result_version) DO UPDATE SET run_json = excluded.run_json; + """; + cmd.Parameters.AddWithValue("$id", document.SessionId.ToString("D")); + cmd.Parameters.AddWithValue("$version", run.ResultVersion); + cmd.Parameters.AddWithValue("$json", JsonSerializer.Serialize(run, JsonOptions)); + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + foreach (var speaker in document.Speakers) + { + await using var cmd = conn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = """ + INSERT INTO speakers (session_id, id, default_label, display_name) VALUES ($sid, $id, $label, $name) + ON CONFLICT(session_id, id) DO UPDATE SET default_label = excluded.default_label, display_name = excluded.display_name; + """; + cmd.Parameters.AddWithValue("$sid", document.SessionId.ToString("D")); + cmd.Parameters.AddWithValue("$id", speaker.Id); + cmd.Parameters.AddWithValue("$label", speaker.DefaultLabel); + cmd.Parameters.AddWithValue("$name", (object?)speaker.DisplayName ?? DBNull.Value); + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + await using (var clear = conn.CreateCommand()) + { + clear.Transaction = tx; + clear.CommandText = "DELETE FROM audio_refs WHERE session_id = $id;"; + clear.Parameters.AddWithValue("$id", document.SessionId.ToString("D")); + await clear.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + foreach (var audio in document.Audio) + { + await using var cmd = conn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = "INSERT INTO audio_refs (session_id, kind, path, sha256) VALUES ($id, $kind, $path, $sha);"; + cmd.Parameters.AddWithValue("$id", document.SessionId.ToString("D")); + cmd.Parameters.AddWithValue("$kind", (int)audio.Kind); + cmd.Parameters.AddWithValue("$path", audio.Path); + cmd.Parameters.AddWithValue("$sha", (object?)audio.Sha256 ?? DBNull.Value); + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + foreach (var segment in document.Segments.Where(s => s.State == SegmentState.Final)) + { + await using var cmd = conn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = """ + INSERT INTO segments (session_id, result_version, id, start_ticks, end_ticks, raw_text, edited_text, + revision, confidence, provenance, words_json, speakers_json) + VALUES ($sid, $version, $id, $start, $end, $raw, $edited, $revision, $confidence, $provenance, $words, $speakers) + ON CONFLICT(session_id, result_version, id) DO UPDATE SET + start_ticks = CASE WHEN excluded.revision > segments.revision THEN excluded.start_ticks ELSE segments.start_ticks END, + end_ticks = CASE WHEN excluded.revision > segments.revision THEN excluded.end_ticks ELSE segments.end_ticks END, + raw_text = CASE WHEN excluded.revision > segments.revision THEN excluded.raw_text ELSE segments.raw_text END, + confidence = CASE WHEN excluded.revision > segments.revision THEN excluded.confidence ELSE segments.confidence END, + provenance = CASE WHEN excluded.revision > segments.revision THEN excluded.provenance ELSE segments.provenance END, + words_json = CASE WHEN excluded.revision > segments.revision THEN excluded.words_json ELSE segments.words_json END, + speakers_json = CASE WHEN excluded.revision > segments.revision THEN excluded.speakers_json ELSE segments.speakers_json END, + revision = MAX(excluded.revision, segments.revision), + edited_text = excluded.edited_text; + """; + cmd.Parameters.AddWithValue("$sid", document.SessionId.ToString("D")); + cmd.Parameters.AddWithValue("$version", segment.ResultVersion); + cmd.Parameters.AddWithValue("$id", segment.Id); + cmd.Parameters.AddWithValue("$start", segment.Start.Ticks); + cmd.Parameters.AddWithValue("$end", segment.End.Ticks); + cmd.Parameters.AddWithValue("$raw", segment.RawText); + cmd.Parameters.AddWithValue("$edited", (object?)segment.EditedText ?? DBNull.Value); + cmd.Parameters.AddWithValue("$revision", segment.Revision); + cmd.Parameters.AddWithValue("$confidence", (object?)segment.Confidence ?? DBNull.Value); + cmd.Parameters.AddWithValue("$provenance", (int)segment.TimingProvenance); + cmd.Parameters.AddWithValue("$words", segment.Words is null ? DBNull.Value : JsonSerializer.Serialize(segment.Words, JsonOptions)); + cmd.Parameters.AddWithValue("$speakers", JsonSerializer.Serialize(segment.Speakers, JsonOptions)); + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + this.gate.Release(); + } + } + + public async ValueTask LoadAsync(Guid sessionId, CancellationToken cancellationToken) + { + await this.gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var conn = this.RequireConnection(); + var id = sessionId.ToString("D"); + TranscriptDocument? document = null; + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = """ + SELECT schema_version, title, source_type, status, created_at, updated_at, duration_ticks, language, + active_result_version, media_json + FROM sessions WHERE id = $id; + """; + cmd.Parameters.AddWithValue("$id", id); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + document = new TranscriptDocument + { + SessionId = sessionId, + SchemaVersion = reader.GetInt32(0), + Title = reader.GetString(1), + SourceType = (TranscriptSourceType)reader.GetInt32(2), + Status = (TranscriptSessionStatus)reader.GetInt32(3), + CreatedAt = DateTimeOffset.Parse(reader.GetString(4), System.Globalization.CultureInfo.InvariantCulture), + UpdatedAt = DateTimeOffset.Parse(reader.GetString(5), System.Globalization.CultureInfo.InvariantCulture), + Duration = reader.IsDBNull(6) ? null : TimeSpan.FromTicks(reader.GetInt64(6)), + Language = reader.IsDBNull(7) ? null : reader.GetString(7), + ActiveResultVersion = reader.GetInt32(8), + Media = reader.IsDBNull(9) ? null : JsonSerializer.Deserialize(reader.GetString(9), JsonOptions) + }; + } + + var runs = new List(); + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = "SELECT run_json FROM runs WHERE session_id = $id ORDER BY result_version;"; + cmd.Parameters.AddWithValue("$id", id); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + runs.Add(JsonSerializer.Deserialize(reader.GetString(0), JsonOptions)!); + } + } + + var speakers = new List(); + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = "SELECT id, default_label, display_name FROM speakers WHERE session_id = $id ORDER BY rowid;"; + cmd.Parameters.AddWithValue("$id", id); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + speakers.Add(new TranscriptSpeaker(reader.GetString(0), reader.GetString(1), reader.IsDBNull(2) ? null : reader.GetString(2))); + } + } + + var audio = await this.ReadAudioRefsAsync(conn, id, cancellationToken).ConfigureAwait(false); + + var segments = new List(); + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = """ + SELECT result_version, id, start_ticks, end_ticks, raw_text, edited_text, revision, confidence, + provenance, words_json, speakers_json + FROM segments WHERE session_id = $id ORDER BY result_version, start_ticks, id; + """; + cmd.Parameters.AddWithValue("$id", id); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + segments.Add(new TranscriptSegment + { + ResultVersion = reader.GetInt32(0), + Id = reader.GetString(1), + Start = TimeSpan.FromTicks(reader.GetInt64(2)), + End = TimeSpan.FromTicks(reader.GetInt64(3)), + RawText = reader.GetString(4), + EditedText = reader.IsDBNull(5) ? null : reader.GetString(5), + Revision = reader.GetInt64(6), + Confidence = reader.IsDBNull(7) ? null : reader.GetDouble(7), + TimingProvenance = (TimingProvenance)reader.GetInt32(8), + Words = reader.IsDBNull(9) ? null : JsonSerializer.Deserialize>(reader.GetString(9), JsonOptions), + Speakers = JsonSerializer.Deserialize>(reader.GetString(10), JsonOptions) ?? [], + State = SegmentState.Final + }); + } + } + + return document with { Runs = runs, Speakers = speakers, Audio = audio, Segments = segments }; + } + finally + { + this.gate.Release(); + } + } + + public async ValueTask> ListAsync(int skip, int take, CancellationToken cancellationToken) + { + ArgumentOutOfRangeException.ThrowIfNegative(skip); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(take); + await this.gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return await this.QuerySummariesAsync( + this.RequireConnection(), + null, + "ORDER BY s.updated_at DESC LIMIT $take OFFSET $skip", + p => + { + p.AddWithValue("$take", take); + p.AddWithValue("$skip", skip); + }, + cancellationToken).ConfigureAwait(false); + } + finally + { + this.gate.Release(); + } + } + + public async ValueTask> MarkInterruptedSessionsAsync(CancellationToken cancellationToken) + { + await this.gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var conn = this.RequireConnection(); + await using var tx = conn.BeginTransaction(); + var inFlight = new[] { TranscriptSessionStatus.Running, TranscriptSessionStatus.Paused, TranscriptSessionStatus.Finalizing } + .Select(s => ((int)s).ToString(System.Globalization.CultureInfo.InvariantCulture)); + var inList = string.Join(",", inFlight); + await ExecuteAsync( + conn, + tx, + $"UPDATE sessions SET status = {(int)TranscriptSessionStatus.Interrupted} WHERE status IN ({inList});", + cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + return await this.QuerySummariesAsync( + conn, + $"WHERE s.status = {(int)TranscriptSessionStatus.Interrupted}", + "ORDER BY s.updated_at DESC", + null, + cancellationToken).ConfigureAwait(false); + } + finally + { + this.gate.Release(); + } + } + + public async ValueTask> DeleteOwnedAudioAsync(Guid sessionId, CancellationToken cancellationToken) + { + await this.gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var conn = this.RequireConnection(); + var id = sessionId.ToString("D"); + var refs = await this.ReadAudioRefsAsync(conn, id, cancellationToken).ConfigureAwait(false); + var deleted = this.DeleteOwnedFiles(refs); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = $"DELETE FROM audio_refs WHERE session_id = $id AND kind = {(int)AudioReferenceKind.Owned};"; + cmd.Parameters.AddWithValue("$id", id); + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + return deleted; + } + finally + { + this.gate.Release(); + } + } + + public async ValueTask DeleteAsync(Guid sessionId, CancellationToken cancellationToken) + { + await this.gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var conn = this.RequireConnection(); + var id = sessionId.ToString("D"); + var refs = await this.ReadAudioRefsAsync(conn, id, cancellationToken).ConfigureAwait(false); + + int removed; + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = "DELETE FROM sessions WHERE id = $id;"; + cmd.Parameters.AddWithValue("$id", id); + removed = await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + var deleted = this.DeleteOwnedFiles(refs); + var sessionMedia = this.GetSessionMediaDirectory(sessionId); + if (Directory.Exists(sessionMedia)) + { + Directory.Delete(sessionMedia, recursive: true); + deleted.Add(sessionMedia); + } + + var kept = refs.Where(r => r.Kind == AudioReferenceKind.ExternalReference).Select(r => r.Path).ToList(); + return new SessionDeletionResult(removed > 0, deleted, kept); + } + finally + { + this.gate.Release(); + } + } + + /// Directory for audio and caches owned by one session. + public string GetSessionMediaDirectory(Guid sessionId) => Path.Combine(this.mediaRoot, sessionId.ToString("N")); + + public async ValueTask DisposeAsync() + { + await this.gate.WaitAsync().ConfigureAwait(false); + try + { + if (this.connection is not null) + { + await this.connection.DisposeAsync().ConfigureAwait(false); + this.connection = null; + } + } + finally + { + this.gate.Release(); + } + } + + private List DeleteOwnedFiles(IEnumerable refs) + { + var deleted = new List(); + foreach (var audio in refs.Where(r => r.Kind == AudioReferenceKind.Owned)) + { + var full = Path.GetFullPath(audio.Path); + // Owned audio must live under the media root; anything else is refused, not deleted. + if (!IsUnder(full, this.mediaRoot)) + { + continue; + } + + if (File.Exists(full)) + { + File.Delete(full); + deleted.Add(full); + } + } + + return deleted; + } + + private static bool IsUnder(string path, string root) + { + var relative = Path.GetRelativePath(root, path); + return !relative.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relative); + } + + private async Task> ReadAudioRefsAsync(SqliteConnection conn, string id, CancellationToken cancellationToken) + { + var audio = new List(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT kind, path, sha256 FROM audio_refs WHERE session_id = $id ORDER BY rowid;"; + cmd.Parameters.AddWithValue("$id", id); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + audio.Add(new AudioReference((AudioReferenceKind)reader.GetInt32(0), reader.GetString(1), reader.IsDBNull(2) ? null : reader.GetString(2))); + } + + return audio; + } + + private async Task> QuerySummariesAsync( + SqliteConnection conn, + string? where, + string tail, + Action? bind, + CancellationToken cancellationToken) + { + await using var cmd = conn.CreateCommand(); + cmd.CommandText = $""" + SELECT s.id, s.title, s.source_type, s.status, s.created_at, s.updated_at, s.duration_ticks, + (SELECT COUNT(*) FROM segments g WHERE g.session_id = s.id AND g.result_version = s.active_result_version) + FROM sessions s {where} {tail}; + """; + bind?.Invoke(cmd.Parameters); + var result = new List(); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + result.Add(new TranscriptSessionSummary( + Guid.Parse(reader.GetString(0)), + reader.GetString(1), + (TranscriptSourceType)reader.GetInt32(2), + (TranscriptSessionStatus)reader.GetInt32(3), + DateTimeOffset.Parse(reader.GetString(4), System.Globalization.CultureInfo.InvariantCulture), + DateTimeOffset.Parse(reader.GetString(5), System.Globalization.CultureInfo.InvariantCulture), + reader.IsDBNull(6) ? null : TimeSpan.FromTicks(reader.GetInt64(6)), + reader.GetInt32(7))); + } + + return result; + } + + private SqliteConnection RequireConnection() => + this.connection ?? throw new InvalidOperationException("Call InitializeAsync before using the session store."); + + private static async Task ExecuteAsync(SqliteConnection conn, SqliteTransaction? tx, string sql, CancellationToken cancellationToken) + { + await using var cmd = conn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task ScalarLongAsync(SqliteConnection conn, string sql, CancellationToken cancellationToken) + { + await using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + return (long)(await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) ?? 0L); + } + + private const string SchemaV1 = """ + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + schema_version INTEGER NOT NULL, + title TEXT NOT NULL, + source_type INTEGER NOT NULL, + status INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + duration_ticks INTEGER NULL, + language TEXT NULL, + active_result_version INTEGER NOT NULL, + media_json TEXT NULL + ); + CREATE INDEX ix_sessions_updated ON sessions (updated_at DESC); + CREATE INDEX ix_sessions_status ON sessions (status); + + CREATE TABLE runs ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + result_version INTEGER NOT NULL, + run_json TEXT NOT NULL, + PRIMARY KEY (session_id, result_version) + ); + + CREATE TABLE speakers ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + id TEXT NOT NULL, + default_label TEXT NOT NULL, + display_name TEXT NULL, + PRIMARY KEY (session_id, id) + ); + + CREATE TABLE audio_refs ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + kind INTEGER NOT NULL, + path TEXT NOT NULL, + sha256 TEXT NULL + ); + CREATE INDEX ix_audio_refs_session ON audio_refs (session_id); + + CREATE TABLE segments ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + result_version INTEGER NOT NULL, + id TEXT NOT NULL, + start_ticks INTEGER NOT NULL, + end_ticks INTEGER NOT NULL, + raw_text TEXT NOT NULL, + edited_text TEXT NULL, + revision INTEGER NOT NULL, + confidence REAL NULL, + provenance INTEGER NOT NULL, + words_json TEXT NULL, + speakers_json TEXT NOT NULL, + PRIMARY KEY (session_id, result_version, id) + ); + CREATE INDEX ix_segments_time ON segments (session_id, result_version, start_ticks); + """; +} diff --git a/src/PrimeDictate.Core/Transcripts/TranscriptDocumentReducer.cs b/src/PrimeDictate.Core/Transcripts/TranscriptDocumentReducer.cs new file mode 100644 index 0000000..650ba00 --- /dev/null +++ b/src/PrimeDictate.Core/Transcripts/TranscriptDocumentReducer.cs @@ -0,0 +1,138 @@ +namespace PrimeDictate.Core.Transcripts; + +/// +/// Pure functions that apply events and user edits to a . +/// Recognizer updates never overwrite user edits or speaker names. +/// +public static class TranscriptDocumentReducer +{ + public static TranscriptDocument Apply(TranscriptDocument document, TranscriptionEvent @event, DateTimeOffset now) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(@event); + if (@event.SessionId != document.SessionId) + { + throw new ArgumentException("Event belongs to a different session.", nameof(@event)); + } + + return @event switch + { + SessionStarted started => StartRun(document, started.Run, now), + SegmentUpserted upserted => UpsertSegment(document, upserted.Segment, now), + SegmentFinalized finalized => UpsertSegment(document, finalized.Segment with { State = SegmentState.Final }, now), + SpeakerUpdated speaker => UpsertSpeaker(document, speaker.Speaker, now), + SessionCompleted completed => document with + { + Status = TranscriptSessionStatus.Completed, + Duration = completed.Duration, + UpdatedAt = now + }, + SessionFailed => document with { Status = TranscriptSessionStatus.Failed, UpdatedAt = now }, + ProgressChanged => document, + _ => throw new NotSupportedException($"Unknown event {@event.GetType().Name}.") + }; + } + + public static TranscriptDocument EditSegment(TranscriptDocument document, string segmentId, string? editedText, DateTimeOffset now) + { + var index = FindSegment(document, segmentId, document.ActiveResultVersion); + if (index < 0) + { + throw new KeyNotFoundException($"Segment {segmentId} is not in the active result."); + } + + var segments = document.Segments.ToList(); + var existing = segments[index]; + var normalized = editedText is not null && string.Equals(editedText, existing.RawText, StringComparison.Ordinal) + ? null + : editedText; + segments[index] = existing with { EditedText = normalized }; + return document with { Segments = segments, UpdatedAt = now }; + } + + public static TranscriptDocument RenameSpeaker(TranscriptDocument document, string speakerId, string? displayName, DateTimeOffset now) + { + var speakers = document.Speakers.ToList(); + var index = speakers.FindIndex(s => s.Id == speakerId); + if (index < 0) + { + throw new KeyNotFoundException($"Speaker {speakerId} does not exist."); + } + + speakers[index] = speakers[index] with + { + DisplayName = string.IsNullOrWhiteSpace(displayName) ? null : displayName.Trim() + }; + return document with { Speakers = speakers, UpdatedAt = now }; + } + + private static TranscriptDocument StartRun(TranscriptDocument document, RecognitionRunInfo run, DateTimeOffset now) + { + var runs = document.Runs.Where(r => r.ResultVersion != run.ResultVersion).Append(run).OrderBy(r => r.ResultVersion).ToList(); + return document with + { + Runs = runs, + ActiveResultVersion = run.ResultVersion, + Language = run.Language ?? document.Language, + Status = TranscriptSessionStatus.Running, + UpdatedAt = now + }; + } + + private static TranscriptDocument UpsertSegment(TranscriptDocument document, TranscriptSegment incoming, DateTimeOffset now) + { + if (incoming.End < incoming.Start) + { + throw new ArgumentException($"Segment {incoming.Id} ends before it starts."); + } + + var segments = document.Segments.ToList(); + var index = FindSegment(document, incoming.Id, incoming.ResultVersion); + if (index < 0) + { + segments.Add(incoming with { EditedText = null }); + return document with { Segments = segments, UpdatedAt = now }; + } + + var existing = segments[index]; + if (incoming.Revision <= existing.Revision) + { + // Duplicate or out-of-order delivery. + return document; + } + + segments[index] = incoming with { EditedText = existing.EditedText }; + return document with { Segments = segments, UpdatedAt = now }; + } + + private static TranscriptDocument UpsertSpeaker(TranscriptDocument document, TranscriptSpeaker incoming, DateTimeOffset now) + { + var speakers = document.Speakers.ToList(); + var index = speakers.FindIndex(s => s.Id == incoming.Id); + if (index < 0) + { + speakers.Add(incoming); + } + else + { + // Provider label changes never discard a name the user chose. + speakers[index] = incoming with { DisplayName = speakers[index].DisplayName ?? incoming.DisplayName }; + } + + return document with { Speakers = speakers, UpdatedAt = now }; + } + + private static int FindSegment(TranscriptDocument document, string segmentId, int resultVersion) + { + for (var i = 0; i < document.Segments.Count; i++) + { + var segment = document.Segments[i]; + if (segment.ResultVersion == resultVersion && string.Equals(segment.Id, segmentId, StringComparison.Ordinal)) + { + return i; + } + } + + return -1; + } +} diff --git a/src/PrimeDictate.Core/Transcripts/TranscriptModel.cs b/src/PrimeDictate.Core/Transcripts/TranscriptModel.cs new file mode 100644 index 0000000..ee6b234 --- /dev/null +++ b/src/PrimeDictate.Core/Transcripts/TranscriptModel.cs @@ -0,0 +1,169 @@ +namespace PrimeDictate.Core.Transcripts; + +/// Where a segment's or word's timing came from. +public enum TimingProvenance +{ + /// Timing emitted by the recognition model itself. + Model = 0, + + /// Timing produced by a separate forced-alignment step. + Alignment = 1, + + /// + /// Only the boundaries of the audio window that was recognized are known. The words fall + /// somewhere inside; exports must present this timing as approximate. + /// + ApproximateChunk = 2 +} + +public enum SegmentState +{ + Provisional = 0, + Final = 1 +} + +public enum TranscriptSourceType +{ + ImportedFile = 0, + Microphone = 1 +} + +public enum TranscriptSessionStatus +{ + Created = 0, + Running = 1, + Paused = 2, + Finalizing = 3, + Completed = 4, + Failed = 5, + Canceled = 6, + /// The app stopped while the session was running; finalized content is intact. + Interrupted = 7 +} + +public sealed record WordTiming( + string Text, + TimeSpan Start, + TimeSpan End, + double? Confidence, + TimingProvenance Provenance); + +/// A speaker's share of a segment; several entries represent overlap or uncertainty. +public sealed record SpeakerAttribution(string SpeakerId, TimeSpan Start, TimeSpan End, double? Confidence); + +public sealed record TranscriptSegment +{ + public required string Id { get; init; } + + public required TimeSpan Start { get; init; } + + public required TimeSpan End { get; init; } + + /// Recognizer output exactly as produced, never modified by edits. + public required string RawText { get; init; } + + /// User edit, if any. Null means the raw text is displayed. + public string? EditedText { get; init; } + + public SegmentState State { get; init; } + + /// Monotonic per-segment revision; an update with a lower or equal revision is ignored. + public long Revision { get; init; } + + public IReadOnlyList? Words { get; init; } + + /// Null when the provider does not report confidence. + public double? Confidence { get; init; } + + public IReadOnlyList Speakers { get; init; } = []; + + public required TimingProvenance TimingProvenance { get; init; } + + /// Which recognition run produced this segment. + public int ResultVersion { get; init; } = 1; + + public string DisplayText => this.EditedText ?? this.RawText; + + public bool IsEdited => this.EditedText is not null && !string.Equals(this.EditedText, this.RawText, StringComparison.Ordinal); +} + +/// Anonymous speaker found by diarization. Renaming changes only . +public sealed record TranscriptSpeaker(string Id, string DefaultLabel, string? DisplayName) +{ + public string Name => string.IsNullOrWhiteSpace(this.DisplayName) ? this.DefaultLabel : this.DisplayName; +} + +public sealed record MediaMetadata( + string? OriginalFileName, + string? ContainerFormat, + string? Codec, + int? SampleRate, + int? Channels, + int? SelectedTrack, + string? ChannelMapping, + TimeSpan? Duration); + +/// How the session relates to audio it can play back. +public enum AudioReferenceKind +{ + /// Audio recorded or copied into app storage, owned and deleted with the session. + Owned = 0, + + /// User's original file, referenced in place. Never deleted by the app. + ExternalReference = 1 +} + +public sealed record AudioReference(AudioReferenceKind Kind, string Path, string? Sha256); + +/// Everything that determines a run's result, captured when the run starts. +public sealed record RecognitionRunInfo( + int ResultVersion, + string AsrModelId, + string? AsrModelRevision, + string? DiarizerModelId, + string? DiarizerModelRevision, + string? Language, + string RuntimeName, + string? RuntimeVersion, + string RequestedBackend, + string EffectiveBackend, + DateTimeOffset StartedAt); + +public sealed record TranscriptDocument +{ + public const int CurrentSchemaVersion = 1; + + public int SchemaVersion { get; init; } = CurrentSchemaVersion; + + public required Guid SessionId { get; init; } + + public required string Title { get; init; } + + public required TranscriptSourceType SourceType { get; init; } + + public required DateTimeOffset CreatedAt { get; init; } + + public required DateTimeOffset UpdatedAt { get; init; } + + public MediaMetadata? Media { get; init; } + + public TimeSpan? Duration { get; init; } + + public string? Language { get; init; } + + public TranscriptSessionStatus Status { get; init; } + + public IReadOnlyList Audio { get; init; } = []; + + public IReadOnlyList Runs { get; init; } = []; + + /// The run whose segments are shown; older runs' segments are kept, not overwritten. + public int ActiveResultVersion { get; init; } = 1; + + public IReadOnlyList Speakers { get; init; } = []; + + public IReadOnlyList Segments { get; init; } = []; + + public IEnumerable ActiveSegments => + this.Segments.Where(s => s.ResultVersion == this.ActiveResultVersion).OrderBy(s => s.Start).ThenBy(s => s.Id, StringComparer.Ordinal); +} diff --git a/src/PrimeDictate.Core/Transcripts/TranscriptionEvents.cs b/src/PrimeDictate.Core/Transcripts/TranscriptionEvents.cs new file mode 100644 index 0000000..1fb99d4 --- /dev/null +++ b/src/PrimeDictate.Core/Transcripts/TranscriptionEvents.cs @@ -0,0 +1,37 @@ +namespace PrimeDictate.Core.Transcripts; + +/// +/// Application-owned session events. Providers' own event names are mapped onto these. +/// +public abstract record TranscriptionEvent(Guid SessionId); + +public sealed record SessionStarted(Guid SessionId, RecognitionRunInfo Run) : TranscriptionEvent(SessionId); + +/// Inserts or replaces a segment. Idempotent by (segment ID, revision). +public sealed record SegmentUpserted(Guid SessionId, TranscriptSegment Segment) : TranscriptionEvent(SessionId); + +public sealed record SegmentFinalized(Guid SessionId, TranscriptSegment Segment) : TranscriptionEvent(SessionId); + +public sealed record SpeakerUpdated(Guid SessionId, TranscriptSpeaker Speaker) : TranscriptionEvent(SessionId); + +public enum TranscriptionPhase +{ + LoadingModel = 0, + Decoding = 1, + Transcribing = 2, + DetectingSpeakers = 3, + Finalizing = 4 +} + +/// Progress; is null for live sessions with no known end. +public sealed record ProgressChanged( + Guid SessionId, + TranscriptionPhase Phase, + double? Fraction, + TimeSpan ProcessedAudio, + TimeSpan? Backlog) : TranscriptionEvent(SessionId); + +public sealed record SessionCompleted(Guid SessionId, TimeSpan Duration) : TranscriptionEvent(SessionId); + +/// Failure. The message is technical and must never contain transcript text. +public sealed record SessionFailed(Guid SessionId, string ErrorCode, string Message, bool Recoverable) : TranscriptionEvent(SessionId); diff --git a/tests/PrimeDictate.Core.Tests/AudioTests.cs b/tests/PrimeDictate.Core.Tests/AudioTests.cs new file mode 100644 index 0000000..f697a38 --- /dev/null +++ b/tests/PrimeDictate.Core.Tests/AudioTests.cs @@ -0,0 +1,178 @@ +using PrimeDictate.Core.Audio; + +namespace PrimeDictate.Core.Tests; + +public class AudioTests +{ + [Fact] + public void Sample_offsets_convert_to_exact_time() + { + var format = AudioFormat.SpeechTimeline; + Assert.Equal(TimeSpan.FromSeconds(1), format.ToTime(16_000)); + Assert.Equal(TimeSpan.FromHours(3), format.ToTime(16_000L * 3 * 3600)); + Assert.Equal(16_000L * 7200, format.ToSampleOffset(TimeSpan.FromHours(2))); + } + + [Fact] + public void Frame_owns_a_copy_of_callback_memory() + { + var pooled = new float[] { 0.1f, 0.2f, 0.3f, 0.4f }; + var frame = AudioFrame.CopyFrom(pooled, new AudioFormat(48_000, 2, AudioSampleFormat.Float32), 7, 96_000); + Array.Clear(pooled); + Assert.Equal(new[] { 0.1f, 0.2f, 0.3f, 0.4f }, frame.Samples.ToArray()); + Assert.Equal(2, frame.SamplesPerChannel); + Assert.Equal(TimeSpan.FromSeconds(2), frame.Start); + Assert.Equal(96_002, frame.EndSampleOffset); + } + + [Fact] + public void Frame_rejects_partial_interleaved_frames() + { + Assert.Throws(() => + AudioFrame.CopyFrom(new float[3], new AudioFormat(48_000, 2, AudioSampleFormat.Float32), 0, 0)); + } + + [Fact] + public void Pcm16_round_trips_through_float() + { + short[] source = [0, 1000, -1000, short.MaxValue, short.MinValue + 1]; + var bytes = new byte[source.Length * 2]; + Buffer.BlockCopy(source, 0, bytes, 0, bytes.Length); + var floats = new float[source.Length]; + AudioConversion.Pcm16ToFloat(bytes, floats); + var back = new byte[bytes.Length]; + AudioConversion.FloatToPcm16(floats, back); + var result = new short[source.Length]; + Buffer.BlockCopy(back, 0, result, 0, back.Length); + for (var i = 0; i < source.Length; i++) + { + Assert.InRange(result[i] - source[i], -1, 1); + } + } + + [Fact] + public void Downmix_averages_and_channel_selection_is_explicit() + { + float[] stereo = [1f, 0f, 0.5f, -0.5f]; + Assert.Equal(new[] { 0.5f, 0f }, AudioConversion.DownmixToMono(stereo, 2)); + Assert.Equal(new[] { 0f, -0.5f }, AudioConversion.SelectChannel(stereo, 2, 1)); + Assert.Throws(() => AudioConversion.SelectChannel(stereo, 2, 2)); + } + + [Theory] + [InlineData(48_000)] + [InlineData(44_100)] + [InlineData(22_050)] + [InlineData(8_000)] + public void Resampler_output_length_matches_rate_ratio(int inputRate) + { + var resampler = new StreamingResampler(inputRate, 16_000); + var input = new float[inputRate * 3 + 17]; + var produced = resampler.Process(input).Length + resampler.Flush().Length; + var expected = (long)Math.Ceiling(input.Length * 16_000.0 / inputRate); + Assert.Equal(expected, produced); + Assert.Equal(expected, resampler.OutputSamplesProduced); + } + + [Theory] + [InlineData(48_000)] + [InlineData(44_100)] + public void Resampler_result_does_not_depend_on_chunk_boundaries(int inputRate) + { + var input = Sine(inputRate, 440, inputRate * 2); + var whole = new StreamingResampler(inputRate, 16_000); + var expected = whole.Process(input).Concat(whole.Flush()).ToArray(); + + var chunked = new StreamingResampler(inputRate, 16_000); + var random = new Random(1234); + var output = new List(); + var position = 0; + while (position < input.Length) + { + var size = Math.Min(random.Next(1, 3000), input.Length - position); + output.AddRange(chunked.Process(input.AsSpan(position, size))); + position += size; + } + + output.AddRange(chunked.Flush()); + Assert.Equal(expected.Length, output.Count); + for (var i = 0; i < expected.Length; i++) + { + Assert.Equal(expected[i], output[i], 5); + } + } + + [Fact] + public void Resampler_keeps_speech_band_and_rejects_aliasing_tones() + { + const int inputRate = 48_000; + var passband = Resample(Sine(inputRate, 1_000, inputRate), inputRate); + var alias = Resample(Sine(inputRate, 12_000, inputRate), inputRate); + + // Skip filter warm-up and tail. + Assert.InRange(Rms(passband.AsSpan(1000, 14_000)), 0.69, 0.72); // 1/sqrt(2) + Assert.True(Rms(alias.AsSpan(1000, 14_000)) < 0.01, "12 kHz must not fold into the 16 kHz timeline."); + } + + [Fact] + public void Resampler_preserves_timing_of_an_impulse() + { + const int inputRate = 48_000; + var input = new float[inputRate]; + input[24_000] = 1f; // exactly 0.5 s + var output = Resample(input, inputRate); + var peak = Array.IndexOf(output, output.Max()); + Assert.Equal(8_000, peak); + } + + [Fact] + public void Resampler_passthrough_when_rates_match() + { + var resampler = new StreamingResampler(16_000, 16_000); + float[] input = [0.1f, 0.2f]; + Assert.Equal(input, resampler.Process(input)); + Assert.Empty(resampler.Flush()); + } + + [Fact] + public void Paused_time_is_not_recorded_and_maps_back_to_wall_clock() + { + var timeline = new RecordedAudioTimeline(AudioFormat.SpeechTimeline); + var t0 = new DateTimeOffset(2026, 9, 29, 12, 0, 0, TimeSpan.Zero); + timeline.BeginInterval(t0); + Assert.Equal(0, timeline.Append(16_000 * 10)); + timeline.EndInterval(CaptureIntervalEnd.Paused); + + Assert.Throws(() => timeline.Append(1)); + + var resumedAt = t0.AddMinutes(5); + timeline.BeginInterval(resumedAt); + Assert.Equal(16_000 * 10, timeline.Append(16_000 * 4)); + timeline.EndInterval(CaptureIntervalEnd.Stopped); + + Assert.Equal(TimeSpan.FromSeconds(14), timeline.RecordedDuration); + Assert.Equal(t0.AddSeconds(3), timeline.ToWallClock(TimeSpan.FromSeconds(3))); + Assert.Equal(resumedAt.AddSeconds(1), timeline.ToWallClock(TimeSpan.FromSeconds(11))); + Assert.Null(timeline.ToWallClock(TimeSpan.FromSeconds(14))); + } + + private static float[] Resample(float[] input, int inputRate) + { + var resampler = new StreamingResampler(inputRate, 16_000); + return resampler.Process(input).Concat(resampler.Flush()).ToArray(); + } + + private static float[] Sine(int rate, double frequency, int count) => + Enumerable.Range(0, count).Select(i => (float)Math.Sin(2 * Math.PI * frequency * i / rate)).ToArray(); + + private static double Rms(ReadOnlySpan samples) + { + double sum = 0; + foreach (var s in samples) + { + sum += s * s; + } + + return Math.Sqrt(sum / samples.Length); + } +} diff --git a/tests/PrimeDictate.Core.Tests/CoordinationTests.cs b/tests/PrimeDictate.Core.Tests/CoordinationTests.cs new file mode 100644 index 0000000..954b2b6 --- /dev/null +++ b/tests/PrimeDictate.Core.Tests/CoordinationTests.cs @@ -0,0 +1,138 @@ +using PrimeDictate.Core.Coordination; + +namespace PrimeDictate.Core.Tests; + +public class CoordinationTests +{ + private sealed class FakeConsumer(string name, bool active, bool busy = false, bool failSuspend = false) : IMicrophoneConsumer + { + public string Name { get; } = name; + + public bool IsBusy { get; set; } = busy; + + public bool IsActive { get; set; } = active; + + public List Calls { get; } = []; + + public ValueTask SuspendAsync(string reason, CancellationToken cancellationToken) + { + if (failSuspend) + { + throw new InvalidOperationException("device error"); + } + + this.Calls.Add("suspend"); + this.IsActive = false; + return ValueTask.CompletedTask; + } + + public ValueTask ResumeAsync(CancellationToken cancellationToken) + { + this.Calls.Add("resume"); + this.IsActive = true; + return ValueTask.CompletedTask; + } + } + + [Fact] + public async Task Transcription_suspends_active_consumers_and_restores_only_those() + { + var coordinator = new MicrophoneCoordinator(); + var wake = new FakeConsumer("Wake word", active: true); + var dictation = new FakeConsumer("Dictation", active: false); + coordinator.Register(wake); + coordinator.Register(dictation); + + await using (var lease = await coordinator.AcquireAsync("Transcription", CancellationToken.None)) + { + Assert.Equal("Transcription", coordinator.CurrentOwner); + Assert.Contains("Transcription", coordinator.SuspensionReason); + Assert.Equal(["suspend"], wake.Calls); + Assert.Empty(dictation.Calls); + } + + Assert.Null(coordinator.CurrentOwner); + Assert.Equal(["suspend", "resume"], wake.Calls); + Assert.Empty(dictation.Calls); + Assert.False(dictation.IsActive); + } + + [Fact] + public async Task Busy_dictation_or_existing_owner_blocks_acquire() + { + var coordinator = new MicrophoneCoordinator(); + var dictation = new FakeConsumer("Dictation", active: true, busy: true); + coordinator.Register(dictation); + var ex = await Assert.ThrowsAsync(async () => await coordinator.AcquireAsync("Transcription", CancellationToken.None)); + Assert.Contains("Dictation", ex.Message); + Assert.Empty(dictation.Calls); + + dictation.IsBusy = false; + await using var lease = await coordinator.AcquireAsync("Transcription", CancellationToken.None); + await Assert.ThrowsAsync(async () => await coordinator.AcquireAsync("Another session", CancellationToken.None)); + } + + [Fact] + public async Task Failed_suspend_rolls_back_and_releases() + { + var coordinator = new MicrophoneCoordinator(); + var ok = new FakeConsumer("Wake word", active: true); + var broken = new FakeConsumer("Dictation", active: true, failSuspend: true); + coordinator.Register(ok); + coordinator.Register(broken); + + await Assert.ThrowsAsync(async () => await coordinator.AcquireAsync("Transcription", CancellationToken.None)); + Assert.Null(coordinator.CurrentOwner); + Assert.Equal(["suspend", "resume"], ok.Calls); + } + + [Fact] + public async Task Double_dispose_releases_once() + { + var coordinator = new MicrophoneCoordinator(); + var wake = new FakeConsumer("Wake word", active: true); + coordinator.Register(wake); + var lease = await coordinator.AcquireAsync("Transcription", CancellationToken.None); + await lease.DisposeAsync(); + await lease.DisposeAsync(); + Assert.Equal(["suspend", "resume"], wake.Calls); + } + + [Fact] + public async Task Live_work_is_granted_before_queued_file_windows() + { + var scheduler = new ModelLeaseScheduler(); + var order = new List(); + var first = await scheduler.AcquireAsync("whisper", ModelLeasePriority.Background, CancellationToken.None); + + var file2 = scheduler.AcquireAsync("whisper", ModelLeasePriority.Background, CancellationToken.None).AsTask() + .ContinueWith(t => { lock (order) { order.Add("file2"); } return t.Result; }, TaskScheduler.Default); + var file3 = scheduler.AcquireAsync("whisper", ModelLeasePriority.Background, CancellationToken.None).AsTask() + .ContinueWith(t => { lock (order) { order.Add("file3"); } return t.Result; }, TaskScheduler.Default); + var live = scheduler.AcquireAsync("whisper", ModelLeasePriority.Live, CancellationToken.None).AsTask() + .ContinueWith(t => { lock (order) { order.Add("live"); } return t.Result; }, TaskScheduler.Default); + Assert.Equal(3, scheduler.QueuedCount("whisper")); + + first.Dispose(); + (await live).Dispose(); + (await file2).Dispose(); + (await file3).Dispose(); + + Assert.Equal(["live", "file2", "file3"], order); + Assert.Equal(0, scheduler.QueuedCount("whisper")); + } + + [Fact] + public async Task Different_models_do_not_block_each_other_and_cancel_removes_waiter() + { + var scheduler = new ModelLeaseScheduler(); + using var a = await scheduler.AcquireAsync("whisper", ModelLeasePriority.Background, CancellationToken.None); + using var b = await scheduler.AcquireAsync("parakeet", ModelLeasePriority.Background, CancellationToken.None); + + using var cts = new CancellationTokenSource(); + var waiting = scheduler.AcquireAsync("whisper", ModelLeasePriority.Live, cts.Token).AsTask(); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => waiting); + Assert.Equal(0, scheduler.QueuedCount("whisper")); + } +} diff --git a/tests/PrimeDictate.Core.Tests/PrimeDictate.Core.Tests.csproj b/tests/PrimeDictate.Core.Tests/PrimeDictate.Core.Tests.csproj new file mode 100644 index 0000000..5536fbd --- /dev/null +++ b/tests/PrimeDictate.Core.Tests/PrimeDictate.Core.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + enable + enable + false + true + + + + + + + + + + + + + + + + + diff --git a/tests/PrimeDictate.Core.Tests/StorageTests.cs b/tests/PrimeDictate.Core.Tests/StorageTests.cs new file mode 100644 index 0000000..163e462 --- /dev/null +++ b/tests/PrimeDictate.Core.Tests/StorageTests.cs @@ -0,0 +1,283 @@ +using System.Text.Json.Nodes; +using Microsoft.Data.Sqlite; +using PrimeDictate.Core.Sessions; +using PrimeDictate.Core.Settings; +using PrimeDictate.Core.Storage; +using PrimeDictate.Core.Transcripts; +using static PrimeDictate.Core.Tests.TestData; + +namespace PrimeDictate.Core.Tests; + +public sealed class StorageTests : IDisposable +{ + private readonly string root = Path.Combine(Path.GetTempPath(), "pd-core-tests", Guid.NewGuid().ToString("N")); + + public void Dispose() + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(this.root)) + { + Directory.Delete(this.root, recursive: true); + } + } + + private AppDataPaths Paths => new(this.root); + + private async Task OpenStoreAsync() + { + var store = SqliteTranscriptionSessionStore.Create(this.Paths); + await store.InitializeAsync(CancellationToken.None); + return store; + } + + [Fact] + public async Task Checkpoint_round_trips_final_content_but_not_partials() + { + await using var store = await this.OpenStoreAsync(); + var doc = NewDocument(); + doc = TranscriptDocumentReducer.Apply(doc, new SessionStarted(doc.SessionId, Run()), Now); + doc = TranscriptDocumentReducer.Apply(doc, new SpeakerUpdated(doc.SessionId, new TranscriptSpeaker("spk0", "Speaker 1", null)), Now); + var final = Segment("s1", "Größe — 日本語 \"quoted\"\nline", 0, 2.5) with + { + Confidence = 0.8, + Words = [new WordTiming("Größe", TimeSpan.Zero, TimeSpan.FromSeconds(0.4), 0.9, TimingProvenance.Model)], + Speakers = [new SpeakerAttribution("spk0", TimeSpan.Zero, TimeSpan.FromSeconds(2.5), null)], + TimingProvenance = TimingProvenance.Model + }; + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, final), Now); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(doc.SessionId, Segment("s2", "partial", 2.5, 3, state: SegmentState.Provisional)), Now); + doc = TranscriptDocumentReducer.RenameSpeaker(doc, "spk0", "Justin", Now); + doc = doc with { Media = new MediaMetadata("call.m4a", "mov", "aac", 48_000, 2, 0, "downmix", TimeSpan.FromMinutes(3)) }; + + await store.SaveCheckpointAsync(doc, CancellationToken.None); + var loaded = await store.LoadAsync(doc.SessionId, CancellationToken.None); + + Assert.NotNull(loaded); + var segment = Assert.Single(loaded!.Segments); + Assert.Equal(final.RawText, segment.RawText); + Assert.Equal(final.Start, segment.Start); + Assert.Equal(final.End, segment.End); + Assert.Equal(0.8, segment.Confidence); + Assert.Equal(TimingProvenance.Model, segment.TimingProvenance); + Assert.Equal("Größe", Assert.Single(segment.Words!).Text); + Assert.Equal("spk0", Assert.Single(segment.Speakers).SpeakerId); + Assert.Equal("Justin", Assert.Single(loaded.Speakers).Name); + Assert.Equal("whisper-onnx/base.en", Assert.Single(loaded.Runs).AsrModelId); + Assert.Equal(48_000, loaded.Media!.SampleRate); + Assert.Equal(TranscriptSessionStatus.Running, loaded.Status); + } + + [Fact] + public async Task Edits_persist_and_stale_revisions_do_not_overwrite() + { + await using var store = await this.OpenStoreAsync(); + var doc = NewDocument(); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("s1", "new text", 0, 1, revision: 5)), Now); + await store.SaveCheckpointAsync(doc, CancellationToken.None); + + var stale = doc with { Segments = [Segment("s1", "old text", 0, 1, revision: 3) with { EditedText = "edited" }] }; + await store.SaveCheckpointAsync(stale, CancellationToken.None); + + var segment = Assert.Single((await store.LoadAsync(doc.SessionId, CancellationToken.None))!.Segments); + Assert.Equal("new text", segment.RawText); + Assert.Equal(5, segment.Revision); + Assert.Equal("edited", segment.EditedText); + } + + [Fact] + public async Task Startup_marks_in_flight_sessions_interrupted() + { + await using var store = await this.OpenStoreAsync(); + var running = NewDocument() with { Status = TranscriptSessionStatus.Running }; + var done = NewDocument() with { Status = TranscriptSessionStatus.Completed }; + await store.SaveCheckpointAsync(running, CancellationToken.None); + await store.SaveCheckpointAsync(done, CancellationToken.None); + + var interrupted = await store.MarkInterruptedSessionsAsync(CancellationToken.None); + Assert.Equal(running.SessionId, Assert.Single(interrupted).SessionId); + Assert.Equal(TranscriptSessionStatus.Interrupted, (await store.LoadAsync(running.SessionId, CancellationToken.None))!.Status); + Assert.Equal(TranscriptSessionStatus.Completed, (await store.LoadAsync(done.SessionId, CancellationToken.None))!.Status); + } + + [Fact] + public async Task Delete_removes_owned_audio_and_never_the_users_original() + { + await using var store = await this.OpenStoreAsync(); + var doc = NewDocument(); + var mediaDir = store.GetSessionMediaDirectory(doc.SessionId); + Directory.CreateDirectory(mediaDir); + var owned = Path.Combine(mediaDir, "recording.flac"); + File.WriteAllText(owned, "audio"); + var original = Path.Combine(this.root, "Documents", "interview.m4a"); + Directory.CreateDirectory(Path.GetDirectoryName(original)!); + File.WriteAllText(original, "user file"); + var outsideButMarkedOwned = Path.Combine(this.root, "Documents", "not-ours.wav"); + File.WriteAllText(outsideButMarkedOwned, "user file"); + + doc = doc with + { + Audio = + [ + new AudioReference(AudioReferenceKind.Owned, owned, null), + new AudioReference(AudioReferenceKind.ExternalReference, original, null), + new AudioReference(AudioReferenceKind.Owned, outsideButMarkedOwned, null) + ] + }; + await store.SaveCheckpointAsync(doc, CancellationToken.None); + + var result = await store.DeleteAsync(doc.SessionId, CancellationToken.None); + + Assert.True(result.Existed); + Assert.False(File.Exists(owned)); + Assert.False(Directory.Exists(mediaDir)); + Assert.True(File.Exists(original)); + Assert.True(File.Exists(outsideButMarkedOwned)); + Assert.Contains(original, result.KeptExternalFiles); + Assert.Null(await store.LoadAsync(doc.SessionId, CancellationToken.None)); + } + + [Fact] + public async Task Transcript_only_retention_deletes_audio_and_keeps_text() + { + await using var store = await this.OpenStoreAsync(); + var doc = NewDocument(); + var mediaDir = store.GetSessionMediaDirectory(doc.SessionId); + Directory.CreateDirectory(mediaDir); + var owned = Path.Combine(mediaDir, "recording.flac"); + File.WriteAllText(owned, "audio"); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("s1", "keep me", 0, 1)), Now); + doc = doc with { Audio = [new AudioReference(AudioReferenceKind.Owned, owned, null)] }; + await store.SaveCheckpointAsync(doc, CancellationToken.None); + + var deleted = await store.DeleteOwnedAudioAsync(doc.SessionId, CancellationToken.None); + + Assert.Equal([Path.GetFullPath(owned)], deleted); + var loaded = (await store.LoadAsync(doc.SessionId, CancellationToken.None))!; + Assert.Empty(loaded.Audio); + Assert.Equal("keep me", Assert.Single(loaded.Segments).RawText); + } + + [Fact] + public async Task List_is_paged_newest_first_with_segment_counts() + { + await using var store = await this.OpenStoreAsync(); + for (var i = 0; i < 5; i++) + { + var doc = NewDocument() with { Title = $"S{i}", UpdatedAt = Now.AddMinutes(i) }; + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("a", "x", 0, 1)), Now.AddMinutes(i)); + await store.SaveCheckpointAsync(doc, CancellationToken.None); + } + + var page = await store.ListAsync(1, 2, CancellationToken.None); + Assert.Equal(["S3", "S2"], page.Select(p => p.Title)); + Assert.All(page, p => Assert.Equal(1, p.SegmentCount)); + } + + [Fact] + public async Task A_database_from_a_newer_version_is_not_modified() + { + Directory.CreateDirectory(Path.GetDirectoryName(this.Paths.SessionDatabasePath)!); + await using (var conn = new SqliteConnection($"Data Source={this.Paths.SessionDatabasePath};Pooling=False")) + { + await conn.OpenAsync(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = "PRAGMA user_version = 99;"; + await cmd.ExecuteNonQueryAsync(); + } + + await using var store = SqliteTranscriptionSessionStore.Create(this.Paths); + await Assert.ThrowsAsync(async () => await store.InitializeAsync(CancellationToken.None)); + } + + [Fact] + public async Task Initialize_is_idempotent_across_restarts() + { + var doc = NewDocument(); + await using (var store = await this.OpenStoreAsync()) + { + await store.SaveCheckpointAsync(doc, CancellationToken.None); + } + + await using (var store = await this.OpenStoreAsync()) + { + Assert.NotNull(await store.LoadAsync(doc.SessionId, CancellationToken.None)); + } + } + + [Fact] + public void Unix_permissions_are_user_only() + { + if (OperatingSystem.IsWindows()) + { + return; + } + + var dir = Path.Combine(this.root, "private"); + AppDataPaths.EnsurePrivateDirectory(dir); + Assert.Equal(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute, File.GetUnixFileMode(dir)); + } + + [Fact] + public void Preferences_first_run_uses_seed_without_writing() + { + var store = new TranscriptionPreferencesStore(this.Paths.TranscriptionPreferencesPath); + var result = store.Load(() => new TranscriptionPreferences { AsrModelId = "whisper-onnx/base.en" }); + Assert.Equal("whisper-onnx/base.en", result.Preferences.AsrModelId); + Assert.False(File.Exists(store.FilePath)); + } + + [Fact] + public void Preferences_migration_backs_up_once_and_is_idempotent() + { + var path = this.Paths.TranscriptionPreferencesPath; + Directory.CreateDirectory(this.root); + File.WriteAllText(path, """{ "SchemaVersion": 1, "Model": "parakeet", "AudioRetention": "TranscriptOnly" }"""); + var migrations = new Dictionary> + { + [1] = o => + { + o["AsrModelId"] = o["Model"]?.GetValue(); + o.Remove("Model"); + return o; + } + }; + var store = new TranscriptionPreferencesStore(path, migrations); + + var first = store.Load(() => new TranscriptionPreferences(), targetVersion: 2); + Assert.Equal("parakeet", first.Preferences.AsrModelId); + Assert.Equal(Sessions.AudioRetention.TranscriptOnly, first.Preferences.AudioRetention); + Assert.Equal(2, first.Preferences.SchemaVersion); + Assert.NotNull(first.BackupPath); + Assert.Contains("\"Model\"", File.ReadAllText(first.BackupPath!)); + + var second = store.Load(() => new TranscriptionPreferences(), targetVersion: 2); + Assert.Null(second.BackupPath); + Assert.Equal("parakeet", second.Preferences.AsrModelId); + Assert.Single(Directory.GetFiles(this.root, "*.bak")); + } + + [Fact] + public void Preferences_from_a_newer_version_are_read_only_and_untouched() + { + var path = this.Paths.TranscriptionPreferencesPath; + Directory.CreateDirectory(this.root); + const string newer = """{ "SchemaVersion": 9, "AsrModelId": "future" }"""; + File.WriteAllText(path, newer); + var result = new TranscriptionPreferencesStore(path).Load(() => new TranscriptionPreferences()); + Assert.True(result.IsReadOnly); + Assert.NotNull(result.Warning); + Assert.Equal(newer, File.ReadAllText(path)); + } + + [Fact] + public void Corrupt_preferences_are_kept_as_backup_and_reset() + { + var path = this.Paths.TranscriptionPreferencesPath; + Directory.CreateDirectory(this.root); + File.WriteAllText(path, "{ not json"); + var result = new TranscriptionPreferencesStore(path).Load(() => new TranscriptionPreferences { AsrModelId = "seed" }); + Assert.Equal("seed", result.Preferences.AsrModelId); + Assert.NotNull(result.Warning); + Assert.Equal("{ not json", File.ReadAllText(result.BackupPath!)); + } +} diff --git a/tests/PrimeDictate.Core.Tests/TestData.cs b/tests/PrimeDictate.Core.Tests/TestData.cs new file mode 100644 index 0000000..4d24756 --- /dev/null +++ b/tests/PrimeDictate.Core.Tests/TestData.cs @@ -0,0 +1,40 @@ +using PrimeDictate.Core.Transcripts; + +namespace PrimeDictate.Core.Tests; + +internal static class TestData +{ + public static readonly DateTimeOffset Now = new(2026, 9, 29, 18, 0, 0, TimeSpan.Zero); + + public static TranscriptDocument NewDocument(Guid? id = null) => new() + { + SessionId = id ?? Guid.NewGuid(), + Title = "Weekly sync", + SourceType = TranscriptSourceType.ImportedFile, + CreatedAt = Now, + UpdatedAt = Now, + Status = TranscriptSessionStatus.Created + }; + + public static RecognitionRunInfo Run(int version = 1, string model = "whisper-onnx/base.en") => new( + version, model, "rev1", null, null, "en", "sherpa-onnx", "1.13.0", "cpu", "cpu", Now); + + public static TranscriptSegment Segment( + string id, + string text, + double start, + double end, + long revision = 1, + SegmentState state = SegmentState.Final, + int resultVersion = 1) => new() + { + Id = id, + Start = TimeSpan.FromSeconds(start), + End = TimeSpan.FromSeconds(end), + RawText = text, + Revision = revision, + State = state, + ResultVersion = resultVersion, + TimingProvenance = TimingProvenance.ApproximateChunk + }; +} diff --git a/tests/PrimeDictate.Core.Tests/TranscriptTests.cs b/tests/PrimeDictate.Core.Tests/TranscriptTests.cs new file mode 100644 index 0000000..bfd8026 --- /dev/null +++ b/tests/PrimeDictate.Core.Tests/TranscriptTests.cs @@ -0,0 +1,150 @@ +using PrimeDictate.Core.Sessions; +using PrimeDictate.Core.Transcripts; +using static PrimeDictate.Core.Tests.TestData; + +namespace PrimeDictate.Core.Tests; + +public class TranscriptTests +{ + [Fact] + public void Upserts_are_idempotent_and_ignore_stale_revisions() + { + var doc = NewDocument(); + doc = TranscriptDocumentReducer.Apply(doc, new SessionStarted(doc.SessionId, Run()), Now); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(doc.SessionId, Segment("s1", "hel", 0, 1, 1, SegmentState.Provisional)), Now); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(doc.SessionId, Segment("s1", "hello", 0, 1.2, 2, SegmentState.Provisional)), Now); + var afterDuplicate = TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(doc.SessionId, Segment("s1", "hello", 0, 1.2, 2, SegmentState.Provisional)), Now); + var afterStale = TranscriptDocumentReducer.Apply(afterDuplicate, new SegmentUpserted(doc.SessionId, Segment("s1", "he", 0, 1, 1, SegmentState.Provisional)), Now); + + Assert.Same(doc, afterDuplicate); + Assert.Same(afterDuplicate, afterStale); + var segment = Assert.Single(afterStale.Segments); + Assert.Equal("hello", segment.RawText); + Assert.Equal(2, segment.Revision); + } + + [Fact] + public void Finalizing_replaces_the_provisional_segment_in_place() + { + var doc = NewDocument(); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(doc.SessionId, Segment("s1", "hello wor", 0, 1, 1, SegmentState.Provisional)), Now); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("s1", "hello world", 0, 1.5, 2, SegmentState.Provisional)), Now); + var segment = Assert.Single(doc.Segments); + Assert.Equal(SegmentState.Final, segment.State); + Assert.Equal("hello world", segment.RawText); + } + + [Fact] + public void Edits_are_kept_separate_from_raw_text_and_survive_recognizer_updates() + { + var doc = NewDocument(); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(doc.SessionId, Segment("s1", "my name is just in", 0, 2)), Now); + doc = TranscriptDocumentReducer.EditSegment(doc, "s1", "My name is Justin.", Now); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("s1", "my name is justin", 0, 2, 2)), Now); + + var segment = Assert.Single(doc.Segments); + Assert.Equal("my name is justin", segment.RawText); + Assert.Equal("My name is Justin.", segment.EditedText); + Assert.Equal("My name is Justin.", segment.DisplayText); + Assert.True(segment.IsEdited); + + doc = TranscriptDocumentReducer.EditSegment(doc, "s1", "my name is justin", Now); + Assert.Null(Assert.Single(doc.Segments).EditedText); + } + + [Fact] + public void Provider_cannot_supply_edited_text() + { + var doc = NewDocument(); + var sneaky = Segment("s1", "raw", 0, 1) with { EditedText = "injected" }; + doc = TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(doc.SessionId, sneaky), Now); + Assert.Null(Assert.Single(doc.Segments).EditedText); + } + + [Fact] + public void Renaming_a_speaker_changes_the_mapping_not_segments() + { + var doc = NewDocument(); + doc = TranscriptDocumentReducer.Apply(doc, new SpeakerUpdated(doc.SessionId, new TranscriptSpeaker("spk0", "Speaker 1", null)), Now); + var segment = Segment("s1", "hi", 0, 1) with { Speakers = [new SpeakerAttribution("spk0", TimeSpan.Zero, TimeSpan.FromSeconds(1), null)] }; + doc = TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(doc.SessionId, segment), Now); + var segmentsBefore = doc.Segments; + + doc = TranscriptDocumentReducer.RenameSpeaker(doc, "spk0", " Justin ", Now); + Assert.Same(segmentsBefore, doc.Segments); + Assert.Equal("Justin", Assert.Single(doc.Speakers).Name); + + // A later provider label refresh keeps the user's name. + doc = TranscriptDocumentReducer.Apply(doc, new SpeakerUpdated(doc.SessionId, new TranscriptSpeaker("spk0", "Speaker 1", null)), Now); + Assert.Equal("Justin", Assert.Single(doc.Speakers).Name); + + doc = TranscriptDocumentReducer.RenameSpeaker(doc, "spk0", "", Now); + Assert.Equal("Speaker 1", Assert.Single(doc.Speakers).Name); + } + + [Fact] + public void A_rerun_adds_a_result_version_without_overwriting_the_old_one() + { + var doc = NewDocument(); + doc = TranscriptDocumentReducer.Apply(doc, new SessionStarted(doc.SessionId, Run(1)), Now); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("s1", "first model", 0, 1)), Now); + doc = TranscriptDocumentReducer.EditSegment(doc, "s1", "First model, edited", Now); + + doc = TranscriptDocumentReducer.Apply(doc, new SessionStarted(doc.SessionId, Run(2, "parakeet")), Now); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("s1", "second model", 0, 1, resultVersion: 2)), Now); + + Assert.Equal(2, doc.ActiveResultVersion); + Assert.Equal(2, doc.Runs.Count); + Assert.Equal("second model", Assert.Single(doc.ActiveSegments).DisplayText); + var old = doc.Segments.Single(s => s.ResultVersion == 1); + Assert.Equal("First model, edited", old.EditedText); + } + + [Fact] + public void Repeated_legitimate_phrases_are_kept() + { + var doc = NewDocument(); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("a", "thank you", 0, 1)), Now); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("b", "thank you", 1, 2)), Now); + Assert.Equal(["thank you", "thank you"], doc.ActiveSegments.Select(s => s.DisplayText)); + } + + [Fact] + public void Invalid_intervals_and_foreign_events_are_rejected() + { + var doc = NewDocument(); + Assert.Throws(() => + TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(doc.SessionId, Segment("s1", "x", 2, 1)), Now)); + Assert.Throws(() => + TranscriptDocumentReducer.Apply(doc, new SegmentUpserted(Guid.NewGuid(), Segment("s1", "x", 0, 1)), Now)); + } + + [Fact] + public void Nullable_timing_and_confidence_stay_null() + { + var doc = NewDocument(); + doc = TranscriptDocumentReducer.Apply(doc, new SegmentFinalized(doc.SessionId, Segment("s1", "text only", 0, 12)), Now); + var segment = Assert.Single(doc.Segments); + Assert.Null(segment.Confidence); + Assert.Null(segment.Words); + Assert.Equal(TimingProvenance.ApproximateChunk, segment.TimingProvenance); + } + + [Theory] + [InlineData(TranscriptSessionStatus.Created, TranscriptSessionStatus.Running, true)] + [InlineData(TranscriptSessionStatus.Running, TranscriptSessionStatus.Paused, true)] + [InlineData(TranscriptSessionStatus.Paused, TranscriptSessionStatus.Running, true)] + [InlineData(TranscriptSessionStatus.Running, TranscriptSessionStatus.Completed, false)] + [InlineData(TranscriptSessionStatus.Finalizing, TranscriptSessionStatus.Completed, true)] + [InlineData(TranscriptSessionStatus.Canceled, TranscriptSessionStatus.Running, false)] + [InlineData(TranscriptSessionStatus.Interrupted, TranscriptSessionStatus.Finalizing, true)] + [InlineData(TranscriptSessionStatus.Completed, TranscriptSessionStatus.Running, true)] + public void State_machine_allows_only_defined_transitions(TranscriptSessionStatus from, TranscriptSessionStatus to, bool allowed) + { + Assert.Equal(allowed, TranscriptionSessionStateMachine.CanTransition(from, to)); + if (!allowed) + { + Assert.Throws(() => TranscriptionSessionStateMachine.Transition(from, to)); + } + } +} From f398bee98ea955c899d313b55c3925b6565e783b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 18:55:16 +0000 Subject: [PATCH 04/77] Stage 3 spikes: Avalonia shell, miniaudio capture, NeMo-Speech.cpp findings Adds PrimeDictate.Platforms (SoundFlow/miniaudio capture for macOS and Linux) and a minimal Avalonia 12 desktop shell, plus the spike record with measured results and decisions. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GUzvd3VYM6n8xTXtBdDt5f --- PrimeDictate.sln | 30 +++ docs/architecture/stage-3-spikes.md | 144 ++++++++++ src/PrimeDictate.Desktop/App.axaml | 8 + src/PrimeDictate.Desktop/App.axaml.cs | 31 +++ src/PrimeDictate.Desktop/MainWindow.axaml | 31 +++ src/PrimeDictate.Desktop/MainWindow.axaml.cs | 44 ++++ .../PrimeDictate.Desktop.csproj | 27 ++ src/PrimeDictate.Desktop/Program.cs | 18 ++ .../Audio/MiniAudioCaptureSource.cs | 245 ++++++++++++++++++ .../PrimeDictate.Platforms.csproj | 20 ++ 10 files changed, 598 insertions(+) create mode 100644 docs/architecture/stage-3-spikes.md create mode 100644 src/PrimeDictate.Desktop/App.axaml create mode 100644 src/PrimeDictate.Desktop/App.axaml.cs create mode 100644 src/PrimeDictate.Desktop/MainWindow.axaml create mode 100644 src/PrimeDictate.Desktop/MainWindow.axaml.cs create mode 100644 src/PrimeDictate.Desktop/PrimeDictate.Desktop.csproj create mode 100644 src/PrimeDictate.Desktop/Program.cs create mode 100644 src/PrimeDictate.Platforms/Audio/MiniAudioCaptureSource.cs create mode 100644 src/PrimeDictate.Platforms/PrimeDictate.Platforms.csproj diff --git a/PrimeDictate.sln b/PrimeDictate.sln index 0f5f494..7ec88e5 100644 --- a/PrimeDictate.sln +++ b/PrimeDictate.sln @@ -13,6 +13,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PrimeDictate.Core.Tests", "tests\PrimeDictate.Core.Tests\PrimeDictate.Core.Tests.csproj", "{C0F52246-2A96-4B15-BEC6-B141EC8C61E8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PrimeDictate.Platforms", "src\PrimeDictate.Platforms\PrimeDictate.Platforms.csproj", "{44B68D74-1796-4299-93A5-2B4922DFBA21}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PrimeDictate.Desktop", "src\PrimeDictate.Desktop\PrimeDictate.Desktop.csproj", "{6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -59,6 +63,30 @@ Global {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Release|x64.Build.0 = Release|Any CPU {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Release|x86.ActiveCfg = Release|Any CPU {C0F52246-2A96-4B15-BEC6-B141EC8C61E8}.Release|x86.Build.0 = Release|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Debug|Any CPU.Build.0 = Debug|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Debug|x64.ActiveCfg = Debug|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Debug|x64.Build.0 = Debug|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Debug|x86.ActiveCfg = Debug|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Debug|x86.Build.0 = Debug|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Release|Any CPU.ActiveCfg = Release|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Release|Any CPU.Build.0 = Release|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Release|x64.ActiveCfg = Release|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Release|x64.Build.0 = Release|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Release|x86.ActiveCfg = Release|Any CPU + {44B68D74-1796-4299-93A5-2B4922DFBA21}.Release|x86.Build.0 = Release|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Debug|x64.ActiveCfg = Debug|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Debug|x64.Build.0 = Debug|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Debug|x86.ActiveCfg = Debug|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Debug|x86.Build.0 = Debug|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Release|Any CPU.Build.0 = Release|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Release|x64.ActiveCfg = Release|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Release|x64.Build.0 = Release|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Release|x86.ActiveCfg = Release|Any CPU + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -66,6 +94,8 @@ Global GlobalSection(NestedProjects) = preSolution {4FB02F11-2FF8-413B-A890-1E2FAE755C92} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {C0F52246-2A96-4B15-BEC6-B141EC8C61E8} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {44B68D74-1796-4299-93A5-2B4922DFBA21} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {6A9FC1A1-AAC7-46BB-A75E-F78968F4F78D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9A418AF3-E9F2-40D5-91CA-B3E5B169DADD} diff --git a/docs/architecture/stage-3-spikes.md b/docs/architecture/stage-3-spikes.md new file mode 100644 index 0000000..da07283 --- /dev/null +++ b/docs/architecture/stage-3-spikes.md @@ -0,0 +1,144 @@ +# Stage 3: risk spikes and decisions + +Date: 2026-09-29. Environment: Linux x64 container (Ubuntu 24.04, .NET SDK 10.0.112), no GPU, +no physical audio device, and outbound access to Hugging Face blocked by the environment's network +policy. Everything below says where it ran; nothing here was run on Windows or macOS. + +## Decisions + +| # | Decision | Status | Basis | +|---|----------|--------|-------| +| D1 | Cross-platform shell: **Avalonia 12.1.3**. WPF app stays the shipping Windows shell until parity. | Adopted | Builds and renders on Linux x64 (X11); self-contained publish produces correct natives for win-x64, win-arm64, osx-arm64, linux-x64. | +| D2 | macOS/Linux capture and playback: **SoundFlow 1.4.1 (MIT) over miniaudio**. Windows dictation keeps NAudio/WASAPI. | Adopted for macOS/Linux; not yet for Windows transcription | Captures real audio through PulseAudio here; ships natives for all four targets. PortAudioSharp2 rejected: no win-arm64 native. | +| D3 | **Pause closes the capture device; Resume reopens it.** | Adopted | Measured: Stop/Start leaks paused audio on PulseAudio (see A3). | +| D4 | Existing sherpa-onnx engines are portable candidates for macOS/Linux. | Verified on Linux x64 only | Whisper ONNX (tiny.en) transcribes correctly on Linux x64 CPU with the app's settings. | +| D5 | Nemotron via **NeMo-Speech.cpp `nemo-speech serve`** as an app-managed child process, pinned to a reviewed commit and pinned GGUF artifacts. | Provisional: protocol verified from source, inference not yet run | Builds on Linux x64 CPU; model files could not be downloaded here. | +| D6 | The app passes **only absolute, checksum-verified local GGUF paths** to the worker, never indexed model names. | Adopted | Indexed names trigger a download through `curl` (see N3). | +| D7 | Live Nemotron segments get **client-assigned utterance IDs**; partial text is display-only and the final `transcript` is authoritative. | Adopted | Realtime events carry no item ID, and a delta can be a suffix or a full replacement that the client cannot tell apart (see N5). | + +## A. Audio capture (SoundFlow/miniaudio), Linux x64, PulseAudio 16.1 + +Setup: PulseAudio in the container with a null sink. A 30 s "clock" WAV plays into the sink; its +tone is 300 + 100 × (second mod 15) Hz, so captured content can be matched to wall-clock time. +Capture goes through `MiniAudioCaptureSource` (`src/PrimeDictate.Platforms`) at 48 kHz mono float. + +- **A1. Real-time capture works.** 4.02 s captured in 4.0 s wall; no sequence gaps; a 440 Hz + source measured 440.5 Hz after the 48 to 16 kHz resampler. +- **A2. Open latency is about 2 s** before the first frame in this container, mostly miniaudio + probing JACK and ALSA. Requesting a 20 ms period did not change it. This needs measuring on real + macOS and desktop Linux before any latency claim. +- **A3. Stop/Start leaks paused audio.** With the device stopped for 3 s, about 3 s of audio from + the paused period (the 700 to 976 Hz tones) arrived in one burst on Resume. That violates "Pause + records nothing". +- **A4. Close/reopen does not leak.** No frames arrived while paused, and the first frames after + Resume carried the current tone. The cost is about 1.2 s after Resume before audio flows, so the + UI must show "Resuming…" until the first frame arrives instead of implying it is recording. +- **A5. Device naming.** With the PulseAudio backend, enumeration returned 11 ALSA-style PCM names + ("Playback/recording through the PulseAudio sound server", "Discard all samples…"). miniaudio's + native device IDs are pointers, not stable IDs. The adapter uses names as IDs for now; the picker + needs filtering and a stable-ID strategy before release. +- **A6. No audio backend** (ALSA only, no device) throws from the engine constructor; the adapter + wraps it as `AudioSourceException` so the UI can say the microphone is unavailable while import + keeps working. +- Not tested: macOS CoreAudio and microphone permission prompts, PipeWire, Wayland sessions, + device unplug and default-device change, Windows through miniaudio. + +## B. Existing engines on Linux x64 + +`sherpa-onnx-whisper-tiny.en` (from sherpa-onnx GitHub releases), CPU, with the same recognizer +settings as `WhisperOnnxTranscriptionEngine`: + +| Path | Audio | Result | +|------|-------|--------| +| File: `jfk.wav` (16 kHz PCM16) through `AudioFrame` | 11.0 s | Correct text; decode 1.00 s (RTF 0.09); model load 0.5 s | +| Live: same WAV played into the null sink, captured at 48 kHz, resampled to 16 kHz | 16.5 s captured | Correct text; decode 1.26 s | + +sherpa-onnx Whisper returned no timestamps, so these segments map to `ApproximateChunk` timing, as +designed. `org.k2fsa.sherpa.onnx` 1.13.0 declares runtime packages for win-x64, win-arm64, +osx-arm64, osx-x64, linux-x64, and linux-arm64. Parakeet, Moonshine, and Whisper.net were not run +here. + +## C. Avalonia shell + +`src/PrimeDictate.Desktop` is a minimal shell with Transcription and Dictation tabs. It states the +mode boundary, says microphone capture does not include remote call participants, disables global +dictation where it isn't implemented, and probes the audio backend. + +- Rendered under Xvfb (X11) on Linux x64, both from `dotnet run` and from the self-contained + `linux-x64` publish. +- Self-contained publishes: win-x64 210 MB, win-arm64 219 MB, osx-arm64 117 MB, linux-x64 107 MB. + About 105 MB of the Windows size is SkiaSharp and HarfBuzz native `.pdb` files, which packaging + should exclude. Each output contains matching-architecture `miniaudio`, `SkiaSharp`, `HarfBuzz`, + and `e_sqlite3` natives (checked with `file`). +- Not tested: Wayland, macOS app bundle, notarization, Windows rendering, tray integration. + +## N. NeMo-Speech.cpp (native Nemotron runtime) + +Pinned commit: `0f706e43cf1fbc031bad1423e05460d3acaeaa1c` (VERSION 0.1.0), Apache-2.0 runtime code +with third-party notices. Built with `scripts/configure.sh cpu-server` and +`cmake --build --preset cpu-server` on Linux x64 (Ubuntu CMake 3.28, `libsentencepiece-dev`); the +binary reports `nemo-speech 0.1.0`. + +Artifacts pinned by the runtime's `models/index.json` at that commit (not downloaded here): + +| Model | HF revision | File | Size | SHA-256 | +|-------|-------------|------|------|---------| +| nvidia/nemotron-3.5-asr-streaming-0.6b | `1c8deaecc64b91f034d73e08dd8b64625eb3395d` | `nemotron-3.5-asr-streaming-0.6b.q8_0.gguf` | 741,548,352 | `a5c435f2…f429ae` | +| nvidia/nemotron-speech-streaming-en-0.6b | `ebe59e5a817142986528bbbee5dba8db7b38ed50` | `nemotron-speech-streaming-en-0.6b.q8_0.gguf` | 699,872,960 | `d9a01898…3812d` | +| nvidia/Nemotron-3-Diarization | `f667ed73aee57d40cc39428eb768b4fd87a0a29e` | `Nemotron-3-Diarization.q8_0.gguf` | 107,012,128 | `08456d9e…c7a3a1` | + +Findings from running the binary and reading the pinned source (`server/http/http_server.cpp`): + +- **N1. Startup failures are fast and distinct.** No model: exit 1. Missing file: exit 3 with a + validation message. Corrupt GGUF: exit 1 ("invalid magic"). Download failure: exit 2. +- **N2. Security settings exist and match the brief.** Default bind is `127.0.0.1`; `--no-ui` + disables the playground; `NEMO_SPEECH_HTTP_API_KEY` supplies the bearer key without a command-line + argument; CORS is off unless `--cors-origin` is set. `/health`, `/ready`, and `/version` are + unauthenticated by design, and the `/v1` routes and the realtime socket require the key. +- **N3. Indexed model names download silently.** `--asr-model nemotron-3.5` started a 707 MiB + download through `curl` into `~/.cache`. PrimeDictate must pass only local paths it has verified + (D6), so an offline session never triggers a pull. +- **N4. One loaded ASR model per process.** The HTTP `model` field is ignored, as documented. + Switching models means restarting the worker with another path and checking `session.created`'s + `session.model` and `/v1/models`. +- **N5. The realtime protocol differs from what the docs imply.** + - Transcription events have no `item_id`; only `type`, `delta` or `transcript`, + `audio_processed` (seconds), optional `words`, and `event_id`. + - `delta` is the new suffix when the partial extends the previous one, but the **whole partial** + when the recognizer revised it. Clients cannot tell which, and appending deltas (as the bundled + playground does) can duplicate text. PrimeDictate will treat partials as display-only and the + final `transcript` as authoritative (D7). Upstream issue to file: include the full partial on + delta events. + - `input_audio_buffer.commit` finalizes and **resets the stream**, so word times after a commit + restart at 0 unless the server offsets them. Needs a model to confirm; the client must add the + session offset either way. + - `session.update` is rejected once audio has started, and `max_speaker_count` is parsed and + validated despite being documented as ignored. +- **N6. Limits.** The 512 MiB upload limit also caps cumulative audio per realtime stream (about + 4.6 h of 16 kHz PCM16) and resets on commit. WebSocket reads time out after 300 s (cpp-httplib + default; the 30 s `--read-timeout` applies to HTTP), with server pings every 30 s. A Pause that + commits and stops sending audio needs a test to see whether pongs keep the socket open past 300 s. + Otherwise reconnect on Resume with an explicit offset. +- **N7. Diarizer capacity.** The runtime docs list Nemotron 3 Diarization as Sortformer V3 with up to + 8 speakers (Streaming Sortformer v2: 4). +- **N8. Languages.** The runtime docs say "40+ language-locales" for Nemotron 3.5 and allow `auto`. + The model card separates supported locales from adaptation-ready ones, so the picker must use the + card's supported list, not the runtime's. + +### Nemotron blockers + +- **Model files.** `huggingface.co` is denied by this environment's network policy, so neither + GGUF could be downloaded and no Nemotron inference, streaming, flush, timing, or speaker tagging + has run. It can proceed either by allowing `huggingface.co` and `cdn-lfs.huggingface.co` in the + environment's network settings, or by running the spike on a machine that can download them. +- **Accelerated builds.** Only the CPU preset was built. CUDA, Metal, and Vulkan builds and the + Windows build driver are untested. +- **Release binaries.** GitHub release metadata for NeMo-Speech.cpp could not be read from this + environment, so it is unknown whether signed release archives exist for every target. The app will + not run upstream install scripts on users' machines. + +## Next + +Stage 4 (file transcription) can start on the verified path: Core, the resampler, the existing +sherpa engines, and SoundFlow playback. Nemotron work (stage 6) waits on the model-file blocker +above. diff --git a/src/PrimeDictate.Desktop/App.axaml b/src/PrimeDictate.Desktop/App.axaml new file mode 100644 index 0000000..75d01ce --- /dev/null +++ b/src/PrimeDictate.Desktop/App.axaml @@ -0,0 +1,8 @@ + + + + + diff --git a/src/PrimeDictate.Desktop/App.axaml.cs b/src/PrimeDictate.Desktop/App.axaml.cs new file mode 100644 index 0000000..e3b2f80 --- /dev/null +++ b/src/PrimeDictate.Desktop/App.axaml.cs @@ -0,0 +1,31 @@ +using Avalonia; +using Avalonia.Controls.ApplicationLifetimes; +using Avalonia.Markup.Xaml; + +namespace PrimeDictate.Desktop; + +public sealed class App : Application +{ + public override void Initialize() => AvaloniaXamlLoader.Load(this); + + public override void OnFrameworkInitializationCompleted() + { + if (this.ApplicationLifetime is IClassicDesktopStyleApplicationLifetime desktop) + { + var window = new MainWindow(); + desktop.MainWindow = window; + if (desktop.Args is { Length: > 0 } args && args.Contains("--smoke-screenshot")) + { + var path = args.SkipWhile(a => a != "--smoke-screenshot").Skip(1).FirstOrDefault() ?? "smoke.png"; + window.Opened += async (_, _) => + { + await Task.Delay(1500); + await window.SaveScreenshotAsync(path); + desktop.Shutdown(); + }; + } + } + + base.OnFrameworkInitializationCompleted(); + } +} diff --git a/src/PrimeDictate.Desktop/MainWindow.axaml b/src/PrimeDictate.Desktop/MainWindow.axaml new file mode 100644 index 0000000..14e1f70 --- /dev/null +++ b/src/PrimeDictate.Desktop/MainWindow.axaml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + +