From f3ace73a81ce745e84a8245fbc4e2d585fd2fe26 Mon Sep 17 00:00:00 2001 From: Alessandro Olivero Date: Fri, 31 Jul 2026 22:37:55 +0200 Subject: [PATCH] feat(transcription): add Whisper engine support --- Package.resolved | 11 +- Package.swift | 2 + README.md | 30 ++++- Sources/quill/Config.swift | 28 ++++- Sources/quill/Doctor.swift | 33 +++++- .../TranscriptionCoordinator.swift | 10 +- .../quill/Transcription/WhisperEngine.swift | 108 ++++++++++++++++++ 7 files changed, 210 insertions(+), 12 deletions(-) create mode 100644 Sources/quill/Transcription/WhisperEngine.swift diff --git a/Package.resolved b/Package.resolved index 726a7aa..42badd0 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,6 +1,15 @@ { - "originHash" : "64d631bc12fe3e0b97d15edf738c24f24db21298d1ded6f5392325824c797da5", + "originHash" : "2012cac3f33a09ebd94994b995b5ff53f9888d09ddef40caf0650e144de5008f", "pins" : [ + { + "identity" : "argmax-oss-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/argmaxinc/argmax-oss-swift.git", + "state" : { + "revision" : "25c62997041c134b03ca82731ce2f6fd2cae1eb9", + "version" : "1.0.0" + } + }, { "identity" : "fluidaudio", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index e6b9706..691f781 100644 --- a/Package.swift +++ b/Package.swift @@ -7,6 +7,7 @@ let package = Package( dependencies: [ .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.3.0"), .package(url: "https://github.com/FluidInference/FluidAudio.git", from: "0.7.0"), + .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"), ], targets: [ .executableTarget( @@ -14,6 +15,7 @@ let package = Package( dependencies: [ .product(name: "ArgumentParser", package: "swift-argument-parser"), .product(name: "FluidAudio", package: "FluidAudio"), + .product(name: "WhisperKit", package: "argmax-oss-swift"), ], exclude: ["Info.plist"], linkerSettings: [ diff --git a/README.md b/README.md index 14192fa..0c5b437 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,13 @@ Core ML port — roughly 20 seconds per hour of audio on Apple Silicon. Models whether they're already cached so you're never downloading after an important meeting. +For multilingual audio, switch to the **Whisper** engine +([WhisperKit](https://github.com/argmaxinc/argmax-oss-swift) / Core ML, running +on the Apple Neural Engine). Set `transcription.engine` to `"whisper"` in +config; the model defaults to `large-v3-v20240930_turbo` (~626 MB) and is +configurable. Whisper auto-detects the language per file, or you can force one +with `transcription.language`. + Each track is transcribed separately, shifted by its start offset so both share one clock, and merged by timestamp. Jobs run in a serial queue — you can start a new recording while the last one transcribes. Unfinished jobs resume @@ -65,8 +72,7 @@ on next launch (the filesystem is the queue: a session with `meta.json` but no `transcript.json` is pending). Failures append to the session's `transcribe.log` and never block later jobs. -The engine sits behind a small protocol; a Whisper engine (WhisperKit -large-v3-turbo) is planned as the fallback / re-transcription option. +The engine sits behind a small protocol; `parakeet` and `whisper` ship today. ## Config @@ -75,7 +81,12 @@ Optional, at `~/.config/quill/config.json`: ```json { "recordings_dir": "~/Recordings", - "transcription": { "enabled": true, "engine": "parakeet" }, + "transcription": { + "enabled": true, + "engine": "parakeet", + "model": "large-v3-v20240930_turbo", + "language": "it" + }, "on_stop": "my-hook" } ``` @@ -83,6 +94,15 @@ Optional, at `~/.config/quill/config.json`: - `recordings_dir` — where sessions land. Resolution order: `--out` flag > config > `~/Recordings`. - `transcription.enabled` — set `false` to just record. +- `transcription.engine` — `parakeet` (default, English-only, fastest) or + `whisper` (multilingual via WhisperKit / Core ML). Unknown values warn and + fall back to parakeet. +- `transcription.model` — WhisperKit model name, only used when + `engine == "whisper"`. Defaults to `large-v3-v20240930_turbo`. Any model in + the `argmaxinc/whisperkit-coreml*` HuggingFace family works. +- `transcription.language` — optional ISO 639-1 code (e.g. `"it"`, `"en"`) to + force for the whisper engine. Omit for per-file auto-detection. Ignored by + parakeet (English-only). - `mic_voice_processing` — Apple's echo cancellation on the mic (default off). Set `true` when recording meetings through the speakers, so playback doesn't bleed into the mic track and get transcribed twice as "me". The trade: while @@ -112,6 +132,7 @@ quill install --uninstall - **AVAudioEngine** — mic capture - **AVAudioFile** — streaming AAC encode into CAF - **FluidAudio / Parakeet** — on-device Core ML transcription +- **WhisperKit / argmax-oss-swift** — on-device Core ML transcription (whisper engine) - **NSStatusItem** — the whole UI ## Gotchas @@ -121,7 +142,6 @@ quill install --uninstall per-process picker if it bothers you). - If recordings come out silent, check System Settings → Privacy & Security → Screen & System Audio Recording. -- Parakeet v2 is English-only. Other languages will come with the Whisper - engine. +- Parakeet v2 is English-only. Use the `whisper` engine for other languages. - The binary embeds its Info.plist (`__TEXT,__info_plist`) so TCC can attribute permissions to quill itself when running as a LaunchAgent. diff --git a/Sources/quill/Config.swift b/Sources/quill/Config.swift index 5db668f..6199814 100644 --- a/Sources/quill/Config.swift +++ b/Sources/quill/Config.swift @@ -4,7 +4,12 @@ import Foundation /// /// { /// "recordings_dir": "~/Recordings", -/// "transcription": { "enabled": true, "engine": "parakeet" }, +/// "transcription": { +/// "enabled": true, +/// "engine": "parakeet", // or "whisper" +/// "model": "large-v3-v20240930_turbo", // whisper only +/// "language": "it" // whisper only; omit for auto-detect +/// }, /// "mic_voice_processing": true, /// "on_stop": "my-hook" /// } @@ -38,12 +43,29 @@ enum Config { transcription()?["enabled"] as? Bool ?? true } - /// Configured engine name. Only "parakeet" ships today; the coordinator - /// warns and falls back for anything else. + /// Configured engine name. "parakeet" ships as the default; "whisper" + /// (WhisperKit / Core ML) is the multilingual fallback. Anything else + /// warns and falls back to parakeet. static func transcriptionEngine() -> String { transcription()?["engine"] as? String ?? "parakeet" } + /// WhisperKit model name (only used when engine == "whisper"). Defaults + /// to large-v3-turbo — the best speed/accuracy balance on macOS per + /// Argmax's recommendation. Any HuggingFace model in the + /// `argmaxinc/whisperkit-coreml*` family works. + static func whisperModel() -> String { + transcription()?["model"] as? String ?? "large-v3-v20240930_turbo" + } + + /// Optional ISO 639-1 language code (e.g. "it", "en", "fr") to force for + /// the whisper engine. nil → Whisper auto-detects per file. Ignored by + /// parakeet (English-only). + static func whisperLanguage() -> String? { + guard let lang = transcription()?["language"] as? String, !lang.isEmpty else { return nil } + return lang + } + private static func transcription() -> [String: Any]? { load()?["transcription"] as? [String: Any] } diff --git a/Sources/quill/Doctor.swift b/Sources/quill/Doctor.swift index 8af8125..230ff97 100644 --- a/Sources/quill/Doctor.swift +++ b/Sources/quill/Doctor.swift @@ -77,7 +77,9 @@ enum DoctorReport { } /// Never discover a missing model after an important meeting: report - /// whether the parakeet models are already in FluidAudio's cache. + /// whether the configured engine's models are already cached. Parakeet + /// uses FluidAudio's cache; Whisper uses WhisperKit's HuggingFace cache + /// under ~/Documents/huggingface. static func checkTranscription() -> Check { guard Config.transcriptionEnabled() else { return Check( @@ -86,6 +88,15 @@ enum DoctorReport { remediation: nil ) } + switch Config.transcriptionEngine() { + case "whisper": + return checkWhisperModels() + default: + return checkParakeetModels() + } + } + + private static func checkParakeetModels() -> Check { let cache = AsrModels.defaultCacheDirectory(for: .v2) if AsrModels.modelsExist(at: cache, version: .v2) { return Check(name: "transcription", status: .ok, remediation: nil) @@ -97,6 +108,26 @@ enum DoctorReport { ) } + /// WhisperKit downloads Core ML models from HuggingFace into + /// ~/Documents/huggingface/models--argmaxinc--whisperkit-coreml. We can't + /// cheaply tell whether the *specific* configured model is present (the + /// cache is keyed by commit hash), so we report on the repo folder as a + /// best-effort "has anything been downloaded yet" signal. + private static func checkWhisperModels() -> Check { + let model = Config.whisperModel() + let repoDir = FileManager.default + .homeDirectoryForCurrentUser + .appendingPathComponent("Documents/huggingface/models--argmaxinc--whisperkit-coreml") + if FileManager.default.fileExists(atPath: repoDir.path) { + return Check(name: "transcription", status: .ok, remediation: nil) + } + return Check( + name: "transcription", + status: .warn("whisper models not downloaded (model: \(model), ~600 MB+)"), + remediation: "downloads automatically on first transcription — record a short test session while online" + ) + } + static func print(_ checks: [Check]) { for c in checks { let (mark, label): (String, String) = { diff --git a/Sources/quill/Transcription/TranscriptionCoordinator.swift b/Sources/quill/Transcription/TranscriptionCoordinator.swift index 5300fbd..79ee694 100644 --- a/Sources/quill/Transcription/TranscriptionCoordinator.swift +++ b/Sources/quill/Transcription/TranscriptionCoordinator.swift @@ -143,12 +143,18 @@ actor TranscriptionCoordinator { private func preparedEngine() async throws -> TranscriptionEngine { if let engine { return engine } let configured = Config.transcriptionEngine() - if configured != "parakeet" { + let engine: TranscriptionEngine + switch configured { + case "whisper": + engine = WhisperEngine(model: Config.whisperModel()) + case "parakeet": + engine = ParakeetEngine() + default: FileHandle.standardError.write(Data( "warning: unknown transcription engine \"\(configured)\" — using parakeet\n".utf8 )) + engine = ParakeetEngine() } - let engine = ParakeetEngine() try await engine.prepare() self.engine = engine return engine diff --git a/Sources/quill/Transcription/WhisperEngine.swift b/Sources/quill/Transcription/WhisperEngine.swift new file mode 100644 index 0000000..bc71b8f --- /dev/null +++ b/Sources/quill/Transcription/WhisperEngine.swift @@ -0,0 +1,108 @@ +import AVFoundation +import Foundation +import WhisperKit + +/// Whisper (large-v3-turbo by default) via WhisperKit's Core ML port, running +/// on the Apple Neural Engine. Models download once into WhisperKit's +/// HuggingFace cache (~626 MB for turbo); after that, transcription runs +/// entirely on-device. Multilingual — set `transcription.language` in config +/// to force a language, or leave it unset for auto-detection. +/// +/// Slower than Parakeet (English-only) but covers every language Whisper +/// supports, so it's the fallback / multilingual option behind the same +/// `TranscriptionEngine` protocol. +actor WhisperEngine: TranscriptionEngine { + enum EngineError: Error, CustomStringConvertible { + case notPrepared + case unreadableAudio(URL, Error?) + case emptyResult + + var description: String { + switch self { + case .notPrepared: return "whisper engine used before prepare()" + case .unreadableAudio(let url, let e): + return "unreadable or empty audio \(url.lastPathComponent)" + + (e.map { ": \($0)" } ?? "") + case .emptyResult: return "transcription returned no results" + } + } + } + + nonisolated let name = "whisper" + nonisolated let model: String + + private var pipe: WhisperKit? + + init(model: String) { + self.model = model + } + + func prepare() async throws { + guard pipe == nil else { return } + let config = WhisperKitConfig( + model: model, + verbose: false, + logLevel: .error, + download: true + ) + let pipe = try await WhisperKit(config) + self.pipe = pipe + } + + func transcribe(_ audio: URL) async throws -> [TranscriptSegment] { + guard let pipe else { throw EngineError.notPrepared } + + // A track with no frames (recorder died before its first buffer) + // makes AVFoundation raise an ObjC exception deep inside the + // resampler — uncatchable from Swift, so it takes the whole daemon + // down. Check readability up front instead, mirroring ParakeetEngine. + do { + let probe = try AVAudioFile(forReading: audio) + guard probe.length > 0 else { throw EngineError.unreadableAudio(audio, nil) } + } catch let error as EngineError { + throw error + } catch { + throw EngineError.unreadableAudio(audio, error) + } + + var options = DecodingOptions() + // Strip Whisper's special tokens (<|startoftranscript|>, <|it|>, + // <|transcribe|>, timestamp markers) from the segment text — they're + // metadata, not speech. + options.skipSpecialTokens = true + if let language = Config.whisperLanguage() { + options.language = language + // Pin the language via a prefill prompt token instead of + // auto-detection. + options.usePrefillPrompt = true + } + + let results = try await pipe.transcribe( + audioPath: audio.path, + decodeOptions: options + ) + guard let result = results.first else { throw EngineError.emptyResult } + + // WhisperKit emits one segment per ~30s window plus sentence breaks; + // drop no-speech windows and empty text so the transcript stays tidy. + return result.segments.compactMap { seg in + let text = seg.text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return nil } + // noSpeechProb is 0..1; WhisperKit's default noSpeechThreshold is + // 0.6 — skip windows the model considers silence. + if seg.noSpeechProb > 0.6 { return nil } + return TranscriptSegment( + start: TimeInterval(seg.start), + end: TimeInterval(seg.end), + text: text + ) + } + } + + func release() async { + // WhisperKit holds Core ML models via ARC; dropping the reference + // lets them deallocate from unified memory, mirroring ParakeetEngine's + // manager.cleanup() + nil. + pipe = nil + } +}