feat(transcription): add Whisper engine support

Package.swift

@@ -7,13 +7,15 @@ let package = Package(
     dependencies: [
         .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.3.0"),
         .package(url: "https://github.com/FluidInference/FluidAudio.git", from: "0.7.0"),
+        .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
     ],
     targets: [
         .executableTarget(
             name: "quill",
             dependencies: [
                 .product(name: "ArgumentParser", package: "swift-argument-parser"),
                 .product(name: "FluidAudio", package: "FluidAudio"),
+                .product(name: "WhisperKit", package: "argmax-oss-swift"),
             ],
             exclude: ["Info.plist"],
             linkerSettings: [

README.md

@@ -58,15 +58,21 @@ Core ML port — roughly 20 seconds per hour of audio on Apple Silicon. Models
 whether they're already cached so you're never downloading after an important
 meeting.
 
+For multilingual audio, switch to the **Whisper** engine
+([WhisperKit](https://github.com/argmaxinc/argmax-oss-swift) / Core ML, running
+on the Apple Neural Engine). Set `transcription.engine` to `"whisper"` in
+config; the model defaults to `large-v3-v20240930_turbo` (~626 MB) and is
+configurable. Whisper auto-detects the language per file, or you can force one
+with `transcription.language`.
+
 Each track is transcribed separately, shifted by its start offset so both
 share one clock, and merged by timestamp. Jobs run in a serial queue — you can
 start a new recording while the last one transcribes. Unfinished jobs resume
 on next launch (the filesystem is the queue: a session with `meta.json` but no
 `transcript.json` is pending). Failures append to the session's
 `transcribe.log` and never block later jobs.
 
-The engine sits behind a small protocol; a Whisper engine (WhisperKit
-large-v3-turbo) is planned as the fallback / re-transcription option.
+The engine sits behind a small protocol; `parakeet` and `whisper` ship today.
 
 ## Config
 
@@ -75,14 +81,28 @@ Optional, at `~/.config/quill/config.json`:
 ```json
 {
   "recordings_dir": "~/Recordings",
-  "transcription": { "enabled": true, "engine": "parakeet" },
+  "transcription": {
+    "enabled": true,
+    "engine": "parakeet",
+    "model": "large-v3-v20240930_turbo",
+    "language": "it"
+  },
   "on_stop": "my-hook"
 }
 ```
 
 - `recordings_dir` — where sessions land. Resolution order: `--out` flag >
   config > `~/Recordings`.
 - `transcription.enabled` — set `false` to just record.
+- `transcription.engine` — `parakeet` (default, English-only, fastest) or
+  `whisper` (multilingual via WhisperKit / Core ML). Unknown values warn and
+  fall back to parakeet.
+- `transcription.model` — WhisperKit model name, only used when
+  `engine == "whisper"`. Defaults to `large-v3-v20240930_turbo`. Any model in
+  the `argmaxinc/whisperkit-coreml*` HuggingFace family works.
+- `transcription.language` — optional ISO 639-1 code (e.g. `"it"`, `"en"`) to
+  force for the whisper engine. Omit for per-file auto-detection. Ignored by
+  parakeet (English-only).
 - `mic_voice_processing` — Apple's echo cancellation on the mic (default off).
   Set `true` when recording meetings through the speakers, so playback doesn't
   bleed into the mic track and get transcribed twice as "me". The trade: while
@@ -112,6 +132,7 @@ quill install --uninstall
 - **AVAudioEngine** — mic capture
 - **AVAudioFile** — streaming AAC encode into CAF
 - **FluidAudio / Parakeet** — on-device Core ML transcription
+- **WhisperKit / argmax-oss-swift** — on-device Core ML transcription (whisper engine)
 - **NSStatusItem** — the whole UI
 
 ## Gotchas
@@ -121,7 +142,6 @@ quill install --uninstall
   per-process picker if it bothers you).
 - If recordings come out silent, check System Settings → Privacy & Security →
   Screen & System Audio Recording.
-- Parakeet v2 is English-only. Other languages will come with the Whisper
-  engine.
+- Parakeet v2 is English-only. Use the `whisper` engine for other languages.
 - The binary embeds its Info.plist (`__TEXT,__info_plist`) so TCC can
   attribute permissions to quill itself when running as a LaunchAgent.

Sources/quill/Config.swift

@@ -4,7 +4,12 @@ import Foundation
 ///
 ///     {
 ///       "recordings_dir": "~/Recordings",
-///       "transcription": { "enabled": true, "engine": "parakeet" },
+///       "transcription": {
+///         "enabled": true,
+///         "engine": "parakeet",          // or "whisper"
+///         "model": "large-v3-v20240930_turbo",  // whisper only
+///         "language": "it"                // whisper only; omit for auto-detect
+///       },
 ///       "mic_voice_processing": true,
 ///       "on_stop": "my-hook"
 ///     }
@@ -38,12 +43,29 @@ enum Config {
         transcription()?["enabled"] as? Bool ?? true
     }
 
-    /// Configured engine name. Only "parakeet" ships today; the coordinator
-    /// warns and falls back for anything else.
+    /// Configured engine name. "parakeet" ships as the default; "whisper"
+    /// (WhisperKit / Core ML) is the multilingual fallback. Anything else
+    /// warns and falls back to parakeet.
     static func transcriptionEngine() -> String {
         transcription()?["engine"] as? String ?? "parakeet"
     }
 
+    /// WhisperKit model name (only used when engine == "whisper"). Defaults
+    /// to large-v3-turbo — the best speed/accuracy balance on macOS per
+    /// Argmax's recommendation. Any HuggingFace model in the
+    /// `argmaxinc/whisperkit-coreml*` family works.
+    static func whisperModel() -> String {
+        transcription()?["model"] as? String ?? "large-v3-v20240930_turbo"
+    }
+
+    /// Optional ISO 639-1 language code (e.g. "it", "en", "fr") to force for
+    /// the whisper engine. nil → Whisper auto-detects per file. Ignored by
+    /// parakeet (English-only).
+    static func whisperLanguage() -> String? {
+        guard let lang = transcription()?["language"] as? String, !lang.isEmpty else { return nil }
+        return lang
+    }
+
     private static func transcription() -> [String: Any]? {
         load()?["transcription"] as? [String: Any]
     }

Sources/quill/Doctor.swift

@@ -77,7 +77,9 @@ enum DoctorReport {
     }
 
     /// Never discover a missing model after an important meeting: report
-    /// whether the parakeet models are already in FluidAudio's cache.
+    /// whether the configured engine's models are already cached. Parakeet
+    /// uses FluidAudio's cache; Whisper uses WhisperKit's HuggingFace cache
+    /// under ~/Documents/huggingface.
     static func checkTranscription() -> Check {
         guard Config.transcriptionEnabled() else {
             return Check(
@@ -86,6 +88,15 @@ enum DoctorReport {
                 remediation: nil
             )
         }
+        switch Config.transcriptionEngine() {
+        case "whisper":
+            return checkWhisperModels()
+        default:
+            return checkParakeetModels()
+        }
+    }
+
+    private static func checkParakeetModels() -> Check {
         let cache = AsrModels.defaultCacheDirectory(for: .v2)
         if AsrModels.modelsExist(at: cache, version: .v2) {
             return Check(name: "transcription", status: .ok, remediation: nil)
@@ -97,6 +108,26 @@ enum DoctorReport {
         )
     }
 
+    /// WhisperKit downloads Core ML models from HuggingFace into
+    /// ~/Documents/huggingface/models--argmaxinc--whisperkit-coreml. We can't
+    /// cheaply tell whether the *specific* configured model is present (the
+    /// cache is keyed by commit hash), so we report on the repo folder as a
+    /// best-effort "has anything been downloaded yet" signal.
+    private static func checkWhisperModels() -> Check {
+        let model = Config.whisperModel()
+        let repoDir = FileManager.default
+            .homeDirectoryForCurrentUser
+            .appendingPathComponent("Documents/huggingface/models--argmaxinc--whisperkit-coreml")
+        if FileManager.default.fileExists(atPath: repoDir.path) {
+            return Check(name: "transcription", status: .ok, remediation: nil)
+        }
+        return Check(
+            name: "transcription",
+            status: .warn("whisper models not downloaded (model: \(model), ~600 MB+)"),
+            remediation: "downloads automatically on first transcription — record a short test session while online"
+        )
+    }
+
     static func print(_ checks: [Check]) {
         for c in checks {
             let (mark, label): (String, String) = {

Sources/quill/Transcription/TranscriptionCoordinator.swift

@@ -143,12 +143,18 @@ actor TranscriptionCoordinator {
     private func preparedEngine() async throws -> TranscriptionEngine {
         if let engine { return engine }
         let configured = Config.transcriptionEngine()
-        if configured != "parakeet" {
+        let engine: TranscriptionEngine
+        switch configured {
+        case "whisper":
+            engine = WhisperEngine(model: Config.whisperModel())
+        case "parakeet":
+            engine = ParakeetEngine()
+        default:
             FileHandle.standardError.write(Data(
                 "warning: unknown transcription engine \"\(configured)\" — using parakeet\n".utf8
             ))
+            engine = ParakeetEngine()
         }
-        let engine = ParakeetEngine()
         try await engine.prepare()
         self.engine = engine
         return engine

Sources/quill/Transcription/WhisperEngine.swift

@@ -0,0 +1,108 @@
+import AVFoundation
+import Foundation
+import WhisperKit
+
+/// Whisper (large-v3-turbo by default) via WhisperKit's Core ML port, running
+/// on the Apple Neural Engine. Models download once into WhisperKit's
+/// HuggingFace cache (~626 MB for turbo); after that, transcription runs
+/// entirely on-device. Multilingual — set `transcription.language` in config
+/// to force a language, or leave it unset for auto-detection.
+///
+/// Slower than Parakeet (English-only) but covers every language Whisper
+/// supports, so it's the fallback / multilingual option behind the same
+/// `TranscriptionEngine` protocol.
+actor WhisperEngine: TranscriptionEngine {
+    enum EngineError: Error, CustomStringConvertible {
+        case notPrepared
+        case unreadableAudio(URL, Error?)
+        case emptyResult
+
+        var description: String {
+            switch self {
+            case .notPrepared: return "whisper engine used before prepare()"
+            case .unreadableAudio(let url, let e):
+                return "unreadable or empty audio \(url.lastPathComponent)"
+                    + (e.map { ": \($0)" } ?? "")
+            case .emptyResult: return "transcription returned no results"
+            }
+        }
+    }
+
+    nonisolated let name = "whisper"
+    nonisolated let model: String
+
+    private var pipe: WhisperKit?
+
+    init(model: String) {
+        self.model = model
+    }
+
+    func prepare() async throws {
+        guard pipe == nil else { return }
+        let config = WhisperKitConfig(
+            model: model,
+            verbose: false,
+            logLevel: .error,
+            download: true
+        )
+        let pipe = try await WhisperKit(config)
+        self.pipe = pipe
+    }
+
+    func transcribe(_ audio: URL) async throws -> [TranscriptSegment] {
+        guard let pipe else { throw EngineError.notPrepared }
+
+        // A track with no frames (recorder died before its first buffer)
+        // makes AVFoundation raise an ObjC exception deep inside the
+        // resampler — uncatchable from Swift, so it takes the whole daemon
+        // down. Check readability up front instead, mirroring ParakeetEngine.
+        do {
+            let probe = try AVAudioFile(forReading: audio)
+            guard probe.length > 0 else { throw EngineError.unreadableAudio(audio, nil) }
+        } catch let error as EngineError {
+            throw error
+        } catch {
+            throw EngineError.unreadableAudio(audio, error)
+        }
+
+        var options = DecodingOptions()
+        // Strip Whisper's special tokens (<|startoftranscript|>, <|it|>,
+        // <|transcribe|>, timestamp markers) from the segment text — they're
+        // metadata, not speech.
+        options.skipSpecialTokens = true
+        if let language = Config.whisperLanguage() {
+            options.language = language
+            // Pin the language via a prefill prompt token instead of
+            // auto-detection.
+            options.usePrefillPrompt = true
+        }
+
+        let results = try await pipe.transcribe(
+            audioPath: audio.path,
+            decodeOptions: options
+        )
+        guard let result = results.first else { throw EngineError.emptyResult }
+
+        // WhisperKit emits one segment per ~30s window plus sentence breaks;
+        // drop no-speech windows and empty text so the transcript stays tidy.
+        return result.segments.compactMap { seg in
+            let text = seg.text.trimmingCharacters(in: .whitespacesAndNewlines)
+            guard !text.isEmpty else { return nil }
+            // noSpeechProb is 0..1; WhisperKit's default noSpeechThreshold is
+            // 0.6 — skip windows the model considers silence.
+            if seg.noSpeechProb > 0.6 { return nil }
+            return TranscriptSegment(
+                start: TimeInterval(seg.start),
+                end: TimeInterval(seg.end),
+                text: text
+            )
+        }
+    }
+
+    func release() async {
+        // WhisperKit holds Core ML models via ARC; dropping the reference
+        // lets them deallocate from unified memory, mirroring ParakeetEngine's
+        // manager.cleanup() + nil.
+        pipe = nil
+    }
+}