Add automatic speaker diarization

.github/workflows/test.yml

@@ -0,0 +1,38 @@
+name: Test
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  portable-diarization-tests:
+    name: Portable RTTM / DER tests (Ubuntu)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      # Quill itself is macOS-only, but its fixture integrity, RTTM parser,
+      # and DER-scoring suite is deliberately Foundation-only and runs here.
+      - name: Run portable Swift tests in a pinned Linux toolchain
+        run: |
+          docker run --rm \
+            --volume "$GITHUB_WORKSPACE:/src" \
+            --workdir /src \
+            swift:6.0 \
+            swift test
+
+  macos-build-and-tests:
+    name: macOS build and tests
+    runs-on: macos-15
+    steps:
+      - uses: actions/checkout@v4
+      - name: Show Swift toolchain
+        run: swift --version
+      # The Core ML quality benchmark is intentionally opt-in: it requires a
+      # pre-provisioned FluidAudio model cache. This command still builds the
+      # macOS app and runs all deterministic portable tests.
+      - name: Build Quill and run deterministic tests
+        run: swift test

Package.swift

@@ -1,32 +1,71 @@
 // swift-tools-version:6.0
 import PackageDescription
 
+var targets: [Target] = [
+    // Foundation-only support lets attribution and RTTM/DER tests run in
+    // Linux CI even though Quill inference itself is macOS/Core-ML-only.
+    .target(name: "quillDiarizationSupport", path: "Sources/quillDiarizationSupport"),
+    .target(
+        name: "quillDiarizationTestSupport",
+        path: "Tests/quillTests",
+        exclude: ["Fixtures", "DiarizationBenchmarkTests.swift", "PortableDiarizationScorerTests.swift", "DiarizationAttributionTests.swift"],
+        sources: ["PortableDiarizationScorer.swift"]
+    ),
+]
+
+#if os(macOS)
+targets += [
+    .executableTarget(
+        name: "quill",
+        dependencies: [
+            "quillDiarizationSupport",
+            .product(name: "ArgumentParser", package: "swift-argument-parser"),
+            .product(name: "FluidAudio", package: "FluidAudio"),
+        ],
+        exclude: ["Info.plist"],
+        linkerSettings: [
+            // Embed Info.plist into the binary so TCC can attribute the
+            // system-audio-capture permission to quill itself when it runs as
+            // a LaunchAgent (no .app bundle to carry a plist).
+            .unsafeFlags([
+                "-Xlinker", "-sectcreate",
+                "-Xlinker", "__TEXT",
+                "-Xlinker", "__info_plist",
+                "-Xlinker", "Sources/quill/Info.plist",
+            ]),
+        ]
+    ),
+    .testTarget(
+        name: "quillTests",
+        dependencies: [
+            "quillDiarizationTestSupport",
+            "quillDiarizationSupport",
+            "quill",
+            .product(name: "FluidAudio", package: "FluidAudio"),
+        ],
+        path: "Tests/quillTests",
+        exclude: ["Fixtures", "PortableDiarizationScorer.swift"],
+        sources: ["PortableDiarizationScorerTests.swift", "DiarizationAttributionTests.swift", "DiarizationBenchmarkTests.swift"]
+    ),
+]
+#else
+targets.append(
+    .testTarget(
+        name: "quillTests",
+        dependencies: ["quillDiarizationTestSupport", "quillDiarizationSupport"],
+        path: "Tests/quillTests",
+        exclude: ["Fixtures", "PortableDiarizationScorer.swift", "DiarizationBenchmarkTests.swift"],
+        sources: ["PortableDiarizationScorerTests.swift", "DiarizationAttributionTests.swift"]
+    )
+)
+#endif
+
 let package = Package(
     name: "quill",
     platforms: [.macOS(.v15)],
     dependencies: [
         .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.3.0"),
         .package(url: "https://github.com/FluidInference/FluidAudio.git", from: "0.7.0"),
     ],
-    targets: [
-        .executableTarget(
-            name: "quill",
-            dependencies: [
-                .product(name: "ArgumentParser", package: "swift-argument-parser"),
-                .product(name: "FluidAudio", package: "FluidAudio"),
-            ],
-            exclude: ["Info.plist"],
-            linkerSettings: [
-                // Embed Info.plist into the binary so TCC can attribute the
-                // system-audio-capture permission to quill itself when it
-                // runs as a LaunchAgent (no .app bundle to carry a plist).
-                .unsafeFlags([
-                    "-Xlinker", "-sectcreate",
-                    "-Xlinker", "__TEXT",
-                    "-Xlinker", "__info_plist",
-                    "-Xlinker", "Sources/quill/Info.plist",
-                ]),
-            ]
-        ),
-    ]
+    targets: targets
 )

README.md

@@ -43,11 +43,13 @@ Each session lands in `~/Recordings/<yyyy.MM.dd-HHmm>/`:
 | `transcript.md` | the same transcript rendered for reading |
 | `transcribe.log` | transcription progress/errors for this session |
 
-Two tracks on purpose: speech models do better on clean single-source audio,
-and mic-vs-system is free two-party diarization — `me` vs `them` with no
-speaker-identification model. CAF on purpose: unlike m4a, it needs no
-finalization pass — if the process dies mid-meeting, everything already
-written is still readable.
+Two tracks on purpose: speech models do better on clean single-source audio.
+The mic is labeled `me`; the system track is automatically diarized into
+anonymous, session-local `speaker-1`, `speaker-2`, … labels. This resolves
+multiple remote participants without trying to identify people or sending
+any audio off-device. CAF on purpose: unlike m4a, it needs no finalization
+pass — if the process dies mid-meeting, everything already written is still
+readable.
 
 ## Transcription
 
@@ -59,8 +61,16 @@ whether they're already cached so you're never downloading after an important
 meeting.
 
 Each track is transcribed separately, shifted by its start offset so both
-share one clock, and merged by timestamp. Jobs run in a serial queue — you can
-start a new recording while the last one transcribes. Unfinished jobs resume
+share one clock, and merged by timestamp. Before transcribing the mixed system
+track, quill runs FluidAudio's offline Core ML Community-1/VBx diarizer.
+Parakeet word timestamps are aligned one word at a time, so a transcript span
+that crosses a turn boundary is split rather than assigned wholesale. A word
+must substantially overlap one diarized range before it receives an anonymous
+`speaker-N` label; otherwise it remains the safe generic `them` label. The
+canonical JSON records the speaker source, time-alignment confidence, and an
+overlap flag. If diarization or its model download fails, transcription still
+completes with `them`. Jobs run in a serial queue — you can start a new
+recording while the last one transcribes. Unfinished jobs resume
 on next launch (the filesystem is the queue: a session with `meta.json` but no
 `transcript.json` is pending). Failures append to the session's
 `transcribe.log` and never block later jobs.
@@ -76,13 +86,22 @@ Optional, at `~/.config/quill/config.json`:
 {
   "recordings_dir": "~/Recordings",
   "transcription": { "enabled": true, "engine": "parakeet" },
+  "diarization": { "enabled": true, "minimum_confidence": 0.55 },
   "on_stop": "my-hook"
 }
 ```
 
 - `recordings_dir` — where sessions land. Resolution order: `--out` flag >
   config > `~/Recordings`.
 - `transcription.enabled` — set `false` to just record.
+- `diarization.enabled` — split the mixed system track into anonymous
+  `speaker-N` labels (default `true`). Set `false` to retain one `them` label
+  for the full system track. The diarization models are downloaded once on
+  their first use and run locally thereafter.
+- `diarization.minimum_confidence` — the minimum 0–1 fraction of an ASR
+  word/span that must overlap a diarization range before Quill labels it
+  `speaker-N` (default `0.55`). Lower values label more speech but raise the
+  risk of a wrong attribution; unmatched/low-confidence speech remains `them`.
 - `mic_voice_processing` — Apple's echo cancellation on the mic (default off).
   Set `true` when recording meetings through the speakers, so playback doesn't
   bleed into the mic track and get transcribed twice as "me". The trade: while
@@ -112,8 +131,41 @@ quill install --uninstall
 - **AVAudioEngine** — mic capture
 - **AVAudioFile** — streaming AAC encode into CAF
 - **FluidAudio / Parakeet** — on-device Core ML transcription
+- **FluidAudio / Community-1 + VBx** — on-device, offline speaker diarization
 - **NSStatusItem** — the whole UI
 
+## Diarization acceptance tests
+
+The repository includes 12 short, pinned WAV/RTTM fixtures from the
+[DimQ1 Sortformer Diarization Test Set](https://huggingface.co/datasets/DimQ1/sortformer-diarization-test-set)
+(CC-BY-4.0). They cover two- and three-speaker turns with known annotations;
+see the fixture [attribution notice](Tests/quillTests/Fixtures/diarization/NOTICE.md).
+They are a short, non-overlapping read-speech smoke corpus—not a substitute
+for consented meeting-style regression fixtures.
+
+- **Any Swift platform (including Linux):** `swift test` runs the
+  Foundation-only RTTM parser and DER scorer. It verifies all 12 known
+  fixtures, expected speaker counts, anonymous-label matching, the error
+  accounting, and the 250 ms speaker-change collar.
+- **macOS:** opt into Quill's **actual** offline Core ML diarizer benchmark
+  only on a provisioned runner with its model cache preloaded:
+  ```sh
+  QUILL_RUN_DIARIZATION_BENCHMARK=1 swift test
+  ```
+  It requires each clip’s DER to be at most 35% and its detected speaker count
+  to be within one of the annotation. The test deliberately does not download
+  models, so a missing cache produces an actionable failure rather than making
+  normal test runs network-dependent. GitHub Actions runs the deterministic
+  portable suite on Ubuntu and the normal macOS build/test suite on `macos-15`.
+  Run the opt-in Core ML benchmark only from a separately provisioned macOS
+  runner that has the FluidAudio model cache.
+
+Cluster IDs are intentionally anonymous, so both suites use optimal
+cluster-to-reference matching over the same collar-defined scoring region
+before calculating DER. See [evaluation guidance](docs/diarization-evaluation.md)
+for the required consent, meeting-style corpus design, and a release benchmark
+checklist.
+
 ## Gotchas
 
 - A global tap records *everything* the Mac plays — notification dings,

Sources/quill/Config.swift

@@ -5,6 +5,7 @@ import Foundation
 ///     {
 ///       "recordings_dir": "~/Recordings",
 ///       "transcription": { "enabled": true, "engine": "parakeet" },
+///       "diarization": { "enabled": true, "minimum_confidence": 0.55 },
 ///       "mic_voice_processing": true,
 ///       "on_stop": "my-hook"
 ///     }
@@ -44,10 +45,29 @@ enum Config {
         transcription()?["engine"] as? String ?? "parakeet"
     }
 
+    /// Whether the mixed system-audio track is split into anonymous speakers.
+    /// Default on; set `diarization.enabled` to false to retain one "them"
+    /// speaker for that entire track.
+    static func diarizationEnabled() -> Bool {
+        diarization()?["enabled"] as? Bool ?? true
+    }
+
+    /// Minimum fraction of a word/span that must overlap one diarized range
+    /// before Quill emits an anonymous `speaker-N` label. Out-of-range values
+    /// are ignored in favour of the conservative default.
+    static func diarizationMinimumConfidence() -> Double {
+        let value = diarization()?["minimum_confidence"] as? Double ?? 0.55
+        return (0...1).contains(value) ? value : 0.55
+    }
+
     private static func transcription() -> [String: Any]? {
         load()?["transcription"] as? [String: Any]
     }
 
+    private static func diarization() -> [String: Any]? {
+        load()?["diarization"] as? [String: Any]
+    }
+
     /// Apple voice processing (acoustic echo cancellation) on the mic, so
     /// speaker playback doesn't bleed into the mic track and get transcribed
     /// as "me". Default off — the live voice unit ducks all other playback,

Sources/quill/Doctor.swift

@@ -21,6 +21,7 @@ enum DoctorReport {
             checkSystemAudio(),
             checkRecordingsRoot(recordingsRoot),
             checkTranscription(),
+            checkDiarization(),
         ]
     }
 
@@ -97,6 +98,21 @@ enum DoctorReport {
         )
     }
 
+    /// Report model readiness without loading or downloading the diarizer.
+    static func checkDiarization() -> Check {
+        guard Config.transcriptionEnabled(), Config.diarizationEnabled() else {
+            return Check(name: "diarization", status: .warn("disabled in config"), remediation: nil)
+        }
+        if DiarizationEngine.modelsAvailable() {
+            return Check(name: "diarization", status: .ok, remediation: nil)
+        }
+        return Check(
+            name: "diarization",
+            status: .warn("offline models not downloaded"),
+            remediation: "record a short test session while online to prime the local Core ML cache"
+        )
+    }
+
     static func print(_ checks: [Check]) {
         for c in checks {
             let (mark, label): (String, String) = {

Sources/quill/Transcription/DiarizationEngine.swift

@@ -0,0 +1,88 @@
+import FluidAudio
+import Foundation
+
+/// Offline, on-device speaker diarization for the mixed system-audio track.
+/// FluidAudio's Community-1 Core ML pipeline returns anonymous, stable speaker
+/// IDs (S1, S2, …) and their time ranges; it never identifies a person or
+/// sends recording data off-device.
+actor DiarizationEngine {
+    struct Segment: Sendable {
+        let speaker: String
+        let start: TimeInterval
+        let end: TimeInterval
+    }
+
+    nonisolated let name = "pyannote-community-1-coreml-vbx"
+
+    /// Community-1's practical default for meetings. A 0.25 s collar is
+    /// standard when assessing diarization: it excludes inherently ambiguous
+    /// change boundaries from a reference comparison.
+    static let evaluationCollar: TimeInterval = 0.25
+    static let maximumFixtureDER = 0.35
+    static let maximumFixtureSpeakerCountError = 1
+
+    private var manager: OfflineDiarizerManager?
+    private var displayNames: [String: String] = [:]
+
+    func prepare() async throws {
+        guard manager == nil else { return }
+        let manager = OfflineDiarizerManager()
+        try await manager.prepareModels()
+        self.manager = manager
+    }
+
+    /// Run an audio file through the same configured pipeline used by quill.
+    /// `internal` so the benchmark target can test the integration boundary.
+    func diarize(_ audio: URL) async throws -> [Segment] {
+        guard let manager else { throw EngineError.notPrepared }
+        let result = try await manager.process(audio)
+        return result.segments.map {
+            Segment(
+                speaker: displayName(for: $0.speakerId),
+                start: TimeInterval($0.startTimeSeconds),
+                end: TimeInterval($0.endTimeSeconds)
+            )
+        }
+    }
+
+    func release() async {
+        // OfflineDiarizerManager has no explicit cleanup API. Releasing our
+        // reference lets Core ML reclaim its model resources between jobs.
+        manager = nil
+        displayNames = [:]
+    }
+
+    private func displayName(for id: String) -> String {
+        if let name = displayNames[id] { return name }
+        // FluidAudio currently emits S1, S2, …, but preserve one-to-one
+        // labeling if a future dependency revision changes that convention.
+        let name = "speaker-\(displayNames.count + 1)"
+        displayNames[id] = name
+        return name
+    }
+
+    static func modelsAvailable() -> Bool {
+        let root = OfflineDiarizerModels.defaultModelsDirectory()
+        let required = [
+            "Segmentation.mlmodelc", "FBank.mlmodelc", "Embedding.mlmodelc",
+            "PldaRho.mlmodelc", "plda-parameters.json",
+        ]
+        let candidateRoots = [
+            root,
+            root.appendingPathComponent("speaker-diarization", isDirectory: true),
+            root.appendingPathComponent("speaker-diarization-coreml", isDirectory: true),
+            root.appendingPathComponent("speaker-diarization-offline", isDirectory: true),
+        ]
+        return candidateRoots.contains { directory in
+            required.allSatisfy {
+                FileManager.default.fileExists(atPath: directory.appendingPathComponent($0).path)
+            }
+        }
+    }
+
+    enum EngineError: Error, CustomStringConvertible {
+        case notPrepared
+
+        var description: String { "diarization engine used before prepare()" }
+    }
+}