diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..958cf62 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,38 @@ +name: Test + +on: + pull_request: + push: + branches: [master] + +permissions: + contents: read + +jobs: + portable-diarization-tests: + name: Portable RTTM / DER tests (Ubuntu) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + # Quill itself is macOS-only, but its fixture integrity, RTTM parser, + # and DER-scoring suite is deliberately Foundation-only and runs here. + - name: Run portable Swift tests in a pinned Linux toolchain + run: | + docker run --rm \ + --volume "$GITHUB_WORKSPACE:/src" \ + --workdir /src \ + swift:6.0 \ + swift test + + macos-build-and-tests: + name: macOS build and tests + runs-on: macos-15 + steps: + - uses: actions/checkout@v4 + - name: Show Swift toolchain + run: swift --version + # The Core ML quality benchmark is intentionally opt-in: it requires a + # pre-provisioned FluidAudio model cache. This command still builds the + # macOS app and runs all deterministic portable tests. + - name: Build Quill and run deterministic tests + run: swift test diff --git a/Package.swift b/Package.swift index e6b9706..52bb5c5 100644 --- a/Package.swift +++ b/Package.swift @@ -1,6 +1,65 @@ // swift-tools-version:6.0 import PackageDescription +var targets: [Target] = [ + // Foundation-only support lets attribution and RTTM/DER tests run in + // Linux CI even though Quill inference itself is macOS/Core-ML-only. + .target(name: "quillDiarizationSupport", path: "Sources/quillDiarizationSupport"), + .target( + name: "quillDiarizationTestSupport", + path: "Tests/quillTests", + exclude: ["Fixtures", "DiarizationBenchmarkTests.swift", "PortableDiarizationScorerTests.swift", "DiarizationAttributionTests.swift"], + sources: ["PortableDiarizationScorer.swift"] + ), +] + +#if os(macOS) +targets += [ + .executableTarget( + name: "quill", + dependencies: [ + "quillDiarizationSupport", + .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "FluidAudio", package: "FluidAudio"), + ], + exclude: ["Info.plist"], + linkerSettings: [ + // Embed Info.plist into the binary so TCC can attribute the + // system-audio-capture permission to quill itself when it runs as + // a LaunchAgent (no .app bundle to carry a plist). + .unsafeFlags([ + "-Xlinker", "-sectcreate", + "-Xlinker", "__TEXT", + "-Xlinker", "__info_plist", + "-Xlinker", "Sources/quill/Info.plist", + ]), + ] + ), + .testTarget( + name: "quillTests", + dependencies: [ + "quillDiarizationTestSupport", + "quillDiarizationSupport", + "quill", + .product(name: "FluidAudio", package: "FluidAudio"), + ], + path: "Tests/quillTests", + exclude: ["Fixtures", "PortableDiarizationScorer.swift"], + sources: ["PortableDiarizationScorerTests.swift", "DiarizationAttributionTests.swift", "DiarizationBenchmarkTests.swift"] + ), +] +#else +targets.append( + .testTarget( + name: "quillTests", + dependencies: ["quillDiarizationTestSupport", "quillDiarizationSupport"], + path: "Tests/quillTests", + exclude: ["Fixtures", "PortableDiarizationScorer.swift", "DiarizationBenchmarkTests.swift"], + sources: ["PortableDiarizationScorerTests.swift", "DiarizationAttributionTests.swift"] + ) +) +#endif + let package = Package( name: "quill", platforms: [.macOS(.v15)], @@ -8,25 +67,5 @@ let package = Package( .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.3.0"), .package(url: "https://github.com/FluidInference/FluidAudio.git", from: "0.7.0"), ], - targets: [ - .executableTarget( - name: "quill", - dependencies: [ - .product(name: "ArgumentParser", package: "swift-argument-parser"), - .product(name: "FluidAudio", package: "FluidAudio"), - ], - exclude: ["Info.plist"], - linkerSettings: [ - // Embed Info.plist into the binary so TCC can attribute the - // system-audio-capture permission to quill itself when it - // runs as a LaunchAgent (no .app bundle to carry a plist). - .unsafeFlags([ - "-Xlinker", "-sectcreate", - "-Xlinker", "__TEXT", - "-Xlinker", "__info_plist", - "-Xlinker", "Sources/quill/Info.plist", - ]), - ] - ), - ] + targets: targets ) diff --git a/README.md b/README.md index 14192fa..3a18f69 100644 --- a/README.md +++ b/README.md @@ -43,11 +43,13 @@ Each session lands in `~/Recordings//`: | `transcript.md` | the same transcript rendered for reading | | `transcribe.log` | transcription progress/errors for this session | -Two tracks on purpose: speech models do better on clean single-source audio, -and mic-vs-system is free two-party diarization — `me` vs `them` with no -speaker-identification model. CAF on purpose: unlike m4a, it needs no -finalization pass — if the process dies mid-meeting, everything already -written is still readable. +Two tracks on purpose: speech models do better on clean single-source audio. +The mic is labeled `me`; the system track is automatically diarized into +anonymous, session-local `speaker-1`, `speaker-2`, … labels. This resolves +multiple remote participants without trying to identify people or sending +any audio off-device. CAF on purpose: unlike m4a, it needs no finalization +pass — if the process dies mid-meeting, everything already written is still +readable. ## Transcription @@ -59,8 +61,16 @@ whether they're already cached so you're never downloading after an important meeting. Each track is transcribed separately, shifted by its start offset so both -share one clock, and merged by timestamp. Jobs run in a serial queue — you can -start a new recording while the last one transcribes. Unfinished jobs resume +share one clock, and merged by timestamp. Before transcribing the mixed system +track, quill runs FluidAudio's offline Core ML Community-1/VBx diarizer. +Parakeet word timestamps are aligned one word at a time, so a transcript span +that crosses a turn boundary is split rather than assigned wholesale. A word +must substantially overlap one diarized range before it receives an anonymous +`speaker-N` label; otherwise it remains the safe generic `them` label. The +canonical JSON records the speaker source, time-alignment confidence, and an +overlap flag. If diarization or its model download fails, transcription still +completes with `them`. Jobs run in a serial queue — you can start a new +recording while the last one transcribes. Unfinished jobs resume on next launch (the filesystem is the queue: a session with `meta.json` but no `transcript.json` is pending). Failures append to the session's `transcribe.log` and never block later jobs. @@ -76,6 +86,7 @@ Optional, at `~/.config/quill/config.json`: { "recordings_dir": "~/Recordings", "transcription": { "enabled": true, "engine": "parakeet" }, + "diarization": { "enabled": true, "minimum_confidence": 0.55 }, "on_stop": "my-hook" } ``` @@ -83,6 +94,14 @@ Optional, at `~/.config/quill/config.json`: - `recordings_dir` — where sessions land. Resolution order: `--out` flag > config > `~/Recordings`. - `transcription.enabled` — set `false` to just record. +- `diarization.enabled` — split the mixed system track into anonymous + `speaker-N` labels (default `true`). Set `false` to retain one `them` label + for the full system track. The diarization models are downloaded once on + their first use and run locally thereafter. +- `diarization.minimum_confidence` — the minimum 0–1 fraction of an ASR + word/span that must overlap a diarization range before Quill labels it + `speaker-N` (default `0.55`). Lower values label more speech but raise the + risk of a wrong attribution; unmatched/low-confidence speech remains `them`. - `mic_voice_processing` — Apple's echo cancellation on the mic (default off). Set `true` when recording meetings through the speakers, so playback doesn't bleed into the mic track and get transcribed twice as "me". The trade: while @@ -112,8 +131,41 @@ quill install --uninstall - **AVAudioEngine** — mic capture - **AVAudioFile** — streaming AAC encode into CAF - **FluidAudio / Parakeet** — on-device Core ML transcription +- **FluidAudio / Community-1 + VBx** — on-device, offline speaker diarization - **NSStatusItem** — the whole UI +## Diarization acceptance tests + +The repository includes 12 short, pinned WAV/RTTM fixtures from the +[DimQ1 Sortformer Diarization Test Set](https://huggingface.co/datasets/DimQ1/sortformer-diarization-test-set) +(CC-BY-4.0). They cover two- and three-speaker turns with known annotations; +see the fixture [attribution notice](Tests/quillTests/Fixtures/diarization/NOTICE.md). +They are a short, non-overlapping read-speech smoke corpus—not a substitute +for consented meeting-style regression fixtures. + +- **Any Swift platform (including Linux):** `swift test` runs the + Foundation-only RTTM parser and DER scorer. It verifies all 12 known + fixtures, expected speaker counts, anonymous-label matching, the error + accounting, and the 250 ms speaker-change collar. +- **macOS:** opt into Quill's **actual** offline Core ML diarizer benchmark + only on a provisioned runner with its model cache preloaded: + ```sh + QUILL_RUN_DIARIZATION_BENCHMARK=1 swift test + ``` + It requires each clip’s DER to be at most 35% and its detected speaker count + to be within one of the annotation. The test deliberately does not download + models, so a missing cache produces an actionable failure rather than making + normal test runs network-dependent. GitHub Actions runs the deterministic + portable suite on Ubuntu and the normal macOS build/test suite on `macos-15`. + Run the opt-in Core ML benchmark only from a separately provisioned macOS + runner that has the FluidAudio model cache. + +Cluster IDs are intentionally anonymous, so both suites use optimal +cluster-to-reference matching over the same collar-defined scoring region +before calculating DER. See [evaluation guidance](docs/diarization-evaluation.md) +for the required consent, meeting-style corpus design, and a release benchmark +checklist. + ## Gotchas - A global tap records *everything* the Mac plays — notification dings, diff --git a/Sources/quill/Config.swift b/Sources/quill/Config.swift index 5db668f..befcab1 100644 --- a/Sources/quill/Config.swift +++ b/Sources/quill/Config.swift @@ -5,6 +5,7 @@ import Foundation /// { /// "recordings_dir": "~/Recordings", /// "transcription": { "enabled": true, "engine": "parakeet" }, +/// "diarization": { "enabled": true, "minimum_confidence": 0.55 }, /// "mic_voice_processing": true, /// "on_stop": "my-hook" /// } @@ -44,10 +45,29 @@ enum Config { transcription()?["engine"] as? String ?? "parakeet" } + /// Whether the mixed system-audio track is split into anonymous speakers. + /// Default on; set `diarization.enabled` to false to retain one "them" + /// speaker for that entire track. + static func diarizationEnabled() -> Bool { + diarization()?["enabled"] as? Bool ?? true + } + + /// Minimum fraction of a word/span that must overlap one diarized range + /// before Quill emits an anonymous `speaker-N` label. Out-of-range values + /// are ignored in favour of the conservative default. + static func diarizationMinimumConfidence() -> Double { + let value = diarization()?["minimum_confidence"] as? Double ?? 0.55 + return (0...1).contains(value) ? value : 0.55 + } + private static func transcription() -> [String: Any]? { load()?["transcription"] as? [String: Any] } + private static func diarization() -> [String: Any]? { + load()?["diarization"] as? [String: Any] + } + /// Apple voice processing (acoustic echo cancellation) on the mic, so /// speaker playback doesn't bleed into the mic track and get transcribed /// as "me". Default off — the live voice unit ducks all other playback, diff --git a/Sources/quill/Doctor.swift b/Sources/quill/Doctor.swift index 8af8125..2d1e82a 100644 --- a/Sources/quill/Doctor.swift +++ b/Sources/quill/Doctor.swift @@ -21,6 +21,7 @@ enum DoctorReport { checkSystemAudio(), checkRecordingsRoot(recordingsRoot), checkTranscription(), + checkDiarization(), ] } @@ -97,6 +98,21 @@ enum DoctorReport { ) } + /// Report model readiness without loading or downloading the diarizer. + static func checkDiarization() -> Check { + guard Config.transcriptionEnabled(), Config.diarizationEnabled() else { + return Check(name: "diarization", status: .warn("disabled in config"), remediation: nil) + } + if DiarizationEngine.modelsAvailable() { + return Check(name: "diarization", status: .ok, remediation: nil) + } + return Check( + name: "diarization", + status: .warn("offline models not downloaded"), + remediation: "record a short test session while online to prime the local Core ML cache" + ) + } + static func print(_ checks: [Check]) { for c in checks { let (mark, label): (String, String) = { diff --git a/Sources/quill/Transcription/DiarizationEngine.swift b/Sources/quill/Transcription/DiarizationEngine.swift new file mode 100644 index 0000000..bf4ed12 --- /dev/null +++ b/Sources/quill/Transcription/DiarizationEngine.swift @@ -0,0 +1,88 @@ +import FluidAudio +import Foundation + +/// Offline, on-device speaker diarization for the mixed system-audio track. +/// FluidAudio's Community-1 Core ML pipeline returns anonymous, stable speaker +/// IDs (S1, S2, …) and their time ranges; it never identifies a person or +/// sends recording data off-device. +actor DiarizationEngine { + struct Segment: Sendable { + let speaker: String + let start: TimeInterval + let end: TimeInterval + } + + nonisolated let name = "pyannote-community-1-coreml-vbx" + + /// Community-1's practical default for meetings. A 0.25 s collar is + /// standard when assessing diarization: it excludes inherently ambiguous + /// change boundaries from a reference comparison. + static let evaluationCollar: TimeInterval = 0.25 + static let maximumFixtureDER = 0.35 + static let maximumFixtureSpeakerCountError = 1 + + private var manager: OfflineDiarizerManager? + private var displayNames: [String: String] = [:] + + func prepare() async throws { + guard manager == nil else { return } + let manager = OfflineDiarizerManager() + try await manager.prepareModels() + self.manager = manager + } + + /// Run an audio file through the same configured pipeline used by quill. + /// `internal` so the benchmark target can test the integration boundary. + func diarize(_ audio: URL) async throws -> [Segment] { + guard let manager else { throw EngineError.notPrepared } + let result = try await manager.process(audio) + return result.segments.map { + Segment( + speaker: displayName(for: $0.speakerId), + start: TimeInterval($0.startTimeSeconds), + end: TimeInterval($0.endTimeSeconds) + ) + } + } + + func release() async { + // OfflineDiarizerManager has no explicit cleanup API. Releasing our + // reference lets Core ML reclaim its model resources between jobs. + manager = nil + displayNames = [:] + } + + private func displayName(for id: String) -> String { + if let name = displayNames[id] { return name } + // FluidAudio currently emits S1, S2, …, but preserve one-to-one + // labeling if a future dependency revision changes that convention. + let name = "speaker-\(displayNames.count + 1)" + displayNames[id] = name + return name + } + + static func modelsAvailable() -> Bool { + let root = OfflineDiarizerModels.defaultModelsDirectory() + let required = [ + "Segmentation.mlmodelc", "FBank.mlmodelc", "Embedding.mlmodelc", + "PldaRho.mlmodelc", "plda-parameters.json", + ] + let candidateRoots = [ + root, + root.appendingPathComponent("speaker-diarization", isDirectory: true), + root.appendingPathComponent("speaker-diarization-coreml", isDirectory: true), + root.appendingPathComponent("speaker-diarization-offline", isDirectory: true), + ] + return candidateRoots.contains { directory in + required.allSatisfy { + FileManager.default.fileExists(atPath: directory.appendingPathComponent($0).path) + } + } + } + + enum EngineError: Error, CustomStringConvertible { + case notPrepared + + var description: String { "diarization engine used before prepare()" } + } +} diff --git a/Sources/quill/Transcription/ParakeetEngine.swift b/Sources/quill/Transcription/ParakeetEngine.swift index ff6cad5..f0b60ff 100644 --- a/Sources/quill/Transcription/ParakeetEngine.swift +++ b/Sources/quill/Transcription/ParakeetEngine.swift @@ -80,7 +80,10 @@ actor ParakeetEngine: TranscriptionEngine { out.append(TranscriptSegment( start: first.startTime, end: last.endTime, - text: current.map(\.word).joined(separator: " ") + text: current.map(\.word).joined(separator: " "), + words: current.map { + TimedTranscriptWord(text: $0.word, start: $0.startTime, end: $0.endTime) + } )) current = [] } diff --git a/Sources/quill/Transcription/TranscriptionCoordinator.swift b/Sources/quill/Transcription/TranscriptionCoordinator.swift index 5300fbd..507b2eb 100644 --- a/Sources/quill/Transcription/TranscriptionCoordinator.swift +++ b/Sources/quill/Transcription/TranscriptionCoordinator.swift @@ -1,4 +1,5 @@ import Foundation +import quillDiarizationSupport /// Post-recording pipeline: a serial queue of session folders to transcribe. /// mic.caf → "me", system.caf → "them"; each track's segments are shifted by @@ -17,6 +18,7 @@ actor TranscriptionCoordinator { private var queue: [URL] = [] private var draining = false private var engine: TranscriptionEngine? + private var diarizer: DiarizationEngine? private var lastFailure: String? private var statusHandler: (@Sendable (Status) -> Void)? @@ -90,6 +92,8 @@ actor TranscriptionCoordinator { } await engine?.release() engine = nil + await diarizer?.release() + diarizer = nil publish(lastFailure.map { .failed(session: $0) } ?? .idle) draining = false // An enqueue that landed between the loop exiting and the release @@ -102,12 +106,34 @@ actor TranscriptionCoordinator { let engine = try await preparedEngine() var merged: [Transcript.Segment] = [] + var diarizationEngine: String? for track in meta.tracks { let audio = dir.appendingPathComponent(track.file) guard FileManager.default.fileExists(atPath: audio.path) else { log(dir, "skipping missing track \(track.file)") continue } + + // The mic is already its own speaker. System audio is usually a + // mixed remote track, so attribute each ASR segment to the + // anonymous diarization cluster with the largest time overlap. + var speakerTimeline: [DiarizedSpeakerRange] = [] + if track.speaker == "them", Config.diarizationEnabled() { + do { + let diarizer = try await preparedDiarizer() + log(dir, "diarizing \(track.file) (\(diarizer.name))") + speakerTimeline = try await diarizer.diarize(audio).map { + DiarizedSpeakerRange(speaker: $0.speaker, start: $0.start, end: $0.end) + } + diarizationEngine = diarizer.name + log(dir, "diarized \(track.file) — \(speakerTimeline.count) ranges") + } catch { + // A diarization failure must not cost the transcript. The + // original track-level "them" label remains the fallback. + log(dir, "diarization skipped for \(track.file): \(error)") + } + } + log(dir, "transcribing \(track.file) (\(engine.name))") // One bad track (empty, truncated) shouldn't cost us the other's // transcript — log it and keep going. @@ -119,13 +145,22 @@ actor TranscriptionCoordinator { continue } let offset = TimeInterval(track.offsetMs) / 1000 - merged += segments.map { - Transcript.Segment( - speaker: track.speaker, - start_ms: Int(($0.start + offset) * 1000), - end_ms: Int(($0.end + offset) * 1000), - text: $0.text + for segment in segments { + let turns = Self.attributedTurns( + for: segment, defaultSpeaker: track.speaker, timeline: speakerTimeline, + isMicrophone: track.speaker == "me" ) + merged += turns.map { + Transcript.Segment( + speaker: $0.attribution.speaker, + speaker_source: $0.attribution.source.rawValue, + speaker_confidence: $0.attribution.confidence, + overlap: $0.attribution.overlap, + start_ms: Int(($0.start + offset) * 1000), + end_ms: Int(($0.end + offset) * 1000), + text: $0.text + ) + } } } merged.sort { $0.start_ms < $1.start_ms } @@ -133,6 +168,7 @@ actor TranscriptionCoordinator { let transcript = Transcript( engine: engine.name, model: engine.model, + diarization_engine: diarizationEngine, created_at: ISO8601DateFormatter().string(from: Date()), segments: merged ) @@ -154,6 +190,43 @@ actor TranscriptionCoordinator { return engine } + private func preparedDiarizer() async throws -> DiarizationEngine { + if let diarizer { return diarizer } + let diarizer = DiarizationEngine() + try await diarizer.prepare() + self.diarizer = diarizer + return diarizer + } + + private static func attributedTurns( + for asr: TranscriptSegment, + defaultSpeaker: String, + timeline: [DiarizedSpeakerRange], + isMicrophone: Bool + ) -> [AttributedTranscriptTurn] { + if isMicrophone { + return [AttributedTranscriptTurn( + text: asr.text, start: asr.start, end: asr.end, + attribution: SpeakerAttribution( + speaker: defaultSpeaker, source: .microphone, confidence: 1, overlap: false + ) + )] + } + if !asr.words.isEmpty { + return DiarizationAttributor.turns( + words: asr.words, defaultSpeaker: defaultSpeaker, ranges: timeline, + minimumConfidence: Config.diarizationMinimumConfidence() + ) + } + let attribution = DiarizationAttributor.attribution( + start: asr.start, end: asr.end, defaultSpeaker: defaultSpeaker, ranges: timeline, + minimumConfidence: Config.diarizationMinimumConfidence() + ) + return [AttributedTranscriptTurn( + text: asr.text, start: asr.start, end: asr.end, attribution: attribution + )] + } + /// Fires the configured on_stop shell command with the session directory /// as its sole argument, after the transcript exists (or immediately after /// recording when transcription is disabled). @@ -234,6 +307,9 @@ private struct SessionMeta { private struct Transcript: Codable { struct Segment: Codable { let speaker: String + let speaker_source: String + let speaker_confidence: Double + let overlap: Bool let start_ms: Int let end_ms: Int let text: String @@ -241,6 +317,7 @@ private struct Transcript: Codable { let engine: String let model: String + let diarization_engine: String? let created_at: String let segments: [Segment] @@ -257,9 +334,16 @@ private struct Transcript: Codable { } private func rendered(title: String) -> String { - var lines = ["# \(title)", "", "engine: \(engine) (\(model))", ""] + var lines = ["# \(title)", "", "engine: \(engine) (\(model))"] + if let diarization_engine { + lines.append("diarization: \(diarization_engine)") + } + lines.append("") for seg in segments { - lines.append("**[\(Self.clock(seg.start_ms))] \(seg.speaker):** \(seg.text)") + let confidence = seg.speaker_source == "diarization" + ? " · \(Int((seg.speaker_confidence * 100).rounded()))% aligned" : "" + let overlap = seg.overlap ? " · overlap" : "" + lines.append("**[\(Self.clock(seg.start_ms))] \(seg.speaker)\(confidence)\(overlap):** \(seg.text)") lines.append("") } return lines.joined(separator: "\n") diff --git a/Sources/quill/Transcription/TranscriptionEngine.swift b/Sources/quill/Transcription/TranscriptionEngine.swift index 51a1890..d3b800b 100644 --- a/Sources/quill/Transcription/TranscriptionEngine.swift +++ b/Sources/quill/Transcription/TranscriptionEngine.swift @@ -6,6 +6,16 @@ struct TranscriptSegment: Sendable { let start: TimeInterval let end: TimeInterval let text: String + /// Present when the ASR engine exposes word timing. Diarization can then + /// split a sentence at a speaker handoff instead of assigning it wholesale. + let words: [TimedTranscriptWord] + + init(start: TimeInterval, end: TimeInterval, text: String, words: [TimedTranscriptWord] = []) { + self.start = start + self.end = end + self.text = text + self.words = words + } } /// A speech-to-text engine quill can run locally. Engines are prepared lazily diff --git a/Sources/quillDiarizationSupport/Attribution.swift b/Sources/quillDiarizationSupport/Attribution.swift new file mode 100644 index 0000000..6894d9a --- /dev/null +++ b/Sources/quillDiarizationSupport/Attribution.swift @@ -0,0 +1,147 @@ +import Foundation + +/// Platform-independent time-alignment policy shared by Quill's macOS +/// transcription pipeline and its Linux test suite. It deliberately does not +/// identify a person: all speaker names are anonymous diarization clusters. +public struct TimedTranscriptWord: Sendable, Equatable { + public let text: String + public let start: TimeInterval + public let end: TimeInterval + + public init(text: String, start: TimeInterval, end: TimeInterval) { + self.text = text + self.start = start + self.end = end + } +} + +public struct DiarizedSpeakerRange: Sendable, Equatable { + public let speaker: String + public let start: TimeInterval + public let end: TimeInterval + + public init(speaker: String, start: TimeInterval, end: TimeInterval) { + self.speaker = speaker + self.start = start + self.end = end + } +} + +public struct SpeakerAttribution: Sendable, Equatable { + public enum Source: String, Sendable { + case diarization + case fallback + case microphone + } + + public let speaker: String + public let source: Source + /// Fraction of the word/segment duration that overlaps the chosen range. + /// It is a time-alignment score, not a model-calibrated identity score. + public let confidence: Double + public let overlap: Bool + + public init(speaker: String, source: Source, confidence: Double, overlap: Bool) { + self.speaker = speaker + self.source = source + self.confidence = confidence + self.overlap = overlap + } +} + +public struct AttributedTranscriptTurn: Sendable, Equatable { + public let text: String + public let start: TimeInterval + public let end: TimeInterval + public let attribution: SpeakerAttribution + + public init(text: String, start: TimeInterval, end: TimeInterval, attribution: SpeakerAttribution) { + self.text = text + self.start = start + self.end = end + self.attribution = attribution + } +} + +public enum DiarizationAttributor { + /// Attribute word-timed ASR to anonymous diarization ranges. A word only + /// receives a diarized speaker when enough of it overlaps that range; + /// otherwise it safely retains the caller's generic fallback label. + public static func turns( + words: [TimedTranscriptWord], + defaultSpeaker: String, + ranges: [DiarizedSpeakerRange], + minimumConfidence: Double = 0.55, + maximumGap: TimeInterval = 1.0 + ) -> [AttributedTranscriptTurn] { + guard !words.isEmpty else { return [] } + let attributed = words.map { + ($0, attribution(start: $0.start, end: $0.end, defaultSpeaker: defaultSpeaker, + ranges: ranges, minimumConfidence: minimumConfidence)) + } + var turns: [AttributedTranscriptTurn] = [] + var wordsInTurn: [TimedTranscriptWord] = [] + var current: SpeakerAttribution? + + func flush() { + guard let current, let first = wordsInTurn.first, let last = wordsInTurn.last else { return } + let confidence = current.confidence + turns.append(AttributedTranscriptTurn( + text: wordsInTurn.map(\.text).joined(separator: " "), start: first.start, end: last.end, + attribution: SpeakerAttribution( + speaker: current.speaker, source: current.source, confidence: confidence, + overlap: current.overlap + ) + )) + wordsInTurn = [] + } + + for (word, next) in attributed { + if let existing = current, let last = wordsInTurn.last, + (existing.speaker != next.speaker || existing.source != next.source + || word.start - last.end > maximumGap) { + flush() + current = next + } else if current == nil { + current = next + } + wordsInTurn.append(word) + } + flush() + return turns + } + + /// Attribute an arbitrary span for ASR engines that do not provide word + /// timing. This is deliberately a fallback; word-level `turns` is more + /// accurate when a speech segment crosses a diarization boundary. + public static func attribution( + start: TimeInterval, + end: TimeInterval, + defaultSpeaker: String, + ranges: [DiarizedSpeakerRange], + minimumConfidence: Double = 0.55 + ) -> SpeakerAttribution { + let duration = max(0.001, end - start) + let matches = ranges.map { range in + (range, max(0, min(end, range.end) - max(start, range.start))) + }.filter { $0.1 > 0 } + guard let best = matches.max(by: { $0.1 < $1.1 }) else { + return SpeakerAttribution(speaker: defaultSpeaker, source: .fallback, confidence: 0, overlap: false) + } + let confidence = min(1, best.1 / duration) + guard confidence >= minimumConfidence else { + return SpeakerAttribution(speaker: defaultSpeaker, source: .fallback, confidence: confidence, + overlap: matches.count > 1) + } + return SpeakerAttribution(speaker: best.0.speaker, source: .diarization, confidence: confidence, + overlap: matches.count > 1) + } + + private static func assign( + _ word: TimedTranscriptWord, attribution: SpeakerAttribution, + to words: inout [TimedTranscriptWord], current: inout SpeakerAttribution? + ) { + current = attribution + words.append(word) + } +} diff --git a/Tests/quillTests/DiarizationAttributionTests.swift b/Tests/quillTests/DiarizationAttributionTests.swift new file mode 100644 index 0000000..ee4e06b --- /dev/null +++ b/Tests/quillTests/DiarizationAttributionTests.swift @@ -0,0 +1,60 @@ +import XCTest +@testable import quillDiarizationSupport + +final class DiarizationAttributionTests: XCTestCase { + func testWordTimingSplitsASRSegmentAtSpeakerHandoff() { + let turns = DiarizationAttributor.turns( + words: [word("Hello", 0, 0.4), word("there", 0.45, 0.8), word("yes", 1.05, 1.3)], + defaultSpeaker: "them", + ranges: [range("speaker-1", 0, 0.9), range("speaker-2", 1, 1.5)] + ) + XCTAssertEqual(turns.map(\.text), ["Hello there", "yes"]) + XCTAssertEqual(turns.map(\.attribution.speaker), ["speaker-1", "speaker-2"]) + XCTAssertTrue(turns.allSatisfy { $0.attribution.source == .diarization }) + } + + func testMissingTimelineFallsBackWithoutInventingASpeaker() { + let attributed = DiarizationAttributor.attribution( + start: 0, end: 1, defaultSpeaker: "them", ranges: [] + ) + XCTAssertEqual(attributed.speaker, "them") + XCTAssertEqual(attributed.source, .fallback) + XCTAssertEqual(attributed.confidence, 0) + } + + func testLowOverlapFallsBackToThem() { + let attributed = DiarizationAttributor.attribution( + start: 0, end: 1, defaultSpeaker: "them", ranges: [range("speaker-1", 0, 0.4)] + ) + XCTAssertEqual(attributed.speaker, "them") + XCTAssertEqual(attributed.source, .fallback) + XCTAssertEqual(attributed.confidence, 0.4, accuracy: 0.001) + } + + func testEqualOverlapUsesStableTimelineOrder() { + let attributed = DiarizationAttributor.attribution( + start: 0, end: 1, defaultSpeaker: "them", + ranges: [range("speaker-1", 0, 0.8), range("speaker-2", 0.2, 1)] + ) + XCTAssertEqual(attributed.speaker, "speaker-1") + XCTAssertEqual(attributed.source, .diarization) + XCTAssertEqual(attributed.confidence, 0.8, accuracy: 0.001) + XCTAssertTrue(attributed.overlap) + } + + func testGapStartsANewTurnEvenForSameSpeaker() { + let turns = DiarizationAttributor.turns( + words: [word("first", 0, 0.3), word("second", 2, 2.3)], + defaultSpeaker: "them", ranges: [range("speaker-1", 0, 3)] + ) + XCTAssertEqual(turns.map(\.text), ["first", "second"]) + } + + private func word(_ text: String, _ start: Double, _ end: Double) -> TimedTranscriptWord { + .init(text: text, start: start, end: end) + } + + private func range(_ speaker: String, _ start: Double, _ end: Double) -> DiarizedSpeakerRange { + .init(speaker: speaker, start: start, end: end) + } +} diff --git a/Tests/quillTests/DiarizationBenchmarkTests.swift b/Tests/quillTests/DiarizationBenchmarkTests.swift new file mode 100644 index 0000000..f6cb6e3 --- /dev/null +++ b/Tests/quillTests/DiarizationBenchmarkTests.swift @@ -0,0 +1,128 @@ +import Foundation +import XCTest +@testable import quill +@testable import quillDiarizationTestSupport + +/// End-to-end acceptance tests for quill's actual diarizer, not a mocked +/// implementation. The fixtures are a pinned, representative 12-clip slice +/// of DimQ1's CC-BY-4.0 Sortformer Diarization Test Set. Each WAV has an RTTM +/// reference with known turn times and speaker identities. +/// +/// We score by DER with a 250 ms change-boundary collar, the standard +/// diarization convention. Cluster IDs are anonymous/arbitrary, so +/// the scorer finds the optimal one-to-one cluster-to-reference mapping before +/// it calculates errors. The thresholds are intentionally permissive +/// enough to avoid platform/Core ML numerical flakes while catching a broken +/// model integration, bad sample-rate handling, or loss of speaker clustering. +final class DiarizationBenchmarkTests: XCTestCase { + private static let fixtureNames = (0...11).map { String(format: "ls_real_%03d", $0) } + /// A deliberate, reviewable expectation for this pinned corpus slice. + /// RTTM remains the authority for turn timing; this catches accidental + /// fixture replacement with a different recording/reference pairing. + private static let expectedReferenceSpeakerCounts = [ + "ls_real_000": 2, "ls_real_001": 2, "ls_real_002": 2, + "ls_real_003": 3, "ls_real_004": 2, "ls_real_005": 2, + "ls_real_006": 3, "ls_real_007": 2, "ls_real_008": 2, + "ls_real_009": 2, "ls_real_010": 2, "ls_real_011": 3, + ] + + func testFixtureCorpusIsCompleteAndInternallyConsistent() throws { + let fixtureDirectory = try fixtureDirectory() + for name in Self.fixtureNames { + let wav = fixtureDirectory.appendingPathComponent("\(name).wav") + let rttm = fixtureDirectory.appendingPathComponent("\(name).rttm") + XCTAssertTrue(FileManager.default.fileExists(atPath: wav.path), "missing \(wav.lastPathComponent)") + XCTAssertTrue(FileManager.default.fileExists(atPath: rttm.path), "missing \(rttm.lastPathComponent)") + let reference = try referenceSegments(from: rttm) + XCTAssertFalse(reference.isEmpty, "empty RTTM: \(name)") + XCTAssertEqual( + Set(reference.map(\.speaker)).count, + try XCTUnwrap(Self.expectedReferenceSpeakerCounts[name]), + "unexpected known-speaker count in \(name)" + ) + } + } + + func testAutomaticDiarizationMeetsQualityExpectationAcrossTwelveKnownClips() async throws { + guard ProcessInfo.processInfo.environment["QUILL_RUN_DIARIZATION_BENCHMARK"] == "1" else { + throw XCTSkip( + "Set QUILL_RUN_DIARIZATION_BENCHMARK=1 on a provisioned macOS runner to run Core ML inference" + ) + } + guard DiarizationEngine.modelsAvailable() else { + XCTFail( + "Offline diarizer models are not cached. Prime the FluidAudio cache before enabling this benchmark." + ) + return + } + let fixtureDirectory = try fixtureDirectory() + let diarizer = DiarizationEngine() + try await diarizer.prepare() + defer { Task { await diarizer.release() } } + + var failures: [String] = [] + var totalReferenceSpeech: Double = 0 + var totalError: Double = 0 + + for name in Self.fixtureNames { + let reference = try referenceSegments( + from: fixtureDirectory.appendingPathComponent("\(name).rttm") + ) + let actual = try await diarizer.diarize( + fixtureDirectory.appendingPathComponent("\(name).wav") + ) + let score = PortableDiarizationScorer.score( + reference: reference, + hypothesis: actual.map { + .init(speaker: $0.speaker, start: $0.start, end: $0.end) + }, + collar: DiarizationEngine.evaluationCollar + ) + let referenceSpeakers = Set(reference.map(\.speaker)).count + let actualSpeakers = Set(actual.map(\.speaker)).count + let speakerCountError = abs(referenceSpeakers - actualSpeakers) + totalReferenceSpeech += score.referenceSpeech + totalError += score.referenceSpeech * score.der + + if score.der > DiarizationEngine.maximumFixtureDER + || speakerCountError > DiarizationEngine.maximumFixtureSpeakerCountError { + failures.append( + "\(name): DER \(percent(score.der)); speakers ref=\(referenceSpeakers), actual=\(actualSpeakers)" + ) + } + } + + XCTAssertTrue(failures.isEmpty, "\n" + failures.joined(separator: "\n")) + let corpusDER = totalReferenceSpeech > 0 ? totalError / totalReferenceSpeech : 0 + XCTAssertLessThanOrEqual( + corpusDER, + DiarizationEngine.maximumFixtureDER, + "aggregate DER \(percent(corpusDER)) exceeded \(percent(DiarizationEngine.maximumFixtureDER))" + ) + } + + private func fixtureDirectory() throws -> URL { + let source = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .appendingPathComponent("Fixtures/diarization", isDirectory: true) + guard FileManager.default.fileExists(atPath: source.path) else { + throw FixtureError.missing(source) + } + return source + } + + private func referenceSegments(from url: URL) throws -> [PortableDiarizationScorer.Segment] { + try PortableDiarizationScorer.parseRTTM(String(contentsOf: url, encoding: .utf8)) + } + + private func percent(_ value: Double) -> String { + String(format: "%.1f%%", value * 100) + } + + private enum FixtureError: Error, CustomStringConvertible { + case missing(URL) + var description: String { + switch self { case .missing(let url): return "missing fixture directory: \(url.path)" } + } + } +} diff --git a/Tests/quillTests/Fixtures/diarization/NOTICE.md b/Tests/quillTests/Fixtures/diarization/NOTICE.md new file mode 100644 index 0000000..42f22f9 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/NOTICE.md @@ -0,0 +1,18 @@ +# Fixture attribution and license notice + +This directory redistributes the unmodified first 12 WAV/RTTM pairs +(`ls_real_000` through `ls_real_011`) selected from +[`DimQ1/sortformer-diarization-test-set`](https://huggingface.co/datasets/DimQ1/sortformer-diarization-test-set) +at revision [`92c3f79cf16148c3420766b7af2e47fb2842ec59`](https://huggingface.co/datasets/DimQ1/sortformer-diarization-test-set/tree/92c3f79cf16148c3420766b7af2e47fb2842ec59). +The upstream dataset card identifies these samples as derived from the +[LibriSpeech ASR corpus](https://www.openslr.org/12), **test-clean** subset. + +- **Audio and annotations:** LibriSpeech / Vassil Panayotov with Daniel Povey, + Gautham Chen, Sanjeev Khudanpur, and Vijayaditya Khudanpur; “LibriSpeech: + an ASR corpus based on public domain audio books.” +- **Dataset packaging:** DimQ1, *Sortformer Diarization Test Set*. +- **License:** [Creative Commons Attribution 4.0 International (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/). + +This repository only selected these files as test fixtures and normalized RTTM +line endings from CRLF to LF. The audio and annotation content was otherwise +not modified. See `README.md` for use and hash details. diff --git a/Tests/quillTests/Fixtures/diarization/README.md b/Tests/quillTests/Fixtures/diarization/README.md new file mode 100644 index 0000000..9d595b2 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/README.md @@ -0,0 +1,22 @@ +# Diarization acceptance-test fixtures + +These are the first 12 WAV/RTTM pairs (`ls_real_000` through `ls_real_011`) from +[DimQ1/sortformer-diarization-test-set](https://huggingface.co/datasets/DimQ1/sortformer-diarization-test-set), +pinned to dataset revision `92c3f79cf16148c3420766b7af2e47fb2842ec59`. +See [NOTICE.md](NOTICE.md) for the required LibriSpeech/DimQ1 attribution, +license link, and the exact redistribution statement. + +- **License:** CC-BY-4.0 (as declared by the pinned dataset card) +- **Audio:** 16 kHz mono WAV +- **Reference:** RTTM rows, where field 8 is the speaker ID and fields 4–5 are + the start time and duration. +- **Purpose:** end-to-end quality acceptance tests for Quill's on-device + `OfflineDiarizerManager` integration. They are not sent to any service. + +`DiarizationBenchmarkTests` uses a 250 ms speaker-change collar, optimal +one-to-one anonymous-cluster mapping, and asserts an individual DER <= 35% +plus a speaker-count error <= 1 for all twelve clips. These short, +non-overlapping read-speech clips are an inference smoke corpus, not a +representative meeting-quality benchmark. File hashes are recorded in +`SHA256SUMS` and verified by the portable test suite, so fixture changes are +intentional and reviewable. diff --git a/Tests/quillTests/Fixtures/diarization/SHA256SUMS b/Tests/quillTests/Fixtures/diarization/SHA256SUMS new file mode 100644 index 0000000..60108fd --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/SHA256SUMS @@ -0,0 +1,24 @@ +8d866b557fb6d453df276ab37a0b6c128c7b0aa92863aa880953d4ad2947f5fc ls_real_000.wav +fc9ac3d9147689de0025f6f352b2284e27e612a57f0c6791a6dbd54e53833270 ls_real_001.wav +da96cc5675b73d779692dd5670271e339f61aba505f417ab25077bc6cc9f2204 ls_real_002.wav +005699c3de6c0ed38e1fddf958443e6fed679153909d53be400c222377ec33f4 ls_real_003.wav +22e8fd79a2e72b6a0fa87cdab09d1ded673f6b248af4f19ad7bc95c43e6f2a14 ls_real_004.wav +12b055b395f94e56e355b77af7ef796576673bee8ea3368e74ccefcb03712358 ls_real_005.wav +8a1268ba7d072a60ef194351790593bc13a90a871e9963f92902addd14c8429f ls_real_006.wav +0686098b2193c8a7c830e47428a89dd9dc66c971c2b98bd6a509c45d10902864 ls_real_007.wav +1a7f81c6f85961fe0fb16fecb123a7ee16081f38f4e319bb9c73a37d16ffc834 ls_real_008.wav +4d85db9315014a1ff1921fbdc87a010f0bd933bc1f0db17cda9ab0e9a2c88e06 ls_real_009.wav +3dc9ee25d57c5bb03f0f30eaa323a04760f6c5244f89636c9361bc9f65ed2e66 ls_real_010.wav +6c46b8af006722c86a82193749eab5d181ac4e38394055b46473df4fad4ce6c9 ls_real_011.wav +9eade49e3611c6742e0aa1b66e57a51da13ea22567af74ac061ec213014feb97 ls_real_000.rttm +995000a9d4800f6533fca3e9c1b34248814db4a56bad5e69ab24c88afe0996c2 ls_real_001.rttm +307984a63fd9dd863a8e90ea42dc5fe2bad0bf919f68e68992b376dd59d95ed6 ls_real_002.rttm +48096e759c54ef20eff10a19c4afbdff0f7a66219384bdc525a1c8a1d9b98e0a ls_real_003.rttm +c21e39614bbe26a1c3e3bea234606a9364717637102d281a364c70334382e99b ls_real_004.rttm +5ad1342eaa49db3b2507b7301925b59725620d18f6078f403cd30dbd0245d098 ls_real_005.rttm +7d5dbf9dca502c11b1fbe54ae45f88ffbf97b3fddd3ea70215892b63b3178f31 ls_real_006.rttm +6be42f5472e886b5e0185e94fe56e2d8cd61dcfc8bf48acb8bf278acbcef0505 ls_real_007.rttm +2df2ce1e6e5dc090ecd5ccfe9a9339324514c43cf97609d2fdb9095732b09dbe ls_real_008.rttm +069fbbf03b93db331e01ff7b274d3b9af03ed08ea2d0de02cc90113265ce74a9 ls_real_009.rttm +b92e8fd598781f48b70da8996b77604cb73b48d2ac0da38173f4104e16c351b3 ls_real_010.rttm +d5059dd1173129d4fae5fad3d22d14ccca1c59bf547d480721070172d2994cd5 ls_real_011.rttm diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_000.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_000.rttm new file mode 100644 index 0000000..2989004 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_000.rttm @@ -0,0 +1,13 @@ +SPEAKER ls_real_000 1 0.000 0.964 5639 +SPEAKER ls_real_000 1 1.159 1.794 2961 +SPEAKER ls_real_000 1 3.037 3.335 2961 +SPEAKER ls_real_000 1 6.427 1.958 2961 +SPEAKER ls_real_000 1 8.535 3.023 2961 +SPEAKER ls_real_000 1 11.693 0.730 5639 +SPEAKER ls_real_000 1 12.613 2.338 2961 +SPEAKER ls_real_000 1 15.015 1.817 5639 +SPEAKER ls_real_000 1 16.994 1.062 5639 +SPEAKER ls_real_000 1 18.148 2.799 2961 +SPEAKER ls_real_000 1 21.112 3.977 2961 +SPEAKER ls_real_000 1 25.262 1.174 5639 +SPEAKER ls_real_000 1 26.601 1.978 5639 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_000.wav b/Tests/quillTests/Fixtures/diarization/ls_real_000.wav new file mode 100644 index 0000000..5b6883d Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_000.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_001.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_001.rttm new file mode 100644 index 0000000..d026355 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_001.rttm @@ -0,0 +1,11 @@ +SPEAKER ls_real_001 1 0.000 3.569 1320 +SPEAKER ls_real_001 1 3.759 3.051 1320 +SPEAKER ls_real_001 1 6.928 1.921 1188 +SPEAKER ls_real_001 1 8.980 1.673 1188 +SPEAKER ls_real_001 1 10.840 0.382 1320 +SPEAKER ls_real_001 1 11.272 0.892 1320 +SPEAKER ls_real_001 1 12.238 3.648 1320 +SPEAKER ls_real_001 1 16.040 0.341 1188 +SPEAKER ls_real_001 1 16.457 3.465 1188 +SPEAKER ls_real_001 1 20.033 3.596 1188 +SPEAKER ls_real_001 1 23.753 1.555 1320 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_001.wav b/Tests/quillTests/Fixtures/diarization/ls_real_001.wav new file mode 100644 index 0000000..2c06090 Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_001.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_002.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_002.rttm new file mode 100644 index 0000000..fdc81c2 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_002.rttm @@ -0,0 +1,7 @@ +SPEAKER ls_real_002 1 0.000 0.843 4507 +SPEAKER ls_real_002 1 0.952 0.994 4507 +SPEAKER ls_real_002 1 2.098 3.239 2830 +SPEAKER ls_real_002 1 5.519 1.180 4507 +SPEAKER ls_real_002 1 6.871 0.428 2830 +SPEAKER ls_real_002 1 7.363 2.166 2830 +SPEAKER ls_real_002 1 9.607 1.247 2830 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_002.wav b/Tests/quillTests/Fixtures/diarization/ls_real_002.wav new file mode 100644 index 0000000..90657cc Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_002.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_003.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_003.rttm new file mode 100644 index 0000000..33955c0 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_003.rttm @@ -0,0 +1,6 @@ +SPEAKER ls_real_003 1 0.000 2.322 2830 +SPEAKER ls_real_003 1 2.472 2.028 2830 +SPEAKER ls_real_003 1 4.625 3.055 672 +SPEAKER ls_real_003 1 7.756 1.391 4970 +SPEAKER ls_real_003 1 9.330 1.469 2830 +SPEAKER ls_real_003 1 10.913 0.312 2830 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_003.wav b/Tests/quillTests/Fixtures/diarization/ls_real_003.wav new file mode 100644 index 0000000..687d3d8 Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_003.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_004.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_004.rttm new file mode 100644 index 0000000..5551197 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_004.rttm @@ -0,0 +1,7 @@ +SPEAKER ls_real_004 1 0.000 1.138 4970 +SPEAKER ls_real_004 1 1.202 2.401 4970 +SPEAKER ls_real_004 1 3.676 2.256 8455 +SPEAKER ls_real_004 1 5.987 2.159 8455 +SPEAKER ls_real_004 1 8.216 1.519 4970 +SPEAKER ls_real_004 1 9.930 3.676 4970 +SPEAKER ls_real_004 1 13.737 0.799 4970 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_004.wav b/Tests/quillTests/Fixtures/diarization/ls_real_004.wav new file mode 100644 index 0000000..bd2498e Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_004.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_005.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_005.rttm new file mode 100644 index 0000000..056d252 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_005.rttm @@ -0,0 +1,5 @@ +SPEAKER ls_real_005 1 0.000 0.997 7127 +SPEAKER ls_real_005 1 1.161 1.938 1221 +SPEAKER ls_real_005 1 3.219 2.716 1221 +SPEAKER ls_real_005 1 6.127 2.810 7127 +SPEAKER ls_real_005 1 9.062 1.323 1221 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_005.wav b/Tests/quillTests/Fixtures/diarization/ls_real_005.wav new file mode 100644 index 0000000..8711945 Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_005.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_006.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_006.rttm new file mode 100644 index 0000000..3d30075 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_006.rttm @@ -0,0 +1,13 @@ +SPEAKER ls_real_006 1 0.000 1.376 1221 +SPEAKER ls_real_006 1 1.478 0.771 1221 +SPEAKER ls_real_006 1 2.362 3.077 4077 +SPEAKER ls_real_006 1 5.601 1.579 1221 +SPEAKER ls_real_006 1 7.286 1.749 1221 +SPEAKER ls_real_006 1 9.169 3.005 4077 +SPEAKER ls_real_006 1 12.260 3.107 4992 +SPEAKER ls_real_006 1 15.492 3.653 4077 +SPEAKER ls_real_006 1 19.210 2.609 4077 +SPEAKER ls_real_006 1 21.916 2.319 4992 +SPEAKER ls_real_006 1 24.359 3.826 4077 +SPEAKER ls_real_006 1 28.285 2.826 1221 +SPEAKER ls_real_006 1 31.280 1.459 4077 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_006.wav b/Tests/quillTests/Fixtures/diarization/ls_real_006.wav new file mode 100644 index 0000000..563082a Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_006.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_007.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_007.rttm new file mode 100644 index 0000000..6ed00e7 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_007.rttm @@ -0,0 +1,7 @@ +SPEAKER ls_real_007 1 0.000 1.974 4446 +SPEAKER ls_real_007 1 2.062 1.268 4446 +SPEAKER ls_real_007 1 3.417 0.532 61 +SPEAKER ls_real_007 1 4.017 2.982 4446 +SPEAKER ls_real_007 1 7.194 2.399 4446 +SPEAKER ls_real_007 1 9.677 2.575 61 +SPEAKER ls_real_007 1 12.308 2.019 4446 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_007.wav b/Tests/quillTests/Fixtures/diarization/ls_real_007.wav new file mode 100644 index 0000000..bcffcc8 Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_007.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_008.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_008.rttm new file mode 100644 index 0000000..65274db --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_008.rttm @@ -0,0 +1,8 @@ +SPEAKER ls_real_008 1 0.000 2.267 5142 +SPEAKER ls_real_008 1 2.337 0.657 5142 +SPEAKER ls_real_008 1 3.153 0.301 5142 +SPEAKER ls_real_008 1 3.633 0.363 5142 +SPEAKER ls_real_008 1 4.163 1.790 8463 +SPEAKER ls_real_008 1 6.062 1.623 5142 +SPEAKER ls_real_008 1 7.744 0.844 5142 +SPEAKER ls_real_008 1 8.640 1.098 5142 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_008.wav b/Tests/quillTests/Fixtures/diarization/ls_real_008.wav new file mode 100644 index 0000000..12427ef Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_008.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_009.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_009.rttm new file mode 100644 index 0000000..3520a7c --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_009.rttm @@ -0,0 +1,7 @@ +SPEAKER ls_real_009 1 0.000 1.116 908 +SPEAKER ls_real_009 1 1.257 1.921 8230 +SPEAKER ls_real_009 1 3.241 3.850 8230 +SPEAKER ls_real_009 1 7.186 3.508 8230 +SPEAKER ls_real_009 1 10.766 1.665 8230 +SPEAKER ls_real_009 1 12.491 0.796 908 +SPEAKER ls_real_009 1 13.389 1.051 8230 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_009.wav b/Tests/quillTests/Fixtures/diarization/ls_real_009.wav new file mode 100644 index 0000000..79bd249 Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_009.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_010.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_010.rttm new file mode 100644 index 0000000..0f78150 --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_010.rttm @@ -0,0 +1,14 @@ +SPEAKER ls_real_010 1 0.000 1.186 7176 +SPEAKER ls_real_010 1 1.384 0.845 2830 +SPEAKER ls_real_010 1 2.334 1.347 7176 +SPEAKER ls_real_010 1 3.736 0.620 7176 +SPEAKER ls_real_010 1 4.533 2.828 2830 +SPEAKER ls_real_010 1 7.491 0.809 2830 +SPEAKER ls_real_010 1 8.407 1.448 7176 +SPEAKER ls_real_010 1 9.936 0.590 7176 +SPEAKER ls_real_010 1 10.664 0.816 7176 +SPEAKER ls_real_010 1 11.605 1.198 2830 +SPEAKER ls_real_010 1 12.965 1.295 2830 +SPEAKER ls_real_010 1 14.364 2.458 7176 +SPEAKER ls_real_010 1 16.926 0.538 2830 +SPEAKER ls_real_010 1 17.549 3.480 7176 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_010.wav b/Tests/quillTests/Fixtures/diarization/ls_real_010.wav new file mode 100644 index 0000000..0f616b3 Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_010.wav differ diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_011.rttm b/Tests/quillTests/Fixtures/diarization/ls_real_011.rttm new file mode 100644 index 0000000..f6a349f --- /dev/null +++ b/Tests/quillTests/Fixtures/diarization/ls_real_011.rttm @@ -0,0 +1,5 @@ +SPEAKER ls_real_011 1 0.000 3.136 4077 +SPEAKER ls_real_011 1 3.273 2.785 3575 +SPEAKER ls_real_011 1 6.116 1.340 4077 +SPEAKER ls_real_011 1 7.588 0.630 4077 +SPEAKER ls_real_011 1 8.307 1.501 1580 diff --git a/Tests/quillTests/Fixtures/diarization/ls_real_011.wav b/Tests/quillTests/Fixtures/diarization/ls_real_011.wav new file mode 100644 index 0000000..1d14209 Binary files /dev/null and b/Tests/quillTests/Fixtures/diarization/ls_real_011.wav differ diff --git a/Tests/quillTests/PortableDiarizationScorer.swift b/Tests/quillTests/PortableDiarizationScorer.swift new file mode 100644 index 0000000..670fb50 --- /dev/null +++ b/Tests/quillTests/PortableDiarizationScorer.swift @@ -0,0 +1,161 @@ +import Foundation + +/// Linux-safe RTTM parser and DER scorer used to validate Quill's acceptance +/// corpus and quality thresholds. It deliberately depends on Foundation only: +/// inference is macOS/Core-ML-only, but corpus/scoring regressions should be +/// caught in any CI environment. +enum PortableDiarizationScorer { + struct Segment: Hashable { + let speaker: String + let start: Double + let end: Double + } + + struct Score { + let der: Double + let miss: Double + let falseAlarm: Double + let confusion: Double + let referenceSpeech: Double + } + + enum ParseError: Error, CustomStringConvertible { + case invalidRTTMLine(String) + + var description: String { + switch self { + case .invalidRTTMLine(let line): return "invalid RTTM line: \(line)" + } + } + } + + static func parseRTTM(_ contents: String) throws -> [Segment] { + try contents.split(whereSeparator: \.isNewline).compactMap { line in + let fields = line.split(whereSeparator: { $0 == " " || $0 == "\t" }) + guard !fields.isEmpty else { return nil } + guard fields.count >= 8, fields[0] == "SPEAKER", + let start = Double(fields[3]), let duration = Double(fields[4]), + start >= 0, duration > 0 + else { throw ParseError.invalidRTTMLine(String(line)) } + return Segment(speaker: String(fields[7]), start: start, end: start + duration) + } + } + + /// Frame-wise DER, including an optimal one-to-one label mapping. This is + /// equivalent to normal diarization scoring for our non-overlapping RTTM + /// fixtures, and works with arbitrary anonymous hypothesis labels. + static func score( + reference: [Segment], hypothesis: [Segment], + frameStep: Double = 0.01, collar: Double = 0.25 + ) -> Score { + precondition(frameStep > 0 && collar >= 0) + let referenceLabels = labels(in: reference) + let hypothesisLabels = labels(in: hypothesis) + let end = (reference + hypothesis).map(\.end).max() ?? 0 + let frameCount = Int(ceil(end / frameStep)) + guard frameCount > 0 else { + return Score(der: 0, miss: 0, falseAlarm: 0, confusion: 0, referenceSpeech: 0) + } + + let referenceFrames = activeLabels( + reference, labels: referenceLabels, frameCount: frameCount, step: frameStep + ) + let hypothesisFrames = activeLabels( + hypothesis, labels: hypothesisLabels, frameCount: frameCount, step: frameStep + ) + let boundaries = Set(reference.flatMap { [$0.start, $0.end] }) + let halfCollar = collar / 2 + let scorable = (0.. [String] { + var seen = Set() + return segments.compactMap { seen.insert($0.speaker).inserted ? $0.speaker : nil } + } + + private static func activeLabels( + _ segments: [Segment], labels: [String], frameCount: Int, step: Double + ) -> [Set] { + var frames = Array(repeating: Set(), count: frameCount) + for segment in segments { + let start = max(0, Int(ceil(segment.start / step - 0.5))) + let end = min(frameCount, Int(ceil(segment.end / step - 0.5))) + guard end > start else { continue } + for index in start..], hypothesis: [Set], + referenceLabels: [String], hypothesisLabels: [String], scorable: [Bool] + ) -> [String: String] { + guard !referenceLabels.isEmpty, !hypothesisLabels.isEmpty else { return [:] } + var overlap: [String: [String: Int]] = [:] + for (index, (ref, hyp)) in zip(reference, hypothesis).enumerated() where scorable[index] { + for h in hyp { + for r in ref { overlap[h, default: [:]][r, default: 0] += 1 } + } + } + + var best: [String: String] = [:] + var bestTotal = -1 + func search(_ index: Int, used: Set, mapping: [String: String], total: Int) { + if index == hypothesisLabels.count { + if total > bestTotal { bestTotal = total; best = mapping } + return + } + let hypothesis = hypothesisLabels[index] + // An unmatched system speaker is valid; it maps to no reference. + search(index + 1, used: used, mapping: mapping, total: total) + for reference in referenceLabels where !used.contains(reference) { + var next = mapping + next[hypothesis] = reference + var nextUsed = used + nextUsed.insert(reference) + search( + index + 1, used: nextUsed, mapping: next, + total: total + (overlap[hypothesis]?[reference] ?? 0) + ) + } + } + search(0, used: [], mapping: [:], total: 0) + return best + } +} diff --git a/Tests/quillTests/PortableDiarizationScorerTests.swift b/Tests/quillTests/PortableDiarizationScorerTests.swift new file mode 100644 index 0000000..d58b097 --- /dev/null +++ b/Tests/quillTests/PortableDiarizationScorerTests.swift @@ -0,0 +1,166 @@ +import Foundation +import XCTest +@testable import quillDiarizationTestSupport + +final class PortableDiarizationScorerTests: XCTestCase { + private static let fixtureNames = (0...11).map { String(format: "ls_real_%03d", $0) } + private static let expectedSpeakerCounts = [ + "ls_real_000": 2, "ls_real_001": 2, "ls_real_002": 2, + "ls_real_003": 3, "ls_real_004": 2, "ls_real_005": 2, + "ls_real_006": 3, "ls_real_007": 2, "ls_real_008": 2, + "ls_real_009": 2, "ls_real_010": 2, "ls_real_011": 3, + ] + + func testTwelvePinnedRTTMFixturesParseWithExpectedSpeakerCounts() throws { + let fixtures = try fixtureDirectory() + for name in Self.fixtureNames { + let wav = fixtures.appendingPathComponent("\(name).wav") + XCTAssertTrue(FileManager.default.fileExists(atPath: wav.path), "missing WAV: \(name)") + XCTAssertGreaterThan( + (try? wav.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0, + 1_000, + "unexpectedly small WAV: \(name)" + ) + let segments = try PortableDiarizationScorer.parseRTTM( + String(contentsOf: fixtures.appendingPathComponent("\(name).rttm"), encoding: .utf8) + ) + XCTAssertFalse(segments.isEmpty, "empty fixture \(name)") + XCTAssertEqual( + Set(segments.map(\.speaker)).count, + try XCTUnwrap(Self.expectedSpeakerCounts[name]), + "unexpected annotated speaker count in \(name)" + ) + XCTAssertTrue(segments.allSatisfy { $0.end > $0.start }) + } + } + + func testFixtureChecksumsMatchManifest() throws { + let fixtures = try fixtureDirectory() + let manifest = fixtures.appendingPathComponent("SHA256SUMS") + let entries = try String(contentsOf: manifest, encoding: .utf8).split(whereSeparator: \.isNewline) + let fixtureFiles = Set( + try FileManager.default.contentsOfDirectory(atPath: fixtures.path) + .filter { $0.hasSuffix(".wav") || $0.hasSuffix(".rttm") } + ) + XCTAssertEqual(entries.count, fixtureFiles.count) + + var names = Set() + for entry in entries { + let parts = entry.split(maxSplits: 1, whereSeparator: { $0 == " " || $0 == "\t" }) + XCTAssertEqual(parts.count, 2, "malformed checksum entry: \(entry)") + let name = String(parts[1]).trimmingCharacters(in: .whitespaces) + XCTAssertTrue(fixtureFiles.contains(name), "manifest names missing fixture: \(name)") + XCTAssertTrue(names.insert(name).inserted, "duplicate manifest entry: \(name)") + XCTAssertEqual(try sha256(of: fixtures.appendingPathComponent(name)), String(parts[0])) + } + XCTAssertEqual(names, fixtureFiles) + } + + func testKnownReferenceMatchesItselfExactly() throws { + let fixtures = try fixtureDirectory() + for name in Self.fixtureNames { + let reference = try readRTTM(named: name, in: fixtures) + let result = PortableDiarizationScorer.score(reference: reference, hypothesis: reference) + XCTAssertEqual(result.der, 0, accuracy: 0.000_001, "self score failed for \(name)") + } + } + + func testAnonymousLabelsAreOptimallyMapped() { + let reference = [ + segment("alice", 0, 2), segment("bob", 2, 4), segment("alice", 4, 6), + ] + let hypothesis = [ + segment("speaker-2", 0, 2), segment("speaker-1", 2, 4), segment("speaker-2", 4, 6), + ] + let result = PortableDiarizationScorer.score(reference: reference, hypothesis: hypothesis, collar: 0) + XCTAssertEqual(result.der, 0, accuracy: 0.000_001) + } + + func testScorerSeparatesMissFalseAlarmAndConfusion() { + let reference = [segment("alice", 0, 1), segment("bob", 1, 2)] + let missed = PortableDiarizationScorer.score(reference: reference, hypothesis: [segment("x", 0, 1)], collar: 0) + XCTAssertEqual(missed.miss, 1, accuracy: 0.01) + XCTAssertEqual(missed.der, 0.5, accuracy: 0.01) + + let falseAlarm = PortableDiarizationScorer.score( + reference: [segment("alice", 0, 1)], + hypothesis: [segment("x", 0, 2)], collar: 0 + ) + XCTAssertEqual(falseAlarm.falseAlarm, 1, accuracy: 0.01) + XCTAssertEqual(falseAlarm.der, 1, accuracy: 0.01) + + let confusion = PortableDiarizationScorer.score( + reference: reference, hypothesis: [segment("x", 0, 2)], collar: 0 + ) + XCTAssertEqual(confusion.confusion, 1, accuracy: 0.01) + XCTAssertEqual(confusion.der, 0.5, accuracy: 0.01) + } + + func testCollarExcludesSpeakerChangeBoundaryError() { + let reference = [segment("alice", 0, 1), segment("bob", 1, 2)] + let lateBoundary = [segment("x", 0, 1.1), segment("y", 1.1, 2)] + let withoutCollar = PortableDiarizationScorer.score( + reference: reference, hypothesis: lateBoundary, collar: 0 + ) + let withStandardCollar = PortableDiarizationScorer.score( + reference: reference, hypothesis: lateBoundary, collar: 0.25 + ) + XCTAssertGreaterThan(withoutCollar.der, 0) + XCTAssertEqual(withStandardCollar.der, 0, accuracy: 0.000_001) + } + + func testCollarDefinesBothMappingAndErrorScoringRegion() { + // The only overlap that would favour the swapped labels is inside the + // excluded 250 ms region around 1.0 s. Mapping must ignore it. + let reference = [segment("alice", 0, 1), segment("bob", 1, 2)] + let hypothesis = [segment("x", 0, 1.1), segment("y", 1.1, 2)] + let score = PortableDiarizationScorer.score( + reference: reference, hypothesis: hypothesis, collar: 0.25 + ) + XCTAssertEqual(score.der, 0, accuracy: 0.000_001) + } + + private func sha256(of url: URL) throws -> String { + let command: (String, [String]) = { + #if os(macOS) + ("/usr/bin/shasum", ["-a", "256", url.path]) + #else + ("/usr/bin/sha256sum", [url.path]) + #endif + }() + let process = Process() + process.executableURL = URL(fileURLWithPath: command.0) + process.arguments = command.1 + let output = Pipe() + process.standardOutput = output + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { throw FixtureError.hashFailed(url) } + guard let digest = String( + data: output.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8 + )?.split(whereSeparator: \.isWhitespace).first else { throw FixtureError.hashFailed(url) } + return String(digest) + } + + private func fixtureDirectory() throws -> URL { + let source = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .appendingPathComponent("Fixtures/diarization", isDirectory: true) + guard FileManager.default.fileExists(atPath: source.path) else { + throw FixtureError.missing(source) + } + return source + } + + private func readRTTM(named name: String, in directory: URL) throws -> [PortableDiarizationScorer.Segment] { + try PortableDiarizationScorer.parseRTTM( + String(contentsOf: directory.appendingPathComponent("\(name).rttm"), encoding: .utf8) + ) + } + + private func segment(_ speaker: String, _ start: Double, _ end: Double) -> PortableDiarizationScorer.Segment { + .init(speaker: speaker, start: start, end: end) + } + + private enum FixtureError: Error { case missing(URL), hashFailed(URL) } +} diff --git a/docs/diarization-evaluation.md b/docs/diarization-evaluation.md new file mode 100644 index 0000000..2580657 --- /dev/null +++ b/docs/diarization-evaluation.md @@ -0,0 +1,64 @@ +# Diarization evaluation and release checklist + +Quill's diarizer produces anonymous, session-local labels (`speaker-N`). It +answers *who spoke when*, not a person's identity. Do not present a model label +as a known attendee, and do not build or match voice profiles without explicit, +revocable user consent and a separate privacy/security review. + +## What ships and what it does not prove + +`Tests/quillTests/Fixtures/diarization` is a pinned 12-clip, non-overlapping +read-speech **smoke corpus**. It detects wiring, timing, fixture, and scoring +regressions. It does not establish quality for actual meetings. + +Before enabling or materially changing diarization, evaluate a separate, +consented, access-controlled meeting-style corpus. Do not commit private +meeting recordings to this repository. + +## Required corpus coverage + +Include at least these slices, with RTTM ground truth and documented consent: + +- system-track captures from the supported meeting applications; +- 2, 3, and 4+ remote participants; +- 10–30 minute conversations to measure cluster stability over time; +- overlap/crosstalk and short backchannels; +- silence/no-speech, notification/music contamination, and poor network audio; +- both clean headphone capture and speaker-playback/echo-cancellation paths; +- representative AAC-in-CAF capture, not only 16 kHz PCM WAV. + +## Metrics and acceptance decisions + +Record, per corpus slice and overall: + +1. collar-aware DER (0.25 s collar, document overlap policy); +2. speaker-count error; +3. turn-boundary error and rate of `them` fallback labels; +4. long-session cluster consistency; +5. manual review of the worst-scoring samples. + +Pin a baseline by Quill commit, FluidAudio revision, model cache version, and +runner hardware. Treat a statistically/materially worse score versus that +baseline as a release blocker even when an absolute threshold still passes. +The current 35% DER smoke threshold is an integration tripwire, **not** a +meeting-quality target. + +## Running the real-model benchmark + +Use a provisioned Apple Silicon macOS runner with the offline FluidAudio model +cache already installed. Regular CI deliberately never downloads models. + +```sh +QUILL_RUN_DIARIZATION_BENCHMARK=1 swift test +``` + +Archive the console result with the hardware, macOS version, and model/cache +version. If the cache is absent, the test fails with an actionable message. + +## Correctability and safe failure + +The transcript JSON exposes `speaker_source`, `speaker_confidence`, and +`overlap`. Consumers should preserve those fields, show anonymous labels as +editable names, support merge/correction workflows, and never convert a low +confidence alignment into a personal identity. Generic `them` is preferable to +a confident but wrong speaker label.