diff --git a/Sources/HeardCore/AppModel.swift b/Sources/HeardCore/AppModel.swift index 15ee142..e06c81a 100644 --- a/Sources/HeardCore/AppModel.swift +++ b/Sources/HeardCore/AppModel.swift @@ -66,6 +66,11 @@ public final class AppModel: ObservableObject { downloadManager.transcriptionModel = savedVersion dictationManager.modelVersion = savedVersion + // Propagate the user's selected mic to both managers + let selectedMicUID = settingsStore.settings.selectedInputDeviceUID + dictationManager.inputDeviceUID = selectedMicUID + recordingManager.inputDeviceUID = selectedMicUID + let model = AppModel( settingsStore: settingsStore, speakerStore: speakerStore, @@ -218,6 +223,7 @@ public final class AppModel: ObservableObject { // Stop dictation before recording starts — mic should not transcribe // remote participants' audio (which the mic picks up from speakers). self.stopDictationIfActive() + self.recordingManager.inputDeviceUID = self.settingsStore.settings.selectedInputDeviceUID do { try self.recordingManager.startRecording( title: snapshot.title, @@ -319,6 +325,7 @@ public var filteredSpeakers: [SpeakerProfile] { dictationManager.formattingCommands = settingsStore.settings.formattingCommands dictationManager.modelVersion = settingsStore.settings.transcriptionModel dictationManager.modelKeepAliveSeconds = TimeInterval(settingsStore.settings.modelKeepAlive * 60) + dictationManager.inputDeviceUID = settingsStore.settings.selectedInputDeviceUID try await dictationManager.start() // Push-to-talk race: if the key was released before loading finished, // stop immediately rather than leaving dictation stuck on. @@ -468,6 +475,17 @@ public var filteredSpeakers: [SpeakerProfile] { objectWillChange.send() } + /// Set the input device used for dictation and meeting mic capture. + /// Passing nil reverts to the system default input. The change takes + /// effect on the next dictation start / recording start — already-running + /// sessions keep their current device until restarted. + public func setInputDeviceUID(_ uid: String?) { + settingsStore.settings.selectedInputDeviceUID = uid + dictationManager.inputDeviceUID = uid + recordingManager.inputDeviceUID = uid + objectWillChange.send() + } + private func setupHotkeyManager() { let pushToTalk = settingsStore.settings.pushToTalk hotkeyManager = HotkeyManager( @@ -555,6 +573,7 @@ public var filteredSpeakers: [SpeakerProfile] { public func startManualRecording() { guard recordingManager.activeSession == nil else { return } stopDictationIfActive() + recordingManager.inputDeviceUID = settingsStore.settings.selectedInputDeviceUID do { try recordingManager.startRecording(title: "Manual Recording", meetingPID: nil, rosterNames: []) phase = .recording diff --git a/Sources/HeardCore/AudioInputDevices.swift b/Sources/HeardCore/AudioInputDevices.swift new file mode 100644 index 0000000..65f4258 --- /dev/null +++ b/Sources/HeardCore/AudioInputDevices.swift @@ -0,0 +1,147 @@ +import AudioToolbox +import AVFoundation +import CoreAudio +import Foundation + +/// A microphone-capable CoreAudio device the user can select for dictation and +/// meeting recording. +public struct AudioInputDevice: Identifiable, Hashable, Sendable { + /// CoreAudio device UID — stable across launches and unplug/replug. + public let uid: String + /// Transient CoreAudio device ID, only valid for the current process lifetime. + public let deviceID: AudioDeviceID + public let name: String + + public var id: String { uid } +} + +/// Enumerates input devices via CoreAudio and applies a user-chosen device to +/// an `AVAudioEngine`. UID is persisted in settings; the transient device ID +/// is resolved at engine-start time, so devices that have been unplugged or +/// renumbered between launches are handled gracefully. +public enum AudioInputDevices { + + /// All connected audio devices that expose at least one input stream. + public static func list() -> [AudioInputDevice] { + var addr = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var size: UInt32 = 0 + guard + AudioObjectGetPropertyDataSize( + AudioObjectID(kAudioObjectSystemObject), &addr, 0, nil, &size + ) == noErr, + size > 0 + else { return [] } + + let count = Int(size) / MemoryLayout.size + var ids = [AudioDeviceID](repeating: 0, count: count) + guard + AudioObjectGetPropertyData( + AudioObjectID(kAudioObjectSystemObject), &addr, 0, nil, &size, &ids + ) == noErr + else { return [] } + + return ids.compactMap { deviceID -> AudioInputDevice? in + guard hasInputStreams(deviceID) else { return nil } + guard + let uid = stringProperty(deviceID, selector: kAudioDevicePropertyDeviceUID), + let name = stringProperty(deviceID, selector: kAudioObjectPropertyName) + ?? stringProperty(deviceID, selector: kAudioDevicePropertyDeviceNameCFString) + else { return nil } + return AudioInputDevice(uid: uid, deviceID: deviceID, name: name) + } + } + + /// Resolve a saved UID to the current `AudioDeviceID`, or nil if the + /// device isn't connected. + public static func deviceID(forUID uid: String) -> AudioDeviceID? { + list().first { $0.uid == uid }?.deviceID + } + + /// The current system default input device, if any. + public static func defaultInputDevice() -> AudioInputDevice? { + var addr = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDefaultInputDevice, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var deviceID: AudioDeviceID = 0 + var size = UInt32(MemoryLayout.size) + guard + AudioObjectGetPropertyData( + AudioObjectID(kAudioObjectSystemObject), &addr, 0, nil, &size, &deviceID + ) == noErr, + deviceID != 0, + let uid = stringProperty(deviceID, selector: kAudioDevicePropertyDeviceUID), + let name = stringProperty(deviceID, selector: kAudioObjectPropertyName) + ?? stringProperty(deviceID, selector: kAudioDevicePropertyDeviceNameCFString) + else { return nil } + return AudioInputDevice(uid: uid, deviceID: deviceID, name: name) + } + + /// Set the chosen device on the engine's input audio unit. Must be called + /// BEFORE `inputNode.outputFormat(forBus:)` or any tap install, since + /// changing the device changes the format. A nil UID, an unknown UID, or + /// an unplugged device leaves the system default in place. + @discardableResult + public static func apply(uid: String?, to engine: AVAudioEngine) -> Bool { + guard let uid, !uid.isEmpty, let deviceID = deviceID(forUID: uid) else { + return false + } + guard let audioUnit = engine.inputNode.audioUnit else { + NSLog("Heard: AudioInputDevices.apply — input node has no audio unit") + return false + } + var id = deviceID + let size = UInt32(MemoryLayout.size) + let status = AudioUnitSetProperty( + audioUnit, + kAudioOutputUnitProperty_CurrentDevice, + kAudioUnitScope_Global, + 0, + &id, + size + ) + if status != noErr { + NSLog("Heard: AudioInputDevices.apply failed (uid=%@): %d", uid, status) + return false + } + return true + } + + // MARK: - Private + + private static func hasInputStreams(_ deviceID: AudioDeviceID) -> Bool { + var addr = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreams, + mScope: kAudioDevicePropertyScopeInput, + mElement: kAudioObjectPropertyElementMain + ) + var size: UInt32 = 0 + guard AudioObjectGetPropertyDataSize(deviceID, &addr, 0, nil, &size) == noErr else { + return false + } + return size > 0 + } + + private static func stringProperty( + _ deviceID: AudioDeviceID, + selector: AudioObjectPropertySelector + ) -> String? { + var addr = AudioObjectPropertyAddress( + mSelector: selector, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var cfString: Unmanaged? + var size = UInt32(MemoryLayout?>.size) + let status = AudioObjectGetPropertyData( + deviceID, &addr, 0, nil, &size, &cfString + ) + guard status == noErr, let value = cfString else { return nil } + return value.takeRetainedValue() as String + } +} diff --git a/Sources/HeardCore/CoreModels.swift b/Sources/HeardCore/CoreModels.swift index 07e660d..b583ea3 100644 --- a/Sources/HeardCore/CoreModels.swift +++ b/Sources/HeardCore/CoreModels.swift @@ -362,6 +362,9 @@ public struct AppSettings: Codable, Equatable { /// How many days of inactivity before a speaker profile is automatically deleted. /// 0 means never delete automatically. public var speakerRetentionDays: Int + /// CoreAudio UID of the user-selected input device for dictation and meeting + /// recording. nil means follow the system default input device. + public var selectedInputDeviceUID: String? public static let `default` = AppSettings( userName: "", @@ -389,7 +392,8 @@ public struct AppSettings: Codable, Equatable { diarizationClusteringSimilarity: 0.65, appearance: .system, lowMemoryMode: false, - speakerRetentionDays: 90 + speakerRetentionDays: 90, + selectedInputDeviceUID: nil ) public init( @@ -415,7 +419,8 @@ public struct AppSettings: Codable, Equatable { diarizationClusteringSimilarity: Double = 0.65, appearance: AppAppearance = .system, lowMemoryMode: Bool = false, - speakerRetentionDays: Int = 90 + speakerRetentionDays: Int = 90, + selectedInputDeviceUID: String? = nil ) { self.userName = userName self.launchAtLogin = launchAtLogin @@ -440,6 +445,7 @@ public struct AppSettings: Codable, Equatable { self.appearance = appearance self.lowMemoryMode = lowMemoryMode self.speakerRetentionDays = speakerRetentionDays + self.selectedInputDeviceUID = selectedInputDeviceUID } } diff --git a/Sources/HeardCore/DictationManager.swift b/Sources/HeardCore/DictationManager.swift index 859605e..1d2d74b 100644 --- a/Sources/HeardCore/DictationManager.swift +++ b/Sources/HeardCore/DictationManager.swift @@ -10,11 +10,14 @@ public enum DictationState: String { public enum DictationError: Error, LocalizedError { case notIdle(current: DictationState) + case microphoneDenied public var errorDescription: String? { switch self { case .notIdle(let s): return "Cannot start dictation: already \(s.rawValue). Please wait for the current operation to finish." + case .microphoneDenied: + return "Microphone access is required for dictation. Grant it in System Settings → Privacy & Security → Microphone." } } } @@ -35,6 +38,9 @@ public final class DictationManager: ObservableObject { private var micEngine: AVAudioEngine? private var updateConsumerTask: Task? private var unloadTask: Task? + private var micBufferContinuation: AsyncStream.Continuation? + private var micForwardTask: Task? + private var bufferCount = 0 /// Called when new transcribed text is ready for injection. public var onUtterance: ((String) -> Void)? @@ -51,6 +57,10 @@ public final class DictationManager: ObservableObject { /// How long to keep the model loaded after dictation stops (seconds). public var modelKeepAliveSeconds: TimeInterval = 120 + /// CoreAudio UID of the input device to capture from. nil = follow the + /// system default input device. + public var inputDeviceUID: String? + /// Full text injected so far in the current session (confirmed deltas only). private var injectedText: String = "" @@ -61,6 +71,18 @@ public final class DictationManager: ObservableObject { public func start() async throws { guard state == .idle else { throw DictationError.notIdle(current: state) } + // Surface mic-permission denial up front rather than letting the user + // stare at a silently-failing HUD — without mic access, AVAudioEngine + // starts but no tap callbacks fire. Accessibility access is checked + // mid-session by AppModel.startAXPolling() rather than here, because + // AXIsProcessTrusted() can lag behind a freshly-granted permission + // (especially with ad-hoc signed builds) and we'd rather attempt + // dictation than reject on a stale check. + let micStatus = AVCaptureDevice.authorizationStatus(for: .audio) + if micStatus == .denied || micStatus == .restricted { + throw DictationError.microphoneDenied + } + unloadTask?.cancel() unloadTask = nil @@ -226,14 +248,47 @@ public final class DictationManager: ObservableObject { private func startMicCapture(mgr: SlidingWindowAsrManager) throws { let engine = AVAudioEngine() + // Apply the user-selected input device BEFORE querying the input + // format — changing the device changes sample rate / channel count. + let applied = AudioInputDevices.apply(uid: inputDeviceUID, to: engine) let inputNode = engine.inputNode let hwFormat = inputNode.outputFormat(forBus: 0) - // SlidingWindowAsrManager handles format conversion internally. - inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat) { [weak mgr] buffer, _ in - guard let mgr else { return } - // Hop to the actor from the tap thread. - Task { await mgr.streamAudio(buffer) } + NSLog( + "Heard: Dictation mic format — sampleRate=%.0f channels=%u device=%@", + hwFormat.sampleRate, + hwFormat.channelCount, + applied ? (inputDeviceUID ?? "default") : "default" + ) + + // Forward tap buffers through an AsyncStream so a single consumer task + // can feed them to streamAudio() in arrival order. Spawning an + // unstructured Task per callback doesn't preserve ordering, and the + // tap-provided buffer is only valid for the duration of the callback + // (AVAudioEngine reuses the underlying memory), so we deep-copy + // synchronously before yielding. + let (stream, continuation) = AsyncStream.makeStream( + bufferingPolicy: .bufferingNewest(64) + ) + micBufferContinuation = continuation + bufferCount = 0 + + micForwardTask = Task { [weak self, weak mgr] in + for await buffer in stream { + guard let mgr, !Task.isCancelled else { break } + await mgr.streamAudio(buffer) + if let self { + await self.tickBufferCount() + } + } + } + + // continuation is a struct; capture by value. Once stopMicCapture() + // calls finish(), subsequent yields are no-ops, so we don't need to + // null it out from the tap. + inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat) { buffer, _ in + guard let copy = buffer.heardDeepCopy() else { return } + continuation.yield(copy) } engine.prepare() @@ -241,10 +296,25 @@ public final class DictationManager: ObservableObject { micEngine = engine } + /// Logs progress on the first buffer and every ~5s thereafter (at 48 kHz × 4096 + /// frames ≈ 11.7 Hz) so silent-mic regressions surface in logs. + private func tickBufferCount() { + bufferCount += 1 + if bufferCount == 1 { + NSLog("Heard: Dictation received first mic buffer") + } else if bufferCount % 60 == 0 { + NSLog("Heard: Dictation buffers streamed=%d", bufferCount) + } + } + private func stopMicCapture() { micEngine?.inputNode.removeTap(onBus: 0) micEngine?.stop() micEngine = nil + micBufferContinuation?.finish() + micBufferContinuation = nil + micForwardTask?.cancel() + micForwardTask = nil } // MARK: - Model lifecycle @@ -265,3 +335,39 @@ public final class DictationManager: ObservableObject { NSLog("Heard: Dictation models unloaded") } } + +// MARK: - Buffer copy + +private extension AVAudioPCMBuffer { + /// Allocates a standalone copy of the buffer. The buffer handed to an + /// AVAudioEngine tap callback only owns its sample memory for the duration + /// of the callback; copying lets us hand it off to a consumer task safely. + func heardDeepCopy() -> AVAudioPCMBuffer? { + guard let copy = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCapacity) else { + return nil + } + copy.frameLength = frameLength + let frames = Int(frameLength) + let channels = Int(format.channelCount) + + if let src = floatChannelData, let dst = copy.floatChannelData { + for ch in 0...size) + } + return copy + } + if let src = int16ChannelData, let dst = copy.int16ChannelData { + for ch in 0...size) + } + return copy + } + if let src = int32ChannelData, let dst = copy.int32ChannelData { + for ch in 0...size) + } + return copy + } + return nil + } +} diff --git a/Sources/HeardCore/Services.swift b/Sources/HeardCore/Services.swift index cd00992..253fd10 100644 --- a/Sources/HeardCore/Services.swift +++ b/Sources/HeardCore/Services.swift @@ -427,6 +427,10 @@ public final class RecordingManager: ObservableObject { /// True when the app-audio process tap failed — recording is mic-only. @Published public private(set) var appAudioTapFailed: Bool = false + /// CoreAudio UID of the input device to record the mic track from. nil = + /// follow the system default input device. Set by AppModel from settings. + public var inputDeviceUID: String? + public init() {} // Context object shared between the main actor and the IOProc dispatch queue. @@ -578,6 +582,9 @@ public final class RecordingManager: ObservableObject { private func setupMicRecording(to url: URL) throws { let engine = AVAudioEngine() + // Apply user-selected input device BEFORE reading the format. Changing + // the device changes its sample rate and channel count. + AudioInputDevices.apply(uid: inputDeviceUID, to: engine) let inputNode = engine.inputNode let hwFormat = inputNode.outputFormat(forBus: 0) diff --git a/Sources/HeardCore/Stores.swift b/Sources/HeardCore/Stores.swift index 96e9447..6b32d27 100644 --- a/Sources/HeardCore/Stores.swift +++ b/Sources/HeardCore/Stores.swift @@ -190,7 +190,8 @@ public final class SettingsStore: ObservableObject { enableZoomDetection: defaults.object(forKey: "enableZoomDetection") as? Bool ?? base.enableZoomDetection, enableWebexDetection: defaults.object(forKey: "enableWebexDetection") as? Bool ?? base.enableWebexDetection, diarizationClusteringSimilarity: diarizationClusteringSimilarity, - speakerRetentionDays: speakerRetentionDays + speakerRetentionDays: speakerRetentionDays, + selectedInputDeviceUID: defaults.string(forKey: "selectedInputDeviceUID") ) } @@ -221,6 +222,11 @@ public final class SettingsStore: ObservableObject { defaults.set(settings.enableWebexDetection, forKey: "enableWebexDetection") defaults.set(settings.diarizationClusteringSimilarity, forKey: "diarizationClusteringSimilarity") defaults.set(settings.speakerRetentionDays, forKey: "speakerRetentionDays") + if let uid = settings.selectedInputDeviceUID { + defaults.set(uid, forKey: "selectedInputDeviceUID") + } else { + defaults.removeObject(forKey: "selectedInputDeviceUID") + } } } diff --git a/Sources/HeardCore/Views.swift b/Sources/HeardCore/Views.swift index 207ed71..b4295c7 100644 --- a/Sources/HeardCore/Views.swift +++ b/Sources/HeardCore/Views.swift @@ -1063,6 +1063,17 @@ public struct SettingsView: View { } } + sectionGroup("Microphone") { + SettingsCard { + CardRow(isLast: true) { + MicrophonePickerRow( + selectedUID: model.settingsStore.settings.selectedInputDeviceUID, + onSelect: { model.setInputDeviceUID($0) } + ) + } + } + } + sectionGroup("Output") { SettingsCard { CardRow { @@ -1722,6 +1733,65 @@ public struct SettingsView: View { // MARK: - Model Status Row +// MARK: - Microphone Picker Row + +private struct MicrophonePickerRow: View { + let selectedUID: String? + let onSelect: (String?) -> Void + + @State private var devices: [AudioInputDevice] = [] + @State private var defaultDevice: AudioInputDevice? + + var body: some View { + HStack { + Text("Input Device") + .font(.system(size: 12, weight: .medium)) + .foregroundStyle(HeardTheme.Paper.ink) + Spacer() + Picker("", selection: pickerBinding) { + Text(systemDefaultLabel).tag(String?.none) + ForEach(devices) { device in + Text(device.name).tag(String?.some(device.uid)) + } + // Keep the stored UID selectable even if the device is currently + // unplugged, so the user doesn't silently lose their preference. + if let uid = selectedUID, devices.first(where: { $0.uid == uid }) == nil { + Text("Unavailable — last used \(shortUID(uid))") + .tag(String?.some(uid)) + } + } + .labelsHidden() + .controlSize(.small) + .frame(maxWidth: .infinity) + .frame(width: 220) + } + .onAppear { refresh() } + } + + private var pickerBinding: Binding { + Binding( + get: { selectedUID }, + set: { onSelect($0) } + ) + } + + private var systemDefaultLabel: String { + if let name = defaultDevice?.name { + return "System Default (\(name))" + } + return "System Default" + } + + private func refresh() { + devices = AudioInputDevices.list() + defaultDevice = AudioInputDevices.defaultInputDevice() + } + + private func shortUID(_ uid: String) -> String { + uid.count <= 16 ? uid : String(uid.prefix(13)) + "…" + } +} + private struct ModelStatusRow: View { let item: ModelStatusItem @ObservedObject var downloadManager: ModelDownloadManager