From e388802cbb42a8a9d0a22b564aadd2f3ec2a9355 Mon Sep 17 00:00:00 2001 From: Antony Njoroge Mutharia Date: Fri, 3 Jul 2026 15:28:39 -0600 Subject: [PATCH 1/7] fix(hotkey): release watchdog polls session flags, not per-keycode state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two field bugs, one root: missed/misread trigger-key release. - Fn/Globe: macOS swallows the key-up (globe action), the listen-only tap never sees the release, and the app stays stuck in .recording — the next tap then pastes the previous recording's audio. - Right Shift: CGEventSource.keyState() misreports held modifiers (reads "up" mid-hold), so the earlier watchdog draft force-stopped recording after its first poll. The watchdog now polls CGEventSource.flagsState() — the session's live modifier flags, the same domain as the flagsChanged events the tap consumes. Each TriggerKey carries its family mask (maskShift, maskSecondaryFn, ...) for that check. Held key keeps its flag set (no false stop); a swallowed release still clears the flag, so the watchdog ends the recording within ~0.12s. Co-Authored-By: Claude Fable 5 --- ZeroGSwift/ZeroG/Core/KeyMonitor.swift | 44 ++++++++++++++++++++++++++ ZeroGSwift/ZeroG/Core/TriggerKey.swift | 17 ++++++---- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/ZeroGSwift/ZeroG/Core/KeyMonitor.swift b/ZeroGSwift/ZeroG/Core/KeyMonitor.swift index 0f70ad0..d6385d7 100644 --- a/ZeroGSwift/ZeroG/Core/KeyMonitor.swift +++ b/ZeroGSwift/ZeroG/Core/KeyMonitor.swift @@ -44,6 +44,16 @@ final class KeyMonitor { private let maxRecordingDuration: TimeInterval = 120.0 private var timeoutTimer: Timer? + /// Watchdog that polls the physical trigger-key state while recording. + /// Some keys — notably Fn/Globe when macOS maps it to a system action like + /// "Show Emoji & Symbols" — get their key-up event swallowed by the OS before + /// it reaches our listen-only tap. Without this, a swallowed release strands + /// the mic on (no `flagsChanged` ever clears `isTriggerKeyPressed`). The + /// watchdog detects the key is no longer physically down and forces release. + /// Active only between press and release, so idle CPU is unaffected. + private var releaseWatchdog: Timer? + private let releaseWatchdogInterval: TimeInterval = 0.12 + // MARK: Lifecycle init( @@ -149,6 +159,7 @@ final class KeyMonitor { func stop() { timeoutTimer?.invalidate() timeoutTimer = nil + stopReleaseWatchdog() NotificationCenter.default.removeObserver(self) @@ -179,6 +190,7 @@ final class KeyMonitor { isTriggerKeyPressed = false timeoutTimer?.invalidate() timeoutTimer = nil + stopReleaseWatchdog() DispatchQueue.main.async { [weak self] in self?.onStopRecording() } @@ -254,6 +266,7 @@ final class KeyMonitor { self.stateMachine.transition(to: .recording) self.onStartRecording() self.startTimeoutTimer() + self.startReleaseWatchdog() default: break } @@ -264,6 +277,7 @@ final class KeyMonitor { isTriggerKeyPressed = false timeoutTimer?.invalidate() timeoutTimer = nil + stopReleaseWatchdog() DispatchQueue.main.async { [weak self] in self?.onStopRecording() @@ -282,7 +296,37 @@ final class KeyMonitor { Log.debug("KeyMonitor", "Recording timeout (\(self.maxRecordingDuration)s) — forcing stop.") self.isTriggerKeyPressed = false + self.stopReleaseWatchdog() self.onStopRecording() } } + + // MARK: - Release Watchdog + + /// Begin polling the physical trigger-key state. Catches a swallowed key-up + /// (e.g. OS-intercepted Fn/Globe) that would otherwise leave the mic stuck on. + private func startReleaseWatchdog() { + releaseWatchdog?.invalidate() + releaseWatchdog = Timer.scheduledTimer(withTimeInterval: releaseWatchdogInterval, repeats: true) { [weak self] _ in + guard let self else { return } + guard self.isTriggerKeyPressed else { + self.stopReleaseWatchdog() + return + } + // Live session modifier flags — same domain as the flagsChanged + // events we consume. Per-keycode keyState is unreliable for held + // modifiers (Right Shift reads "up" mid-hold → false stop). + let flags = CGEventSource.flagsState(.combinedSessionState) + let stillDown = flags.contains(self.triggerKey.familyFlagMask) + if !stillDown { + Log.debug("KeyMonitor", "Watchdog: \(self.triggerKey.displayName) release was missed — forcing stop.") + self.triggerReleased() + } + } + } + + private func stopReleaseWatchdog() { + releaseWatchdog?.invalidate() + releaseWatchdog = nil + } } diff --git a/ZeroGSwift/ZeroG/Core/TriggerKey.swift b/ZeroGSwift/ZeroG/Core/TriggerKey.swift index b8e403b..9585ce0 100644 --- a/ZeroGSwift/ZeroG/Core/TriggerKey.swift +++ b/ZeroGSwift/ZeroG/Core/TriggerKey.swift @@ -10,14 +10,19 @@ struct TriggerKey: Equatable { /// Device bits distinguish left from right, preventing stuck-recording /// when both sides of the same modifier family are held simultaneously. let deviceFlagMask: UInt64 + /// Generic CGEventFlags mask for the key's modifier family (maskControl, + /// maskShift, …). Used by the release watchdog to poll the session's live + /// flags via `CGEventSource.flagsState` — per-keycode `keyState` misreports + /// held modifiers (e.g. Right Shift reads "up" while physically held). + let familyFlagMask: CGEventFlags static let allOptions: [TriggerKey] = [ - TriggerKey(id: "leftControl", displayName: "Left Control", keyCode: 59, deviceFlagMask: 0x00000001), - TriggerKey(id: "rightControl", displayName: "Right Control", keyCode: 62, deviceFlagMask: 0x00002000), - TriggerKey(id: "leftOption", displayName: "Left Option", keyCode: 58, deviceFlagMask: 0x00000020), - TriggerKey(id: "rightOption", displayName: "Right Option", keyCode: 61, deviceFlagMask: 0x00000040), - TriggerKey(id: "rightShift", displayName: "Right Shift", keyCode: 60, deviceFlagMask: 0x00000004), - TriggerKey(id: "fn", displayName: "Fn / Globe", keyCode: 63, deviceFlagMask: 0x00800000), + TriggerKey(id: "leftControl", displayName: "Left Control", keyCode: 59, deviceFlagMask: 0x00000001, familyFlagMask: .maskControl), + TriggerKey(id: "rightControl", displayName: "Right Control", keyCode: 62, deviceFlagMask: 0x00002000, familyFlagMask: .maskControl), + TriggerKey(id: "leftOption", displayName: "Left Option", keyCode: 58, deviceFlagMask: 0x00000020, familyFlagMask: .maskAlternate), + TriggerKey(id: "rightOption", displayName: "Right Option", keyCode: 61, deviceFlagMask: 0x00000040, familyFlagMask: .maskAlternate), + TriggerKey(id: "rightShift", displayName: "Right Shift", keyCode: 60, deviceFlagMask: 0x00000004, familyFlagMask: .maskShift), + TriggerKey(id: "fn", displayName: "Fn / Globe", keyCode: 63, deviceFlagMask: 0x00800000, familyFlagMask: .maskSecondaryFn), ] static let defaultKey = allOptions[0] From dc53db584ba648691eea34f2ecd0ab1331c477d6 Mon Sep 17 00:00:00 2001 From: Antony Njoroge Mutharia Date: Fri, 3 Jul 2026 15:38:35 -0600 Subject: [PATCH 2/7] feat(diagnostics): field breadcrumbs + inference timeout + busy feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stuck-recording report turned out flaky and the release build is unobservable: every pipeline breadcrumb was Log.debug, compiled out of release, so 'log show' had nothing. Three changes: - Log.info: always-on NSLog breadcrumbs (unified log) for press/release/ watchdog/timeout, audio + inference durations, and paste outcome. Transcript content stays debug-only; release logs lengths, not text. - ParakeetTranscriptionEngine.transcribe now runs under a 30s hard deadline. A hung ANE call surfaced as indefinite .processing — the UI sat on 'transcribing' forever and silently dropped the next press. Now it errors and the state machine recovers. - A trigger press while still .processing plays the Basso error sound. Previously the press was swallowed silently, the user spoke into a dead mic, and the late-arriving previous transcription pasted 'on its own' — the exact reported symptom. Co-Authored-By: Claude Fable 5 --- ZeroGSwift/ZeroG/Core/AudioRecorder.swift | 21 ++++++++++++------- ZeroGSwift/ZeroG/Core/KeyMonitor.swift | 15 ++++++++++--- ZeroGSwift/ZeroG/Core/Log.swift | 9 ++++++++ .../Core/ParakeetTranscriptionEngine.swift | 20 ++++++++++++++---- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/ZeroGSwift/ZeroG/Core/AudioRecorder.swift b/ZeroGSwift/ZeroG/Core/AudioRecorder.swift index 40de504..9b64860 100644 --- a/ZeroGSwift/ZeroG/Core/AudioRecorder.swift +++ b/ZeroGSwift/ZeroG/Core/AudioRecorder.swift @@ -220,7 +220,7 @@ final class AudioRecorder: @unchecked Sendable { // Safety-only silence detection. Normal recording still ends when Control is released. if silenceTracker.observe(rms: rms, at: Date()) == .stop { - Log.debug("AudioRecorder", "Safety silence detected (>\(Config.silenceDuration)s). Auto-stopping.") + Log.info("AudioRecorder", "Safety silence detected (>\(Config.silenceDuration)s). Auto-stopping.") DispatchQueue.main.async { [weak self] in self?.beginProcessing() } @@ -232,24 +232,28 @@ final class AudioRecorder: @unchecked Sendable { /// Transcribe audio data on-device and inject the result. private func transcribeAndInject(audioData: [Float]) async { guard !audioData.isEmpty else { + Log.info("AudioRecorder", "No audio captured — nothing to transcribe.") DispatchQueue.main.async { [weak self] in self?.stateMachine.transition(to: .idle) } return } - + do { let startTime = CFAbsoluteTimeGetCurrent() + let audioDuration = Double(audioData.count) / AudioConstants.sampleRate + Log.info("AudioRecorder", "Transcribing \(String(format: "%.1f", audioDuration))s of audio...") // Transcribe let text = try await transcriptionEngine.transcribe(audioData) - + let transcriptionDuration = CFAbsoluteTimeGetCurrent() - startTime - - let audioDuration = Double(audioData.count) / AudioConstants.sampleRate - Log.debug("AudioRecorder", "Transcribed \(String(format: "%.1f", audioDuration))s audio in \(String(format: "%.2f", transcriptionDuration))s: \(text)") - + + Log.info("AudioRecorder", "Transcribed \(String(format: "%.1f", audioDuration))s audio in \(String(format: "%.2f", transcriptionDuration))s (\(text.count) chars)") + Log.debug("AudioRecorder", "Transcript: \(text)") + guard !text.isEmpty else { + Log.info("AudioRecorder", "Empty transcription — nothing to paste.") DispatchQueue.main.async { [weak self] in self?.stateMachine.transition(to: .idle) } @@ -266,6 +270,7 @@ final class AudioRecorder: @unchecked Sendable { // Inject text. False = Accessibility missing (the injector bailed // before touching the pasteboard) — never show fake success. let injected = TextInjector.injectText(finalText) + Log.info("AudioRecorder", "Paste \(injected ? "OK" : "FAILED (Accessibility missing)")") DispatchQueue.main.async { [weak self] in guard let self else { return } @@ -286,7 +291,7 @@ final class AudioRecorder: @unchecked Sendable { } } catch { - Log.debug("AudioRecorder", "Transcription error: \(error)") + Log.error("AudioRecorder", "Transcription error: \(error)") DispatchQueue.main.async { [weak self] in self?.stateMachine.transition(to: .error("Processing Failed")) self?.stateMachine.resetToIdle(after: Config.Timing.errorReset) diff --git a/ZeroGSwift/ZeroG/Core/KeyMonitor.swift b/ZeroGSwift/ZeroG/Core/KeyMonitor.swift index d6385d7..a9c4aeb 100644 --- a/ZeroGSwift/ZeroG/Core/KeyMonitor.swift +++ b/ZeroGSwift/ZeroG/Core/KeyMonitor.swift @@ -256,13 +256,19 @@ final class KeyMonitor { let state = self.stateMachine.currentState guard state.isReady else { - Log.debug("KeyMonitor", "Ignoring \(self.triggerKey.displayName) press — app not ready (state: \(state))") + Log.info("KeyMonitor", "Press ignored — app busy (state: \(state))") + // The user's speech after this press is NOT being recorded. + // Make that audible instead of silently eating it. + if case .processing = state { + NSSound(named: "Basso")?.play() + } return } switch state { case .idle, .success, .error, .needsPermission: self.recordingStartTime = Date() + Log.info("KeyMonitor", "Press → recording (\(self.triggerKey.displayName))") self.stateMachine.transition(to: .recording) self.onStartRecording() self.startTimeoutTimer() @@ -279,6 +285,9 @@ final class KeyMonitor { timeoutTimer = nil stopReleaseWatchdog() + let held = recordingStartTime.map { String(format: "%.1fs", Date().timeIntervalSince($0)) } ?? "?" + Log.info("KeyMonitor", "Release → stop recording (held \(held))") + DispatchQueue.main.async { [weak self] in self?.onStopRecording() } @@ -293,7 +302,7 @@ final class KeyMonitor { timeoutTimer = Timer.scheduledTimer(withTimeInterval: maxRecordingDuration, repeats: false) { [weak self] _ in guard let self else { return } - Log.debug("KeyMonitor", "Recording timeout (\(self.maxRecordingDuration)s) — forcing stop.") + Log.info("KeyMonitor", "Recording timeout (\(self.maxRecordingDuration)s) — forcing stop.") self.isTriggerKeyPressed = false self.stopReleaseWatchdog() @@ -319,7 +328,7 @@ final class KeyMonitor { let flags = CGEventSource.flagsState(.combinedSessionState) let stillDown = flags.contains(self.triggerKey.familyFlagMask) if !stillDown { - Log.debug("KeyMonitor", "Watchdog: \(self.triggerKey.displayName) release was missed — forcing stop.") + Log.info("KeyMonitor", "Watchdog: \(self.triggerKey.displayName) release was missed — forcing stop.") self.triggerReleased() } } diff --git a/ZeroGSwift/ZeroG/Core/Log.swift b/ZeroGSwift/ZeroG/Core/Log.swift index 56037f2..ac47d60 100644 --- a/ZeroGSwift/ZeroG/Core/Log.swift +++ b/ZeroGSwift/ZeroG/Core/Log.swift @@ -18,6 +18,15 @@ enum Log { #endif } + /// Always-on breadcrumb for the dictation pipeline (press/release/stop + /// reason, audio + inference durations, paste outcome). Sparse by design — + /// a handful of lines per dictation, enough to reconstruct a flaky field + /// failure from `log show` without a debug build. Never log transcript + /// content here — lengths and durations only. + static func info(_ tag: String, _ message: String) { + NSLog("%{public}@", "[\(tag)] \(message)") + } + /// Always-on logging for failures and operationally significant events. /// Uses NSLog so the message lands in the unified log (visible in Console /// filtered by process "ZeroG") — `print` only reaches stdout, which Console diff --git a/ZeroGSwift/ZeroG/Core/ParakeetTranscriptionEngine.swift b/ZeroGSwift/ZeroG/Core/ParakeetTranscriptionEngine.swift index e2bca63..b65be57 100644 --- a/ZeroGSwift/ZeroG/Core/ParakeetTranscriptionEngine.swift +++ b/ZeroGSwift/ZeroG/Core/ParakeetTranscriptionEngine.swift @@ -91,9 +91,16 @@ final class ParakeetTranscriptionEngine: Transcribing { throw TranscriptionError.emptyAudio } - // Fresh decoder state per push-to-talk utterance (no cross-utterance carryover). - var state = try TdtDecoderState(decoderLayers: models.version.decoderLayers) - let result = try await manager.transcribe(audioArray, decoderState: &state) + // Hard deadline: a hung inference must surface as an error the state + // machine can recover from, never an indefinite .processing (the UI + // would otherwise sit "transcribing" forever and silently drop the + // user's next press). + let decoderLayers = models.version.decoderLayers + let result = try await Self.withTimeout(seconds: Self.transcribeTimeout) { + // Fresh decoder state per push-to-talk utterance (no cross-utterance carryover). + var state = try TdtDecoderState(decoderLayers: decoderLayers) + return try await manager.transcribe(audioArray, decoderState: &state) + } // Parakeet is verbatim by training ("um"/"uh" kept); strip fillers in the text // domain. Whisper needs no equivalent — its caption-trained decoder self-cleans. @@ -132,6 +139,11 @@ final class ParakeetTranscriptionEngine: Transcribing { /// give up and surface a retryable error. private static let downloadTimeout: Double = 180 + /// Hard deadline for a single utterance's inference. Push-to-talk audio is + /// capped at 120s of recording; anything past this is a hung ANE call, not + /// a slow one. + private static let transcribeTimeout: Double = 30 + /// Run `op` with a hard deadline. Whichever finishes first wins; the loser is /// cancelled. A timeout throws a transcription error the caller can retry. private static func withTimeout( @@ -142,7 +154,7 @@ final class ParakeetTranscriptionEngine: Transcribing { group.addTask { try await op() } group.addTask { try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) - throw TranscriptionError.transcriptionFailed("Model download timed out") + throw TranscriptionError.transcriptionFailed("Timed out after \(Int(seconds))s") } defer { group.cancelAll() } return try await group.next()! From 4cc94403f4d07c332c22054800f9615b72dad299 Mon Sep 17 00:00:00 2001 From: Antony Njoroge Mutharia Date: Fri, 3 Jul 2026 15:47:55 -0600 Subject: [PATCH 3/7] fix(diagnostics): os_log breadcrumbs (NSLog gets redacted) + busy sound on all not-ready presses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field capture 2026-07-03 15:44 confirmed two gaps: - NSLog dynamic content is redacted to in the unified log even with %{public}@ — that specifier is honored by os_log only. Log.info/ Log.error now use os.Logger with privacy: .public interpolation, so 'log show --predicate' finally yields readable breadcrumbs from a release build. - The captured incident was a press swallowed ~40s after relaunch while the model was still loading — a state the Basso feedback didn't cover (it was gated to .processing). The busy sound now plays for every not-ready press, so a dead-feeling hotkey is always audible. Co-Authored-By: Claude Fable 5 --- ZeroGSwift/ZeroG/Core/KeyMonitor.swift | 10 +++++----- ZeroGSwift/ZeroG/Core/Log.swift | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ZeroGSwift/ZeroG/Core/KeyMonitor.swift b/ZeroGSwift/ZeroG/Core/KeyMonitor.swift index a9c4aeb..c1c82bf 100644 --- a/ZeroGSwift/ZeroG/Core/KeyMonitor.swift +++ b/ZeroGSwift/ZeroG/Core/KeyMonitor.swift @@ -257,11 +257,11 @@ final class KeyMonitor { guard state.isReady else { Log.info("KeyMonitor", "Press ignored — app busy (state: \(state))") - // The user's speech after this press is NOT being recorded. - // Make that audible instead of silently eating it. - if case .processing = state { - NSSound(named: "Basso")?.play() - } + // The user's speech after this press is NOT being recorded — + // whether we're still loading the model (~45s after launch) or + // mid-transcription. Make that audible instead of silently + // eating it. + NSSound(named: "Basso")?.play() return } diff --git a/ZeroGSwift/ZeroG/Core/Log.swift b/ZeroGSwift/ZeroG/Core/Log.swift index ac47d60..5fde25e 100644 --- a/ZeroGSwift/ZeroG/Core/Log.swift +++ b/ZeroGSwift/ZeroG/Core/Log.swift @@ -1,4 +1,5 @@ import Foundation +import os // MARK: - Logging @@ -7,10 +8,15 @@ import Foundation /// /// - `debug` is compiled out of release builds and takes its message as an /// `@autoclosure`, so interpolation cost is never paid when DEBUG is off. -/// - `error` always prints — it's for failures the user/operator should see in -/// any build. +/// - `info`/`error` always log — via os.Logger, NOT NSLog: the unified log +/// redacts NSLog's dynamic content to `` regardless of format +/// specifiers (`%{public}@` is honored by os_log only — verified in the +/// field 2026-07-03). `privacy: .public` on the interpolation is the only +/// thing that keeps `log show` output readable. enum Log { + private static let logger = os.Logger(subsystem: "com.zerog.app", category: "app") + /// Verbose / developer-only logging. No-op in release builds. static func debug(_ tag: String, _ message: @autoclosure () -> String) { #if DEBUG @@ -24,15 +30,12 @@ enum Log { /// failure from `log show` without a debug build. Never log transcript /// content here — lengths and durations only. static func info(_ tag: String, _ message: String) { - NSLog("%{public}@", "[\(tag)] \(message)") + logger.log("[\(tag, privacy: .public)] \(message, privacy: .public)") } /// Always-on logging for failures and operationally significant events. - /// Uses NSLog so the message lands in the unified log (visible in Console - /// filtered by process "ZeroG") — `print` only reaches stdout, which Console - /// does not capture for a GUI app launched via Finder/`open`. + /// Lands in the unified log (Console, filtered by process "ZeroG"). static func error(_ tag: String, _ message: String) { - // %{public}@ so the message isn't redacted to in the unified log. - NSLog("%{public}@", "[\(tag)] \(message)") + logger.error("[\(tag, privacy: .public)] \(message, privacy: .public)") } } From 895a0b99861fa16559dc3765e4e18c6a962de730 Mon Sep 17 00:00:00 2001 From: Antony Njoroge Mutharia Date: Fri, 3 Jul 2026 15:56:50 -0600 Subject: [PATCH 4/7] fix(state): stale resetToIdle timer stomped an in-flight recording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field-captured root cause of the flaky stuck-recording / stale-paste bug (breadcrumbs, 2026-07-03 15:53): resetToIdle schedules an unconditional success → idle transition ~2s after a paste. Pressing the trigger again inside that window went success → recording, then the stale timer fired and forced recording → idle. On release, beginProcessing's 'are we recording?' state check failed, so the mic was never stopped — the session kept capturing and merged into the next tap's audio, which then pasted both utterances late ('pastes what I said earlier'). Reproduced only when the next press came within ~2s of the previous paste, hence the flakiness. - resetToIdle now captures the state it was scheduled from and no-ops if the state has moved on by the time the timer fires. - beginProcessing gates on the recorder's own isRecording instead of the display state, so a missed stop can't recur even if some future transition knocks the state machine out of .recording mid-capture. - Regression test: resetToIdle must not stomp a recording that started inside the delay window. Co-Authored-By: Claude Fable 5 --- ZeroGSwift/ZeroG/Core/AppStateMachine.swift | 11 +++++++++-- ZeroGSwift/ZeroG/Core/AudioRecorder.swift | 6 +++++- ZeroGSwift/ZeroGTests/AppStateMachineTests.swift | 13 +++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/ZeroGSwift/ZeroG/Core/AppStateMachine.swift b/ZeroGSwift/ZeroG/Core/AppStateMachine.swift index 8c53f25..1e4c3b4 100644 --- a/ZeroGSwift/ZeroG/Core/AppStateMachine.swift +++ b/ZeroGSwift/ZeroG/Core/AppStateMachine.swift @@ -70,10 +70,17 @@ final class AppStateMachine: ObservableObject { Log.debug("StateMachine", "\(previous) → \(newState)") } - /// Convenience: transition to `.idle` after a delay. + /// Convenience: transition to `.idle` after a delay — but only if the state + /// hasn't moved on in the meantime. An unconditional delayed reset stomped + /// a recording that started within the delay window (success → recording → + /// timer fires → idle), which left the mic running: the release handler's + /// "are we recording?" check failed and the session never stopped, merging + /// into the next one (field-captured 2026-07-03 15:53). func resetToIdle(after delay: TimeInterval = Config.Timing.successReset) { + let scheduledFrom = currentState DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [weak self] in - self?.transition(to: .idle) + guard let self, self.currentState == scheduledFrom else { return } + self.transition(to: .idle) } } } diff --git a/ZeroGSwift/ZeroG/Core/AudioRecorder.swift b/ZeroGSwift/ZeroG/Core/AudioRecorder.swift index 9b64860..c6f6b56 100644 --- a/ZeroGSwift/ZeroG/Core/AudioRecorder.swift +++ b/ZeroGSwift/ZeroG/Core/AudioRecorder.swift @@ -112,7 +112,11 @@ final class AudioRecorder: @unchecked Sendable { /// stop" sequence lives in exactly one place. Must be called on the main thread /// (it touches the state machine). func beginProcessing() { - guard stateMachine.currentState == .recording else { return } + // Gate on the recorder's own truth, not the display state: if a stray + // UI transition ever knocks the state machine out of .recording while + // the engine is still capturing, the stop must still happen — a missed + // stop leaves the mic running and merges this session into the next. + guard isRecording else { return } stateMachine.transition(to: .processing) stopRecording() } diff --git a/ZeroGSwift/ZeroGTests/AppStateMachineTests.swift b/ZeroGSwift/ZeroGTests/AppStateMachineTests.swift index a0de5ac..b8f9f16 100644 --- a/ZeroGSwift/ZeroGTests/AppStateMachineTests.swift +++ b/ZeroGSwift/ZeroGTests/AppStateMachineTests.swift @@ -133,4 +133,17 @@ struct AppStateMachineTests { #expect(machine.currentState == .idle) } + + @Test("resetToIdle is a no-op when a new recording started inside the delay window") + func resetToIdleDoesNotStompNewRecording() async throws { + let machine = AppStateMachine() + machine.transition(to: .success) + + machine.resetToIdle(after: 0.01) + // The user pressed the trigger again before the reset fired. + machine.transition(to: .recording) + try await Task.sleep(for: .milliseconds(50)) + + #expect(machine.currentState == .recording) + } } From cf616d55caa2ef8a80681a7ef398a1a47d88edf6 Mon Sep 17 00:00:00 2001 From: Antony Njoroge Mutharia Date: Fri, 3 Jul 2026 16:09:23 -0600 Subject: [PATCH 5/7] fix(onboarding): show the OS Accessibility prompt before deep-linking Settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clicking the Accessibility step's button fired the AX prompt and opened the Settings pane in the same breath. The pane landed on top (hiding the OS 'ZeroG would like to…' dialog) and could render before the prompt's pre-listing registered — users arrived at an Accessibility list with no ZeroG row to toggle. First click now fires only the native prompt; its own Open System Settings button lands on a pane where ZeroG is already listed. macOS shows that prompt at most once per app record, so subsequent clicks fall through to the existing request-then-deep-link path, with the button retitled accordingly. Co-Authored-By: Claude Fable 5 --- ZeroGSwift/ZeroG/GUI/OnboardingWindow.swift | 37 ++++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/ZeroGSwift/ZeroG/GUI/OnboardingWindow.swift b/ZeroGSwift/ZeroG/GUI/OnboardingWindow.swift index 886105a..6639620 100644 --- a/ZeroGSwift/ZeroG/GUI/OnboardingWindow.swift +++ b/ZeroGSwift/ZeroG/GUI/OnboardingWindow.swift @@ -71,6 +71,10 @@ final class OnboardingViewModel: ObservableObject { /// Set when Accessibility was granted but the key tap could not be created /// live — the step then asks the user to relaunch. @Published var relaunchRequired = false + /// Whether the one-shot OS Accessibility prompt has been fired this session. + /// Drives the button label (first click shows the prompt, later clicks + /// deep-link to Settings). + @Published private(set) var didPromptAccessibility = false let permissions: PermissionsManager @@ -147,15 +151,25 @@ final class OnboardingViewModel: ObservableObject { } } - /// Fire the native request (pre-lists ZeroG in the pane + shows the OS - /// "…would like to…" prompt) and then open the pane — so the app is already - /// listed with a toggle when the user arrives, no "+ and hunt" friction. + /// First click: fire ONLY the native OS prompt ("ZeroG would like to…"). + /// That dialog is what pre-lists ZeroG in the Accessibility pane, and its + /// own "Open System Settings" button lands the user on a pane where the + /// toggle already exists. Opening the pane ourselves in the same breath + /// covered the prompt and could race the pre-listing — the user arrived at + /// a list with no ZeroG entry. + /// + /// macOS shows that prompt at most once per app record, so later clicks + /// fall through to the Settings deep link (request() stays: it's a silent + /// no-op then, but still pre-lists after a tccutil reset). Grant detection + /// is the 1s poll reading AXIsProcessTrusted — never the tap (tapCreate can + /// succeed with events withheld, so it proves nothing about the grant). func requestAndOpenSettings(for step: OnboardingStep) { guard let kind = step.permission else { return } - // request() (AXIsProcessTrustedWithOptions prompt) pre-lists ZeroG in the - // Accessibility pane so the toggle is already there. Grant detection is the - // 1s poll reading AXIsProcessTrusted — never the tap (tapCreate can succeed - // with events withheld, so it proves nothing about the grant). + if kind == .accessibility && !didPromptAccessibility { + didPromptAccessibility = true + permissions.request(kind) + return + } permissions.request(kind) permissions.openSettings(for: kind) } @@ -754,9 +768,14 @@ struct OnboardingWizardView: View { } else if granted { PrimaryButton(title: "Continue") { model.advance() }.riseIn(0.30) } else { + let promptFirst = model.step == .accessibility && !model.didPromptAccessibility VStack(spacing: 0) { - PrimaryButton(title: "Open System Settings") { model.requestAndOpenSettings(for: model.step) }.riseIn(0.30) - Text("Turn on the ZeroG toggle, then come back. We'll detect it for you.") + PrimaryButton(title: promptFirst ? "Allow Accessibility" : "Open System Settings") { + model.requestAndOpenSettings(for: model.step) + }.riseIn(0.30) + Text(promptFirst + ? "macOS will ask for permission — use its Open System Settings button." + : "Turn on the ZeroG toggle, then come back. We'll detect it for you.") .font(.system(size: 11)).foregroundColor(OB.textFaint) .padding(.top, 12).riseIn(0.36) } From 043b0b8b2866911bb1aaa9c8bfe389e880208b10 Mon Sep 17 00:00:00 2001 From: Antony Njoroge Mutharia Date: Wed, 15 Jul 2026 18:43:03 -0600 Subject: [PATCH 6/7] feat: improve microphone routing and app guidance --- README.md | 150 ++++++++++++---- ZeroGSwift/ZeroG/Config.swift | 6 + .../ZeroG/Core/AVAudioCaptureSession.swift | 72 ++++++++ .../ZeroG/Core/AudioCaptureSession.swift | 14 ++ ZeroGSwift/ZeroG/Core/AudioInputDevice.swift | 11 ++ .../ZeroG/Core/AudioInputDeviceManager.swift | 165 ++++++++++++++++++ ZeroGSwift/ZeroG/Core/AudioRecorder.swift | 95 ++++++---- ZeroGSwift/ZeroG/Core/KeyMonitor.swift | 6 +- .../ZeroG/GUI/StatusBarController.swift | 50 ++++++ ZeroGSwift/ZeroG/ZeroGApp.swift | 6 +- .../AudioInputDeviceManagerTests.swift | 82 +++++++++ .../AudioRecorderRecoveryTests.swift | 64 +++++++ ZeroGSwift/ZeroGTests/ConfigTests.swift | 1 + ZeroGSwift/ZeroGTests/TranscribingTests.swift | 6 +- 14 files changed, 658 insertions(+), 70 deletions(-) create mode 100644 ZeroGSwift/ZeroG/Core/AVAudioCaptureSession.swift create mode 100644 ZeroGSwift/ZeroG/Core/AudioCaptureSession.swift create mode 100644 ZeroGSwift/ZeroG/Core/AudioInputDevice.swift create mode 100644 ZeroGSwift/ZeroG/Core/AudioInputDeviceManager.swift create mode 100644 ZeroGSwift/ZeroGTests/AudioInputDeviceManagerTests.swift create mode 100644 ZeroGSwift/ZeroGTests/AudioRecorderRecoveryTests.swift diff --git a/README.md b/README.md index 7e45757..b9b4962 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,142 @@ -# ZeroG +

+ ZeroG logo +

-Privacy-focused voice typing for macOS. +# 🧑‍🚀 ZeroG -Hold **Left Control** to record, release to transcribe and paste into the active app. Hold **Control + Q** while recording to optionally polish the transcription with Gemini. +**Open Source Voice Typing for macOS** -## Requirements +> **"The voice typing tool so good, you'll forget how to type."** -- macOS 14+ -- Xcode with Swift 5.9+ toolchain -- Apple Silicon recommended for WhisperKit performance +--- -## Development +## 🛰️ The Manifesto -```bash -cd ZeroGSwift -swift test -swift run -``` +**We don't type. We transmit.** + +ZeroG was born from the realization that typing is a bottleneck. It is a terrestrial limitation. We spent decades training our fingers to hit 100 Words Per Minute (WPM), only to realize that the speed of thought is infinite. + +ZeroG is not just a dictation tool. It is an evolutionary step. Just as an astronaut in orbit unlearns the physics of gravity and expects a pen to float rather than fall, ZeroG users unlearn the friction of the keyboard. + +We are building the "Air Gap" for your thoughts: **Private. Local. Weightless.** + +--- + +## 🚀 Flight Systems (Features) -## Build +- **Zero Friction**: Native Swift running NVIDIA Parakeet on the **Apple Neural Engine** for near-instant transcription. 0 WPM. 100% Output. +- **Vacuum Sealed**: In space, sound doesn't travel. In ZeroG, your voice doesn't travel either. Audio and text are processed entirely on your Mac. No data leaves the ship. +- **Universal Comms**: Hold `Left Control` to transmit thought into *any* application. Release to paste. +- **Gravity Assist** (Optional): Press `⌃⌥P` to polish your last transmission with **Apple Foundation Models**, fully on-device (macOS 26 with Apple Intelligence). +- **Debris Filter**: Strips the "um"s and "uh"s from the feed before payload delivery. +- **Auto-Cut**: A silence sensor cuts the feed if the mic is left open on dead air. +- **Flight Recorder**: A native "Glass" HUD that floats above your dock. +![ZeroG HUD floating above the macOS dock](assets/zerog-hud.png) + +--- + +## 🛠️ Pre-Flight Check (Installation) + +### Prerequisites +- macOS 14+ (Apple Silicon recommended for optimal thrust) +- Xcode with a Swift 5.9+ toolchain +- [Git](https://git-scm.com/) + +### 1. Board the Ship ```bash -cd ZeroGSwift -swift build -c release +git clone https://github.com/antonynjoro/ZeroG.git +cd ZeroG/ZeroGSwift ./build_app.sh ``` -The packaged app is written to: +The packaged app lands at `ZeroGSwift/build/ZeroG.app`. Move it to `/Applications` and launch. -```text -ZeroGSwift/build/ZeroG.app -``` +First takeoff downloads the transcription model (roughly 460 MB, one time). After that, every flight is fully offline. + +### 2. Clearance Codes (Permissions) +First launch runs a guided pre-flight sequence that requests exactly two clearances: + +- **Microphone**: Audio input feed. +- **Accessibility**: Detects the trigger key and injects the payload (text) into target fields. + +That is the full list. No Input Monitoring. No network clearance. Nothing else. -## Configuration +### 3. Takeoff +Look for the microphone icon in your status bar. -ZeroG reads configuration from environment variables and `UserDefaults`. +- **Transmit**: Hold `Left Control`. The HUD appears. Speak. Release to paste. +- **Polished Transmission**: Press `⌃⌥P` after a transmission to clean it up and paste the refined version. -- `DEBUG=true` enables debug logging. -- `GOOGLE_API_KEY=...` enables optional Gemini polishing. +--- -In packaged app use, the Gemini key can be set from the menu bar item. +## 🕹️ Flight Controls (Configuration) -## Permissions +Everything lives in the status bar menu: -Grant these in **System Settings -> Privacy & Security**: +- **Trigger Key**: Default is `Left Control`. Swappable. +- **Polish Shortcut**: Default is `⌃⌥P`. Swappable. -- Microphone -- Accessibility -- Input Monitoring +### Flight Recorder (Logging) +The Black Box is **OFF** by default for maximum privacy. For development builds, create a `.env` with `DEBUG=true` and reboot systems. -## Testing and CI +--- -The active implementation is Swift-only. CI runs on macOS and executes: +## ⚠️ Turbulence (Troubleshooting) +### Payload Failure (Not Pasting) +- Check **System Settings > Privacy & Security > Accessibility**. +- If ZeroG is listed but pasting fails, remove it (-) and re-add it. Old clearance codes expire, especially after rebuilding the app. + +### Dead Air (No Audio) +- Check **System Settings > Privacy & Security > Microphone**. +- Ensure we have a lock on your comms. + +--- + +## 🧪 R&D (Development) + +### Run From Source +```bash +cd ZeroGSwift +swift run +``` + +### Run Diagnostics ```bash cd ZeroGSwift -swift package resolve swift test -swift build -c release -./build_app.sh ``` -The legacy Python implementation and Python tests have been removed from the active tree. +### Blueprint +- `ZeroGSwift/ZeroG/Core`: Core physics engine (state machine, key monitor, recorder, engines, polish). +- `ZeroGSwift/ZeroG/GUI`: Visual interface (HUD, status bar, onboarding). +- `ZeroGSwift/ZeroGTests`: Simulation scenarios. + +--- + +## 📅 Captain's Log + +ZeroG's first prototype flew on Python. In 2026 the whole ship was rebuilt in native Swift. + +### [The Swift Era] - 2026 +- **Full Rewrite**: Python systems decommissioned. ZeroG is now a native Swift bird: lighter, faster, silent on the pad. +- **New Engine**: NVIDIA Parakeet TDT running on the Apple Neural Engine for near-instant transcription. +- **Gravity Assist 2.0**: Cloud thrusters removed. Polish now runs on-device via Apple Foundation Models. Nothing leaves the ship, ever. +- **Guided Pre-Flight**: New onboarding sequence walks both permission grants on first launch. + +### [v0.10.0] - 2026-02-20 +- **Parallel Chunk Transcription**: Audio chunked and transcribed in the background while you speak, cutting wait-time for long dictations. + +### [v0.9.0] - 2026-01-03 +- **Rebrand**: Initiated "ZeroG" protocol. +- **Brand Guide**: Published `BRANDING.md` for all contributors. + +### [v0.8.1] - 2026-01-02 +- **Hands-Free**: Auto-stop on silence. + +### [v0.8.0] - 2026-01-02 +- **Universal Injection**: Compatibility across all sectors. + +--- +*ZeroG: Don't let gravity hold back your thoughts.* diff --git a/ZeroGSwift/ZeroG/Config.swift b/ZeroGSwift/ZeroG/Config.swift index db3209f..a0e5a5c 100644 --- a/ZeroGSwift/ZeroG/Config.swift +++ b/ZeroGSwift/ZeroG/Config.swift @@ -39,6 +39,12 @@ enum Config { /// Interacts with the trailing-audio knobs — see `TranscriptionQuality`. static let silenceDuration: TimeInterval = 12.0 + /// Last-resort cap for a recording whose trigger key appears to stay held. + /// The release watchdog and silence detector normally stop abandoned sessions + /// much sooner; this only bounds audio memory if both safeguards miss one. + /// Ten minutes of 16 kHz mono Float audio is roughly 38 MB. + static let recordingSafetyTimeout: TimeInterval = 10 * 60 + /// Seconds to keep recording after key release to capture trailing speech. /// Sourced from `TranscriptionQuality.recordingTailSeconds`. static let recordingTailDuration: TimeInterval = TranscriptionQuality.recordingTailSeconds diff --git a/ZeroGSwift/ZeroG/Core/AVAudioCaptureSession.swift b/ZeroGSwift/ZeroG/Core/AVAudioCaptureSession.swift new file mode 100644 index 0000000..aa2effc --- /dev/null +++ b/ZeroGSwift/ZeroG/Core/AVAudioCaptureSession.swift @@ -0,0 +1,72 @@ +import AudioToolbox +import AVFoundation + +/// A fresh AVAudioEngine configured for one recording and one concrete input device. +/// Recreating this object avoids retaining a dead audio route after calls or device changes. +final class AVAudioCaptureSession: AudioCaptureSession { + private enum CaptureError: LocalizedError { + case audioUnitUnavailable + case deviceSelectionFailed(OSStatus) + + var errorDescription: String? { + switch self { + case .audioUnitUnavailable: + "The microphone audio unit is unavailable." + case .deviceSelectionFailed(let status): + "The microphone could not be selected (Core Audio error \(status))." + } + } + } + + private let audioEngine: AVAudioEngine + private let inputNode: AVAudioInputNode + + let recordingFormat: AVAudioFormat + + init(device: AudioInputDevice) throws { + let audioEngine = AVAudioEngine() + let inputNode = audioEngine.inputNode + guard let audioUnit = inputNode.audioUnit else { + throw CaptureError.audioUnitUnavailable + } + + var deviceID = device.deviceID + let status = AudioUnitSetProperty( + audioUnit, + kAudioOutputUnitProperty_CurrentDevice, + kAudioUnitScope_Global, + 0, + &deviceID, + UInt32(MemoryLayout.size) + ) + guard status == noErr else { + throw CaptureError.deviceSelectionFailed(status) + } + + self.audioEngine = audioEngine + self.inputNode = inputNode + self.recordingFormat = inputNode.inputFormat(forBus: 0) + } + + func installTap( + bufferSize: AVAudioFrameCount, + handler: @escaping (AVAudioPCMBuffer, AVAudioFormat) -> Void + ) { + let format = recordingFormat + inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: format) { buffer, _ in + handler(buffer, format) + } + } + + func start() throws { + try audioEngine.start() + } + + func removeTap() { + inputNode.removeTap(onBus: 0) + } + + func stop() { + audioEngine.stop() + } +} diff --git a/ZeroGSwift/ZeroG/Core/AudioCaptureSession.swift b/ZeroGSwift/ZeroG/Core/AudioCaptureSession.swift new file mode 100644 index 0000000..f80f26a --- /dev/null +++ b/ZeroGSwift/ZeroG/Core/AudioCaptureSession.swift @@ -0,0 +1,14 @@ +import AVFoundation + +/// Testable boundary around the AVAudioEngine capture operations used per recording. +protocol AudioCaptureSession: AnyObject { + var recordingFormat: AVAudioFormat { get } + + func installTap( + bufferSize: AVAudioFrameCount, + handler: @escaping (AVAudioPCMBuffer, AVAudioFormat) -> Void + ) + func start() throws + func removeTap() + func stop() +} diff --git a/ZeroGSwift/ZeroG/Core/AudioInputDevice.swift b/ZeroGSwift/ZeroG/Core/AudioInputDevice.swift new file mode 100644 index 0000000..f0c4d4b --- /dev/null +++ b/ZeroGSwift/ZeroG/Core/AudioInputDevice.swift @@ -0,0 +1,11 @@ +import CoreAudio + +/// A physical or virtual macOS audio input that ZeroG can record from. +struct AudioInputDevice: Equatable, Identifiable { + let deviceID: AudioDeviceID + let uid: String + let name: String + let isBuiltIn: Bool + + var id: String { uid } +} diff --git a/ZeroGSwift/ZeroG/Core/AudioInputDeviceManager.swift b/ZeroGSwift/ZeroG/Core/AudioInputDeviceManager.swift new file mode 100644 index 0000000..4147bd5 --- /dev/null +++ b/ZeroGSwift/ZeroG/Core/AudioInputDeviceManager.swift @@ -0,0 +1,165 @@ +import CoreAudio +import Foundation + +/// Discovers macOS input devices and persists an explicit external-microphone choice. +/// With no explicit choice, the Mac's built-in microphone always wins over the +/// current system default so calls and newly connected devices cannot reroute ZeroG. +final class AudioInputDeviceManager { + private static let selectedUIDDefaultsKey = "AudioInputDeviceUID" + + private let defaults: UserDefaults + private let deviceProvider: () -> [AudioInputDevice] + + init( + defaults: UserDefaults = .standard, + deviceProvider: @escaping () -> [AudioInputDevice] = AudioInputDeviceManager.discoverDevices + ) { + self.defaults = defaults + self.deviceProvider = deviceProvider + } + + var availableDevices: [AudioInputDevice] { + deviceProvider().sorted { left, right in + if left.isBuiltIn != right.isBuiltIn { + return left.isBuiltIn + } + return left.name.localizedStandardCompare(right.name) == .orderedAscending + } + } + + var selectedDevice: AudioInputDevice? { + let devices = availableDevices + if let selectedUID = defaults.string(forKey: Self.selectedUIDDefaultsKey), + let selected = devices.first(where: { $0.uid == selectedUID }) { + return selected + } + return devices.first(where: \.isBuiltIn) ?? devices.first + } + + /// Built-in is the durable default, so selecting it clears any external override. + func select(_ device: AudioInputDevice) { + if device.isBuiltIn { + defaults.removeObject(forKey: Self.selectedUIDDefaultsKey) + } else { + defaults.set(device.uid, forKey: Self.selectedUIDDefaultsKey) + } + } + + private static func discoverDevices() -> [AudioInputDevice] { + allDeviceIDs().compactMap { deviceID in + guard hasInputStreams(deviceID), + let uid = stringProperty( + kAudioDevicePropertyDeviceUID, + deviceID: deviceID + ), + let name = stringProperty( + kAudioObjectPropertyName, + deviceID: deviceID + ) else { + return nil + } + + return AudioInputDevice( + deviceID: deviceID, + uid: uid, + name: name, + isBuiltIn: transportType(deviceID) == kAudioDeviceTransportTypeBuiltIn + ) + } + } + + private static func allDeviceIDs() -> [AudioDeviceID] { + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var dataSize: UInt32 = 0 + guard AudioObjectGetPropertyDataSize( + AudioObjectID(kAudioObjectSystemObject), + &address, + 0, + nil, + &dataSize + ) == noErr else { + return [] + } + + let count = Int(dataSize) / MemoryLayout.stride + guard count > 0 else { return [] } + var deviceIDs = [AudioDeviceID](repeating: 0, count: count) + let status = deviceIDs.withUnsafeMutableBytes { buffer in + guard let baseAddress = buffer.baseAddress else { return OSStatus(-50) } + return AudioObjectGetPropertyData( + AudioObjectID(kAudioObjectSystemObject), + &address, + 0, + nil, + &dataSize, + baseAddress + ) + } + return status == noErr ? deviceIDs : [] + } + + private static func hasInputStreams(_ deviceID: AudioDeviceID) -> Bool { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreams, + mScope: kAudioObjectPropertyScopeInput, + mElement: kAudioObjectPropertyElementMain + ) + var dataSize: UInt32 = 0 + return AudioObjectGetPropertyDataSize( + deviceID, + &address, + 0, + nil, + &dataSize + ) == noErr && dataSize >= MemoryLayout.stride + } + + private static func stringProperty( + _ selector: AudioObjectPropertySelector, + deviceID: AudioDeviceID + ) -> String? { + var address = AudioObjectPropertyAddress( + mSelector: selector, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var value: Unmanaged? + var dataSize = UInt32(MemoryLayout?>.size) + guard AudioObjectGetPropertyData( + deviceID, + &address, + 0, + nil, + &dataSize, + &value + ) == noErr, let value else { + return nil + } + return value.takeRetainedValue() as String + } + + private static func transportType(_ deviceID: AudioDeviceID) -> UInt32 { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyTransportType, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var value: UInt32 = 0 + var dataSize = UInt32(MemoryLayout.size) + guard AudioObjectGetPropertyData( + deviceID, + &address, + 0, + nil, + &dataSize, + &value + ) == noErr else { + return 0 + } + return value + } +} diff --git a/ZeroGSwift/ZeroG/Core/AudioRecorder.swift b/ZeroGSwift/ZeroG/Core/AudioRecorder.swift index c6f6b56..55222d6 100644 --- a/ZeroGSwift/ZeroG/Core/AudioRecorder.swift +++ b/ZeroGSwift/ZeroG/Core/AudioRecorder.swift @@ -35,7 +35,11 @@ final class AudioRecorder: @unchecked Sendable { // MARK: Audio Engine - private let audioEngine = AVAudioEngine() + typealias CaptureSessionFactory = (AudioInputDevice) throws -> AudioCaptureSession + + private let inputDeviceProvider: () -> AudioInputDevice? + private let captureSessionFactory: CaptureSessionFactory + private var captureSession: AudioCaptureSession? private var isRecording = false // MARK: Audio Accumulation @@ -57,14 +61,21 @@ final class AudioRecorder: @unchecked Sendable { // MARK: Lifecycle - init(stateMachine: AppStateMachine, transcriptionEngine: Transcribing) { + init( + stateMachine: AppStateMachine, + transcriptionEngine: Transcribing, + inputDeviceProvider: @escaping () -> AudioInputDevice?, + captureSessionFactory: @escaping CaptureSessionFactory = AVAudioCaptureSession.init(device:) + ) { self.stateMachine = stateMachine self.transcriptionEngine = transcriptionEngine + self.inputDeviceProvider = inputDeviceProvider + self.captureSessionFactory = captureSessionFactory } // MARK: - Recording Control - /// Begin capturing audio from the default input device. + /// Begin capturing audio from the preferred input device. func startRecording() { guard !isRecording else { return } @@ -75,34 +86,57 @@ final class AudioRecorder: @unchecked Sendable { } transcribedTexts.removeAll() - let inputNode = audioEngine.inputNode - let recordingFormat = inputNode.inputFormat(forBus: 0) - - // Ensure we have a valid format - guard recordingFormat.sampleRate > 0 else { - DispatchQueue.main.async { [weak self] in - self?.stateMachine.transition(to: .error("No microphone available")) - self?.stateMachine.resetToIdle(after: Config.Timing.errorReset) + var lastError: Error? + for attempt in 1...2 { + guard let inputDevice = inputDeviceProvider() else { break } + + do { + let session = try captureSessionFactory(inputDevice) + let recordingFormat = session.recordingFormat + guard recordingFormat.sampleRate > 0, recordingFormat.channelCount > 0 else { + Log.info( + "AudioRecorder", + "Input route for \(inputDevice.name) was invalid; rebuilding (attempt \(attempt)/2)." + ) + continue + } + + session.installTap(bufferSize: AudioConstants.bufferSize) { [weak self] buffer, format in + self?.processAudioBuffer(buffer, format: format) + } + + do { + try session.start() + } catch { + session.removeTap() + session.stop() + throw error + } + + captureSession = session + isRecording = true + playFeedbackSound() + Log.debug( + "AudioRecorder", + "Recording started with \(inputDevice.name). Format: \(recordingFormat)" + ) + return + } catch { + lastError = error + Log.error( + "AudioRecorder", + "Could not start input route (attempt \(attempt)/2): \(error.localizedDescription)" + ) } - return - } - - // Install tap on the input node for raw audio capture - inputNode.installTap(onBus: 0, bufferSize: AudioConstants.bufferSize, format: recordingFormat) { [weak self] buffer, _ in - self?.processAudioBuffer(buffer, format: recordingFormat) } - - do { - try audioEngine.start() - isRecording = true - playFeedbackSound() - Log.debug("AudioRecorder", "Recording started. Format: \(recordingFormat)") - } catch { - DispatchQueue.main.async { [weak self] in - self?.stateMachine.transition(to: .error("Mic Error: \(error.localizedDescription)")) - self?.stateMachine.resetToIdle(after: Config.Timing.errorReset) - } + let diagnostic = lastError.map { ": \($0.localizedDescription)" } ?? "" + Log.error("AudioRecorder", "No usable microphone was available\(diagnostic)") + DispatchQueue.main.async { [weak self] in + self?.stateMachine.transition( + to: .error("Microphone unavailable. Choose one from the ZeroG menu") + ) + self?.stateMachine.resetToIdle(after: Config.Timing.errorReset) } } @@ -131,8 +165,9 @@ final class AudioRecorder: @unchecked Sendable { DispatchQueue.main.asyncAfter(deadline: .now() + Config.recordingTailDuration) { [weak self] in guard let self else { return } - self.audioEngine.inputNode.removeTap(onBus: 0) - self.audioEngine.stop() + self.captureSession?.removeTap() + self.captureSession?.stop() + self.captureSession = nil let rawAudio: [Float] = self.sampleQueue.sync { self.accumulatedSamples } let audioData = Self.trimTrailingSilence(rawAudio) diff --git a/ZeroGSwift/ZeroG/Core/KeyMonitor.swift b/ZeroGSwift/ZeroG/Core/KeyMonitor.swift index c1c82bf..70e94ed 100644 --- a/ZeroGSwift/ZeroG/Core/KeyMonitor.swift +++ b/ZeroGSwift/ZeroG/Core/KeyMonitor.swift @@ -40,8 +40,10 @@ final class KeyMonitor { private var isTriggerKeyPressed = false private var recordingStartTime: Date? - /// Safety timeout to prevent stuck recording state (2 minutes). - private let maxRecordingDuration: TimeInterval = 120.0 + /// Last-resort timeout for a stuck recording. The physical-key watchdog and + /// silence detector are the primary safeguards, so legitimate long-form + /// dictation gets a much wider window before this backstop fires. + private let maxRecordingDuration = Config.recordingSafetyTimeout private var timeoutTimer: Timer? /// Watchdog that polls the physical trigger-key state while recording. diff --git a/ZeroGSwift/ZeroG/GUI/StatusBarController.swift b/ZeroGSwift/ZeroG/GUI/StatusBarController.swift index 9a9842e..1082c23 100644 --- a/ZeroGSwift/ZeroG/GUI/StatusBarController.swift +++ b/ZeroGSwift/ZeroG/GUI/StatusBarController.swift @@ -22,11 +22,13 @@ final class StatusBarController: NSObject, NSMenuDelegate { private var polishReasonMenuItem: NSMenuItem! private var polishShortcutSubmenu: NSMenu! private var triggerKeySubmenu: NSMenu! + private var audioInputSubmenu: NSMenu! private var cancellables = Set() // MARK: Dependencies private let stateMachine: AppStateMachine + private let audioInputManager: AudioInputDeviceManager /// Opens the permissions / setup wizard. private let onShowPermissions: () -> Void @@ -40,10 +42,12 @@ final class StatusBarController: NSObject, NSMenuDelegate { // MARK: Initialization init(stateMachine: AppStateMachine, + audioInputManager: AudioInputDeviceManager, onShowPermissions: @escaping () -> Void = {}, onCopyPolished: @escaping () -> Void = {}, onRetryModel: @escaping () -> Void = {}) { self.stateMachine = stateMachine + self.audioInputManager = audioInputManager self.onShowPermissions = onShowPermissions self.onCopyPolished = onCopyPolished self.onRetryModel = onRetryModel @@ -104,6 +108,14 @@ final class StatusBarController: NSObject, NSMenuDelegate { triggerKeyItem.submenu = triggerKeySubmenu menu.addItem(triggerKeyItem) + // Audio input submenu. Rebuilt whenever the menu opens so newly connected + // microphones appear without relaunching the app. + let audioInputItem = NSMenuItem(title: "Microphone", action: nil, keyEquivalent: "") + audioInputSubmenu = NSMenu() + rebuildAudioInputSubmenu() + audioInputItem.submenu = audioInputSubmenu + menu.addItem(audioInputItem) + menu.addItem(NSMenuItem.separator()) // Copy Last Transcription @@ -164,6 +176,8 @@ final class StatusBarController: NSObject, NSMenuDelegate { /// Refresh dynamic enablement each time the menu opens — polish availability /// (Apple Intelligence can be toggled) and whether there's a transcription to act on. func menuWillOpen(_ menu: NSMenu) { + rebuildAudioInputSubmenu() + let hasText = stateMachine.lastTranscription != nil copyTranscriptionMenuItem.isEnabled = hasText @@ -226,6 +240,42 @@ final class StatusBarController: NSObject, NSMenuDelegate { } } + // MARK: - Audio Input Selection + + private func rebuildAudioInputSubmenu() { + audioInputSubmenu.removeAllItems() + let devices = audioInputManager.availableDevices + guard !devices.isEmpty else { + let item = NSMenuItem(title: "No microphones available", action: nil, keyEquivalent: "") + item.isEnabled = false + audioInputSubmenu.addItem(item) + return + } + + let selectedUID = audioInputManager.selectedDevice?.uid + for device in devices { + let suffix = device.isBuiltIn ? " (Default)" : "" + let item = NSMenuItem( + title: device.name + suffix, + action: #selector(audioInputSelected(_:)), + keyEquivalent: "" + ) + item.target = self + item.representedObject = device.uid + item.state = device.uid == selectedUID ? .on : .off + audioInputSubmenu.addItem(item) + } + } + + @objc private func audioInputSelected(_ sender: NSMenuItem) { + guard let uid = sender.representedObject as? String, + let device = audioInputManager.availableDevices.first(where: { $0.uid == uid }) else { + return + } + audioInputManager.select(device) + rebuildAudioInputSubmenu() + } + // MARK: - Permissions / Setup @objc private func showPermissions() { diff --git a/ZeroGSwift/ZeroG/ZeroGApp.swift b/ZeroGSwift/ZeroG/ZeroGApp.swift index b5fd15e..f0acabb 100644 --- a/ZeroGSwift/ZeroG/ZeroGApp.swift +++ b/ZeroGSwift/ZeroG/ZeroGApp.swift @@ -23,6 +23,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, @unchecked Sendable { private var stateMachine: AppStateMachine! private var transcriptionEngine: Transcribing! private var audioRecorder: AudioRecorder! + private var audioInputManager: AudioInputDeviceManager! private var keyMonitor: KeyMonitor! // MARK: GUI Components @@ -45,11 +46,13 @@ final class AppDelegate: NSObject, NSApplicationDelegate, @unchecked Sendable { // Initialize core stateMachine = AppStateMachine() transcriptionEngine = Self.makeTranscriptionEngine(for: Config.sttBackend) + audioInputManager = AudioInputDeviceManager() Log.debug("ZeroGApp", "STT backend: \(Config.sttBackend.rawValue)") audioRecorder = AudioRecorder( stateMachine: stateMachine, - transcriptionEngine: transcriptionEngine + transcriptionEngine: transcriptionEngine, + inputDeviceProvider: { [weak self] in self?.audioInputManager.selectedDevice } ) keyMonitor = KeyMonitor( @@ -114,6 +117,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, @unchecked Sendable { // Initialize GUI statusBarController = StatusBarController( stateMachine: stateMachine, + audioInputManager: audioInputManager, onShowPermissions: { [weak self] in self?.onboardingController.show() }, onCopyPolished: { [weak self] in self?.copyPolished() }, onRetryModel: { [weak self] in self?.startModelLoad() } diff --git a/ZeroGSwift/ZeroGTests/AudioInputDeviceManagerTests.swift b/ZeroGSwift/ZeroGTests/AudioInputDeviceManagerTests.swift new file mode 100644 index 0000000..a8dd566 --- /dev/null +++ b/ZeroGSwift/ZeroGTests/AudioInputDeviceManagerTests.swift @@ -0,0 +1,82 @@ +import Foundation +import Testing +@testable import ZeroG + +@Suite(.serialized) +struct AudioInputDeviceManagerTests { + private let builtIn = AudioInputDevice( + deviceID: 1, + uid: "built-in", + name: "MacBook Pro Microphone", + isBuiltIn: true + ) + + private let external = AudioInputDevice( + deviceID: 2, + uid: "external", + name: "Studio Microphone", + isBuiltIn: false + ) + + @Test("Built-in microphone is preferred when no selection is stored") + func builtInIsTheDefault() { + let defaults = makeDefaults() + let manager = AudioInputDeviceManager( + defaults: defaults, + deviceProvider: { [external, builtIn] } + ) + + #expect(manager.selectedDevice == builtIn) + } + + @Test("An external microphone can be selected and remembered") + func externalSelectionIsRemembered() { + let defaults = makeDefaults() + let manager = AudioInputDeviceManager( + defaults: defaults, + deviceProvider: { [builtIn, external] } + ) + + manager.select(external) + + #expect(manager.selectedDevice == external) + } + + @Test("Missing selected microphone falls back to the built-in microphone") + func missingSelectionFallsBackToBuiltIn() { + var devices = [builtIn, external] + let defaults = makeDefaults() + let manager = AudioInputDeviceManager( + defaults: defaults, + deviceProvider: { devices } + ) + manager.select(external) + + devices = [builtIn] + + #expect(manager.selectedDevice == builtIn) + } + + @Test("Choosing the built-in microphone restores automatic built-in preference") + func choosingBuiltInRestoresDefault() { + let defaults = makeDefaults() + let manager = AudioInputDeviceManager( + defaults: defaults, + deviceProvider: { [builtIn, external] } + ) + manager.select(external) + + manager.select(builtIn) + + #expect(manager.selectedDevice == builtIn) + } + + private func makeDefaults() -> UserDefaults { + let suiteName = "AudioInputDeviceManagerTests.\(UUID().uuidString)" + guard let defaults = UserDefaults(suiteName: suiteName) else { + fatalError("Could not create isolated UserDefaults suite") + } + defaults.removePersistentDomain(forName: suiteName) + return defaults + } +} diff --git a/ZeroGSwift/ZeroGTests/AudioRecorderRecoveryTests.swift b/ZeroGSwift/ZeroGTests/AudioRecorderRecoveryTests.swift new file mode 100644 index 0000000..4e691c2 --- /dev/null +++ b/ZeroGSwift/ZeroGTests/AudioRecorderRecoveryTests.swift @@ -0,0 +1,64 @@ +import AVFoundation +import Testing +@testable import ZeroG + +@Suite +struct AudioRecorderRecoveryTests { + @Test("Recorder rebuilds a stale audio route instead of reporting no microphone") + func staleRouteIsRebuilt() throws { + let staleSession = MockAudioCaptureSession(recordingFormat: AVAudioFormat()) + let healthyFormat = try #require( + AVAudioFormat(standardFormatWithSampleRate: 48_000, channels: 1) + ) + let healthySession = MockAudioCaptureSession(recordingFormat: healthyFormat) + var sessions: [MockAudioCaptureSession] = [staleSession, healthySession] + var factoryCallCount = 0 + let device = AudioInputDevice( + deviceID: 1, + uid: "built-in", + name: "MacBook Pro Microphone", + isBuiltIn: true + ) + + let recorder = AudioRecorder( + stateMachine: AppStateMachine(), + transcriptionEngine: MockTranscriber(), + inputDeviceProvider: { device }, + captureSessionFactory: { _ in + factoryCallCount += 1 + return sessions.removeFirst() + } + ) + + recorder.startRecording() + + #expect(factoryCallCount == 2) + #expect(staleSession.startCallCount == 0) + #expect(healthySession.installTapCallCount == 1) + #expect(healthySession.startCallCount == 1) + } +} + +private final class MockAudioCaptureSession: AudioCaptureSession { + let recordingFormat: AVAudioFormat + private(set) var installTapCallCount = 0 + private(set) var startCallCount = 0 + + init(recordingFormat: AVAudioFormat) { + self.recordingFormat = recordingFormat + } + + func installTap( + bufferSize: AVAudioFrameCount, + handler: @escaping (AVAudioPCMBuffer, AVAudioFormat) -> Void + ) { + installTapCallCount += 1 + } + + func start() throws { + startCallCount += 1 + } + + func removeTap() {} + func stop() {} +} diff --git a/ZeroGSwift/ZeroGTests/ConfigTests.swift b/ZeroGSwift/ZeroGTests/ConfigTests.swift index 55ded19..d9f7d93 100644 --- a/ZeroGSwift/ZeroGTests/ConfigTests.swift +++ b/ZeroGSwift/ZeroGTests/ConfigTests.swift @@ -8,6 +8,7 @@ struct ConfigTests { func defaultConstants() { #expect(Config.silenceThreshold == 0.003) #expect(Config.silenceDuration == 12.0) + #expect(Config.recordingSafetyTimeout == 10 * 60) #expect(Config.recordingTailDuration == 0.3) #expect(Config.whisperModel == "large-v3-v20240930_turbo") } diff --git a/ZeroGSwift/ZeroGTests/TranscribingTests.swift b/ZeroGSwift/ZeroGTests/TranscribingTests.swift index a70ceb6..ccd0c78 100644 --- a/ZeroGSwift/ZeroGTests/TranscribingTests.swift +++ b/ZeroGSwift/ZeroGTests/TranscribingTests.swift @@ -31,7 +31,11 @@ struct TranscribingTests { func audioRecorderTakesProtocol() { let mock = MockTranscriber() // Compiles only if AudioRecorder's dependency is the protocol, not a concrete engine. - _ = AudioRecorder(stateMachine: AppStateMachine(), transcriptionEngine: mock) + _ = AudioRecorder( + stateMachine: AppStateMachine(), + transcriptionEngine: mock, + inputDeviceProvider: { nil } + ) } @Test("The concrete WhisperKit engine conforms to Transcribing") From cfafe62d9370bc1845faebf58554000c114625b8 Mon Sep 17 00:00:00 2001 From: Antony Njoroge Mutharia Date: Wed, 15 Jul 2026 18:59:59 -0600 Subject: [PATCH 7/7] ci: build with macOS 26 runner --- .github/workflows/swift-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/swift-ci.yml b/.github/workflows/swift-ci.yml index e26d7ba..1484da9 100644 --- a/.github/workflows/swift-ci.yml +++ b/.github/workflows/swift-ci.yml @@ -15,7 +15,7 @@ on: jobs: test: name: Build and Test Swift App - runs-on: macos-15 + runs-on: macos-26 defaults: run: working-directory: ZeroGSwift