From e82cabb5612a7fddd6e6595b765e24b5a817eb0d Mon Sep 17 00:00:00 2001
From: Ryan Vogel <89211796+R44VC0RP@users.noreply.github.com>
Date: Sat, 21 Nov 2026 09:35:17 -0500
Subject: [PATCH 1/4] iOS keyboard: a failure from Hex shows for seconds, not
forever, and the Hex key retries it
X testers on build 56 and earlier: "The keyboard keeps either bringing me back to
the app or telling me another app is using the microphone" (@itsjling) and "The
another app is using audio message seems inaccurate cause it's showing for me all
the time although it's working fine" (@RedaSjakur).
Cause, reproduced on build 56 (2ef08aa4) in an iOS 27.2 simulator on the Mac Studio
(no microphone input): Hex's dictation failed ("Hex couldn't start the microphone.
Try again."), Hex was then closed, and a minute later the Hex keyboard's strip in
Settings' search field still said so. Hex writes its snapshot only while it runs,
so a failed snapshot (heartbeat 0, no time) stayed in the App Group, and
KeyboardViewModel read any .failed snapshot as failed whatever its age. The code
path is unchanged on origin/dev. With a warm session up it was worse: Hex stayed
in .failed, consumeKeyboardCommand ignored every start unless the phase was .ready,
and the full keyboard's Hex key (isWarm = Full Access && warm available) kept
sending starts to it, so nothing recorded until Hex was opened. The keyboard's own
failure snapshot also reads as available for its 6 s, so a Hex key tap right after
an in-keyboard failure went to a warm session that doesn't exist.
Now:
- KeyboardSnapshot.messageAt: when Hex wrote the message (DictationController
stamps a failure when its phase becomes failed, and keyboardMessage when set; the
keyboard's own failure stamps its snapshot). The keyboard drops a message, and
reads a failure as offline, 8 s after it (messageShownFor), and at once for a
snapshot with no time (an earlier build's). Whether a warm session is up and any
result stay as they were.
- KeyboardViewModel.dictatesThroughHex (Full Access, warm available, not failed),
used by the full keyboard's Hex key and its touch-down capture: after a failure
both keyboards dictate in the keyboard, as the Dictation bar already did.
- Hex takes a keyboard start while failed when the failure was the microphone for
a keyboard dictation with the model loaded (captureFailed): the next Hex key is
its Try again. Model failures still wait for Hex to get ready again.
Not changed: the 32 MB memory threshold and keyboardNeedsHex, the two ways the Hex
key opens Hex. In the simulator the handoffs logged "speech model missing here"
after the keyboard's own asset check failed ("anomaly.hex.mobile.keyboard is not
subscribed to assistant.en_US") on a sim whose Hex had never installed a model,
which is the intended path; a device's reason needs its probe log.
Verified: new KeyboardBridgeTests.testAFailureHexWroteAWhileAgoIsOver and
DictationControllerTests.testAWarmSessionsMicrophoneFailureIsRetriedFromTheKeyboard;
KeyboardBridgeTests and DictationControllerTests pass (79 tests, 0 failures, iPhone
17 Pro iOS 27.2 simulator). In the simulator with this build, the same failure no
longer shows in the keyboard a minute later.
---
ios/HEXKeyboard/KeyboardLocalDictation.swift | 2 +-
ios/HEXKeyboard/KeyboardViewController.swift | 2 +-
ios/HEXMobile/DictationController.swift | 25 +++++-
.../DictationControllerTests.swift | 48 ++++++++++++
ios/HEXMobileTests/KeyboardBridgeTests.swift | 78 +++++++++++++++++++
.../FullKeyboard/FullKeyboardView.swift | 2 +-
ios/HEXShared/KeyboardBridge.swift | 44 ++++++++++-
ios/HEXShared/KeyboardSurface+ViewModel.swift | 9 ++-
ios/README.md | 6 +-
9 files changed, 207 insertions(+), 9 deletions(-)
diff --git a/ios/HEXKeyboard/KeyboardLocalDictation.swift b/ios/HEXKeyboard/KeyboardLocalDictation.swift
index 039daff54..ca353e5ab 100644
--- a/ios/HEXKeyboard/KeyboardLocalDictation.swift
+++ b/ios/HEXKeyboard/KeyboardLocalDictation.swift
@@ -152,7 +152,7 @@ final class KeyboardLocalDictation {
phase = .idle
return nil
}
- return KeyboardSnapshot(state: .failed, heartbeat: now, expiresAt: until, message: message)
+ return KeyboardSnapshot(state: .failed, heartbeat: now, expiresAt: until, message: message, messageAt: now)
}
}
diff --git a/ios/HEXKeyboard/KeyboardViewController.swift b/ios/HEXKeyboard/KeyboardViewController.swift
index 6c9031409..817b686f7 100644
--- a/ios/HEXKeyboard/KeyboardViewController.swift
+++ b/ios/HEXKeyboard/KeyboardViewController.swift
@@ -1181,7 +1181,7 @@ final class KeyboardViewController: UIInputViewController {
/// the Hex key's own choice of in-keyboard dictation; a warm Hex session records instead.
private func hexKeyPressed(at time: TimeInterval) {
hexKeyPressedAt = time
- guard viewModel.hasFullAccess, !viewModel.isWarmAvailable, viewModel.onLocalDictateTap != nil,
+ guard viewModel.hasFullAccess, !viewModel.dictatesThroughHex, viewModel.onLocalDictateTap != nil,
viewModel.stagePhase == .idle || viewModel.stagePhase == .failed else { return }
localDictation.arm(in: view, microphone: bridge.microphone, pressedAt: time)
}
diff --git a/ios/HEXMobile/DictationController.swift b/ios/HEXMobile/DictationController.swift
index 9d8f5ec4e..ae5744472 100644
--- a/ios/HEXMobile/DictationController.swift
+++ b/ios/HEXMobile/DictationController.swift
@@ -112,7 +112,9 @@ final class DictationController {
private var keyboardResultTranslationFailure: TranslationFailure?
/// Record typing logs: each step of the result, for the keyboard's log; nil while off.
private var keyboardResultStages: DictationStages?
- private var keyboardMessage: String?
+ private var keyboardMessage: String? {
+ didSet { keyboardMessageAt = keyboardMessage == nil ? nil : now() }
+ }
private var pendingHandoff: PendingKeyboardHandoff?
var keyboardSessionExpiresAt: TimeInterval = 0
private var suspendedWarmSessionOnInterruption = false
@@ -129,6 +131,8 @@ final class DictationController {
var phase = Phase.modelRequired {
didSet {
+ captureFailed = false
+ if case .failed = phase { failedAt = now() }
guard phase.keepsScreenAwake != oldValue.keepsScreenAwake else { return }
// Only applies while Hex is in front; nothing else in Hex sets it, so ending a
// dictation hands the screen back to Auto-Lock.
@@ -167,6 +171,13 @@ final class DictationController {
}
@ObservationIgnored private var lastPublishedSnapshotSignature: String?
@ObservationIgnored private var lastPublishedSnapshotAt: TimeInterval = -100
+ /// When the phase last became failed, and when `keyboardMessage` was last set: the
+ /// keyboard shows a message only for a few seconds after (`KeyboardSnapshot.messageAt`).
+ @ObservationIgnored private var failedAt: TimeInterval?
+ @ObservationIgnored private var keyboardMessageAt: TimeInterval?
+ /// The failure on screen is a microphone that didn't start for a keyboard's dictation, with
+ /// the model still loaded: the keyboard's next Hex key tries again.
+ @ObservationIgnored private var captureFailed = false
var warmDuration: WarmSessionDuration
private(set) var style: DictationStyle
private(set) var microphone: MicrophoneChoice
@@ -876,6 +887,7 @@ final class DictationController {
let message = UserFacingError.message(for: error, context: .microphone)
keyboardMessage = keyboardJobID == nil ? nil : message
phase = .failed(message)
+ captureFailed = isModelReady
restoreAudioAfterCapture()
publishKeyboardSnapshot()
}
@@ -1190,7 +1202,10 @@ final class DictationController {
switch command.kind {
case .start:
- guard phase == .ready, activeKeyboardJobID == nil else { return }
+ // A warm session whose microphone failed for the last dictation stays marked failed:
+ // the keyboard's next Hex key is its Try again. Without this Hex ignored every
+ // start until it was opened, while the keyboard kept sending them.
+ guard phase == .ready || captureFailed, activeKeyboardJobID == nil else { return }
setStyle(command.style)
startRecording(
keyboardJobID: command.jobID,
@@ -1240,10 +1255,13 @@ final class DictationController {
}
let failureMessage: String?
+ let messageAt: TimeInterval?
if case .failed(let message) = phase {
failureMessage = message
+ messageAt = failedAt
} else {
failureMessage = keyboardMessage
+ messageAt = keyboardMessageAt
}
let signature = "\(state.rawValue)|\(activeKeyboardJobID ?? "")|\(keyboardResultID ?? "")|\(failureMessage ?? "")"
@@ -1277,7 +1295,8 @@ final class DictationController {
message: failureMessage,
heard: keyboardResultHeard,
stages: keyboardResultStages,
- translationFailure: keyboardResultTranslationFailure
+ translationFailure: keyboardResultTranslationFailure,
+ messageAt: failureMessage == nil ? nil : messageAt
)
)
}
diff --git a/ios/HEXMobileTests/DictationControllerTests.swift b/ios/HEXMobileTests/DictationControllerTests.swift
index ce4a30a99..fb6b52828 100644
--- a/ios/HEXMobileTests/DictationControllerTests.swift
+++ b/ios/HEXMobileTests/DictationControllerTests.swift
@@ -1712,6 +1712,54 @@ final class DictationControllerTests: XCTestCase {
controller.cancelCurrentJob()
}
+ /// A warm session's microphone failed for a keyboard's dictation (another app held it):
+ /// Hex stays marked failed, and ignored every start the keyboard sent until it was opened,
+ /// while its snapshot kept saying so to every keyboard. The failure carries when it
+ /// happened, so the keyboard stops showing it, and the next Hex key is Hex's Try again.
+ func testAWarmSessionsMicrophoneFailureIsRetriedFromTheKeyboard() async throws {
+ var now = 52_000.0
+ let audio = FakeAudioManager()
+ let transcriber = ControlledTranscriber()
+ let bridge = KeyboardBridge(defaults: defaults)
+ let controller = DictationController(
+ audio: audio,
+ keyboardBridge: bridge,
+ transcriber: transcriber,
+ transcriberFactory: { transcriber },
+ now: { now },
+ autoPrepareOnInit: false
+ )
+ controller.prepareModel()
+ try await settle()
+ controller.selectWarmDuration(.fifteenMinutes)
+ try await settle()
+ XCTAssertTrue(audio.isKeepaliveRunning)
+
+ audio.captureShouldThrow = true
+ bridge.send(.start, jobID: "busy-mic", style: .polished, sentAt: now)
+ try await settle()
+ let message = "Hex couldn't start the microphone. Try again."
+ XCTAssertEqual(controller.phase, .failed(message))
+ let failed = bridge.snapshot()
+ XCTAssertEqual(failed.state, .failed)
+ XCTAssertEqual(failed.message, message)
+ XCTAssertEqual(failed.messageAt, now)
+ XCTAssertFalse(failed.hasStaleMessage(at: now + 2))
+ XCTAssertTrue(failed.hasStaleMessage(at: now + KeyboardSnapshot.messageShownFor + 1))
+ XCTAssertTrue(audio.isKeepaliveRunning, "the warm session goes on")
+
+ // The keyboard's next Hex key, with the microphone free again: Hex records.
+ audio.captureShouldThrow = false
+ now += 12
+ bridge.send(.start, jobID: "try-again", style: .polished, sentAt: now)
+ try await settle()
+ XCTAssertEqual(controller.phase, .recording)
+ XCTAssertEqual(bridge.snapshot().jobID, "try-again")
+ XCTAssertNil(bridge.snapshot().message)
+ XCTAssertNil(bridge.snapshot().messageAt)
+ controller.endWarmSession()
+ }
+
/// A recorder's end can arrive after Hex already tore its capture down (Cancel, or the
/// next dictation started): it changes nothing, and the dictation running now goes on.
func testAnEndFromATornDownCaptureChangesNothing() async throws {
diff --git a/ios/HEXMobileTests/KeyboardBridgeTests.swift b/ios/HEXMobileTests/KeyboardBridgeTests.swift
index 1773250f1..0fa3a5c21 100644
--- a/ios/HEXMobileTests/KeyboardBridgeTests.swift
+++ b/ios/HEXMobileTests/KeyboardBridgeTests.swift
@@ -105,6 +105,84 @@ final class KeyboardBridgeTests: XCTestCase {
XCTAssertEqual(later.stagePhase, .recording)
}
+ /// TestFlight: "The another app is using audio message seems inaccurate cause it's showing
+ /// for me all the time although it's working fine", and "telling me another app is using
+ /// the microphone". Hex writes its snapshot only while it runs, so a failure stayed in the
+ /// App Group: every keyboard after showed it, and the full keyboard's Hex key sent its
+ /// dictations to Hex's warm session, which ignored them while marked failed. A message now
+ /// shows for a few seconds after Hex wrote it; one with no time (an earlier build's) never.
+ func testAFailureHexWroteAWhileAgoIsOver() {
+ let t0 = 20_000.0
+ let busy = "Another app is using the microphone. Try again."
+ func show(_ snapshot: KeyboardSnapshot, at now: TimeInterval, on model: KeyboardViewModel) {
+ model.applyBridgeState(
+ hasFullAccess: true,
+ snapshot: snapshot,
+ isModelInstalled: true,
+ style: .polished,
+ now: now,
+ animated: false
+ )
+ }
+
+ // A warm session's dictation failed just now: the keyboard says so and dictates itself.
+ let model = KeyboardViewModel()
+ show(
+ KeyboardSnapshot(state: .failed, heartbeat: t0 + 1, expiresAt: t0 + 900, message: busy, messageAt: t0),
+ at: t0 + 1,
+ on: model
+ )
+ XCTAssertEqual(model.stagePhase, .failed)
+ XCTAssertEqual(model.statusMessage, busy)
+ XCTAssertTrue(model.isWarmAvailable)
+ XCTAssertFalse(model.dictatesThroughHex, "the Hex key dictates in the keyboard, as Dictate does")
+
+ // Seconds later, with the session still up, it's over and the Hex key uses the session.
+ let later = KeyboardSnapshot(state: .failed, heartbeat: t0 + 9, expiresAt: t0 + 900, message: busy, messageAt: t0)
+ XCTAssertTrue(later.hasStaleMessage(at: t0 + 9))
+ show(later, at: t0 + 9, on: model)
+ XCTAssertEqual(model.stagePhase, .idle)
+ XCTAssertNil(model.statusMessage)
+ XCTAssertTrue(model.dictatesThroughHex)
+
+ // Hex suspended since, or an earlier build's snapshot with no time: never shown.
+ for stale in [
+ KeyboardSnapshot(state: .failed, heartbeat: 0, expiresAt: 0, message: busy, messageAt: t0),
+ KeyboardSnapshot(state: .failed, heartbeat: 0, expiresAt: 0, message: busy),
+ ] {
+ let next = KeyboardViewModel()
+ show(stale, at: t0 + 3_600, on: next)
+ XCTAssertEqual(next.stagePhase, .idle)
+ XCTAssertNil(next.statusMessage)
+ }
+
+ // A note beside a live session ("Interrupted") goes the same way, the session staying.
+ let noted = KeyboardViewModel()
+ show(KeyboardSnapshot(state: .ready, heartbeat: t0 + 1, expiresAt: t0 + 900, message: "Interrupted", messageAt: t0), at: t0 + 1, on: noted)
+ XCTAssertEqual(noted.statusMessage, "Interrupted")
+ show(KeyboardSnapshot(state: .ready, heartbeat: t0 + 9, expiresAt: t0 + 900, message: "Interrupted", messageAt: t0), at: t0 + 9, on: noted)
+ XCTAssertNil(noted.statusMessage)
+ XCTAssertTrue(noted.dictatesThroughHex)
+
+ // The keyboard's own failure keeps its snapshot live for its 6 s, which is no session:
+ // the full keyboard's Hex key tries again in the keyboard instead of sending to no one.
+ let own = KeyboardViewModel()
+ show(
+ KeyboardSnapshot(state: .failed, heartbeat: t0 + 1, expiresAt: t0 + 6, message: busy, messageAt: t0 + 1),
+ at: t0 + 1,
+ on: own
+ )
+ XCTAssertEqual(own.stagePhase, .failed)
+ XCTAssertTrue(own.isWarmAvailable)
+ XCTAssertFalse(own.dictatesThroughHex)
+
+ let encoded = KeyboardSnapshot(state: .failed, heartbeat: t0, expiresAt: 0, message: busy, messageAt: t0)
+ XCTAssertEqual(
+ try JSONDecoder().decode(KeyboardSnapshot.self, from: JSONEncoder().encode(encoded)).messageAt,
+ t0
+ )
+ }
+
/// The same keyboard's own dictation: one still recording or transcribing when the keyboard
/// was dismissed finishes without it, goes back into its field when the keyboard returns
/// there while it's fresh, and is offered as Paste dictation anywhere else or later. A
diff --git a/ios/HEXShared/FullKeyboard/FullKeyboardView.swift b/ios/HEXShared/FullKeyboard/FullKeyboardView.swift
index bf0437199..8e11f5163 100644
--- a/ios/HEXShared/FullKeyboard/FullKeyboardView.swift
+++ b/ios/HEXShared/FullKeyboard/FullKeyboardView.swift
@@ -690,7 +690,7 @@ final class FullKeyboardView: UIView {
}
private func refreshFromModel() {
- hexKey.isWarm = model.hasFullAccess && model.isWarmAvailable
+ hexKey.isWarm = model.dictatesThroughHex
hexKey.translatingTo = model.translatingTo
if engine.showsGlobe != model.needsGlobeKey {
engine.showsGlobe = model.needsGlobeKey
diff --git a/ios/HEXShared/KeyboardBridge.swift b/ios/HEXShared/KeyboardBridge.swift
index 33c7934e6..b887669cc 100644
--- a/ios/HEXShared/KeyboardBridge.swift
+++ b/ios/HEXShared/KeyboardBridge.swift
@@ -201,6 +201,9 @@ nonisolated struct KeyboardSnapshot: Codable, Equatable, Sendable {
/// Why `transcript` went in as spoken while Translate what I say was on: the keyboard
/// says so as it inserts it.
let translationFailure: TranslationFailure?
+ /// When `message` was written (wall clock). The keyboard says it only for
+ /// `messageShownFor` after that; nil (an earlier build's) reads as long past.
+ let messageAt: TimeInterval?
init(
state: KeyboardDictationState,
@@ -216,7 +219,8 @@ nonisolated struct KeyboardSnapshot: Codable, Equatable, Sendable {
message: String? = nil,
heard: String? = nil,
stages: DictationStages? = nil,
- translationFailure: TranslationFailure? = nil
+ translationFailure: TranslationFailure? = nil,
+ messageAt: TimeInterval? = nil
) {
self.state = state
self.heartbeat = heartbeat
@@ -232,6 +236,7 @@ nonisolated struct KeyboardSnapshot: Codable, Equatable, Sendable {
self.heard = heard
self.stages = stages
self.translationFailure = translationFailure
+ self.messageAt = messageAt
}
var effectiveLoudnessDB: Float {
@@ -304,6 +309,43 @@ nonisolated struct KeyboardSnapshot: Codable, Equatable, Sendable {
message: message,
heard: heard,
stages: stages,
+ translationFailure: translationFailure,
+ messageAt: messageAt
+ )
+ }
+
+ /// How long the keyboard says why Hex's last dictation failed ("Another app is using the
+ /// microphone. Try again."), a little longer than its own failures show.
+ static let messageShownFor: TimeInterval = 8
+
+ /// A message from more than `messageShownFor` ago, or with no time. Hex writes the
+ /// snapshot only while it runs, so a failure stayed in the App Group after Hex was
+ /// suspended: every keyboard after showed it, and the full keyboard's Hex key sent its
+ /// dictations to a Hex still marked failed, which ignored them, though dictation itself
+ /// worked (TestFlight: "showing for me all the time although it's working fine").
+ func hasStaleMessage(at now: TimeInterval) -> Bool {
+ guard message != nil || state == .failed else { return false }
+ guard let messageAt else { return true }
+ return now - messageAt > Self.messageShownFor
+ }
+
+ /// The snapshot as the keyboard shows it: a stale failure reads as offline and a stale
+ /// message is dropped. Whether a warm session is up, and any result, stay as they are.
+ func droppingStaleMessage(at now: TimeInterval) -> KeyboardSnapshot {
+ guard hasStaleMessage(at: now) else { return self }
+ return KeyboardSnapshot(
+ state: state == .failed ? .offline : state,
+ heartbeat: heartbeat,
+ expiresAt: expiresAt,
+ recordingStartedAt: recordingStartedAt,
+ audioLevel: audioLevel,
+ loudnessDB: loudnessDB,
+ jobID: jobID,
+ resultID: resultID,
+ resultExpiresAt: resultExpiresAt,
+ transcript: transcript,
+ heard: heard,
+ stages: stages,
translationFailure: translationFailure
)
}
diff --git a/ios/HEXShared/KeyboardSurface+ViewModel.swift b/ios/HEXShared/KeyboardSurface+ViewModel.swift
index 25b49de47..79f1d3fdd 100644
--- a/ios/HEXShared/KeyboardSurface+ViewModel.swift
+++ b/ios/HEXShared/KeyboardSurface+ViewModel.swift
@@ -40,6 +40,12 @@ final class KeyboardViewModel: ObservableObject {
@Published private(set) var isWarmAvailable = false
@Published private(set) var stagePhase: StagePhase = .idle
@Published private(set) var statusMessage: String?
+ /// A tap on Dictate or the Hex key records through Hex's warm session. Not while the last
+ /// dictation shows as failed: the keyboard's own failure keeps its snapshot live for a few
+ /// seconds, which isn't a session, and the bar already dictates in the keyboard then.
+ var dictatesThroughHex: Bool {
+ hasFullAccess && isWarmAvailable && stagePhase != .failed
+ }
@Published var style: DictationStyle = .polished
@Published var activeTray: PickerTray = .none
/// Settings › Dictation language (nil for Automatic), what Automatic resolves to, and the
@@ -206,7 +212,8 @@ final class KeyboardViewModel: ObservableObject {
// A job Hex stopped heartbeating for is over (Hex ended, crashed or was suspended
// mid-dictation): without this its last snapshot held every keyboard in recording or
// transcribing for good, with Done and Cancel sent to no one.
- let newSnapshot = newSnapshot.droppingOrphanedJob(at: now)
+ // A failure Hex wrote a while ago is over too (`KeyboardSnapshot.hasStaleMessage`).
+ let newSnapshot = newSnapshot.droppingOrphanedJob(at: now).droppingStaleMessage(at: now)
currentTimestamp = now
currentSnapshot = newSnapshot
liveLoudnessDB = newSnapshot.state == .recording ? newSnapshot.effectiveLoudnessDB : -62.0
diff --git a/ios/README.md b/ios/README.md
index 3bb5c6ddb..e726a75c9 100644
--- a/ios/README.md
+++ b/ios/README.md
@@ -1095,7 +1095,11 @@ never at a launch by itself.
(`KeyboardSnapshot.isOrphanedJob`): a Hex that ended, crashed or was
suspended mid-job used to hold every keyboard in recording for good. A
result that missed its 30 s to go in by itself stays offered in the full
- keyboard's strip as **Paste dictation** for 10 minutes.
+ keyboard's strip as **Paste dictation** for 10 minutes. A failure or note
+ from Hex ("Another app is using the microphone. Try again.") shows for 8 s
+ after Hex wrote it (`KeyboardSnapshot.messageAt`), not for as long as it
+ stays in the App Group; meanwhile both keyboards dictate in the keyboard, and
+ after it a warm session's next start is Hex's Try again.
- An in-keyboard dictation the keyboard is dismissed from (or another app
comes forward) stops its capture at once, keeps what was said, and
finishes without the keyboard; its text is held in the App Group
From ee31663e2d9a6735b0ee2c5a4806eba9c89899f4 Mon Sep 17 00:00:00 2001
From: Ryan Vogel <89211796+R44VC0RP@users.noreply.github.com>
Date: Sat, 21 Nov 2026 09:38:12 -0500
Subject: [PATCH 2/4] iOS keyboard: the Hex key's audio session calls leave the
main thread
X tester @leeladhar (iPhone 16 Pro Max, build 56 or earlier): "the keyboard freezes
while using apps sometimes ... the only way to close the keyboard is to close the
app completely".
Not reproduced in the iOS 27.2 simulator on build 56 (2ef08aa4) or origin/dev: 24
host visits (Messages, Contacts, Settings search) with typing, the Hex key and globe
switches kept one keyboard process, no crash or hang report, footprint 29-40 MB with
no growth. The device cause isn't confirmed. The likeliest one left on origin/dev is
the last synchronous audio server round trips on the keyboard's main thread: every
Hex key touch-down set the session's category, read and preferred its inputs
(KeyboardLocalDictation.microphone(for:)), asked isOtherAudioPlaying, and read the
Speech Recognition status for the probe log, all before the page could start. These
wait on the audio server (and TCC) with no timeout; a build 56 crash report from
TestFlight shows Hex's own main thread parked in such a call (setPreferredSampleRate
waiting on SetBatchProperties). #2185 moved the one on the launch path for the same
reason. While blocked, no key, Done or globe responds, and the keyboard can't
dismiss.
Now begin() runs those on the keyboard's audio session queue (after any earlier
capture's release there) and starts the page when they return, unless the job was
dropped, stopped or replaced meanwhile. Done before the page was asked to start
ends quietly at once instead of leaving the page to start a capture nobody wants.
Verified: builds clean; in an iPhone 17 Pro iOS 27.2 simulator (WebKit's mock audio
device) the Hex key's touch-down capture still starts and is kept: "first audio
+124 ms after touch-down (... getUserMedia +83 ...)", 13.5 s recorded as 643,072
frames at 48 kHz (no gap), then transcription as before.
---
ios/HEXKeyboard/KeyboardLocalDictation.swift | 46 +++++++++++++++-----
1 file changed, 36 insertions(+), 10 deletions(-)
diff --git a/ios/HEXKeyboard/KeyboardLocalDictation.swift b/ios/HEXKeyboard/KeyboardLocalDictation.swift
index ca353e5ab..2291947cc 100644
--- a/ios/HEXKeyboard/KeyboardLocalDictation.swift
+++ b/ios/HEXKeyboard/KeyboardLocalDictation.swift
@@ -57,6 +57,8 @@ final class KeyboardLocalDictation {
private var armedFailure: (message: String, detail: String?)?
/// Another app played audio as this capture began, which WebKit's capture interrupts.
private var otherAudioAtStart = false
+ /// The job whose page start waits on the audio session's queue (`begin`).
+ private var startPending: String?
/// After a capture ends, long enough for WebKit to have let its audio session go.
static let resumeOtherAudioDelay: TimeInterval = 0.5
private static let audioSessionQueue = DispatchQueue(label: "anomaly.hex.keyboard.audio-session", qos: .userInitiated)
@@ -283,8 +285,8 @@ final class KeyboardLocalDictation {
private func begin(in host: UIView, microphone: MicrophoneChoice, pressedAt: TimeInterval?, armed: Bool) -> String? {
let log = KeyboardProbeLog.shared
let appAllowed = KeyboardBridge().appMicrophoneAllowed
- log.write("start(web\(armed ? ", touch-down" : "")): mic=\(Self.describe(AVAudioApplication.shared.recordPermission)) app=\(appAllowed ? "granted" : "no") speech=\(Self.describe(SFSpeechRecognizer.authorizationStatus())) \(KeyboardProbeLog.memory())")
guard Self.microphoneAllowed else {
+ log.write("start(web\(armed ? ", touch-down" : "")): mic=\(Self.describe(AVAudioApplication.shared.recordPermission)) app=no")
fail("Allow the microphone in Hex first")
return nil
}
@@ -307,10 +309,28 @@ final class KeyboardLocalDictation {
capture.onEvent = { [weak self] event in
self?.handle(event, jobID: jobID, requestedAt: requestedAt)
}
- let input = Self.microphone(for: microphone)
- otherAudioAtStart = otherAudioAtStart || AVAudioSession.sharedInstance().isOtherAudioPlaying
- // Without a name to ask for, every choice but Bluetooth asks for the iPhone's own.
- capture.start(microphone: input?.name, builtInFallback: microphone != .bluetooth)
+ // The audio session's calls are round trips to the audio server, which can be slow to
+ // answer (a call, another app's audio, a capture that just ended): on the main thread
+ // they held every key until it did. They run on the session's queue, after any earlier
+ // capture's release there, and the page starts once they're back, unless the job was
+ // dropped, stopped or replaced meanwhile.
+ startPending = jobID
+ let label = armed ? ", touch-down" : ""
+ Self.audioSessionQueue.async { [weak self, weak capture] in
+ log.write("start(web\(label)): mic=\(Self.describe(AVAudioApplication.shared.recordPermission)) app=\(appAllowed ? "granted" : "no") speech=\(Self.describe(SFSpeechRecognizer.authorizationStatus())) \(KeyboardProbeLog.memory())")
+ let input = Self.microphone(for: microphone)
+ let otherAudio = AVAudioSession.sharedInstance().isOtherAudioPlaying
+ DispatchQueue.main.async {
+ MainActor.assumeIsolated {
+ guard let self, let capture, self.capture === capture,
+ self.startPending == jobID, self.isCapturing(jobID) else { return }
+ self.startPending = nil
+ self.otherAudioAtStart = self.otherAudioAtStart || otherAudio
+ // Without a name to ask for, every choice but Bluetooth asks for the iPhone's own.
+ capture.start(microphone: input?.name, builtInFallback: microphone != .bluetooth)
+ }
+ }
+ }
return jobID
}
@@ -351,7 +371,7 @@ final class KeyboardLocalDictation {
/// Bluetooth hands-free inputs are listed only there. WebKit records in its own session,
/// which only getUserMedia's `deviceId` steers (WebKit then prefers that input), so the
/// page asks for this input by its name, the label WebKit gives it.
- private static func microphone(for choice: MicrophoneChoice) -> Microphone? {
+ nonisolated private static func microphone(for choice: MicrophoneChoice) -> Microphone? {
let session = AVAudioSession.sharedInstance()
do {
try session.setCategory(
@@ -442,7 +462,13 @@ final class KeyboardLocalDictation {
}
func stop() {
- guard case .recording = phase else { return }
+ guard case .recording(let jobID, _) = phase else { return }
+ guard startPending != jobID else {
+ // Done before the page was even asked to start: nothing was recorded.
+ startPending = nil
+ finishRecording(jobID: jobID)
+ return
+ }
capture?.stop()
}
@@ -641,12 +667,12 @@ final class KeyboardLocalDictation {
phase = .idle
}
- private static func describe(_ error: Error) -> String {
+ nonisolated private static func describe(_ error: Error) -> String {
let ns = error as NSError
return "\(ns.domain) \(ns.code) \(ns.localizedDescription)"
}
- private static func describe(_ permission: AVAudioApplication.recordPermission) -> String {
+ nonisolated private static func describe(_ permission: AVAudioApplication.recordPermission) -> String {
switch permission {
case .granted: "granted"
case .denied: "denied"
@@ -655,7 +681,7 @@ final class KeyboardLocalDictation {
}
}
- private static func describe(_ status: SFSpeechRecognizerAuthorizationStatus) -> String {
+ nonisolated private static func describe(_ status: SFSpeechRecognizerAuthorizationStatus) -> String {
switch status {
case .authorized: "authorized"
case .denied: "denied"
From b035d6b2d0152aafb2bde12956adbf41d462dc22 Mon Sep 17 00:00:00 2001
From: Ryan Vogel <89211796+R44VC0RP@users.noreply.github.com>
Date: Sat, 21 Nov 2026 09:46:53 -0500
Subject: [PATCH 3/4] iOS keyboard: the capture page resumes an audio context
iOS suspended mid-dictation
TestFlight tester on X (iPhone 13 Pro Max, iOS 27, build 56 or earlier): "in progress
recording dropped when I took a screenshot".
What a screenshot does to an in-keyboard dictation, in an iPhone 17 Pro iOS 27.2
simulator on the Mac Studio (WebKit's mock audio device records there):
- The host only going inactive doesn't stop it: with Control Center pulled down for
4 s mid-dictation the page kept sending audio (700,416 frames at 48 kHz over a
14.8 s recording, no gap).
- A screenshot taken with Cmd-Shift-3 hid the keyboard and the probe log said
"keyboard hidden while recording: finishing with what was said" at the shutter.
The simulator can only take one through its hardware keyboard, which also turns
on keyboard minimization, so it doesn't show which of the two hid it. On an
iPhone, iOS 26 and 27 open the screenshot's full-screen preview over the app, and
a keyboard that hides is torn down and its WebKit processes suspended, so a
dictation can't keep recording through that.
- On build 56 a keyboard hidden mid-recording lost the dictation:
viewWillDisappear only dropped a touch-down capture, and the recording went with
the view controller (reproduced in #2163). Since #2163 on origin/dev it stops,
keeps what was said, and the text goes into the same field when the keyboard
comes back within 30 s, else is offered as Paste dictation.
The one way left for a capture to end early with the keyboard still up is the
page's audio context: iOS may suspend or interrupt it for other audio and the page
never resumed it, so no more audio came and the 6 s watchdog ended the dictation.
The page now resumes a context that isn't running every 250 ms while it records,
and stops trying when the capture stops.
Not changed: a keyboard hidden by a screenshot's preview still ends the recording
(with its text kept, as above).
Verified: builds clean; in the simulator the touch-down capture still starts and
records through Control Center with no gap (numbers above).
---
ios/HEXKeyboard/KeyboardLocalDictation.swift | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/ios/HEXKeyboard/KeyboardLocalDictation.swift b/ios/HEXKeyboard/KeyboardLocalDictation.swift
index 2291947cc..7ae973046 100644
--- a/ios/HEXKeyboard/KeyboardLocalDictation.swift
+++ b/ios/HEXKeyboard/KeyboardLocalDictation.swift
@@ -925,7 +925,7 @@ private final class KeyboardWebCapture: NSObject, WKNavigationDelegate, WKUIDele
private static let page = """