Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 45 additions & 12 deletions ios/HEXKeyboard/KeyboardLocalDictation.swift
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ final class KeyboardLocalDictation {
private var armedFailure: (message: String, detail: String?)?
/// Another app played audio as this capture began, which WebKit's capture interrupts.
private var otherAudioAtStart = false
/// The job whose page start waits on the audio session's queue (`begin`).
private var startPending: String?
/// After a capture ends, long enough for WebKit to have let its audio session go.
static let resumeOtherAudioDelay: TimeInterval = 0.5
private static let audioSessionQueue = DispatchQueue(label: "anomaly.hex.keyboard.audio-session", qos: .userInitiated)
Expand Down Expand Up @@ -152,7 +154,7 @@ final class KeyboardLocalDictation {
phase = .idle
return nil
}
return KeyboardSnapshot(state: .failed, heartbeat: now, expiresAt: until, message: message)
return KeyboardSnapshot(state: .failed, heartbeat: now, expiresAt: until, message: message, messageAt: now)
}
}

Expand Down Expand Up @@ -283,8 +285,8 @@ final class KeyboardLocalDictation {
private func begin(in host: UIView, microphone: MicrophoneChoice, pressedAt: TimeInterval?, armed: Bool) -> String? {
let log = KeyboardProbeLog.shared
let appAllowed = KeyboardBridge().appMicrophoneAllowed
log.write("start(web\(armed ? ", touch-down" : "")): mic=\(Self.describe(AVAudioApplication.shared.recordPermission)) app=\(appAllowed ? "granted" : "no") speech=\(Self.describe(SFSpeechRecognizer.authorizationStatus())) \(KeyboardProbeLog.memory())")
guard Self.microphoneAllowed else {
log.write("start(web\(armed ? ", touch-down" : "")): mic=\(Self.describe(AVAudioApplication.shared.recordPermission)) app=no")
fail("Allow the microphone in Hex first")
return nil
}
Expand All @@ -307,10 +309,28 @@ final class KeyboardLocalDictation {
capture.onEvent = { [weak self] event in
self?.handle(event, jobID: jobID, requestedAt: requestedAt)
}
let input = Self.microphone(for: microphone)
otherAudioAtStart = otherAudioAtStart || AVAudioSession.sharedInstance().isOtherAudioPlaying
// Without a name to ask for, every choice but Bluetooth asks for the iPhone's own.
capture.start(microphone: input?.name, builtInFallback: microphone != .bluetooth)
// The audio session's calls are round trips to the audio server, which can be slow to
// answer (a call, another app's audio, a capture that just ended): on the main thread
// they held every key until it did. They run on the session's queue, after any earlier
// capture's release there, and the page starts once they're back, unless the job was
// dropped, stopped or replaced meanwhile.
startPending = jobID
let label = armed ? ", touch-down" : ""
Self.audioSessionQueue.async { [weak self, weak capture] in
log.write("start(web\(label)): mic=\(Self.describe(AVAudioApplication.shared.recordPermission)) app=\(appAllowed ? "granted" : "no") speech=\(Self.describe(SFSpeechRecognizer.authorizationStatus())) \(KeyboardProbeLog.memory())")
let input = Self.microphone(for: microphone)
let otherAudio = AVAudioSession.sharedInstance().isOtherAudioPlaying
DispatchQueue.main.async {
MainActor.assumeIsolated {
guard let self, let capture, self.capture === capture,
self.startPending == jobID, self.isCapturing(jobID) else { return }
self.startPending = nil
self.otherAudioAtStart = self.otherAudioAtStart || otherAudio
// Without a name to ask for, every choice but Bluetooth asks for the iPhone's own.
capture.start(microphone: input?.name, builtInFallback: microphone != .bluetooth)
}
}
}
return jobID
}

Expand Down Expand Up @@ -351,7 +371,7 @@ final class KeyboardLocalDictation {
/// Bluetooth hands-free inputs are listed only there. WebKit records in its own session,
/// which only getUserMedia's `deviceId` steers (WebKit then prefers that input), so the
/// page asks for this input by its name, the label WebKit gives it.
private static func microphone(for choice: MicrophoneChoice) -> Microphone? {
nonisolated private static func microphone(for choice: MicrophoneChoice) -> Microphone? {
let session = AVAudioSession.sharedInstance()
do {
try session.setCategory(
Expand Down Expand Up @@ -442,7 +462,13 @@ final class KeyboardLocalDictation {
}

func stop() {
guard case .recording = phase else { return }
guard case .recording(let jobID, _) = phase else { return }
guard startPending != jobID else {
// Done before the page was even asked to start: nothing was recorded.
startPending = nil
finishRecording(jobID: jobID)
return
}
capture?.stop()
}

Expand Down Expand Up @@ -641,12 +667,12 @@ final class KeyboardLocalDictation {
phase = .idle
}

private static func describe(_ error: Error) -> String {
nonisolated private static func describe(_ error: Error) -> String {
let ns = error as NSError
return "\(ns.domain) \(ns.code) \(ns.localizedDescription)"
}

private static func describe(_ permission: AVAudioApplication.recordPermission) -> String {
nonisolated private static func describe(_ permission: AVAudioApplication.recordPermission) -> String {
switch permission {
case .granted: "granted"
case .denied: "denied"
Expand All @@ -655,7 +681,7 @@ final class KeyboardLocalDictation {
}
}

private static func describe(_ status: SFSpeechRecognizerAuthorizationStatus) -> String {
nonisolated private static func describe(_ status: SFSpeechRecognizerAuthorizationStatus) -> String {
switch status {
case .authorized: "authorized"
case .denied: "denied"
Expand Down Expand Up @@ -899,7 +925,7 @@ private final class KeyboardWebCapture: NSObject, WKNavigationDelegate, WKUIDele

private static let page = """
<!doctype html><meta charset="utf-8"><script>
let ctx, stream, source, node, running = false, generation = 0;
let ctx, stream, source, node, running = false, generation = 0, revive = null;
const post = (m) => webkit.messageHandlers.hex.postMessage(m);
const release = (s, c) => {
if (s) s.getTracks().forEach((t) => t.stop());
Expand Down Expand Up @@ -964,6 +990,12 @@ private final class KeyboardWebCapture: NSObject, WKNavigationDelegate, WKUIDele
};
source.connect(node);
node.connect(ctx.destination);
// iOS may suspend or interrupt a running context for other audio (an alert, a call's
// ring, another app's session) without resuming it: the page then sends no audio and the
// keyboard's watchdog ends the dictation 6 s later. Resume it as soon as iOS allows.
revive = setInterval(() => {
if (running && ctx && ctx.state !== "running" && ctx.state !== "closed") ctx.resume().catch(() => {});
}, 250);
post({ type: "started", rate: ctx.sampleRate, device: stream.getAudioTracks()[0].label, listed, opened, mic, audio: Date.now() });
} catch (error) {
if (stale()) return release(s, c);
Expand All @@ -977,6 +1009,7 @@ private final class KeyboardWebCapture: NSObject, WKNavigationDelegate, WKUIDele
generation++;
const wasRunning = running;
running = false;
if (revive) { clearInterval(revive); revive = null; }
try { node && node.disconnect(); source && source.disconnect(); } catch (e) {}
if (stream) stream.getTracks().forEach((t) => t.stop());
if (ctx) { try { await ctx.close(); } catch (e) {} }
Expand Down
2 changes: 1 addition & 1 deletion ios/HEXKeyboard/KeyboardViewController.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1181,7 +1181,7 @@ final class KeyboardViewController: UIInputViewController {
/// the Hex key's own choice of in-keyboard dictation; a warm Hex session records instead.
private func hexKeyPressed(at time: TimeInterval) {
hexKeyPressedAt = time
guard viewModel.hasFullAccess, !viewModel.isWarmAvailable, viewModel.onLocalDictateTap != nil,
guard viewModel.hasFullAccess, !viewModel.dictatesThroughHex, viewModel.onLocalDictateTap != nil,
viewModel.stagePhase == .idle || viewModel.stagePhase == .failed else { return }
localDictation.arm(in: view, microphone: bridge.microphone, pressedAt: time)
}
Expand Down
25 changes: 22 additions & 3 deletions ios/HEXMobile/DictationController.swift
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,9 @@ final class DictationController {
private var keyboardResultTranslationFailure: TranslationFailure?
/// Record typing logs: each step of the result, for the keyboard's log; nil while off.
private var keyboardResultStages: DictationStages?
private var keyboardMessage: String?
private var keyboardMessage: String? {
didSet { keyboardMessageAt = keyboardMessage == nil ? nil : now() }
}
private var pendingHandoff: PendingKeyboardHandoff?
var keyboardSessionExpiresAt: TimeInterval = 0
private var suspendedWarmSessionOnInterruption = false
Expand All @@ -129,6 +131,8 @@ final class DictationController {

var phase = Phase.modelRequired {
didSet {
captureFailed = false
if case .failed = phase { failedAt = now() }
guard phase.keepsScreenAwake != oldValue.keepsScreenAwake else { return }
// Only applies while Hex is in front; nothing else in Hex sets it, so ending a
// dictation hands the screen back to Auto-Lock.
Expand Down Expand Up @@ -167,6 +171,13 @@ final class DictationController {
}
@ObservationIgnored private var lastPublishedSnapshotSignature: String?
@ObservationIgnored private var lastPublishedSnapshotAt: TimeInterval = -100
/// When the phase last became failed, and when `keyboardMessage` was last set: the
/// keyboard shows a message only for a few seconds after (`KeyboardSnapshot.messageAt`).
@ObservationIgnored private var failedAt: TimeInterval?
@ObservationIgnored private var keyboardMessageAt: TimeInterval?
/// The failure on screen is a microphone that didn't start for a keyboard's dictation, with
/// the model still loaded: the keyboard's next Hex key tries again.
@ObservationIgnored private var captureFailed = false
var warmDuration: WarmSessionDuration
private(set) var style: DictationStyle
private(set) var microphone: MicrophoneChoice
Expand Down Expand Up @@ -876,6 +887,7 @@ final class DictationController {
let message = UserFacingError.message(for: error, context: .microphone)
keyboardMessage = keyboardJobID == nil ? nil : message
phase = .failed(message)
captureFailed = isModelReady
restoreAudioAfterCapture()
publishKeyboardSnapshot()
}
Expand Down Expand Up @@ -1190,7 +1202,10 @@ final class DictationController {

switch command.kind {
case .start:
guard phase == .ready, activeKeyboardJobID == nil else { return }
// A warm session whose microphone failed for the last dictation stays marked failed:
// the keyboard's next Hex key is its Try again. Without this Hex ignored every
// start until it was opened, while the keyboard kept sending them.
guard phase == .ready || captureFailed, activeKeyboardJobID == nil else { return }
setStyle(command.style)
startRecording(
keyboardJobID: command.jobID,
Expand Down Expand Up @@ -1240,10 +1255,13 @@ final class DictationController {
}

let failureMessage: String?
let messageAt: TimeInterval?
if case .failed(let message) = phase {
failureMessage = message
messageAt = failedAt
} else {
failureMessage = keyboardMessage
messageAt = keyboardMessageAt
}

let signature = "\(state.rawValue)|\(activeKeyboardJobID ?? "")|\(keyboardResultID ?? "")|\(failureMessage ?? "")"
Expand Down Expand Up @@ -1277,7 +1295,8 @@ final class DictationController {
message: failureMessage,
heard: keyboardResultHeard,
stages: keyboardResultStages,
translationFailure: keyboardResultTranslationFailure
translationFailure: keyboardResultTranslationFailure,
messageAt: failureMessage == nil ? nil : messageAt
)
)
}
Expand Down
40 changes: 40 additions & 0 deletions ios/HEXMobileTests/CleanupTaggerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,46 @@ final class CleanupTaggerTests: XCTestCase {
XCTAssertEqual(english, "Let's ship it Thursday.")
}

/// A count said as a microphone test keeps every number and reads the same whichever speech
/// model heard it: Apple's SpeechTranscriber writes "Test one, 2, 3.", DictationTranscriber
/// "Test 123" and Parakeet "Test 1, 2, 3" (macOS 27.2, `say` voices). The tagger alone is
/// sure every count word but the last is a self-correction, which made "Test three." in
/// in-keyboard dictation before `CleanupGuard` (TestFlight, Nov 20).
func testCountsKeepEveryNumberFromEverySpeechModel() async throws {
let suite = "hex.tests.\(UUID().uuidString)"
let defaults = try XCTUnwrap(UserDefaults(suiteName: suite))
defer { defaults.removePersistentDomain(forName: suite) }
let settings = CleanupTaggerSettings(defaults: defaults)
let tagger = CleanupTagger()

let words = CleanupTagger.words(in: "Test one two three.")
let alone = try tagger.deleteProbabilities(for: words).map { $0 > CleanupTagger.deleteThreshold }
XCTAssertEqual(CleanupReconstruction.reconstruct(words, deletes: alone), "Test three.", "the tagger without its guard")
XCTAssertEqual(try tagger.clean("Test one two three."), "Test one two three.", "the guard keeps a count in words")

for enabled in [true, false] {
settings.isEnabled = enabled
for (heard, cleaned) in [
("Test one, 2, 3.", "Test 1, 2, 3."),
("Test one two three.", "Test 1, 2, 3."),
("Test, one, two, three.", "Test, 1, 2, 3."),
("Test 123", "Test 1, 2, 3"),
("Test 1, 2, 3", "Test 1, 2, 3"),
("Testing 123", "Testing 1, 2, 3"),
("Counting one, two, three, four.", "Counting 1, 2, 3, 4."),
("One, two, three, testing.", "1, 2, 3, testing."),
("1, 2, 3, testing", "1, 2, 3, testing"),
] {
let result = await CleanupTagger.cleanIfEnabled(
heard, style: .polished, language: "en-US", settings: settings, tagger: tagger
)
XCTAssertEqual(result, cleaned, "\(heard), cleanup \(enabled)")
}
}
let verbatim = await CleanupTagger.cleanIfEnabled("Test one, 2, 3.", style: .verbatim, settings: settings, tagger: tagger)
XCTAssertEqual(verbatim, "Test one, 2, 3.")
}

func testDictationStartPreparesOnlyWhenOnAndNotForVerbatim() throws {
let suite = "hex.tests.\(UUID().uuidString)"
let defaults = try XCTUnwrap(UserDefaults(suiteName: suite))
Expand Down
Loading