Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions Sources/HeardCore/AppModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ public final class AppModel: ObservableObject {
// Tracks whether the push-to-talk key is currently held. Set on press, cleared on release.
// Used to detect key-up events that arrive before model loading completes.
private var pushToTalkKeyHeld = false
// Where keyboard focus was when the current dictation session started.
// The transcript is pasted back into this field, not wherever focus sits
// once transcription finishes.
var dictationFocusTarget: TextInjector.FocusTarget?

public let settingsStore: SettingsStore
public let speakerStore: SpeakerStore
Expand Down Expand Up @@ -125,9 +129,10 @@ public final class AppModel: ObservableObject {

model.pipelineProcessor.runNextIfNeeded()

// Wire dictation: text injection on finalized utterances
dictationManager.onUtterance = { text in
TextInjector.inject(text)
// Wire dictation: text injection on finalized utterances, targeting the
// field that was focused when the session started.
dictationManager.onUtterance = { [weak model] text in
TextInjector.inject(text, restoringFocusTo: model?.dictationFocusTarget)
}
// Surface silent "no speech captured" stops so a dropped/stalled mic
// isn't an invisible failure. Cleared on the next dictation start.
Expand Down Expand Up @@ -333,6 +338,12 @@ public var filteredSpeakers: [SpeakerProfile] {
DebugFileLog.log("toggleDictation dropped — meeting active and not currently dictating")
return
}
// Snapshot the focused field synchronously at trigger time — model
// loading inside the task can take seconds, during which the user may
// click elsewhere, and the transcript must land where they started.
if !isDictating {
dictationFocusTarget = TextInjector.captureFocusTarget()
}
dictationToggleInFlight = true
Task {
defer {
Expand Down
3 changes: 2 additions & 1 deletion Sources/HeardCore/DictationManager.swift
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ public final class DictationManager: ObservableObject {
private static let maxBufferSamples = 4 * 3600 * 16_000

/// Called when transcription completes for a session with the final text.
public var onUtterance: ((String) -> Void)?
/// Runs on the main actor.
public var onUtterance: (@MainActor (String) -> Void)?

/// Called when a stop produced no usable text despite the user having
/// listened for a non-trivial duration — i.e. the mic captured silence or
Expand Down
92 changes: 89 additions & 3 deletions Sources/HeardCore/TextInjector.swift
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,48 @@ import Foundation
/// Injects text into the focused text field of any app using CGEvent unicode insertion.
public enum TextInjector {

/// Snapshot of where keyboard focus was when dictation started: the owning
/// app's pid plus the focused AX element (when readable). Captured on start
/// and used at paste time so the transcript lands in the field the user was
/// dictating into, even if they switched apps or fields while speaking.
public struct FocusTarget {
let pid: pid_t
let element: AXUIElement?
}

/// Capture the currently focused UI element and its owning app. Returns nil
/// when focus belongs to Heard itself (e.g. dictation toggled from the menu
/// bar panel) or nothing useful can be read — injection then falls back to
/// pasting at whatever is focused at paste time, matching old behavior.
public static func captureFocusTarget() -> FocusTarget? {
var element: AXUIElement?
let sysWide = AXUIElementCreateSystemWide()
var focused: AnyObject?
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focused) == .success,
let focused, CFGetTypeID(focused) == AXUIElementGetTypeID() {
element = (focused as! AXUIElement)
}

// Prefer the pid of the focused element itself; fall back to the
// frontmost app when the element (or its pid) is unreadable.
var pid: pid_t = 0
if let element, AXUIElementGetPid(element, &pid) == .success, pid > 0 {
// pid resolved from the focused element
} else if let front = NSWorkspace.shared.frontmostApplication {
pid = front.processIdentifier
} else {
DebugFileLog.log("TextInjector.captureFocusTarget failed — no focused element and no frontmost app")
return nil
}

guard pid != ProcessInfo.processInfo.processIdentifier else {
DebugFileLog.log("TextInjector.captureFocusTarget skipped — focus is on Heard itself")
return nil
}
DebugFileLog.log("TextInjector.captureFocusTarget pid=\(pid) element=\(element != nil)")
return FocusTarget(pid: pid, element: element)
}

/// Check and prompt for Accessibility permission (needed for text injection).
/// Call this when enabling dictation so the user gets the prompt early.
@discardableResult
Expand All @@ -20,15 +62,16 @@ public enum TextInjector {
return AXIsProcessTrustedWithOptions(options)
}

/// Inject text into the currently focused app via clipboard paste.
/// Inject text via clipboard paste, first restoring focus to `target` (the
/// field the user was in when dictation started) when one was captured.
///
/// We previously had a "short text < 50 chars uses CGEvent unicode insertion"
/// fast path to avoid touching the clipboard, but observed it dropping events
/// silently for short transcripts in Code/Electron apps even at 1–2 chunks.
/// CGEvent.postToPid returned success and the events never landed. Clipboard
/// paste is one Cmd+V regardless of length and works reliably; the original
/// pasteboard contents are restored after a short delay.
public static func inject(_ text: String) {
public static func inject(_ text: String, restoringFocusTo target: FocusTarget? = nil) {
var trusted = AXIsProcessTrusted()
if !trusted {
// Live fallback: AXIsProcessTrusted() returns a stale cached false on macOS 15+
Expand All @@ -48,7 +91,50 @@ public enum TextInjector {
DebugFileLog.log("TextInjector injecting into frontApp=\(frontApp.localizedName ?? "?") pid=\(frontApp.processIdentifier)")
}

insertViaClipboard(text)
if let target, restoreFocus(to: target) {
// App activation and AX focus changes take a beat to land; pasting
// immediately would send Cmd+V to the previously focused app.
DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) {
insertViaClipboard(text)
}
} else {
insertViaClipboard(text)
}
}

// MARK: - Focus restoration

/// Bring the captured app/field back into keyboard focus. Returns true if
/// any focus change was issued (the caller should wait before pasting),
/// false if nothing needed to change or the target is gone.
private static func restoreFocus(to target: FocusTarget) -> Bool {
var changed = false

if NSWorkspace.shared.frontmostApplication?.processIdentifier != target.pid {
guard let app = NSRunningApplication(processIdentifier: target.pid), !app.isTerminated else {
DebugFileLog.log("TextInjector focus target pid=\(target.pid) no longer running — pasting at current focus")
return false
}
DebugFileLog.log("TextInjector re-activating focus target app=\(app.localizedName ?? "?") pid=\(target.pid)")
app.activate()
changed = true
}

if let element = target.element {
// Raise the element's window first — activating the app alone brings
// forward whatever window was last key, which may not be the one the
// user was dictating into.
var windowRef: AnyObject?
if AXUIElementCopyAttributeValue(element, kAXWindowAttribute as CFString, &windowRef) == .success,
let windowRef, CFGetTypeID(windowRef) == AXUIElementGetTypeID() {
AXUIElementPerformAction((windowRef as! AXUIElement), kAXRaiseAction as CFString)
}
let err = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
DebugFileLog.log("TextInjector restore AX focus err=\(err.rawValue)")
if err == .success { changed = true }
}

return changed
}

// MARK: - Clipboard Paste
Expand Down
2 changes: 2 additions & 0 deletions handoff.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ The app builds cleanly with `swift build` and runs as a menu bar app on macOS 15

**Dictation feature is fully functional** — speech recognition, text injection via clipboard paste (Cmd+V), and global hotkey (Ctrl+Shift+D) all working. Requires a stable (non-ad-hoc) code signing identity so Accessibility permission persists across rebuilds; `./scripts/bundle.sh` auto-signs with the available `Developer ID Application` cert (else `Dev Cert`), so a bare `./scripts/bundle.sh` is sufficient.

**Dictation paste targets the field focused at start (not at paste time):** `AppModel.toggleDictation` snapshots the focused AX element + owning app pid (`TextInjector.captureFocusTarget`) synchronously at trigger time; `TextInjector.inject(_:restoringFocusTo:)` re-activates that app, raises the element's window, and re-sets `kAXFocused` before sending Cmd+V (with a 0.2 s settle delay when focus had to change). If the user never moved focus nothing changes; if the capture fails, focus was on Heard itself, or the target app has quit, injection falls back to the old paste-at-current-focus behavior. `DictationManager.onUtterance` is now typed `@MainActor` (its handler reads main-actor state).

**In-meeting notes are fully functional** — global hotkey (Ctrl+Shift+N by default) opens a focused composer panel during recording; notes interleave chronologically into the rendered transcript as italicized `**Note from <userName>:**` lines. See "In-Meeting Notes" below.

**FluidAudio upgraded to 0.15.2** (from 0.14.7). Brings Parakeet v3 ASR throughput gains for free and exposes per-chunk speaker embeddings. Speaker assignment now builds a **duration-weighted, outlier-trimmed centroid per speaker** from `DiarizationResult.chunkEmbeddings` instead of the previous "first segment per speaker" embedding — more stable cross-meeting identity. The aggregation lives in pure, unit-tested code (`SpeakerEmbeddingAggregator` in `SpeakerAssignment.swift`); the diarizer is configured with `exposeChunkEmbeddings = true` in `runDiarization`, and `buildSpeakerEmbeddings(from:)` in `Services.swift` falls back to the legacy per-segment path when chunk embeddings are unavailable (very short audio / older model builds).
Expand Down
Loading