diff --git a/Sources/HeardCore/AppModel.swift b/Sources/HeardCore/AppModel.swift index b421034..b4b5ca3 100644 --- a/Sources/HeardCore/AppModel.swift +++ b/Sources/HeardCore/AppModel.swift @@ -256,7 +256,9 @@ public final class AppModel: ObservableObject { onMeetingEnded: { [weak self] snapshot in guard let self else { return } // Update the recording session with the final accumulated roster names + // and the title (often captured late, after Teams' a11y tree built). self.recordingManager.updateRosterNames(snapshot.rosterNames) + self.recordingManager.updateTitle(snapshot.title) guard let session = self.recordingManager.stopRecording() else { return } self.pipelineProcessor.enqueueFinishedRecording(session, endedAt: Date()) self.phase = .processing diff --git a/Sources/HeardCore/RosterReader.swift b/Sources/HeardCore/RosterReader.swift index 7e4a6a1..8d56174 100644 --- a/Sources/HeardCore/RosterReader.swift +++ b/Sources/HeardCore/RosterReader.swift @@ -265,4 +265,56 @@ public enum RosterReader { return seen.insert(lower).inserted } } + + // MARK: - Diagnostics + + /// Developer-Mode diagnostic. When `readRoster` comes back empty against a real + /// (woken) Teams tree, this dumps a bounded view of the app's AX tree — roles, + /// identifiers, descriptions, and truncated text — so the parser's hardcoded + /// identifiers/heuristics can be retuned against Teams' actual structure. The + /// roster parser has never been validated against live Teams output, so this is + /// the data we need to close that gap. Live entry point: builds the app element + /// and walks it on the caller's (background) thread. + public static func diagnosticTreeDump(pid teamsPID: pid_t?) -> String? { + guard let pid = teamsPID else { return nil } + let element = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(element, 1.0) + return diagnosticTreeDump(root: AXUIElementNode(element)) + } + + /// Testable core: render any AX node tree as a compact, bounded textual outline. + /// Depth-first (mirrors the parser's own traversal) with a shared node budget so + /// a huge Electron tree can't produce an unbounded log. Text is truncated so the + /// dump stays structural (metadata) rather than a verbatim content capture. + public static func diagnosticTreeDump(root: any AXNode, maxDepth: Int = 9, nodeBudget: Int = 500) -> String? { + var lines: [String] = [] + var remaining = nodeBudget + + func truncate(_ s: String, _ limit: Int = 40) -> String { + let flat = s.replacingOccurrences(of: "\n", with: "⏎") + return flat.count <= limit ? flat : String(flat.prefix(limit)) + "…(\(flat.count))" + } + + func walk(_ node: any AXNode, depth: Int) { + guard remaining > 0 else { return } + remaining -= 1 + var parts = ["\(String(repeating: " ", count: depth))\(node.axRole ?? "?")"] + if let id = node.axIdentifier, !id.isEmpty { parts.append("id=\(id)") } + if let d = node.axDescription, !d.isEmpty { parts.append("desc=\(truncate(d))") } + if let t = node.axTitle, !t.isEmpty { parts.append("title=\(truncate(t))") } + if let v = node.axValue, !v.isEmpty { parts.append("value=\(truncate(v))") } + lines.append(parts.joined(separator: " ")) + guard depth < maxDepth, let children = node.axChildren else { return } + for child in children { + if remaining <= 0 { break } + walk(child, depth: depth + 1) + } + } + + walk(root, depth: 0) + if remaining <= 0 { + lines.append("…(node budget \(nodeBudget) reached — tree truncated; raise the cap to see more)") + } + return lines.isEmpty ? nil : lines.joined(separator: "\n") + } } diff --git a/Sources/HeardCore/Services.swift b/Sources/HeardCore/Services.swift index 693c011..731443e 100644 --- a/Sources/HeardCore/Services.swift +++ b/Sources/HeardCore/Services.swift @@ -109,7 +109,7 @@ public struct MeetingSnapshot { } public struct RecordingSession { - public let title: String + public var title: String public let startTime: Date public let appAudioPath: URL public let micAudioPath: URL @@ -299,10 +299,16 @@ public final class MeetingDetector: ObservableObject { // the main thread, then start the meeting once the data is in hand. Task { [weak self] in let (title, rosterNames) = await Task.detached(priority: .userInitiated) { + // Teams is an Electron/Chromium app: its accessibility tree is + // dormant until a client requests it via AXManualAccessibility. + // Nudge it now — the tree builds asynchronously, so this first + // read often still misses; the roster poll retries title+roster. + if source == .teams { Self.enableMeetingAppAccessibility(pid: pid, source: source) } let title = Self.extractMeetingTitle(pid: pid, source: source) ?? "" let roster: [String] = source == .teams ? RosterReader.readRoster(pid: pid) : [] return (title, roster) }.value + DebugFileLog.log("meeting start scrape: source=\(source.rawValue) pid=\(pid) titleCaptured=\(!title.isEmpty) rosterCount=\(rosterNames.count) — roster poll will retry title/roster every 15s") guard let self else { return } // The meeting may have ended (or watching been toggled off) while // we were scraping — the detection state machine is the source of truth. @@ -372,6 +378,25 @@ public final class MeetingDetector: ObservableObject { return nil } + /// Wake a Chromium/Electron meeting app's accessibility tree. + /// + /// Teams keeps its a11y tree off by default to save resources, so AX reads of + /// its windows/titles/roster fail (commonly `kAXErrorAPIDisabled`, -25211) even + /// though Heard itself is trusted. Setting `AXManualAccessibility` on the app + /// element signals Chromium to build the tree. The build is asynchronous, so + /// callers must still retry the actual read after this returns. + /// + /// The `setErr` log line is the diagnostic discriminator: `.success` means the + /// nudge was accepted (any lingering empty reads are tree-build timing); a + /// `.apiDisabled` here would instead mean Heard is genuinely untrusted. + nonisolated private static func enableMeetingAppAccessibility(pid: pid_t?, source: MeetingApp) { + guard let pid else { return } + let app = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(app, 1.0) + let setErr = AXUIElementSetAttributeValue(app, "AXManualAccessibility" as CFString, kCFBooleanTrue) + DebugFileLog.log("enableMeetingAppAccessibility: pid=\(pid) source=\(source.rawValue) setErr=\(setErr.rawValue)") + } + /// Extract the meeting title from a meeting-app window via Accessibility API. /// Strips the trailing app-name suffix (` | Microsoft Teams`, ` - Zoom`, etc.). /// Returns nil if AX is denied, no window matches, or the title is just a placeholder. @@ -390,8 +415,11 @@ public final class MeetingDetector: ObservableObject { var windowsRef: AnyObject? let windowsErr = AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &windowsRef) guard windowsErr == .success, let windows = windowsRef as? [AXUIElement] else { - // .apiDisabled => Accessibility denied; other errors / non-array => Electron - // a11y tree not yet built. Both produce an empty title -> "Meeting" fallback. + // For a per-app Electron query, .apiDisabled (-25211) usually means the + // app's Chromium a11y tree is still dormant (see enableMeetingAppAccessibility), + // NOT that Heard is untrusted — confirm trust via the system-wide element + // instead. Any failure here yields an empty title -> "Meeting" fallback; + // the roster poll retries once the tree wakes. DebugFileLog.log("extractMeetingTitle: windows query failed pid=\(pid) source=\(source.rawValue) axErr=\(windowsErr.rawValue)") return nil } @@ -431,17 +459,41 @@ public final class MeetingDetector: ObservableObject { private func startRosterPolling() { rosterPollingTask?.cancel() rosterPollingTask = Task { [weak self] in + var tick = 0 + // Developer-Mode only: emit at most a few AX tree dumps per meeting when + // the roster keeps coming back empty, so we can retune the parser without + // flooding the log. Reset per meeting (the task is recreated on each start). + var rosterDumpsRemaining = 3 while !Task.isCancelled { try? await Task.sleep(for: .seconds(15)) guard let self, !Task.isCancelled, let snapshot = self.activeSnapshot, snapshot.source == .teams else { break } + tick += 1 // The roster walk is a deep recursive AX scrape of Teams' // Electron tree — keep it off the main thread. let pid = snapshot.meetingPID - let names = await Task.detached(priority: .utility) { - RosterReader.readRoster(pid: pid) + let source = snapshot.source + // Re-extract the title until we get one: the initial capture at + // join time usually fires before Teams' a11y tree has finished + // building, leaving the title empty (-> "Meeting" filename). + let needTitle = (self.activeSnapshot?.title ?? "").isEmpty + // An empty roster means the tree still looks asleep — the join-time + // nudge may have timed out against a busy Teams. Re-nudge before each + // read until names appear (idempotent). Gate on roster, not title: + // window titles can be AppKit-provided and succeed without the woken + // Chromium tree that the roster DOM actually needs. + let rosterEmpty = (self.activeSnapshot?.rosterNames.isEmpty ?? true) + // While roster is still empty, capture a bounded tree dump (Developer + // Mode + budget) so an empty result is self-diagnosing, not silent. + let wantDump = rosterEmpty && rosterDumpsRemaining > 0 && DebugFileLog.isEnabled + let (names, refreshedTitle, treeDump) = await Task.detached(priority: .utility) { + if rosterEmpty { Self.enableMeetingAppAccessibility(pid: pid, source: source) } + let names = RosterReader.readRoster(pid: pid) + let title = needTitle ? Self.extractMeetingTitle(pid: pid, source: source) : nil + let dump = (wantDump && names.isEmpty) ? RosterReader.diagnosticTreeDump(pid: pid) : nil + return (names, title, dump) }.value if !names.isEmpty { let existing = Set(self.activeSnapshot?.rosterNames ?? []) @@ -450,6 +502,14 @@ public final class MeetingDetector: ObservableObject { self.activeSnapshot?.rosterNames.append(contentsOf: newNames) } } + if let refreshedTitle, !refreshedTitle.isEmpty { + self.activeSnapshot?.title = refreshedTitle + } + DebugFileLog.log("roster poll tick=\(tick): read=\(names.count) total=\(self.activeSnapshot?.rosterNames.count ?? 0) titleEmpty=\(needTitle) titleRefreshed=\(refreshedTitle != nil)") + if let treeDump { + rosterDumpsRemaining -= 1 + DebugFileLog.log("roster poll tick=\(tick): readRoster empty — AX tree dump (\(3 - rosterDumpsRemaining)/3):\n\(treeDump)") + } } } } @@ -626,6 +686,18 @@ public final class RecordingManager: ObservableObject { currentRosterNames = names } + /// Update the meeting title on the active session. The title is often captured + /// late (Teams' a11y tree builds after join), so the recording starts with an + /// empty title and this fills it in before the session is enqueued, fixing the + /// transcript filename. No-ops on empty so a failed re-attempt can't clobber a + /// title already captured. + public func updateTitle(_ title: String) { + let trimmed = title.trimmingCharacters(in: .whitespaces) + guard activeSession != nil, !trimmed.isEmpty else { return } + activeSession?.title = trimmed + currentTitle = trimmed + } + /// Append a user-authored note to the active recording session. The offset /// is measured from `session.startTime` using the wall-clock instant at /// which the composer was *opened* (passed in by the caller), not the @@ -1623,35 +1695,42 @@ public final class PermissionCenter: ObservableObject { queue: .main ) { [weak self] notification in guard let self, - self.pendingScreenCaptureCheck, let app = notification.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication, // macOS 13+: "System Settings"; earlier: "System Preferences" — both share this bundle ID app.bundleIdentifier == "com.apple.systempreferences" else { return } - // One-shot: remove the observer and clear the flag immediately. - self.pendingScreenCaptureCheck = false - if let obs = self.systemPrefsObserver { - NSWorkspace.shared.notificationCenter.removeObserver(obs) - self.systemPrefsObserver = nil - } + // The observer is registered with `queue: .main`, so this block always runs + // on the main actor — assert that to access main-actor-isolated state safely + // (the alternative, a bare Task hop, would let a second notification slip in + // before the flag/observer are cleared). + MainActor.assumeIsolated { + guard self.pendingScreenCaptureCheck else { return } + + // One-shot: remove the observer and clear the flag immediately. + self.pendingScreenCaptureCheck = false + if let obs = self.systemPrefsObserver { + NSWorkspace.shared.notificationCenter.removeObserver(obs) + self.systemPrefsObserver = nil + } - Task { @MainActor [weak self] in - guard let self else { return } - // Fast path: sync check (works on macOS < 15 and after any restart). - if CGPreflightScreenCaptureAccess() { - self.applyScreenCaptureProbe(true) + Task { @MainActor [weak self] in + guard let self else { return } + // Fast path: sync check (works on macOS < 15 and after any restart). + if CGPreflightScreenCaptureAccess() { + self.applyScreenCaptureProbe(true) + self.refresh() + return + } + // Bypass macOS 15+ per-process TCC cache with one-shot SCShareableContent. + // Safe here: this is user-initiated and fires exactly once per "Grant…" click, + // only after the user has left System Settings. The result is authoritative, + // so a false clears any stale cached grant immediately. + self.applyScreenCaptureProbe( + await self.checkScreenCapturePermissionLive(), authoritative: true + ) self.refresh() - return } - // Bypass macOS 15+ per-process TCC cache with one-shot SCShareableContent. - // Safe here: this is user-initiated and fires exactly once per "Grant…" click, - // only after the user has left System Settings. The result is authoritative, - // so a false clears any stale cached grant immediately. - self.applyScreenCaptureProbe( - await self.checkScreenCapturePermissionLive(), authoritative: true - ) - self.refresh() } } } diff --git a/Tests/HeardTests/TestRunner.swift b/Tests/HeardTests/TestRunner.swift index 6d1bddb..6e5a62f 100644 --- a/Tests/HeardTests/TestRunner.swift +++ b/Tests/HeardTests/TestRunner.swift @@ -1909,6 +1909,49 @@ func runRosterReaderAXTests() { let names = RosterReader.readRosterFromNode(tree) try expectEqual(names, []) } + + // Diagnostic tree dump (Developer-Mode aid for retuning the parser). + + test("diagnosticTreeDump renders roles, identifiers, and truncated text") { + let tree = MockAXNode(role: "AXApplication", children: [ + MockAXNode(role: "AXWindow", identifier: "main", title: "Sprint Planning", children: [ + MockAXNode(role: "AXGroup", identifier: "roster-list", children: [ + MockAXNode(role: "AXStaticText", value: "Alice Smith"), + ]), + ]), + ]) + let out = RosterReader.diagnosticTreeDump(root: tree) + try expect(out != nil, "expected a dump") + let s = out ?? "" + try expect(s.contains("AXApplication"), "missing root role") + try expect(s.contains("id=roster-list"), "missing identifier") + try expect(s.contains("title=Sprint Planning"), "missing window title") + try expect(s.contains("value=Alice Smith"), "missing leaf value") + } + + test("diagnosticTreeDump truncates long text and notes the full length") { + let long = String(repeating: "x", count: 100) + let tree = MockAXNode(role: "AXStaticText", value: long) + let out = RosterReader.diagnosticTreeDump(root: tree) ?? "" + try expect(out.contains("…(100)"), "expected truncation marker with full length") + try expect(!out.contains(long), "full untruncated text should not appear") + } + + test("diagnosticTreeDump bounds output to the node budget") { + let children = (0..<50).map { MockAXNode(role: "AXRow", identifier: "row-\($0)") } + let tree = MockAXNode(role: "AXList", children: children) + let out = RosterReader.diagnosticTreeDump(root: tree, maxDepth: 9, nodeBudget: 10) ?? "" + let lines = out.split(separator: "\n") + try expect(lines.count <= 11, "expected <= 11 lines (10 nodes + note), got \(lines.count)") + try expect(out.contains("budget 10 reached"), "missing truncation note") + } + + test("diagnosticTreeDump stops descending at maxDepth") { + var node = MockAXNode(role: "AXLeaf", identifier: "deep") + for i in 0..<6 { node = MockAXNode(role: "AXGroup", identifier: "g\(i)", children: [node]) } + let out = RosterReader.diagnosticTreeDump(root: node, maxDepth: 2, nodeBudget: 500) ?? "" + try expect(!out.contains("id=deep"), "leaf below maxDepth should be excluded") + } } // MARK: - SegmentDeduplicator Tests diff --git a/handoff.md b/handoff.md index edc0890..cba43a5 100644 --- a/handoff.md +++ b/handoff.md @@ -15,6 +15,16 @@ The app builds cleanly with `swift build` and runs as a menu bar app on macOS 15 **Robustness/efficiency hardening pass (post-v0.2.3):** - `DebugFileLog` is now gated behind Developer Mode (off by default), rotates at 5 MB, and never logs transcript content (lengths only) — previously it logged every dictated utterance to an unbounded plaintext file and ran a 1 Hz heartbeat in every install. - Meeting-start title/roster AX scraping and the 15 s roster poll now run on detached background tasks with a 1 s `AXUIElementSetMessagingTimeout`, so a busy/hung Teams can no longer stall Heard's main thread. The meeting (and recording) starts when the scrape completes; if the meeting ends mid-scrape, the detection state machine prevents an orphaned start. +- **Teams accessibility-tree wake (fixes empty meeting titles → "Meeting" filename; unblocks — but does not yet verify — the Teams roster scrape).** Teams is an Electron/Chromium app whose accessibility tree is dormant until a client requests it; AX reads of its windows/titles/roster failed (logged `axErr=-25211`, `kAXErrorAPIDisabled`) even though Heard is trusted (stable Developer ID signing + live `AXIsProcessTrusted`). At meeting detection we now set `AXManualAccessibility=true` on the Teams app element (`MeetingDetector.enableMeetingAppAccessibility`) to wake the tree; the build is async and the join-time set can time out against a busy Teams, so the 15 s roster poll **re-nudges while the roster is still empty**, re-extracts the title until non-empty, and the late title is pushed into the live session via `RecordingManager.updateTitle` in `onMeetingEnded` (parallel to `updateRosterNames`) before the filename is built at transcript-write time. `RecordingSession.title` is now `var`. + - **Title — pending real-meeting confirmation:** with Developer Mode on, the next Teams meeting should log `enableMeetingAppAccessibility: ... setErr=0` (success) followed by a `roster poll: title refreshed` line. If `setErr` is non-zero, or the title stays empty after `setErr=0`, capture the log — a `setErr=-25211` here would instead point back at Heard's own AX trust. + - **Roster — unblocked but unverified.** Waking the tree is *necessary* but not *sufficient*: `RosterReader`'s parser (hardcoded identifiers `roster-list`/`people-pane`/… and AXList/AXTable heuristics) has never run against real woken Teams output, so the roster may still return empty due to a parser/structure mismatch independent of the tree-wake. (Today participant names in transcripts come from voice matching, not this scrape.) + - **Diagnostics for the next real meeting (Developer Mode on, tail `~/Library/Application Support/Heard/dict-debug.log`).** The flow is now self-diagnosing end to end: + - `meeting start scrape: source=teams pid=… titleCaptured= rosterCount=N` — the anchor line at join. + - `enableMeetingAppAccessibility: … setErr=` — the wake result. `setErr=0` = nudge accepted (good); `setErr=-25211` = Heard itself untrusted (whole theory wrong, fix the grant); other non-zero = app busy/unsupported. + - `roster poll tick=K: read=N total=M titleEmpty= titleRefreshed=` — one per 15 s poll; the timeline of title/roster recovery. + - `roster poll tick=K: readRoster empty — AX tree dump (n/3): …` — emitted up to **3×** per meeting while the roster stays empty. A bounded (depth ≤ 9, ≤ 500 nodes) outline of Teams' real AX tree — role, identifier, description, truncated title/value per node. **This is the artifact to retune the parser from:** find the container holding participant names and update `RosterReader`'s `rosterIdentifiers` / list-role heuristics to match. The dump core is `RosterReader.diagnosticTreeDump(root:)` (unit-tested for structure/bounds/depth); live entry `diagnosticTreeDump(pid:)`. + - Title extraction also logs its own `extractMeetingTitle: …` lines (windows-query error, window count, each raw→cleaned title) on every attempt. + - **Screen-capture permission observer concurrency fix (unrelated cleanup found during this work):** the `NSWorkspace.didDeactivateApplicationNotification` observer block in `RecordingManager` accessed main-actor state from a `@Sendable` closure (4 warnings, would be Swift 6 errors). Now wrapped in `MainActor.assumeIsolated` (the block already runs on `queue: .main`). Build is warning-free for these. - App-audio process collection is now per-app (`MeetingApp.isProcessFamilyMember`) instead of hard-coded to Teams — Zoom/Webex helper processes are tapped too. `RecordingManager.startRecording` takes a `source:` parameter. - `skipNaming` uses `processingJob` (not `activeJob`) so a stale failed job can't pin the phase at `.processing` after "Skip All". - Removed the dead `partialTranscript` plumbing (the batch dictation engine never produced partials) and its 10 Hz polling loop in `AppModel`.