Fix Teams AX-tree wake and add opt-in meeting-chat scraping

Sources/HeardCore/AppModel.swift

@@ -233,6 +233,9 @@ public final class AppModel: ObservableObject {
                 if s.enableWebexDetection { enabled.insert(.webex) }
                 return enabled
             },
+            chatScrapeEnabled: { [weak self] in
+                self?.settingsStore.settings.includeMeetingChat ?? false
+            },
             onMeetingStarted: { [weak self] snapshot in
                 guard let self else { return }
                 // Stop dictation before recording starts — mic should not transcribe
@@ -262,6 +265,9 @@ public final class AppModel: ObservableObject {
                 guard let session = self.recordingManager.stopRecording() else { return }
                 self.pipelineProcessor.enqueueFinishedRecording(session, endedAt: Date())
                 self.phase = .processing
+            },
+            onChatMessagesObserved: { [weak self] messages in
+                self?.recordingManager.updateChatMessages(messages)
             }
         )
 

Sources/HeardCore/ChatReader.swift

@@ -0,0 +1,150 @@
+import AppKit
+import Foundation
+
+// MARK: - ChatReader
+
+/// Reads visible chat messages from the meeting app's chat panel via Accessibility
+/// APIs. Requires Accessibility permission and — for Teams — the tree-wake nudge
+/// in `MeetingDetector.enableMeetingAppAccessibility` to have already succeeded.
+///
+/// Opt-in only (`AppSettings.includeMeetingChat`, default off): unlike the roster
+/// (names only) or in-meeting notes (the user's own words), this passively
+/// captures other participants' written words without their per-message consent.
+///
+/// Mirrors `RosterReader`'s identifier-heuristic approach, but ships with only
+/// Strategy 1 (known-identifier search): real AX identifiers for Teams' chat pane
+/// are unknown, and unlike the roster panel (a flat list of short names, low risk
+/// of a false-positive container match), guessing a generic "any list container"
+/// fallback risks silently scraping the wrong panel (e.g. the roster) as if it
+/// were chat. Once a live diagnostic dump (`diagnosticChatTreeDump`) shows Teams'
+/// real chat structure, `chatIdentifiers` and `parseMessageRow` should be retuned
+/// against it — same workflow already used to build `RosterReader`.
+///
+/// Known limitations (see handoff.md before improving this):
+/// - Electron uses virtualized/lazy-rendered lists — only on-screen messages
+///   exist in the AX tree. Scrollback and messages sent while the chat panel is
+///   closed are unreachable; there is no way to recover them retroactively.
+/// - Dedup is by exact (sender, text) match, owned by
+///   `RecordingManager.updateChatMessages` — a legitimate repeated identical
+///   short message ("yes", "+1") from the same sender is only captured once.
+/// - Message edits/deletes are not reflected — Heard's copy reflects whatever
+///   text was on screen the moment it was polled.
+public enum ChatReader {
+
+    /// Identifier/description keywords for the chat message list container.
+    /// Unverified against real Teams output — see the type doc above.
+    private static let chatIdentifiers = [
+        "chat-pane-list", "chat-pane-message-list", "chat-list", "messages-list",
+        "message-list", "chat-messages",
+    ]
+
+    /// UI strings to drop when a row's raw text is itself a control, not a message
+    /// (e.g. a "Send" button living inside the same container).
+    private static let controlStrings: Set<String> = [
+        "send", "more options", "reply", "react", "like", "delete", "edit",
+        "copy", "translate", "mark as unread", "add reaction", "attachment",
+    ]
+
+    /// Attempt to read the currently visible chat messages. Returns an empty
+    /// array if Accessibility isn't granted, the chat panel isn't open, or no
+    /// chat container matching a known identifier is found.
+    public static func readChatMessages(pid: pid_t?) -> [(sender: String, text: String)] {
+        guard let pid else { return [] }
+        let element = AXUIElementCreateApplication(pid)
+        AXUIElementSetMessagingTimeout(element, 1.0)
+        let app = AXUIElementNode(element)
+        return readChatMessagesFromNode(app)
+    }
+
+    /// Testable entry point — accepts any AX tree node, including mocks.
+    public static func readChatMessagesFromNode(_ app: any AXNode) -> [(sender: String, text: String)] {
+        guard let windows = app.axChildren else { return [] }
+        for window in windows {
+            if let messages = searchForChatContainer(in: window, depth: 0, maxDepth: 10), !messages.isEmpty {
+                return messages
+            }
+        }
+        return []
+    }
+
+    private static func searchForChatContainer(
+        in element: any AXNode, depth: Int, maxDepth: Int
+    ) -> [(sender: String, text: String)]? {
+        guard depth < maxDepth else { return nil }
+        let combined = ((element.axIdentifier ?? "") + " " + (element.axDescription ?? "")).lowercased()
+        if chatIdentifiers.contains(where: { combined.contains($0) }) {
+            let messages = extractMessages(from: element)
+            if !messages.isEmpty { return messages }
+        }
+        guard let children = element.axChildren else { return nil }
+        for child in children {
+            if let result = searchForChatContainer(in: child, depth: depth + 1, maxDepth: maxDepth) {
+                return result
+            }
+        }
+        return nil
+    }
+
+    /// Extract (sender, text) pairs from a chat container's row children.
+    private static func extractMessages(from container: any AXNode) -> [(sender: String, text: String)] {
+        guard let rows = container.axChildren else { return [] }
+        return rows.compactMap(parseMessageRow)
+    }
+
+    /// Best-effort per-row parse. Two shapes are tried, matching how
+    /// `RosterReader.extractTextChildren` handles Teams' row-container pattern:
+    /// (1) the row has sub-children — treat the first as sender, the rest joined
+    /// as body; (2) the row is a single node whose own value/description/title
+    /// encodes "Sender: text" or "Sender, text" for screen readers.
+    private static func parseMessageRow(_ row: any AXNode) -> (sender: String, text: String)? {
+        if let children = row.axChildren, children.count >= 2 {
+            guard let sender = (children[0].axValue ?? children[0].axTitle)?
+                .trimmingCharacters(in: .whitespacesAndNewlines), !sender.isEmpty
+            else { return nil }
+            let body = children.dropFirst()
+                .compactMap { $0.axValue ?? $0.axTitle }
+                .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
+                .filter { !$0.isEmpty }
+                .joined(separator: " ")
+            return validated(sender: sender, text: body)
+        }
+
+        guard let raw = (row.axDescription ?? row.axValue ?? row.axTitle)?
+            .trimmingCharacters(in: .whitespacesAndNewlines), !raw.isEmpty
+        else { return nil }
+        for separator in [": ", ", "] {
+            if let range = raw.range(of: separator) {
+                let sender = String(raw[raw.startIndex..<range.lowerBound])
+                let text = String(raw[range.upperBound...])
+                if let result = validated(sender: sender, text: text) { return result }
+            }
+        }
+        return nil
+    }
+
+    private static func validated(sender: String, text: String) -> (sender: String, text: String)? {
+        let sender = sender.trimmingCharacters(in: .whitespacesAndNewlines)
+        let text = text.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !sender.isEmpty, !text.isEmpty else { return nil }
+        guard sender.count <= 60, text.count <= 4000 else { return nil }
+        guard !controlStrings.contains(text.lowercased()) else { return nil }
+        // A sender exactly matching a control string means the row's structure
+        // didn't put an actual name first (e.g. a stray button picked up as the
+        // "sender" child) — unlike `text`, a real participant name is always
+        // short, so exact-match here is safe and doesn't risk dropping genuine
+        // messages the way substring-matching `text` would.
+        guard !controlStrings.contains(sender.lowercased()) else { return nil }
+        return (sender, text)
+    }
+
+    // MARK: - Diagnostics
+
+    /// Developer-Mode diagnostic, parallel to `RosterReader.diagnosticTreeDump`.
+    /// Dumps a bounded view of the app's AX tree so the chat parser's
+    /// identifiers/heuristics can be built against Teams' actual structure once a
+    /// real meeting has the chat panel open. Reuses `RosterReader`'s dump core —
+    /// the chat container (if open) shows up in the same bounded tree walk.
+    public static func diagnosticTreeDump(pid teamsPID: pid_t?) -> String? {
+        RosterReader.diagnosticTreeDump(pid: teamsPID)
+    }
+}

Sources/HeardCore/CoreModels.swift

@@ -67,6 +67,28 @@ public struct MeetingNote: Codable, Identifiable, Equatable {
     }
 }
 
+/// A message scraped from the meeting app's chat panel via Accessibility APIs
+/// (see `ChatReader`). Inserted chronologically into the final transcript,
+/// distinct from spoken segments and from the user's own in-meeting notes.
+/// Opt-in only — see `AppSettings.includeMeetingChat` — since this passively
+/// captures other participants' written words without their per-message consent.
+public struct ChatMessage: Codable, Identifiable, Equatable {
+    public let id: UUID
+    /// Offset in seconds from the recording's `startTime`, captured when the
+    /// message was first observed by the poll (not when it was actually sent —
+    /// Teams' AX tree doesn't reliably expose per-message send times).
+    public let offsetSeconds: TimeInterval
+    public let sender: String
+    public let text: String
+
+    public init(id: UUID = UUID(), offsetSeconds: TimeInterval, sender: String, text: String) {
+        self.id = id
+        self.offsetSeconds = offsetSeconds
+        self.sender = sender
+        self.text = text
+    }
+}
+
 public struct PipelineJob: Codable, Identifiable, Equatable {
     public let id: UUID
     public var meetingTitle: String
@@ -81,6 +103,10 @@ public struct PipelineJob: Codable, Identifiable, Equatable {
     public var retryCount: Int
     public var rosterNames: [String]
     public var notes: [MeetingNote]
+    /// Messages scraped from the meeting app's chat panel, if the user opted in
+    /// (`AppSettings.includeMeetingChat`). Empty for meetings before this feature
+    /// existed or where chat scraping is disabled/unavailable.
+    public var chatMessages: [ChatMessage]
     /// `mic.start − app.start` in seconds. Used during speaker assignment to
     /// align mic-track segments with app-track segments for cross-track
     /// deduplication. Defaults to 0 for jobs persisted before this field
@@ -101,6 +127,7 @@ public struct PipelineJob: Codable, Identifiable, Equatable {
         retryCount: Int,
         rosterNames: [String] = [],
         notes: [MeetingNote] = [],
+        chatMessages: [ChatMessage] = [],
         micDelaySeconds: TimeInterval = 0
     ) {
         self.id = id
@@ -116,13 +143,14 @@ public struct PipelineJob: Codable, Identifiable, Equatable {
         self.retryCount = retryCount
         self.rosterNames = rosterNames
         self.notes = notes
+        self.chatMessages = chatMessages
         self.micDelaySeconds = micDelaySeconds
     }
 
     private enum CodingKeys: String, CodingKey {
         case id, meetingTitle, startTime, endTime, appAudioPath, micAudioPath
         case transcriptPath, stage, stageStartTime, error, retryCount
-        case rosterNames, notes, micDelaySeconds
+        case rosterNames, notes, chatMessages, micDelaySeconds
     }
 
     public init(from decoder: Decoder) throws {
@@ -140,6 +168,7 @@ public struct PipelineJob: Codable, Identifiable, Equatable {
         retryCount = try c.decode(Int.self, forKey: .retryCount)
         rosterNames = try c.decodeIfPresent([String].self, forKey: .rosterNames) ?? []
         notes = try c.decodeIfPresent([MeetingNote].self, forKey: .notes) ?? []
+        chatMessages = try c.decodeIfPresent([ChatMessage].self, forKey: .chatMessages) ?? []
         micDelaySeconds = try c.decodeIfPresent(TimeInterval.self, forKey: .micDelaySeconds) ?? 0
     }
 }
@@ -358,6 +387,12 @@ public struct AppSettings: Codable, Equatable {
     /// Reveals the gated Advanced tab (models, performance, diarization, debugging).
     /// Off by default to keep the default settings surface calm.
     public var showAdvancedSettings: Bool
+    /// Scrape the meeting app's chat panel via Accessibility APIs and interleave it
+    /// into the transcript (see `ChatReader`/`ChatMessage`). Off by default: unlike
+    /// the roster (names only) or notes (the user's own words), this passively
+    /// captures other participants' written words without their per-message
+    /// consent, so it's opt-in rather than automatic.
+    public var includeMeetingChat: Bool
 
     public static let `default` = AppSettings(
         userName: "",
@@ -386,7 +421,8 @@ public struct AppSettings: Codable, Equatable {
         memoryMode: .auto,
         speakerRetentionDays: 90,
         selectedInputDeviceUID: nil,
-        showAdvancedSettings: false
+        showAdvancedSettings: false,
+        includeMeetingChat: false
     )
 
     public init(
@@ -413,7 +449,8 @@ public struct AppSettings: Codable, Equatable {
         memoryMode: MemoryMode = .auto,
         speakerRetentionDays: Int = 90,
         selectedInputDeviceUID: String? = nil,
-        showAdvancedSettings: Bool = false
+        showAdvancedSettings: Bool = false,
+        includeMeetingChat: Bool = false
     ) {
         self.userName = userName
         self.launchAtLogin = launchAtLogin
@@ -439,6 +476,7 @@ public struct AppSettings: Codable, Equatable {
         self.speakerRetentionDays = speakerRetentionDays
         self.selectedInputDeviceUID = selectedInputDeviceUID
         self.showAdvancedSettings = showAdvancedSettings
+        self.includeMeetingChat = includeMeetingChat
     }
 }
 
@@ -575,6 +613,9 @@ public struct TranscriptDocument {
     public var notes: [MeetingNote]
     /// Display name to attribute notes to. Falls back to "Me" when empty.
     public var noteAuthor: String
+    /// Messages scraped from the meeting chat panel, when opted in. Rendered
+    /// chronologically alongside speaker segments and notes in the markdown output.
+    public var chatMessages: [ChatMessage]
 
     public init(
         title: String,
@@ -586,7 +627,8 @@ public struct TranscriptDocument {
         diarizationSegments: [(speakerID: String, startTime: TimeInterval, endTime: TimeInterval)] = [],
         unmatchedRosterNames: [String] = [],
         notes: [MeetingNote] = [],
-        noteAuthor: String = "Me"
+        noteAuthor: String = "Me",
+        chatMessages: [ChatMessage] = []
     ) {
         self.title = title
         self.startTime = startTime
@@ -598,6 +640,7 @@ public struct TranscriptDocument {
         self.unmatchedRosterNames = unmatchedRosterNames
         self.notes = notes
         self.noteAuthor = noteAuthor
+        self.chatMessages = chatMessages
     }
 }
 

Sources/HeardCore/RosterReader.swift

@@ -16,7 +16,7 @@ public protocol AXNode {
 
 // MARK: - Live AX adapter
 
-private struct AXUIElementNode: AXNode {
+struct AXUIElementNode: AXNode {
     private let element: AXUIElement
     init(_ element: AXUIElement) { self.element = element }