diff --git a/Sources/HeardCore/AppModel.swift b/Sources/HeardCore/AppModel.swift index c65cf7c..551f726 100644 --- a/Sources/HeardCore/AppModel.swift +++ b/Sources/HeardCore/AppModel.swift @@ -233,6 +233,9 @@ public final class AppModel: ObservableObject { if s.enableWebexDetection { enabled.insert(.webex) } return enabled }, + chatScrapeEnabled: { [weak self] in + self?.settingsStore.settings.includeMeetingChat ?? false + }, onMeetingStarted: { [weak self] snapshot in guard let self else { return } // Stop dictation before recording starts — mic should not transcribe @@ -262,6 +265,9 @@ public final class AppModel: ObservableObject { guard let session = self.recordingManager.stopRecording() else { return } self.pipelineProcessor.enqueueFinishedRecording(session, endedAt: Date()) self.phase = .processing + }, + onChatMessagesObserved: { [weak self] messages in + self?.recordingManager.updateChatMessages(messages) } ) diff --git a/Sources/HeardCore/ChatReader.swift b/Sources/HeardCore/ChatReader.swift new file mode 100644 index 0000000..cd37e54 --- /dev/null +++ b/Sources/HeardCore/ChatReader.swift @@ -0,0 +1,150 @@ +import AppKit +import Foundation + +// MARK: - ChatReader + +/// Reads visible chat messages from the meeting app's chat panel via Accessibility +/// APIs. Requires Accessibility permission and — for Teams — the tree-wake nudge +/// in `MeetingDetector.enableMeetingAppAccessibility` to have already succeeded. +/// +/// Opt-in only (`AppSettings.includeMeetingChat`, default off): unlike the roster +/// (names only) or in-meeting notes (the user's own words), this passively +/// captures other participants' written words without their per-message consent. +/// +/// Mirrors `RosterReader`'s identifier-heuristic approach, but ships with only +/// Strategy 1 (known-identifier search): real AX identifiers for Teams' chat pane +/// are unknown, and unlike the roster panel (a flat list of short names, low risk +/// of a false-positive container match), guessing a generic "any list container" +/// fallback risks silently scraping the wrong panel (e.g. the roster) as if it +/// were chat. Once a live diagnostic dump (`diagnosticChatTreeDump`) shows Teams' +/// real chat structure, `chatIdentifiers` and `parseMessageRow` should be retuned +/// against it — same workflow already used to build `RosterReader`. +/// +/// Known limitations (see handoff.md before improving this): +/// - Electron uses virtualized/lazy-rendered lists — only on-screen messages +/// exist in the AX tree. Scrollback and messages sent while the chat panel is +/// closed are unreachable; there is no way to recover them retroactively. +/// - Dedup is by exact (sender, text) match, owned by +/// `RecordingManager.updateChatMessages` — a legitimate repeated identical +/// short message ("yes", "+1") from the same sender is only captured once. +/// - Message edits/deletes are not reflected — Heard's copy reflects whatever +/// text was on screen the moment it was polled. +public enum ChatReader { + + /// Identifier/description keywords for the chat message list container. + /// Unverified against real Teams output — see the type doc above. + private static let chatIdentifiers = [ + "chat-pane-list", "chat-pane-message-list", "chat-list", "messages-list", + "message-list", "chat-messages", + ] + + /// UI strings to drop when a row's raw text is itself a control, not a message + /// (e.g. a "Send" button living inside the same container). + private static let controlStrings: Set = [ + "send", "more options", "reply", "react", "like", "delete", "edit", + "copy", "translate", "mark as unread", "add reaction", "attachment", + ] + + /// Attempt to read the currently visible chat messages. Returns an empty + /// array if Accessibility isn't granted, the chat panel isn't open, or no + /// chat container matching a known identifier is found. + public static func readChatMessages(pid: pid_t?) -> [(sender: String, text: String)] { + guard let pid else { return [] } + let element = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(element, 1.0) + let app = AXUIElementNode(element) + return readChatMessagesFromNode(app) + } + + /// Testable entry point — accepts any AX tree node, including mocks. + public static func readChatMessagesFromNode(_ app: any AXNode) -> [(sender: String, text: String)] { + guard let windows = app.axChildren else { return [] } + for window in windows { + if let messages = searchForChatContainer(in: window, depth: 0, maxDepth: 10), !messages.isEmpty { + return messages + } + } + return [] + } + + private static func searchForChatContainer( + in element: any AXNode, depth: Int, maxDepth: Int + ) -> [(sender: String, text: String)]? { + guard depth < maxDepth else { return nil } + let combined = ((element.axIdentifier ?? "") + " " + (element.axDescription ?? "")).lowercased() + if chatIdentifiers.contains(where: { combined.contains($0) }) { + let messages = extractMessages(from: element) + if !messages.isEmpty { return messages } + } + guard let children = element.axChildren else { return nil } + for child in children { + if let result = searchForChatContainer(in: child, depth: depth + 1, maxDepth: maxDepth) { + return result + } + } + return nil + } + + /// Extract (sender, text) pairs from a chat container's row children. + private static func extractMessages(from container: any AXNode) -> [(sender: String, text: String)] { + guard let rows = container.axChildren else { return [] } + return rows.compactMap(parseMessageRow) + } + + /// Best-effort per-row parse. Two shapes are tried, matching how + /// `RosterReader.extractTextChildren` handles Teams' row-container pattern: + /// (1) the row has sub-children — treat the first as sender, the rest joined + /// as body; (2) the row is a single node whose own value/description/title + /// encodes "Sender: text" or "Sender, text" for screen readers. + private static func parseMessageRow(_ row: any AXNode) -> (sender: String, text: String)? { + if let children = row.axChildren, children.count >= 2 { + guard let sender = (children[0].axValue ?? children[0].axTitle)? + .trimmingCharacters(in: .whitespacesAndNewlines), !sender.isEmpty + else { return nil } + let body = children.dropFirst() + .compactMap { $0.axValue ?? $0.axTitle } + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .joined(separator: " ") + return validated(sender: sender, text: body) + } + + guard let raw = (row.axDescription ?? row.axValue ?? row.axTitle)? + .trimmingCharacters(in: .whitespacesAndNewlines), !raw.isEmpty + else { return nil } + for separator in [": ", ", "] { + if let range = raw.range(of: separator) { + let sender = String(raw[raw.startIndex.. (sender: String, text: String)? { + let sender = sender.trimmingCharacters(in: .whitespacesAndNewlines) + let text = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !sender.isEmpty, !text.isEmpty else { return nil } + guard sender.count <= 60, text.count <= 4000 else { return nil } + guard !controlStrings.contains(text.lowercased()) else { return nil } + // A sender exactly matching a control string means the row's structure + // didn't put an actual name first (e.g. a stray button picked up as the + // "sender" child) — unlike `text`, a real participant name is always + // short, so exact-match here is safe and doesn't risk dropping genuine + // messages the way substring-matching `text` would. + guard !controlStrings.contains(sender.lowercased()) else { return nil } + return (sender, text) + } + + // MARK: - Diagnostics + + /// Developer-Mode diagnostic, parallel to `RosterReader.diagnosticTreeDump`. + /// Dumps a bounded view of the app's AX tree so the chat parser's + /// identifiers/heuristics can be built against Teams' actual structure once a + /// real meeting has the chat panel open. Reuses `RosterReader`'s dump core — + /// the chat container (if open) shows up in the same bounded tree walk. + public static func diagnosticTreeDump(pid teamsPID: pid_t?) -> String? { + RosterReader.diagnosticTreeDump(pid: teamsPID) + } +} diff --git a/Sources/HeardCore/CoreModels.swift b/Sources/HeardCore/CoreModels.swift index d426297..4934046 100644 --- a/Sources/HeardCore/CoreModels.swift +++ b/Sources/HeardCore/CoreModels.swift @@ -67,6 +67,28 @@ public struct MeetingNote: Codable, Identifiable, Equatable { } } +/// A message scraped from the meeting app's chat panel via Accessibility APIs +/// (see `ChatReader`). Inserted chronologically into the final transcript, +/// distinct from spoken segments and from the user's own in-meeting notes. +/// Opt-in only — see `AppSettings.includeMeetingChat` — since this passively +/// captures other participants' written words without their per-message consent. +public struct ChatMessage: Codable, Identifiable, Equatable { + public let id: UUID + /// Offset in seconds from the recording's `startTime`, captured when the + /// message was first observed by the poll (not when it was actually sent — + /// Teams' AX tree doesn't reliably expose per-message send times). + public let offsetSeconds: TimeInterval + public let sender: String + public let text: String + + public init(id: UUID = UUID(), offsetSeconds: TimeInterval, sender: String, text: String) { + self.id = id + self.offsetSeconds = offsetSeconds + self.sender = sender + self.text = text + } +} + public struct PipelineJob: Codable, Identifiable, Equatable { public let id: UUID public var meetingTitle: String @@ -81,6 +103,10 @@ public struct PipelineJob: Codable, Identifiable, Equatable { public var retryCount: Int public var rosterNames: [String] public var notes: [MeetingNote] + /// Messages scraped from the meeting app's chat panel, if the user opted in + /// (`AppSettings.includeMeetingChat`). Empty for meetings before this feature + /// existed or where chat scraping is disabled/unavailable. + public var chatMessages: [ChatMessage] /// `mic.start − app.start` in seconds. Used during speaker assignment to /// align mic-track segments with app-track segments for cross-track /// deduplication. Defaults to 0 for jobs persisted before this field @@ -101,6 +127,7 @@ public struct PipelineJob: Codable, Identifiable, Equatable { retryCount: Int, rosterNames: [String] = [], notes: [MeetingNote] = [], + chatMessages: [ChatMessage] = [], micDelaySeconds: TimeInterval = 0 ) { self.id = id @@ -116,13 +143,14 @@ public struct PipelineJob: Codable, Identifiable, Equatable { self.retryCount = retryCount self.rosterNames = rosterNames self.notes = notes + self.chatMessages = chatMessages self.micDelaySeconds = micDelaySeconds } private enum CodingKeys: String, CodingKey { case id, meetingTitle, startTime, endTime, appAudioPath, micAudioPath case transcriptPath, stage, stageStartTime, error, retryCount - case rosterNames, notes, micDelaySeconds + case rosterNames, notes, chatMessages, micDelaySeconds } public init(from decoder: Decoder) throws { @@ -140,6 +168,7 @@ public struct PipelineJob: Codable, Identifiable, Equatable { retryCount = try c.decode(Int.self, forKey: .retryCount) rosterNames = try c.decodeIfPresent([String].self, forKey: .rosterNames) ?? [] notes = try c.decodeIfPresent([MeetingNote].self, forKey: .notes) ?? [] + chatMessages = try c.decodeIfPresent([ChatMessage].self, forKey: .chatMessages) ?? [] micDelaySeconds = try c.decodeIfPresent(TimeInterval.self, forKey: .micDelaySeconds) ?? 0 } } @@ -358,6 +387,12 @@ public struct AppSettings: Codable, Equatable { /// Reveals the gated Advanced tab (models, performance, diarization, debugging). /// Off by default to keep the default settings surface calm. public var showAdvancedSettings: Bool + /// Scrape the meeting app's chat panel via Accessibility APIs and interleave it + /// into the transcript (see `ChatReader`/`ChatMessage`). Off by default: unlike + /// the roster (names only) or notes (the user's own words), this passively + /// captures other participants' written words without their per-message + /// consent, so it's opt-in rather than automatic. + public var includeMeetingChat: Bool public static let `default` = AppSettings( userName: "", @@ -386,7 +421,8 @@ public struct AppSettings: Codable, Equatable { memoryMode: .auto, speakerRetentionDays: 90, selectedInputDeviceUID: nil, - showAdvancedSettings: false + showAdvancedSettings: false, + includeMeetingChat: false ) public init( @@ -413,7 +449,8 @@ public struct AppSettings: Codable, Equatable { memoryMode: MemoryMode = .auto, speakerRetentionDays: Int = 90, selectedInputDeviceUID: String? = nil, - showAdvancedSettings: Bool = false + showAdvancedSettings: Bool = false, + includeMeetingChat: Bool = false ) { self.userName = userName self.launchAtLogin = launchAtLogin @@ -439,6 +476,7 @@ public struct AppSettings: Codable, Equatable { self.speakerRetentionDays = speakerRetentionDays self.selectedInputDeviceUID = selectedInputDeviceUID self.showAdvancedSettings = showAdvancedSettings + self.includeMeetingChat = includeMeetingChat } } @@ -575,6 +613,9 @@ public struct TranscriptDocument { public var notes: [MeetingNote] /// Display name to attribute notes to. Falls back to "Me" when empty. public var noteAuthor: String + /// Messages scraped from the meeting chat panel, when opted in. Rendered + /// chronologically alongside speaker segments and notes in the markdown output. + public var chatMessages: [ChatMessage] public init( title: String, @@ -586,7 +627,8 @@ public struct TranscriptDocument { diarizationSegments: [(speakerID: String, startTime: TimeInterval, endTime: TimeInterval)] = [], unmatchedRosterNames: [String] = [], notes: [MeetingNote] = [], - noteAuthor: String = "Me" + noteAuthor: String = "Me", + chatMessages: [ChatMessage] = [] ) { self.title = title self.startTime = startTime @@ -598,6 +640,7 @@ public struct TranscriptDocument { self.unmatchedRosterNames = unmatchedRosterNames self.notes = notes self.noteAuthor = noteAuthor + self.chatMessages = chatMessages } } diff --git a/Sources/HeardCore/RosterReader.swift b/Sources/HeardCore/RosterReader.swift index 8d56174..60f2e2d 100644 --- a/Sources/HeardCore/RosterReader.swift +++ b/Sources/HeardCore/RosterReader.swift @@ -16,7 +16,7 @@ public protocol AXNode { // MARK: - Live AX adapter -private struct AXUIElementNode: AXNode { +struct AXUIElementNode: AXNode { private let element: AXUIElement init(_ element: AXUIElement) { self.element = element } diff --git a/Sources/HeardCore/Services.swift b/Sources/HeardCore/Services.swift index 731443e..78be9c2 100644 --- a/Sources/HeardCore/Services.swift +++ b/Sources/HeardCore/Services.swift @@ -118,8 +118,11 @@ public struct RecordingSession { /// In-flight notes captured during the meeting via the note-composer hotkey. /// Carried into the `PipelineJob` when the session ends. public var notes: [MeetingNote] + /// In-flight chat messages scraped via `ChatReader`, when opted in. Carried + /// into the `PipelineJob` when the session ends. + public var chatMessages: [ChatMessage] - public init(title: String, startTime: Date, appAudioPath: URL, micAudioPath: URL, micDelaySeconds: TimeInterval, rosterNames: [String] = [], notes: [MeetingNote] = []) { + public init(title: String, startTime: Date, appAudioPath: URL, micAudioPath: URL, micDelaySeconds: TimeInterval, rosterNames: [String] = [], notes: [MeetingNote] = [], chatMessages: [ChatMessage] = []) { self.title = title self.startTime = startTime self.appAudioPath = appAudioPath @@ -127,6 +130,7 @@ public struct RecordingSession { self.micDelaySeconds = micDelaySeconds self.rosterNames = rosterNames self.notes = notes + self.chatMessages = chatMessages } } @@ -202,6 +206,13 @@ public final class MeetingDetector: ObservableObject { private let onMeetingStarted: @MainActor (MeetingSnapshot) -> Void private let onMeetingEnded: @MainActor (MeetingSnapshot) -> Void private let enabledSources: @MainActor () -> Set + /// Whether the user has opted into chat scraping (`AppSettings.includeMeetingChat`). + /// Gates both the AX chat walk (privacy: skip entirely when off) and delivery. + private let chatScrapeEnabled: @MainActor () -> Bool + /// Fired from the roster-poll tick with any newly-observed chat messages + /// (already deduplicated against what that poll last saw; cross-poll dedup + /// happens downstream in `RecordingManager.updateChatMessages`). + private let onChatMessagesObserved: @MainActor ([(sender: String, text: String)]) -> Void private var activeSnapshot: MeetingSnapshot? private var pollingTask: Task? private var rosterPollingTask: Task? @@ -229,12 +240,16 @@ public final class MeetingDetector: ObservableObject { public init( enabledSources: @escaping @MainActor () -> Set = { Set(MeetingApp.allCases) }, + chatScrapeEnabled: @escaping @MainActor () -> Bool = { false }, onMeetingStarted: @escaping @MainActor (MeetingSnapshot) -> Void, - onMeetingEnded: @escaping @MainActor (MeetingSnapshot) -> Void + onMeetingEnded: @escaping @MainActor (MeetingSnapshot) -> Void, + onChatMessagesObserved: @escaping @MainActor ([(sender: String, text: String)]) -> Void = { _ in } ) { self.enabledSources = enabledSources + self.chatScrapeEnabled = chatScrapeEnabled self.onMeetingStarted = onMeetingStarted self.onMeetingEnded = onMeetingEnded + self.onChatMessagesObserved = onChatMessagesObserved } public func startWatching() { @@ -254,6 +269,9 @@ public final class MeetingDetector: ObservableObject { if let snapshot = activeSnapshot { stopRosterPolling() activeSnapshot = nil + Task.detached(priority: .utility) { + Self.disableMeetingAppAccessibility(pid: snapshot.meetingPID, source: snapshot.source) + } onMeetingEnded(snapshot) } } @@ -280,6 +298,9 @@ public final class MeetingDetector: ObservableObject { stopRosterPolling() activeSnapshot = nil detectionState = MeetingDetectionState() + Task.detached(priority: .utility) { + Self.disableMeetingAppAccessibility(pid: snapshot.meetingPID, source: snapshot.source) + } onMeetingEnded(snapshot) return } @@ -328,6 +349,9 @@ public final class MeetingDetector: ObservableObject { guard let snapshot = activeSnapshot else { return } stopRosterPolling() activeSnapshot = nil + Task.detached(priority: .utility) { + Self.disableMeetingAppAccessibility(pid: snapshot.meetingPID, source: snapshot.source) + } onMeetingEnded(snapshot) } } @@ -383,18 +407,55 @@ public final class MeetingDetector: ObservableObject { /// Teams keeps its a11y tree off by default to save resources, so AX reads of /// its windows/titles/roster fail (commonly `kAXErrorAPIDisabled`, -25211) even /// though Heard itself is trusted. Setting `AXManualAccessibility` on the app - /// element signals Chromium to build the tree. The build is asynchronous, so - /// callers must still retry the actual read after this returns. + /// element is the documented way to signal Chromium to build the tree — but on + /// current Electron builds this always fails with `setErr=-25205` + /// (`kAXErrorAttributeUnsupported`): Electron doesn't advertise the attribute in + /// `accessibilityAttributeNames` (see electron/electron#37465), so the call is + /// rejected before it ever reaches Chromium's tree-building logic. This is an + /// upstream Electron bug, not a Heard AX-trust problem. + /// + /// Fallback: `AXEnhancedUserInterface` — the same private attribute VoiceOver + /// sets — reliably forces Chromium's full accessibility tree on regardless of + /// the `AXManualAccessibility` bug above. It's a blunter instrument (it also + /// flips the target app into "assistive technology present" mode, which can + /// alter its own UI behavior while set), so it's only tried after + /// `AXManualAccessibility` fails, and `disableMeetingAppAccessibility` unsets it + /// again once the meeting ends. /// - /// The `setErr` log line is the diagnostic discriminator: `.success` means the - /// nudge was accepted (any lingering empty reads are tree-build timing); a - /// `.apiDisabled` here would instead mean Heard is genuinely untrusted. - nonisolated private static func enableMeetingAppAccessibility(pid: pid_t?, source: MeetingApp) { - guard let pid else { return } + /// The `setErr`/`fallbackSetErr` log lines are the diagnostic discriminators: + /// `.success` on either means the nudge was accepted (any lingering empty reads + /// are tree-build timing); `.apiDisabled` on both would instead mean Heard is + /// genuinely untrusted. + @discardableResult + nonisolated private static func enableMeetingAppAccessibility(pid: pid_t?, source: MeetingApp) -> Bool { + guard let pid else { return false } let app = AXUIElementCreateApplication(pid) AXUIElementSetMessagingTimeout(app, 1.0) let setErr = AXUIElementSetAttributeValue(app, "AXManualAccessibility" as CFString, kCFBooleanTrue) - DebugFileLog.log("enableMeetingAppAccessibility: pid=\(pid) source=\(source.rawValue) setErr=\(setErr.rawValue)") + if setErr == .success { + DebugFileLog.log("enableMeetingAppAccessibility: pid=\(pid) source=\(source.rawValue) setErr=\(setErr.rawValue)") + return true + } + let fallbackSetErr = AXUIElementSetAttributeValue(app, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + DebugFileLog.log("enableMeetingAppAccessibility: pid=\(pid) source=\(source.rawValue) setErr=\(setErr.rawValue) fallbackSetErr=\(fallbackSetErr.rawValue)") + return fallbackSetErr == .success + } + + /// Unset `AXEnhancedUserInterface` on meeting end. `AXManualAccessibility` never + /// needs undoing — it either failed to set (the common case, see above) or is a + /// one-way tree-build nudge with no corresponding teardown. This is best-effort + /// hygiene, not correctness-critical: failures are swallowed. + /// + /// Known gap: this only fires from Heard's own meeting-end paths. If Heard is + /// force-quit or crashes mid-meeting, Teams is left with `AXEnhancedUserInterface` + /// set (and its side effects) until Teams itself restarts — there is no process-exit + /// hook that could run this reliably, so it's accepted as-is rather than engineered + /// around. + nonisolated private static func disableMeetingAppAccessibility(pid: pid_t?, source: MeetingApp) { + guard let pid, source == .teams else { return } + let app = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(app, 1.0) + AXUIElementSetAttributeValue(app, "AXEnhancedUserInterface" as CFString, kCFBooleanFalse) } /// Extract the meeting title from a meeting-app window via Accessibility API. @@ -488,12 +549,14 @@ public final class MeetingDetector: ObservableObject { // While roster is still empty, capture a bounded tree dump (Developer // Mode + budget) so an empty result is self-diagnosing, not silent. let wantDump = rosterEmpty && rosterDumpsRemaining > 0 && DebugFileLog.isEnabled - let (names, refreshedTitle, treeDump) = await Task.detached(priority: .utility) { + let wantChat = self.chatScrapeEnabled() + let (names, refreshedTitle, treeDump, chatMessages) = await Task.detached(priority: .utility) { if rosterEmpty { Self.enableMeetingAppAccessibility(pid: pid, source: source) } let names = RosterReader.readRoster(pid: pid) let title = needTitle ? Self.extractMeetingTitle(pid: pid, source: source) : nil let dump = (wantDump && names.isEmpty) ? RosterReader.diagnosticTreeDump(pid: pid) : nil - return (names, title, dump) + let chat = wantChat ? ChatReader.readChatMessages(pid: pid) : [] + return (names, title, dump, chat) }.value if !names.isEmpty { let existing = Set(self.activeSnapshot?.rosterNames ?? []) @@ -510,6 +573,10 @@ public final class MeetingDetector: ObservableObject { rosterDumpsRemaining -= 1 DebugFileLog.log("roster poll tick=\(tick): readRoster empty — AX tree dump (\(3 - rosterDumpsRemaining)/3):\n\(treeDump)") } + if !chatMessages.isEmpty { + DebugFileLog.log("roster poll tick=\(tick): chat read=\(chatMessages.count) messages") + self.onChatMessagesObserved(chatMessages) + } } } } @@ -686,6 +753,28 @@ public final class RecordingManager: ObservableObject { currentRosterNames = names } + /// Append newly observed chat messages to the active session, deduplicating + /// against everything already captured. Each poll re-reads the AX tree's + /// current on-screen chat state (a snapshot, not an incremental delta), so + /// the caller passes every message it currently sees and this filters down + /// to genuinely new ones. `offsetSeconds` is stamped as "now" relative to + /// the session start — the moment Heard observed the message, not when it + /// was actually sent (Teams' AX tree doesn't reliably expose that). No-ops + /// if no session is active. + public func updateChatMessages(_ raw: [(sender: String, text: String)]) { + guard let session = activeSession, !raw.isEmpty else { return } + var seen = Set(session.chatMessages.map { "\($0.sender)\u{0}\($0.text)" }) + let offset = max(0, Date().timeIntervalSince(session.startTime)) + var newMessages: [ChatMessage] = [] + for entry in raw { + let key = "\(entry.sender)\u{0}\(entry.text)" + guard seen.insert(key).inserted else { continue } + newMessages.append(ChatMessage(offsetSeconds: offset, sender: entry.sender, text: entry.text)) + } + guard !newMessages.isEmpty else { return } + activeSession?.chatMessages.append(contentsOf: newMessages) + } + /// Update the meeting title on the active session. The title is often captured /// late (Teams' a11y tree builds after join), so the recording starts with an /// empty title and this fills it in before the session is enqueued, fixing the @@ -2046,21 +2135,29 @@ public enum TranscriptWriter { """ - let body = renderBody(segments: document.segments, notes: document.notes, noteAuthor: document.noteAuthor) + let body = renderBody( + segments: document.segments, + notes: document.notes, + noteAuthor: document.noteAuthor, + chatMessages: document.chatMessages + ) try (header + body + "\n").write(to: candidate, atomically: true, encoding: .utf8) return candidate } - /// Merge spoken segments and user-authored notes into a single chronological - /// markdown body. Notes are rendered in italics with a `**Note from :**` - /// label so they're visually distinct from spoken speaker blocks. Notes use - /// their `offsetSeconds` as the timestamp; for sort stability when a note - /// shares a timestamp with a segment, the note appears immediately after. + /// Merge spoken segments, user-authored notes, and scraped chat messages into + /// a single chronological markdown body. Notes and chat are both rendered in + /// italics so they're visually distinct from spoken speaker blocks, with + /// different labels (`**Note from :**` vs `**Chat — :**`) so + /// the user's own words stay distinguishable from other participants' chat. + /// Both use their `offsetSeconds` as the timestamp; for sort stability when + /// several items share a timestamp with a segment, the segment sorts first. public static func renderBody( segments: [TranscriptSegment], notes: [MeetingNote], - noteAuthor: String + noteAuthor: String, + chatMessages: [ChatMessage] = [] ) -> String { let author = noteAuthor.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? "Me" @@ -2069,24 +2166,26 @@ public enum TranscriptWriter { enum Item { case segment(TranscriptSegment) case note(MeetingNote) + case chat(ChatMessage) var time: TimeInterval { switch self { case .segment(let s): return s.startTime case .note(let n): return n.offsetSeconds + case .chat(let c): return c.offsetSeconds } } - // 0 for segments, 1 for notes — keeps notes after a segment that + // 0 for segments, 1 for notes/chat — keeps them after a segment that // starts at the same instant rather than splitting the speaker block // by a hair. var sortKind: Int { switch self { case .segment: return 0 - case .note: return 1 + case .note, .chat: return 1 } } } - var items: [Item] = segments.map { .segment($0) } + notes.map { .note($0) } + var items: [Item] = segments.map { .segment($0) } + notes.map { .note($0) } + chatMessages.map { .chat($0) } items.sort { lhs, rhs in if lhs.time != rhs.time { return lhs.time < rhs.time } return lhs.sortKind < rhs.sortKind @@ -2100,6 +2199,10 @@ public enum TranscriptWriter { let body = n.text.trimmingCharacters(in: .whitespacesAndNewlines) let timestamp = max(0, n.offsetSeconds).timestampString return "[\(timestamp)] _**Note from \(author):** \(body)_" + case .chat(let c): + let body = c.text.trimmingCharacters(in: .whitespacesAndNewlines) + let timestamp = max(0, c.offsetSeconds).timestampString + return "[\(timestamp)] _**Chat — \(c.sender):** \(body)_" } }.joined(separator: "\n\n") } @@ -2194,6 +2297,7 @@ public final class PipelineProcessor: ObservableObject { retryCount: 0, rosterNames: session.rosterNames, notes: session.notes, + chatMessages: session.chatMessages, micDelaySeconds: session.micDelaySeconds ) queueStore.enqueue(job) @@ -3057,7 +3161,8 @@ public final class PipelineProcessor: ObservableObject { diarizationSegments: diarSegTuples, unmatchedRosterNames: unmatchedRosterNamesForPrompt, notes: job.notes, - noteAuthor: userName.isEmpty ? "Me" : userName + noteAuthor: userName.isEmpty ? "Me" : userName, + chatMessages: job.chatMessages ) } private func buildSegmentsFromTimings( diff --git a/Sources/HeardCore/SettingsTabs.swift b/Sources/HeardCore/SettingsTabs.swift index 720ec79..4985f70 100644 --- a/Sources/HeardCore/SettingsTabs.swift +++ b/Sources/HeardCore/SettingsTabs.swift @@ -181,6 +181,17 @@ extension SettingsView { } } + sectionGroup("Meeting Chat") { + SettingsCard { + ToggleRow( + title: "Include Meeting Chat in Transcript", + subtitle: "Off by default: this captures other participants' chat messages, not just your own notes.", + isLast: true, + isOn: settingsBinding(\.includeMeetingChat) + ) + } + } + sectionGroup("Output") { SettingsCard { CardRow(isLast: true) { diff --git a/Sources/HeardCore/Stores.swift b/Sources/HeardCore/Stores.swift index b67100f..63f87ec 100644 --- a/Sources/HeardCore/Stores.swift +++ b/Sources/HeardCore/Stores.swift @@ -210,7 +210,8 @@ public final class SettingsStore: ObservableObject { memoryMode: memoryMode, speakerRetentionDays: speakerRetentionDays, selectedInputDeviceUID: defaults.string(forKey: "selectedInputDeviceUID"), - showAdvancedSettings: defaults.object(forKey: "showAdvancedSettings") as? Bool ?? base.showAdvancedSettings + showAdvancedSettings: defaults.object(forKey: "showAdvancedSettings") as? Bool ?? base.showAdvancedSettings, + includeMeetingChat: defaults.object(forKey: "includeMeetingChat") as? Bool ?? base.includeMeetingChat ) } @@ -245,6 +246,7 @@ public final class SettingsStore: ObservableObject { defaults.set(settings.speakerRetentionDays, forKey: "speakerRetentionDays") defaults.set(settings.memoryMode.rawValue, forKey: "memoryMode") defaults.set(settings.showAdvancedSettings, forKey: "showAdvancedSettings") + defaults.set(settings.includeMeetingChat, forKey: "includeMeetingChat") if let uid = settings.selectedInputDeviceUID { defaults.set(uid, forKey: "selectedInputDeviceUID") } else { diff --git a/Tests/HeardTests/TestRunner.swift b/Tests/HeardTests/TestRunner.swift index 6e5a62f..a8d5b9f 100644 --- a/Tests/HeardTests/TestRunner.swift +++ b/Tests/HeardTests/TestRunner.swift @@ -694,6 +694,82 @@ func runTranscriptWriterTests() { try expect(content.contains("**Note from John:** side comment"), "Note line untouched by rename") } + + test("Chat messages are interleaved chronologically with segments and notes") { + let tmpDir = FileManager.default.temporaryDirectory + .appendingPathComponent("HeardTests-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: tmpDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmpDir) } + + let doc = TranscriptDocument( + title: "Q1 Review", + startTime: Date(), + endTime: Date().addingTimeInterval(120), + participants: ["Alice", "Me"], + segments: [ + TranscriptSegment(speaker: "Alice", startTime: 0, endTime: 20, text: "We saw FCR jump."), + TranscriptSegment(speaker: "Me", startTime: 60, endTime: 90, text: "Nice work."), + ], + notes: [MeetingNote(offsetSeconds: 22, text: "FCR = First Call Resolution")], + noteAuthor: "John", + chatMessages: [ChatMessage(offsetSeconds: 40, sender: "Bob Jones", text: "Great numbers!")] + ) + let url = try TranscriptWriter.write(document: doc, outputDirectory: tmpDir) + let content = try String(contentsOf: url, encoding: .utf8) + + try expect(content.contains("**Chat — Bob Jones:** Great numbers!"), + "Should render chat line with sender attribution") + try expect(content.contains("[00:40] _**Chat — Bob Jones:**"), + "Chat timestamp uses offsetSeconds") + + // Order: Alice (00:00) < note (00:22) < chat (00:40) < Me (01:00) + let aliceRange = content.range(of: "Alice:")! + let noteRange = content.range(of: "Note from John:")! + let chatRange = content.range(of: "Chat — Bob Jones:")! + let meRange = content.range(of: "Me:** Nice work")! + try expect(aliceRange.lowerBound < noteRange.lowerBound, "note must come after Alice") + try expect(noteRange.lowerBound < chatRange.lowerBound, "chat must come after the note") + try expect(chatRange.lowerBound < meRange.lowerBound, "chat must come before Me") + } + + test("Chat message at same timestamp as segment sorts after the segment") { + let body = TranscriptWriter.renderBody( + segments: [TranscriptSegment(speaker: "Alice", startTime: 10, endTime: 15, text: "Hi.")], + notes: [], + noteAuthor: "John", + chatMessages: [ChatMessage(offsetSeconds: 10, sender: "Bob", text: "tied")] + ) + let aliceIdx = body.range(of: "Alice:")!.lowerBound + let chatIdx = body.range(of: "Chat — Bob:")!.lowerBound + try expect(aliceIdx < chatIdx, "Segment must precede same-timestamp chat message") + } + + test("Empty chatMessages array renders identical to pre-chat output") { + let segs = [TranscriptSegment(speaker: "Alice", startTime: 0, endTime: 5, text: "Hello.")] + let body = TranscriptWriter.renderBody(segments: segs, notes: [], noteAuthor: "John") + try expect(body.contains("**Alice:** Hello."), "Alice present") + try expect(!body.contains("Chat —"), "No chat marker without chat messages") + } + + test("renameSpeakerInDirectory does not mangle Chat lines") { + let tmpDir = FileManager.default.temporaryDirectory + .appendingPathComponent("HeardTests-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: tmpDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmpDir) } + + let doc = TranscriptDocument( + title: "Standup", startTime: Date(), endTime: Date().addingTimeInterval(60), + participants: ["Speaker 7"], + segments: [TranscriptSegment(speaker: "Speaker 7", startTime: 0, endTime: 30, text: "Hey.")], + chatMessages: [ChatMessage(offsetSeconds: 10, sender: "Speaker 7", text: "typed the same as I said")] + ) + let url = try TranscriptWriter.write(document: doc, outputDirectory: tmpDir) + TranscriptWriter.renameSpeakerInDirectory(tmpDir, from: "Speaker 7", to: "Bob") + let content = try String(contentsOf: url, encoding: .utf8) + try expect(content.contains("**Bob:** Hey."), "Speaker rename applied to segment") + try expect(content.contains("**Chat — Speaker 7:** typed the same as I said"), + "Chat sender label is untouched by speaker-segment rename") + } } // MARK: - Store Tests @@ -1954,6 +2030,104 @@ func runRosterReaderAXTests() { } } +// MARK: - ChatReader Tests + +func runChatReaderTests() { + print("\n💬 ChatReader Tests") + + test("Row-container shape: first child is sender, rest joined as body") { + let tree = MockAXNode(role: "AXApplication", children: [ + MockAXNode(role: "AXWindow", children: [ + MockAXNode(role: "AXGroup", identifier: "chat-pane-list", children: [ + MockAXNode(role: "AXGroup", children: [ + MockAXNode(role: "AXStaticText", value: "Alice Smith"), + MockAXNode(role: "AXStaticText", value: "Hello"), + MockAXNode(role: "AXStaticText", value: "everyone."), + ]), + ]), + ]), + ]) + let messages = ChatReader.readChatMessagesFromNode(tree) + try expectEqual(messages.count, 1) + try expectEqual(messages[0].sender, "Alice Smith") + try expectEqual(messages[0].text, "Hello everyone.") + } + + test("Single-string shape: 'Sender: text' splits on first colon") { + let tree = MockAXNode(role: "AXApplication", children: [ + MockAXNode(role: "AXWindow", children: [ + MockAXNode(role: "AXGroup", identifier: "chat-pane-list", children: [ + MockAXNode(role: "AXStaticText", description: "Bob Jones: Sounds good, thanks!"), + ]), + ]), + ]) + let messages = ChatReader.readChatMessagesFromNode(tree) + try expectEqual(messages.count, 1) + try expectEqual(messages[0].sender, "Bob Jones") + try expectEqual(messages[0].text, "Sounds good, thanks!") + } + + test("Single-string shape: 'Sender, text' splits on first comma when no colon") { + let tree = MockAXNode(role: "AXApplication", children: [ + MockAXNode(role: "AXWindow", children: [ + MockAXNode(role: "AXGroup", identifier: "message-list", children: [ + MockAXNode(role: "AXStaticText", value: "Carol Liu, on my way"), + ]), + ]), + ]) + let messages = ChatReader.readChatMessagesFromNode(tree) + try expectEqual(messages.count, 1) + try expectEqual(messages[0].sender, "Carol Liu") + try expectEqual(messages[0].text, "on my way") + } + + test("Control-string rows are dropped") { + let tree = MockAXNode(role: "AXApplication", children: [ + MockAXNode(role: "AXWindow", children: [ + MockAXNode(role: "AXGroup", identifier: "chat-pane-list", children: [ + MockAXNode(role: "AXStaticText", description: "Alice Smith: Hello"), + MockAXNode(role: "AXStaticText", description: "Bob Jones: send"), + ]), + ]), + ]) + let messages = ChatReader.readChatMessagesFromNode(tree) + try expectEqual(messages.count, 1) + try expectEqual(messages[0].sender, "Alice Smith") + } + + test("No known chat identifier in the tree returns empty") { + let tree = MockAXNode(role: "AXApplication", children: [ + MockAXNode(role: "AXWindow", children: [ + MockAXNode(role: "AXGroup", identifier: "roster-list", children: [ + MockAXNode(role: "AXStaticText", value: "Alice Smith"), + ]), + ]), + ]) + let messages = ChatReader.readChatMessagesFromNode(tree) + try expect(messages.isEmpty, "roster panel should not be mistaken for chat") + } + + test("Chat panel not present (empty tree) returns empty, not a crash") { + let tree = MockAXNode(role: "AXApplication", children: []) + let messages = ChatReader.readChatMessagesFromNode(tree) + try expect(messages.isEmpty) + } + + test("Row with only a sender and no body text is dropped") { + let tree = MockAXNode(role: "AXApplication", children: [ + MockAXNode(role: "AXWindow", children: [ + MockAXNode(role: "AXGroup", identifier: "chat-pane-list", children: [ + MockAXNode(role: "AXGroup", children: [ + MockAXNode(role: "AXStaticText", value: "Alice Smith"), + ]), + ]), + ]), + ]) + let messages = ChatReader.readChatMessagesFromNode(tree) + try expect(messages.isEmpty, "a row with no body text should not become an empty message") + } +} + // MARK: - SegmentDeduplicator Tests func runSegmentDeduplicatorTests() { @@ -2303,6 +2477,7 @@ struct TestRunner { runSpeakerEmbeddingAggregatorTests() runRosterReaderTests() runRosterReaderAXTests() + runChatReaderTests() runSegmentDeduplicatorTests() runSystemMemoryTests() runPermissionCenterTests() diff --git a/handoff.md b/handoff.md index cba43a5..05597ec 100644 --- a/handoff.md +++ b/handoff.md @@ -16,7 +16,7 @@ The app builds cleanly with `swift build` and runs as a menu bar app on macOS 15 - `DebugFileLog` is now gated behind Developer Mode (off by default), rotates at 5 MB, and never logs transcript content (lengths only) — previously it logged every dictated utterance to an unbounded plaintext file and ran a 1 Hz heartbeat in every install. - Meeting-start title/roster AX scraping and the 15 s roster poll now run on detached background tasks with a 1 s `AXUIElementSetMessagingTimeout`, so a busy/hung Teams can no longer stall Heard's main thread. The meeting (and recording) starts when the scrape completes; if the meeting ends mid-scrape, the detection state machine prevents an orphaned start. - **Teams accessibility-tree wake (fixes empty meeting titles → "Meeting" filename; unblocks — but does not yet verify — the Teams roster scrape).** Teams is an Electron/Chromium app whose accessibility tree is dormant until a client requests it; AX reads of its windows/titles/roster failed (logged `axErr=-25211`, `kAXErrorAPIDisabled`) even though Heard is trusted (stable Developer ID signing + live `AXIsProcessTrusted`). At meeting detection we now set `AXManualAccessibility=true` on the Teams app element (`MeetingDetector.enableMeetingAppAccessibility`) to wake the tree; the build is async and the join-time set can time out against a busy Teams, so the 15 s roster poll **re-nudges while the roster is still empty**, re-extracts the title until non-empty, and the late title is pushed into the live session via `RecordingManager.updateTitle` in `onMeetingEnded` (parallel to `updateRosterNames`) before the filename is built at transcript-write time. `RecordingSession.title` is now `var`. - - **Title — pending real-meeting confirmation:** with Developer Mode on, the next Teams meeting should log `enableMeetingAppAccessibility: ... setErr=0` (success) followed by a `roster poll: title refreshed` line. If `setErr` is non-zero, or the title stays empty after `setErr=0`, capture the log — a `setErr=-25211` here would instead point back at Heard's own AX trust. + - **RESOLVED (see the dated entry near the top of this section for the full fix): the first real-meeting test came back `titleRefreshed=false` on every tick with `setErr=-25205` (`kAXErrorAttributeUnsupported`) — not `-25211`.** Root cause: `AXManualAccessibility` is a known-broken no-op on current Electron (electron/electron#37465 — Electron never advertises the attribute in `accessibilityAttributeNames`, so the set is rejected before it reaches Chromium's tree-builder). This is an upstream Electron bug, not a Heard AX-trust problem. Fixed by falling back to `AXEnhancedUserInterface` (the same attribute VoiceOver sets) when `AXManualAccessibility` fails — see `enableMeetingAppAccessibility`. **Still pending a second real-meeting confirmation** that the fallback actually wakes the tree end to end (title/roster populate). - **Roster — unblocked but unverified.** Waking the tree is *necessary* but not *sufficient*: `RosterReader`'s parser (hardcoded identifiers `roster-list`/`people-pane`/… and AXList/AXTable heuristics) has never run against real woken Teams output, so the roster may still return empty due to a parser/structure mismatch independent of the tree-wake. (Today participant names in transcripts come from voice matching, not this scrape.) - **Diagnostics for the next real meeting (Developer Mode on, tail `~/Library/Application Support/Heard/dict-debug.log`).** The flow is now self-diagnosing end to end: - `meeting start scrape: source=teams pid=… titleCaptured= rosterCount=N` — the anchor line at join. @@ -69,6 +69,14 @@ All items from the post-v0.2.2 robustness review are now resolved (see `ROADMAP. **UI file split (tech debt):** the ~1.9 kLOC `Views.swift` was split by view area into `DesignSystem.swift`, `MenuBarView.swift`, `SpeakerNamingView.swift`, `SettingsView.swift`, `SettingsTabs.swift`, and `SettingsComponents.swift` (all in `Sources/HeardCore/`). Pure move — no logic or symbol changes; `swift build` is clean and all 181 tests pass. +**AX-tree-wake root cause found + fixed, plus a new opt-in meeting-chat-scraping feature (post-v0.2.6, unreleased).** A real Teams meeting confirmed the tree-wake nudge was failing with `setErr=-25205` (`kAXErrorAttributeUnsupported`), not the `-25211` the original fix assumed. Root cause (confirmed via electron/electron#37465, cross-checked with a second opinion from an independent Antigravity/agy investigation run through the `delegate` skill): `AXManualAccessibility` never actually works on current Electron builds — Electron doesn't advertise the attribute in `accessibilityAttributeNames`, so the set is rejected before Chromium's tree-builder ever sees it. This affects any Electron app, not just Teams, and isn't fixable from Heard's side. + +- **Fix:** `enableMeetingAppAccessibility` now falls back to the private `AXEnhancedUserInterface` attribute (the same one VoiceOver sets) whenever `AXManualAccessibility` fails; it logs `setErr= fallbackSetErr=` so a future log capture shows which path actually woke the tree. `AXEnhancedUserInterface` is confirmed (both via the Electron issue thread and agy's independent research) as the standard, low-risk way third-party AX tools coexist with Electron on macOS — no realistic risk of Microsoft treating it as automation abuse. It does have side effects on the *target app* while set (reported: altered window positioning/sizing, interference with tools like Rectangle/Magnet, and possibly disabled UI animations) — so `disableMeetingAppAccessibility` unsets it again on every meeting-end path (`stopWatching`, disabled-mid-meeting, and normal `.endMeeting`). **Still needs a second real-meeting confirmation** that this actually resolves `titleRefreshed`/roster population end to end — the fallback logic itself is unit-untestable (it's a live AX side effect), so this can only be confirmed from `dict-debug.log` on a real call. +- **New: opt-in Teams-chat scraping, interleaved into the transcript.** `ChatReader.swift` mirrors `RosterReader`'s identifier-heuristic pattern (ships with only the identifier-search strategy — no generic "any list" fallback, to avoid a false-positive match against the roster panel) and is polled from the same 15 s roster-poll tick, gated behind a new `AppSettings.includeMeetingChat` toggle (**default OFF** — a considered call, not an oversight: unlike the roster (names only) or in-meeting notes (the user's own words), this passively captures other participants' written words without their per-message consent, which agy's review flagged as a real enterprise-IT/compliance risk if defaulted on). New `ChatMessage` model (offsetSeconds/sender/text) follows the exact `MeetingNote` carry-through pattern: `RecordingSession.chatMessages` → `RecordingManager.updateChatMessages` (dedups by exact sender+text against everything already captured, stamping `offsetSeconds` as "now" since Teams' AX tree doesn't reliably expose per-message send times) → `PipelineJob.chatMessages` → `TranscriptDocument.chatMessages` → `TranscriptWriter.renderBody` (rendered as `_**Chat — :** ...text..._`, interleaved chronologically alongside segments and notes). Settings UI: new "Meeting Chat" section in the Recording tab. + - **Known, documented limitations** (see `ChatReader`'s doc comment): Electron's chat list is virtualized/lazy-rendered, so only on-screen messages ever reach the AX tree — scrollback and messages sent while the panel is closed are unreachable, permanently. Dedup is exact (sender, text) match, so a genuinely repeated identical short message ("yes", "+1") from the same sender is only captured once. Message edits/deletes are not reflected. **Real AX identifiers for Teams' chat pane are still unknown** — `chatIdentifiers` in `ChatReader.swift` are best-guess placeholders; `ChatReader.diagnosticTreeDump` (reuses `RosterReader`'s dump core) is the tool to retune them from once a real meeting has the chat panel open during the same diagnostic session as the AX-wake retest above. + - 7 new tests (`runChatReaderTests` + chat-interleaving cases in the transcript-writer tests); all 196 tests pass, `swift build` clean. + - This workstream was validated mid-flight by an independent Antigravity CLI (`agy`) investigation, launched via the `delegate` skill for a second opinion on both the AX-fix root cause and the chat-scraping design (privacy default, dedup approach, virtualization gotchas) before implementation — see conversation history for the full report. + ## What's Working ### Meeting Detection @@ -128,6 +136,13 @@ All items from the post-v0.2.2 robustness review are now resolved (see `ROADMAP. - `HotkeyManager` was refactored to support multiple hotkeys: each manager owns a unique `id` (1 = dictation, 2 = notes), all sharing one Carbon event handler that dispatches by `EventHotKeyID`. - Hotkey-pressed with no active recording: opens a standalone composer (no elapsed-time offset shown). On save, writes a Markdown file named `yyyy-MM-dd_HH-mm-ss_note.md` to the user's configured output folder. Write failures surface as `errorMessage` + a beep. +### Meeting Chat Scraping (opt-in, built but not yet verified against a real meeting) +- Off by default (`AppSettings.includeMeetingChat`, Settings → Recording → Meeting Chat). Deliberately not automatic like the roster/notes — see the dated status entry above for why. +- `ChatReader.readChatMessages(pid:)` walks the same woken Teams AX tree as `RosterReader`, polled from the existing 15 s roster-poll tick (`MeetingDetector.startRosterPolling`) only when the setting is on. +- `RecordingManager.updateChatMessages(_:)` dedups incoming (sender, text) pairs against everything already captured for the session and stamps `offsetSeconds` as "now" relative to the recording start. +- `TranscriptWriter.renderBody` interleaves chat chronologically alongside segments and notes: `[mm:ss] _**Chat — :** ...text..._`. +- **Not yet validated against real Teams output** — `ChatReader`'s identifiers (`chat-pane-list`, `message-list`, etc.) are best-guess placeholders, same starting position `RosterReader` was in before its first real-meeting tree dump. Use `ChatReader.diagnosticTreeDump(pid:)` (reuses `RosterReader`'s dump core) on the next real meeting with the chat panel open to retune them. + ### Dictation (Fully Working) The dictation feature captures mic audio, transcribes in real-time, and injects text into the focused app via CGEvent unicode insertion. Requires Accessibility permission granted to a stable-signed build. @@ -199,7 +214,7 @@ The dictation feature captures mic audio, transcribes in real-time, and injects - **v0.1.0 released**: `dist/Heard-0.1.0.dmg` (notarized, stapled). GitHub Release at `github.com/execsumo/Heard/releases/tag/v0.1.0`. Homebrew tap at `github.com/execsumo/homebrew-heard` (`brew tap execsumo/heard && brew install --cask heard`). ### Testing -- `HeardTests` executable target with 100 tests across: VadSegmentMap, cosine distance, SpeakerMatcher (incl. threshold/margin edge cases), SegmentMerger, AudioPreprocessor, TranscriptWriter, SpeakerStore, PipelineQueueStore, pipeline resume/recovery (`prepareForResume`), meeting detection state machine (`MeetingDetectionState`), retry executor (`PipelineProcessor.executeWithRetry`) incl. lifetime cap, Teams identification, MeetingDetector lifecycle, and RosterReader (window-title parser + filter) +- `HeardTests` executable target with 196 tests across: VadSegmentMap, cosine distance, SpeakerMatcher (incl. threshold/margin edge cases), SegmentMerger, AudioPreprocessor, TranscriptWriter (incl. chat/note interleaving), SpeakerStore, PipelineQueueStore, pipeline resume/recovery (`prepareForResume`), meeting detection state machine (`MeetingDetectionState`), retry executor (`PipelineProcessor.executeWithRetry`) incl. lifetime cap, Teams identification, MeetingDetector lifecycle, RosterReader (window-title parser + filter + AX traversal), and ChatReader (AX traversal) - Custom lightweight test harness (no XCTest/Xcode dependency). `test(...)` for sync, `testAsync(...)` for async bodies - Run with `swift run HeardTests` @@ -239,6 +254,7 @@ The dictation feature captures mic audio, transcribes in real-time, and injects | `Sources/HeardCore/AudioProcessing.swift` | AudioPreprocessor, VadSegmentMap, PreprocessedTrack | | `Sources/HeardCore/SpeakerAssignment.swift` | SpeakerMatcher, SegmentMerger, cosineDistance | | `Sources/HeardCore/AudioClipExtractor.swift` | Extract speaker audio clips from WAV for naming prompt | +| `Sources/HeardCore/ChatReader.swift` | Opt-in AX scrape of the meeting app's chat panel (`ChatMessage`, interleaved into transcripts) | | `Sources/HeardCore/ModelDownloadManager.swift` | Pre-download manager for FluidAudio models | | `Sources/HeardCore/DictationManager.swift` | Real-time dictation engine (batch ASR + polling loop) | | `Sources/HeardCore/TextInjector.swift` | Text injection via CGEvent unicode insertion | diff --git a/settings-reorg-v2.md b/settings-reorg-v2.md new file mode 100644 index 0000000..64470fe --- /dev/null +++ b/settings-reorg-v2.md @@ -0,0 +1,173 @@ +# Heard Settings — Reorganization v2 (deletion-first) + +Supersedes [settings-reorg-proposal.md](settings-reorg-proposal.md). v1 rearranged furniture; this pass asks whether each piece should exist, anchors on user flows, pulls non-settings out of Settings, and commits to the calls v1 left open. + +> **Addendum (post-v0.2.6, unreleased):** a new "Include Meeting Chat in Transcript" toggle (`AppSettings.includeMeetingChat`, default **off**) was added to the live Recording tab as its own "Meeting Chat" section, alongside "Meeting Notes." This proposal predates that addition — when executing this reorg, give chat scraping the same "shared engine, opt-in" treatment as Custom Vocabulary (visible, not gated) and update the ~17-setting count accordingly. See `handoff.md`'s dated status entry for the feature's full design and its known limitations. + + + +--- + +## What changed from v1, and why + +| Roast | Fix in v2 | +|---|---| +| "You reduced nothing." | Every setting now has an explicit verdict: **Cut / Default / Keep / Gate / Move.** 38 → ~17 visible. | +| "Progressive disclosure as a filing cabinet." | Split **status** (model health, permissions) from **advanced** (tuning). Status is never gated; only true expert knobs are. | +| "You split General to fix overcrowding — added surface." | Re-anchored on flows, not nouns. Net **4 visible tabs** (was 6), + a separate Library window. | +| "Mic contradiction." | One rule, applied consistently (below). Mic, language, and vocabulary — all shared-engine — now live together. | +| "Speakers is data, not settings." | Promoted to a **Library window**, out of Settings entirely. | +| "No metric." | Concrete targets stated up front. | +| "Where's the user?" | Structure follows *set up once → use daily → review → tune rarely*. Daily use needs zero Settings trips. | + +**The rule that resolves the mic question:** *anything that shapes audio-in or speech-to-text for the whole app is one group, placed in the flow where people think about transcript quality.* Mic, language, and custom vocabulary are all shared by meetings and dictation, so they sit together in **Recording** — not scattered, and not split between General and a feature tab. Hotkeys and feature toggles stay with their feature. No more "same principle, opposite conclusion." + +--- + +## Success metrics (so this is falsifiable) + +- **Visible settings: 38 → ~17.** The rest are cut, folded into defaults, gated, or moved out of Settings. +- **Default-visible tabs: 6 → 4** (General, Recording, Dictation, About). Advanced is gated; Speakers becomes a Library window. +- **A first-run user reaches "my meetings are being recorded" touching ≤1 tab.** +- **A daily user never opens Settings** — start/stop/pause and the active hotkeys all live in the menu-bar dropdown, which already exists. +- **≤7 decisions visible per screen-glance** (per section group, not per tab). + +--- + +## The cut list (the headline) + +| # | Setting | Verdict | Rationale | +|---|---|---|---| +| 1 | **Theme (System/Light/Dark)** | **CUT** | A menu-bar utility should follow the system. Manual theme is vanity surface. Keep the code path (`.system`), drop the picker. | +| 2 | **Filename Format (5 options)** | **DEFAULT + gate** | Five date permutations = a decision nobody made. Ship one default (`YYYY-MM-DD_MeetingName`); move the picker to Advanced for the rare user who cares. | +| 3 | **Low Memory Mode** | **AUTO + gate** | Don't ask the user about RAM. Auto-enable when detected memory ≤ 8 GB; expose a manual override only in Advanced. "Decide, don't gate." | +| 4 | **Models on Disk + Download/Unload** | **MOVE to Status** | This is app *health*, not a preference, and not "advanced" — the app is broken without it. Auto-download on first run; surface a missing/corrupt model via the existing banner pattern with a one-click fix. A manual "Manage models" view stays, gated, for re-download/unload. | +| 5 | **Permissions (×4)** | **MOVE to Status** | Grants aren't settings. Handle them in first-run onboarding and surface revocations via the dropdown/banners (already done for tap, mic, AX). Keep a read-only status strip in General that deep-links to System Settings. | +| 6 | **Speaker Archive retention** | **MOVE to Library** | Data-lifecycle control belongs with the speaker data it governs. | +| 7 | **Speakers tab** | **MOVE to Library window** | It's a management/data view; you admitted as much. Out of Settings. | +| 8 | **Developer Mode** | **GATE** | Can't fully cut — "Simulate Meeting" is intentional test tooling per project rules — but it has no business in the default surface. Bottom of Advanced. | +| 9 | **Diarization sensitivity slider** | **GATE** | Genuine expert knob (cosine-similarity threshold). Stays for fine-tuning, hidden by default. | +| 10 | **Model Keep-Alive (minutes)** | **GATE** | Performance tuning. Sensible default (2 min), hidden. | +| 11 | **Show Dock Icon** | **KEEP** | Cheap, one-time, legitimately wanted by some. Stays in General. | +| 12 | **Custom Vocabulary** | **KEEP (visible)** | Directly improves accuracy on jargon/names — high user value, earns its place in Recording. | +| 13 | **Custom Formatting Commands** | **KEEP** | Ships with good defaults already; the editor is the substance of the Dictation tab. Leave visible there. | +| 14 | Name, detection, auto-watch, mic, language, save location, hotkeys, dictation toggles | **KEEP** | Core. Regrouped by flow below. | + +**Result:** ~17 settings visible by default; ~6 gated for fine-tuning; permissions + model health moved to status surfaces; speakers + retention moved to a Library window; 1 cut outright. + +--- + +## New structure (by flow) + +### Tab 1 — General *(app-level, set once)* +**Profile** +1. Your Name *(speaker label + note author)* + +**Startup** +2. Launch at Login +3. Show Dock Icon + +**Status** *(read-only, not settings)* +4. Permissions strip — Microphone / Audio Capture / Screen Recording / Accessibility, each showing granted-state with a "Fix in System Settings…" link when not. No grant flow lives here; onboarding owns that. + +--- + +### Tab 2 — Recording *(the capture + engine flow — the heart of the app)* +**Detect meetings from** +1. Microsoft Teams +2. Zoom +3. Webex +4. Auto-Watch & Record *(the master switch; mirrors the dropdown's Watching/Paused)* + +**Audio & Language** *(shared by meetings and dictation)* +5. Microphone — input device +6. Language — speech model (rename the misleading "Supported Languages"); note: *applies to dictation too* +7. Custom Vocabulary + +**Output** +8. Save Location +9. Meeting Note Hotkey + +--- + +### Tab 3 — Dictation *(the feature)* +1. Enable Dictation +2. Show Dictation Indicator *(disabled when off)* +3. Push to Talk *(disabled when off)* +4. Dictation Hotkey *(disabled when off)* +5. Custom Formatting Commands + +--- + +### Tab 4 — About +Version • update check / link • on-device badges. Unchanged. + +--- + +### Advanced *(GATED — appears only when "Show Advanced Settings" in General is on)* +1. Manage Models — status + re-download + Unload All +2. Model Keep-Alive +3. Low Memory Mode override +4. Diarization sensitivity + Reset +5. Filename Format picker +6. Developer Mode + +--- + +### Library *(separate window — NOT Settings)* +Opened from the menu-bar dropdown ("Speakers…" / "Library"). +1. Speaker search, table, merge, rename, delete, voice playback *(all current Speakers UI)* +2. **Archive speakers after** — retention dropdown, lives with the data it governs + +> Rationale: Settings is for preferences; a list of people with playable voice clips is a data view. Conflating them is what forced the awkward "Speakers has no settings" tab in the first place. + +--- + +## The gate, refined + +v1's mistake was one undifferentiated drawer. v2 separates two things v1 conflated: + +- **Status** (model health, permissions) — *never gated.* If the app can't transcribe, that's not "advanced," it's a problem the user must see. Surfaced in the dropdown via banners + a General status strip. +- **Advanced** (tuning, dev) — gated behind a visible **"Show Advanced Settings"** toggle in General. Discoverable, reversible, and safe to hide because nothing here is required for the app to work. + +This keeps the "decide, don't gate" discipline where it's cheap (theme cut, RAM auto-detected, filename defaulted) while still letting you **keep and fine-tune** the genuine expert knobs behind the gate — which is what you asked for during this tuning phase. + +--- + +## Migration map + +| Setting | From | To | +|---|---|---| +| Your Name | General | General | +| Launch at Login / Show Dock Icon | General | General | +| Permissions ×4 | General (grant flow) | General (**status strip**) + onboarding/banners | +| Theme | General | **Cut** | +| Teams / Zoom / Webex / Auto-Watch | General | **Recording** | +| Input Device | General | **Recording** (Audio & Language) | +| Language / Model | General ("Language") | **Recording** (renamed) | +| Custom Vocabulary | Transcription | **Recording** | +| Save Location | General | **Recording** | +| Meeting Note Hotkey | Transcription | **Recording** | +| Filename Format | General | **Advanced** (picker) / default otherwise | +| Dictation (all) | Dictation | Dictation | +| Speaker table + management | Speakers tab | **Library window** | +| Speaker Archive retention | Advanced | **Library window** | +| Models on Disk / download | Advanced | **Status** (banner + General strip) + gated Manage Models | +| Model Keep-Alive | Advanced | Advanced (gated) | +| Diarization sensitivity | Advanced | Advanced (gated) | +| Low Memory Mode | Advanced | **Auto-detected** + gated override | +| Developer Mode | Advanced | Advanced (gated) | +| About | About | About | +| — | — | **New:** Show Advanced Settings toggle (General) | + +--- + +## The few genuine open calls + +These are real product decisions, not me dodging: + +1. **Filename Format — default-and-gate, or keep the picker visible?** I recommend default-and-gate, but if your users are filing-system-minded it could earn a visible spot in Recording's Output section. +2. **Low Memory Mode — auto-detect threshold.** ≤8 GB is my recommended trigger; confirm against your actual peak-RAM numbers. +3. **Library — new window vs. a section in the existing transcripts view.** If a transcripts/history window is on the roadmap, Speakers should live there rather than spawning a second window. + +Everything else above is a committed recommendation. diff --git a/testing-ax-chat-fix.md b/testing-ax-chat-fix.md new file mode 100644 index 0000000..ef711e8 --- /dev/null +++ b/testing-ax-chat-fix.md @@ -0,0 +1,64 @@ +# Testing this branch — AX-tree wake fix + meeting chat scraping + +How to validate this branch (`teams-ax-fix-chat-scrape`) against a real Teams meeting. + +## Setup + +1. Pull the branch: + ```bash + git fetch origin + git checkout teams-ax-fix-chat-scrape + ``` +2. Build and install: + ```bash + ./scripts/bundle.sh --install + ``` + This builds, installs to `/Applications/Heard.app`, and relaunches — needed so TCC + permissions (Accessibility, etc.) stay anchored to a stable path. +3. Turn on Developer Mode: Settings → General → "Show Advanced Settings" → Advanced tab + → Developer Mode. This is what makes `dict-debug.log` write at all. +4. Optional — enable the new chat toggle if you want to test that too: Settings → + Recording → "Meeting Chat" → "Include Meeting Chat in Transcript". Leave it off if + you just want to verify the AX-wake fix first (it defaults off). + +## Run the test + +1. Join a real Teams meeting. +2. Tail the log: + ```bash + tail -f ~/Library/Application\ Support/Heard/dict-debug.log + ``` +3. If the chat toggle is on, open Teams' chat panel at some point during the call and + send/receive at least one message. +4. End the meeting and check the generated `.md` transcript in your configured output + folder. + +## What to look for + +- `enableMeetingAppAccessibility: ... setErr= fallbackSetErr=` — `setErr` is + expected to still be `-25205` (`kAXErrorAttributeUnsupported`, the known-broken + `AXManualAccessibility` path). `fallbackSetErr=0` means the `AXEnhancedUserInterface` + fallback succeeded — that's the fix working. +- `roster poll tick=K: ... titleRefreshed=true` — confirms the meeting title actually + populated (vs. staying empty and falling back to a generic "Meeting" filename). +- If the chat toggle is on and the chat panel was open: `roster poll tick=K: chat + read=N messages`. +- If title/roster still come back empty after several ticks, the log emits up to 3 AX + tree dumps (`roster poll tick=K: readRoster empty — AX tree dump (n/3): ...`) — a + bounded outline (role, identifier, description, truncated value) of Teams' real AX + tree. **Capture and share these** — they're exactly what's needed to retune + `RosterReader`'s (and, if chat was on and still empty, `ChatReader`'s) hardcoded + identifiers against real Teams output, the same way `RosterReader` was originally + tuned. +- In the final transcript `.md` file: + - Filename should use the real meeting title, not "Meeting". + - If chat was on and messages were sent, look for lines like + `[mm:ss] _**Chat — :** ...text..._` interleaved chronologically with the + speaker segments. + +## If something looks wrong + +Paste back: +- The full `enableMeetingAppAccessibility` / `roster poll` log lines for that meeting. +- Any AX tree dump lines, if they appeared. +- The relevant portion of the generated transcript (filename + any chat/title lines). diff --git a/transcript-history-plan.md b/transcript-history-plan.md new file mode 100644 index 0000000..b972169 --- /dev/null +++ b/transcript-history-plan.md @@ -0,0 +1,172 @@ +# Transcript History + Speakers — UI Plan + +How to give users a real history of their meetings, and how speakers fit into it. + +This document is intended to be **delegation-ready**: the decisions are made (see +*Resolved decisions*), Phase 1 has a concrete data spec and acceptance criteria, +and the sequencing constraints are spelled out. Phases 2–3 remain a sketch. + +> **Addendum (post-v0.2.6, unreleased):** `PipelineJob`/`TranscriptDocument` gained a +> `chatMessages: [ChatMessage]` field (opt-in meeting-chat scraping — see `handoff.md`'s +> dated status entry). The Phase 1 `TranscriptRecord` schema below should carry a +> `hasChatMessages: Bool` alongside `hasUnnamedSpeakers`, and Phase 1's metadata-card +> detail view should surface a chat-message count the same way it surfaces notes count — +> both are cheap additions now, while the schema is still being drafted, versus a +> migration later. + +--- + +## The core idea: one **Library**, two linked views + +Today there is no history — just `.md` files in a folder and the last 3 in the dropdown. "Open Transcripts" dumps the user into Finder. + +Transcripts and speakers are not two separate features; they're two sides of the same data. A meeting *has* participants; a person *appears in* meetings. So the right model is a **single Library window** with two cross-linked entry points — **Meetings** and **People** — not two disconnected lists (and definitely not a speaker list buried in Settings). + +This also retires the v2 wart where "Speakers" was a settings tab with no settings. Speakers becomes a *view* in the Library; the one actual setting (archive retention) moves with it. + +--- + +## Resolved decisions + +These were the open questions in the prior draft. They are now decided, so a builder does not have to guess: + +1. **Unify speakers into the Library** — People becomes a Library view, not a Settings tab. The archive-retention setting moves with it (Phase 2). *(Removes the empty-Settings-tab wart.)* +2. **Three-pane layout** — sidebar | list | detail. Scales better as smart-groups/filters grow. +3. **Persisted index store + folder reconciliation** as the source of truth — not a live folder scan. A pure folder scan is simpler but fragile to out-of-band edits. +4. **Open transcripts externally in Phase 1** — in-app rendering is **deferred entirely** to a later phase. The transcript `.md` format is custom (`**Speaker:**` blocks, `[mm:ss]` notes, participants header) and does not lay out cleanly via `AttributedString(markdown:)`; a faithful renderer is a multi-day task that should not block the headline value. Phase 1 "Open" launches the `.md` in the default app; "Reveal in Finder" stays as a secondary action. +5. **Index starts empty; only new meetings are indexed** — no backfill parsing of pre-existing `.md` files. Records are written by the pipeline going forward. Existing `.md` remain in the folder and stay reachable via Finder. *(Implication: the Library is empty on first launch after upgrade — Phase 1 must ship a real empty state, see below.)* + +--- + +## Where it lives + +A dedicated **`Window` scene (id `"library"`)**, opened from the menu-bar dropdown — *not* Settings. Settings is for preferences; the Library is content. This matches macOS convention (Music/Photos library vs. Settings). + +Menu-bar changes: +- "Open Transcripts" (opens Finder) → **"Library…"** (opens the window). Keep "Reveal in Finder" as a per-item action inside. +- "Recent Transcripts" stays in the dropdown for quick access. Today it reads completed jobs from the ephemeral `PipelineQueueStore`; **Phase 1 rewires it to read from the new `TranscriptStore`**, and clicking an item opens the Library window focused on that meeting (detail = metadata; "Open" launches the file). + +--- + +## Layout — three-pane `NavigationSplitView` (macOS-idiomatic) + +``` +┌──────────────────────────────────────────────────────────────┐ +│ Heard Library [ search ]│ +├────────────┬─────────────────────────┬───────────────────────┤ +│ MEETINGS │ Q1 Planning │ # Q1 Planning │ +│ ▸ All │ Today · 48m · 4 people │ Jun 13 · 48m │ +│ ▸ This Week│─────────────────────────│ Alice · Bob · Carol·Me │ +│ ▸ Flagged │ Standup │ ────────────────────── │ +│ │ Yesterday · 15m · 3 │ 2 notes · 1 unnamed │ +│ PEOPLE │─────────────────────────│ [Open] [Reveal] [Name…]│ +│ ▸ All │ 1:1 with Bob │ [Rename] [Delete] │ +│ Alice │ Mon · 32m · 2 │ │ +│ Bob │ │ (transcript opens in │ +│ Carol │ │ the default app) │ +└────────────┴─────────────────────────┴───────────────────────┘ +``` + +- **Sidebar** — source switch: Meetings (with smart groups: All / This Week / Flagged) and People (All + individuals). *Flagged/smart-groups beyond All + This Week are Phase 3.* +- **List** — meetings or people, sorted newest-first, searchable. +- **Detail** — in **Phase 1** the meeting detail is a **metadata card + actions** (not the rendered transcript). In-app rendering is a later phase. + +### Meeting detail (Phase 1) +A metadata card: title, date, duration, participant chips (from `rosterNames`), notes count, and an "unnamed speakers" indicator. Actions: **Open** (launch `.md` externally) · **Reveal in Finder** · **Rename** · **Delete** · (**Retry**, if the job failed) · **Name speakers…** when unnamed speakers remain (opens the existing `speaker-naming` window scene — no new naming UI is built). Export/Share is Phase 3. + +A missing-file record (see reconciliation) renders disabled with a "File missing" badge and only a **Remove from Library** action. + +### Person detail — Phase 2 (the cross-link payoff) +``` +┌────────────┬─────────────────────────┬───────────────────────┐ +│ PEOPLE │ Bob │ ▶ ━━━━━━ voice sample │ +│ Alice │ 12 meetings · 6h 20m │ 12 meetings · 6h 20m │ +│ ▸ Bob │ Last seen: yesterday │ First seen: Mar 3 │ +│ │ │ ────────────────────── │ +│ │ │ MEETINGS WITH BOB │ +│ │ │ • Standup — yesterday →│ +│ │ │ • 1:1 with Bob — Mon →│ +│ │ │ • Q1 Planning — Jun 13→│ +│ │ │ [Rename] [Merge] [Del] │ +└────────────┴─────────────────────────┴───────────────────────┘ +``` +Reuses all current Speakers-table data (voice playback, name, meeting count, time, last seen) plus **the list of meetings the person appears in**, each linking back to the meeting's detail. Merge/rename/delete carry over from the existing Speakers tab. This requires the persisted speaker-ID join (below) and is therefore Phase 2. + +--- + +## Data model — `TranscriptStore` + +Cross-linking ("Bob's meetings", "this meeting's people") needs a persisted archive that doesn't exist yet. `PipelineQueueStore` is a *processing* queue, not an archive — jobs are dismissed via `PipelineQueueStore.remove()`, and it stores participant **names** (`rosterNames: [String]`), not speaker **IDs**. + +Introduce a lightweight **`TranscriptStore`** — a persisted JSON index, one record per completed meeting, **decoupled from the ephemeral queue** so dismissing a job from the queue never removes it from the Library. + +### Phase 1 record schema + +```swift +struct TranscriptRecord: Codable, Identifiable, Equatable { + let id: UUID // == PipelineJob.id, for traceability + var title: String + let start: Date + let end: Date + var duration: TimeInterval // stored, not recomputed at read time + var transcriptPath: URL + var rosterNames: [String] // for participant chips; already on the job + var notesCount: Int + var hasUnnamedSpeakers: Bool + var fileMissing: Bool // set by reconciliation; default false + // participantSpeakerIDs: [UUID] ← added in Phase 2 (see below) +} +``` + +- **Storage**: JSON at `~/Library/Application Support/Heard/transcripts.json`. Follow the existing store conventions: corrupt-file quarantine to `.corrupt` (as `SpeakerStore`/`PipelineQueueStore` already do), single-pass writes. +- **Record identity**: `id == PipelineJob.id`. Upsert by `id` so a retry/rewrite updates the existing record rather than duplicating it. + +### Write trigger + +A record is written/updated when a pipeline job reaches **`.complete`** (transcript file written) — at the same point the job would otherwise just sit in the queue. The write is independent of the queue: it must survive `PipelineQueueStore.remove()` and app restart. `Rename`/`Delete` actions in the Library update or remove the corresponding record (and, for Delete, the `.md` file). + +### Folder reconciliation (Phase 1 minimum) + +On launch, for each record check `FileManager.default.fileExists(atPath: transcriptPath.path)`: +- **Present** → `fileMissing = false`. +- **Missing** (user moved/renamed/deleted the `.md` outside the app) → set `fileMissing = true`; the row renders disabled with a "File missing" badge and a **Remove from Library** action. Do **not** crash or show dead rows. + +Out of scope for Phase 1: relinking a moved file, and *importing* `.md` files the app never wrote (consistent with the empty-start decision — the app only tracks what it produced). These are Phase 3 candidates. + +### Phase 2 addition — the speaker-ID join + +Persisting `participantSpeakerIDs: [UUID]` per record powers both directions of the cross-link ("this meeting's people" ↔ "this person's meetings"). The speaker IDs are known at speaker-assignment time but currently discarded; Phase 2 threads them into the record. `SpeakerProfile` gains no back-reference — the join lives entirely in `TranscriptRecord.participantSpeakerIDs`, queried both ways. + +--- + +## Empty & onboarding state (Phase 1, required) + +Because the index starts empty and is not backfilled, **the Library is empty on first launch after upgrade.** Phase 1 must therefore ship a genuine empty state in the Meetings list: a short explainer ("Meetings you record from now on will appear here") plus a **Reveal Transcripts Folder in Finder** button, so existing users can still reach their old `.md` files. Without this, the feature looks broken on day one. + +--- + +## Acceptance criteria & testing + +This repo has a strong pure-logic + unit-test culture (`PipelineQueueStore`, `SpeakerStore`, `MeetingDetectionState` are all pure and tested via the in-house `HeardTests` harness). The new logic must match it: + +- **`TranscriptStore` is pure and unit-tested**: load, save, upsert-by-id (idempotent on retry), remove, and corrupt-file quarantine — mirroring `PipelineQueueStore`'s tests. +- **Reconciliation is pure and unit-tested**: present → not missing; absent → `fileMissing = true`; reconciliation never deletes records. +- **List logic is pure and unit-tested**: search filtering and newest-first sort over a record array, independent of any view. +- **Phase 1 done = green**: `swift build` clean and `swift run HeardTests` all-pass, with new tests covering the three areas above. +- **Manual smoke**: record (or simulate) a meeting → it appears in the Library → Open launches the `.md` → Rename/Delete reflect in both the index and the folder → move the `.md` in Finder, relaunch → row shows "File missing" with Remove. + +--- + +## Build coordination / sequencing + +- **Rebase on the `Views.swift` split.** The UI was just split by view area (`DesignSystem.swift`, `MenuBarView.swift`, `SettingsView.swift`, `SettingsTabs.swift`, `SettingsComponents.swift`, `SpeakerNamingView.swift`). Library views go in **new files** (e.g. `LibraryView.swift`, `LibraryMeetingDetail.swift`), reusing `DesignSystem.swift` primitives — do not reintroduce a monolith. +- **Settings-reorg collision (Phase 2).** There is a settings reorganization in flight (`settings-reorg-v2.md`) that also touches the **Speakers** settings tab that Phase 2 removes. Resolve the sequencing between that reorg and Phase 2 before starting Phase 2, or they will conflict over the same tab. +- **Menu-bar "Recent" rewire (Phase 1).** "Recent" currently reads ephemeral completed jobs from `PipelineQueueStore`; Phase 1 repoints it at `TranscriptStore` and changes its click action to open the Library. + +--- + +## Suggested phasing + +1. **Phase 1 — Meetings history (metadata + open externally).** `TranscriptStore` + write-on-complete + launch reconciliation; `library` Window scene with three-pane layout; Meetings list (search, newest-first) + metadata detail; actions (Open externally / Reveal / Rename / Delete / Retry-if-failed / Name speakers…); empty state; rewire dropdown "Recent" + "Library…". Pure logic unit-tested. *Ships the headline value without the rendering or speaker-join risk.* +2. **Phase 2 — People + cross-link.** Add the People view, persist `participantSpeakerIDs`, build both-direction links, migrate the Speakers settings tab (and retention setting) into the Library. *Retires the Speakers settings tab — coordinate with `settings-reorg-v2.md`.* +3. **Phase 3 — Polish.** In-app transcript rendering (custom speaker-block renderer), filters (date/participant), Flagged/smart groups, export/share, relink-moved-file, optional import of external `.md`.