Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Sources/HeardCore/AppModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,9 @@ public final class AppModel: ObservableObject {
onMeetingEnded: { [weak self] snapshot in
guard let self else { return }
// Update the recording session with the final accumulated roster names
// and the title (often captured late, after Teams' a11y tree built).
self.recordingManager.updateRosterNames(snapshot.rosterNames)
self.recordingManager.updateTitle(snapshot.title)
guard let session = self.recordingManager.stopRecording() else { return }
self.pipelineProcessor.enqueueFinishedRecording(session, endedAt: Date())
self.phase = .processing
Expand Down
52 changes: 52 additions & 0 deletions Sources/HeardCore/RosterReader.swift
Original file line number Diff line number Diff line change
Expand Up @@ -265,4 +265,56 @@ public enum RosterReader {
return seen.insert(lower).inserted
}
}

// MARK: - Diagnostics

/// Developer-Mode diagnostic. When `readRoster` comes back empty against a real
/// (woken) Teams tree, this dumps a bounded view of the app's AX tree — roles,
/// identifiers, descriptions, and truncated text — so the parser's hardcoded
/// identifiers/heuristics can be retuned against Teams' actual structure. The
/// roster parser has never been validated against live Teams output, so this is
/// the data we need to close that gap. Live entry point: builds the app element
/// and walks it on the caller's (background) thread.
public static func diagnosticTreeDump(pid teamsPID: pid_t?) -> String? {
guard let pid = teamsPID else { return nil }
let element = AXUIElementCreateApplication(pid)
AXUIElementSetMessagingTimeout(element, 1.0)
return diagnosticTreeDump(root: AXUIElementNode(element))
}

/// Testable core: render any AX node tree as a compact, bounded textual outline.
/// Depth-first (mirrors the parser's own traversal) with a shared node budget so
/// a huge Electron tree can't produce an unbounded log. Text is truncated so the
/// dump stays structural (metadata) rather than a verbatim content capture.
public static func diagnosticTreeDump(root: any AXNode, maxDepth: Int = 9, nodeBudget: Int = 500) -> String? {
var lines: [String] = []
var remaining = nodeBudget

func truncate(_ s: String, _ limit: Int = 40) -> String {
let flat = s.replacingOccurrences(of: "\n", with: "⏎")
return flat.count <= limit ? flat : String(flat.prefix(limit)) + "…(\(flat.count))"
}

func walk(_ node: any AXNode, depth: Int) {
guard remaining > 0 else { return }
remaining -= 1
var parts = ["\(String(repeating: " ", count: depth))\(node.axRole ?? "?")"]
if let id = node.axIdentifier, !id.isEmpty { parts.append("id=\(id)") }
if let d = node.axDescription, !d.isEmpty { parts.append("desc=\(truncate(d))") }
if let t = node.axTitle, !t.isEmpty { parts.append("title=\(truncate(t))") }
if let v = node.axValue, !v.isEmpty { parts.append("value=\(truncate(v))") }
lines.append(parts.joined(separator: " "))
guard depth < maxDepth, let children = node.axChildren else { return }
for child in children {
if remaining <= 0 { break }
walk(child, depth: depth + 1)
}
}

walk(root, depth: 0)
if remaining <= 0 {
lines.append("…(node budget \(nodeBudget) reached — tree truncated; raise the cap to see more)")
}
return lines.isEmpty ? nil : lines.joined(separator: "\n")
}
}
131 changes: 105 additions & 26 deletions Sources/HeardCore/Services.swift
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ public struct MeetingSnapshot {
}

public struct RecordingSession {
public let title: String
public var title: String
public let startTime: Date
public let appAudioPath: URL
public let micAudioPath: URL
Expand Down Expand Up @@ -299,10 +299,16 @@ public final class MeetingDetector: ObservableObject {
// the main thread, then start the meeting once the data is in hand.
Task { [weak self] in
let (title, rosterNames) = await Task.detached(priority: .userInitiated) {
// Teams is an Electron/Chromium app: its accessibility tree is
// dormant until a client requests it via AXManualAccessibility.
// Nudge it now — the tree builds asynchronously, so this first
// read often still misses; the roster poll retries title+roster.
if source == .teams { Self.enableMeetingAppAccessibility(pid: pid, source: source) }
let title = Self.extractMeetingTitle(pid: pid, source: source) ?? ""
let roster: [String] = source == .teams ? RosterReader.readRoster(pid: pid) : []
return (title, roster)
}.value
DebugFileLog.log("meeting start scrape: source=\(source.rawValue) pid=\(pid) titleCaptured=\(!title.isEmpty) rosterCount=\(rosterNames.count) — roster poll will retry title/roster every 15s")
guard let self else { return }
// The meeting may have ended (or watching been toggled off) while
// we were scraping — the detection state machine is the source of truth.
Expand Down Expand Up @@ -372,6 +378,25 @@ public final class MeetingDetector: ObservableObject {
return nil
}

/// Wake a Chromium/Electron meeting app's accessibility tree.
///
/// Teams keeps its a11y tree off by default to save resources, so AX reads of
/// its windows/titles/roster fail (commonly `kAXErrorAPIDisabled`, -25211) even
/// though Heard itself is trusted. Setting `AXManualAccessibility` on the app
/// element signals Chromium to build the tree. The build is asynchronous, so
/// callers must still retry the actual read after this returns.
///
/// The `setErr` log line is the diagnostic discriminator: `.success` means the
/// nudge was accepted (any lingering empty reads are tree-build timing); a
/// `.apiDisabled` here would instead mean Heard is genuinely untrusted.
nonisolated private static func enableMeetingAppAccessibility(pid: pid_t?, source: MeetingApp) {
guard let pid else { return }
let app = AXUIElementCreateApplication(pid)
AXUIElementSetMessagingTimeout(app, 1.0)
let setErr = AXUIElementSetAttributeValue(app, "AXManualAccessibility" as CFString, kCFBooleanTrue)
DebugFileLog.log("enableMeetingAppAccessibility: pid=\(pid) source=\(source.rawValue) setErr=\(setErr.rawValue)")
}

/// Extract the meeting title from a meeting-app window via Accessibility API.
/// Strips the trailing app-name suffix (` | Microsoft Teams`, ` - Zoom`, etc.).
/// Returns nil if AX is denied, no window matches, or the title is just a placeholder.
Expand All @@ -390,8 +415,11 @@ public final class MeetingDetector: ObservableObject {
var windowsRef: AnyObject?
let windowsErr = AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &windowsRef)
guard windowsErr == .success, let windows = windowsRef as? [AXUIElement] else {
// .apiDisabled => Accessibility denied; other errors / non-array => Electron
// a11y tree not yet built. Both produce an empty title -> "Meeting" fallback.
// For a per-app Electron query, .apiDisabled (-25211) usually means the
// app's Chromium a11y tree is still dormant (see enableMeetingAppAccessibility),
// NOT that Heard is untrusted — confirm trust via the system-wide element
// instead. Any failure here yields an empty title -> "Meeting" fallback;
// the roster poll retries once the tree wakes.
DebugFileLog.log("extractMeetingTitle: windows query failed pid=\(pid) source=\(source.rawValue) axErr=\(windowsErr.rawValue)")
return nil
}
Expand Down Expand Up @@ -431,17 +459,41 @@ public final class MeetingDetector: ObservableObject {
private func startRosterPolling() {
rosterPollingTask?.cancel()
rosterPollingTask = Task { [weak self] in
var tick = 0
// Developer-Mode only: emit at most a few AX tree dumps per meeting when
// the roster keeps coming back empty, so we can retune the parser without
// flooding the log. Reset per meeting (the task is recreated on each start).
var rosterDumpsRemaining = 3
while !Task.isCancelled {
try? await Task.sleep(for: .seconds(15))
guard let self, !Task.isCancelled,
let snapshot = self.activeSnapshot,
snapshot.source == .teams
else { break }
tick += 1
// The roster walk is a deep recursive AX scrape of Teams'
// Electron tree — keep it off the main thread.
let pid = snapshot.meetingPID
let names = await Task.detached(priority: .utility) {
RosterReader.readRoster(pid: pid)
let source = snapshot.source
// Re-extract the title until we get one: the initial capture at
// join time usually fires before Teams' a11y tree has finished
// building, leaving the title empty (-> "Meeting" filename).
let needTitle = (self.activeSnapshot?.title ?? "").isEmpty
// An empty roster means the tree still looks asleep — the join-time
// nudge may have timed out against a busy Teams. Re-nudge before each
// read until names appear (idempotent). Gate on roster, not title:
// window titles can be AppKit-provided and succeed without the woken
// Chromium tree that the roster DOM actually needs.
let rosterEmpty = (self.activeSnapshot?.rosterNames.isEmpty ?? true)
// While roster is still empty, capture a bounded tree dump (Developer
// Mode + budget) so an empty result is self-diagnosing, not silent.
let wantDump = rosterEmpty && rosterDumpsRemaining > 0 && DebugFileLog.isEnabled
let (names, refreshedTitle, treeDump) = await Task.detached(priority: .utility) {
if rosterEmpty { Self.enableMeetingAppAccessibility(pid: pid, source: source) }
let names = RosterReader.readRoster(pid: pid)
let title = needTitle ? Self.extractMeetingTitle(pid: pid, source: source) : nil
let dump = (wantDump && names.isEmpty) ? RosterReader.diagnosticTreeDump(pid: pid) : nil
return (names, title, dump)
}.value
if !names.isEmpty {
let existing = Set(self.activeSnapshot?.rosterNames ?? [])
Expand All @@ -450,6 +502,14 @@ public final class MeetingDetector: ObservableObject {
self.activeSnapshot?.rosterNames.append(contentsOf: newNames)
}
}
if let refreshedTitle, !refreshedTitle.isEmpty {
self.activeSnapshot?.title = refreshedTitle
}
DebugFileLog.log("roster poll tick=\(tick): read=\(names.count) total=\(self.activeSnapshot?.rosterNames.count ?? 0) titleEmpty=\(needTitle) titleRefreshed=\(refreshedTitle != nil)")
if let treeDump {
rosterDumpsRemaining -= 1
DebugFileLog.log("roster poll tick=\(tick): readRoster empty — AX tree dump (\(3 - rosterDumpsRemaining)/3):\n\(treeDump)")
}
}
}
}
Expand Down Expand Up @@ -626,6 +686,18 @@ public final class RecordingManager: ObservableObject {
currentRosterNames = names
}

/// Update the meeting title on the active session. The title is often captured
/// late (Teams' a11y tree builds after join), so the recording starts with an
/// empty title and this fills it in before the session is enqueued, fixing the
/// transcript filename. No-ops on empty so a failed re-attempt can't clobber a
/// title already captured.
public func updateTitle(_ title: String) {
let trimmed = title.trimmingCharacters(in: .whitespaces)
guard activeSession != nil, !trimmed.isEmpty else { return }
activeSession?.title = trimmed
currentTitle = trimmed
}

/// Append a user-authored note to the active recording session. The offset
/// is measured from `session.startTime` using the wall-clock instant at
/// which the composer was *opened* (passed in by the caller), not the
Expand Down Expand Up @@ -1623,35 +1695,42 @@ public final class PermissionCenter: ObservableObject {
queue: .main
) { [weak self] notification in
guard let self,
self.pendingScreenCaptureCheck,
let app = notification.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication,
// macOS 13+: "System Settings"; earlier: "System Preferences" — both share this bundle ID
app.bundleIdentifier == "com.apple.systempreferences"
else { return }

// One-shot: remove the observer and clear the flag immediately.
self.pendingScreenCaptureCheck = false
if let obs = self.systemPrefsObserver {
NSWorkspace.shared.notificationCenter.removeObserver(obs)
self.systemPrefsObserver = nil
}
// The observer is registered with `queue: .main`, so this block always runs
// on the main actor — assert that to access main-actor-isolated state safely
// (the alternative, a bare Task hop, would let a second notification slip in
// before the flag/observer are cleared).
MainActor.assumeIsolated {
guard self.pendingScreenCaptureCheck else { return }

// One-shot: remove the observer and clear the flag immediately.
self.pendingScreenCaptureCheck = false
if let obs = self.systemPrefsObserver {
NSWorkspace.shared.notificationCenter.removeObserver(obs)
self.systemPrefsObserver = nil
}

Task { @MainActor [weak self] in
guard let self else { return }
// Fast path: sync check (works on macOS < 15 and after any restart).
if CGPreflightScreenCaptureAccess() {
self.applyScreenCaptureProbe(true)
Task { @MainActor [weak self] in
guard let self else { return }
// Fast path: sync check (works on macOS < 15 and after any restart).
if CGPreflightScreenCaptureAccess() {
self.applyScreenCaptureProbe(true)
self.refresh()
return
}
// Bypass macOS 15+ per-process TCC cache with one-shot SCShareableContent.
// Safe here: this is user-initiated and fires exactly once per "Grant…" click,
// only after the user has left System Settings. The result is authoritative,
// so a false clears any stale cached grant immediately.
self.applyScreenCaptureProbe(
await self.checkScreenCapturePermissionLive(), authoritative: true
)
self.refresh()
return
}
// Bypass macOS 15+ per-process TCC cache with one-shot SCShareableContent.
// Safe here: this is user-initiated and fires exactly once per "Grant…" click,
// only after the user has left System Settings. The result is authoritative,
// so a false clears any stale cached grant immediately.
self.applyScreenCaptureProbe(
await self.checkScreenCapturePermissionLive(), authoritative: true
)
self.refresh()
}
}
}
Expand Down
43 changes: 43 additions & 0 deletions Tests/HeardTests/TestRunner.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1909,6 +1909,49 @@ func runRosterReaderAXTests() {
let names = RosterReader.readRosterFromNode(tree)
try expectEqual(names, [])
}

// Diagnostic tree dump (Developer-Mode aid for retuning the parser).

test("diagnosticTreeDump renders roles, identifiers, and truncated text") {
let tree = MockAXNode(role: "AXApplication", children: [
MockAXNode(role: "AXWindow", identifier: "main", title: "Sprint Planning", children: [
MockAXNode(role: "AXGroup", identifier: "roster-list", children: [
MockAXNode(role: "AXStaticText", value: "Alice Smith"),
]),
]),
])
let out = RosterReader.diagnosticTreeDump(root: tree)
try expect(out != nil, "expected a dump")
let s = out ?? ""
try expect(s.contains("AXApplication"), "missing root role")
try expect(s.contains("id=roster-list"), "missing identifier")
try expect(s.contains("title=Sprint Planning"), "missing window title")
try expect(s.contains("value=Alice Smith"), "missing leaf value")
}

test("diagnosticTreeDump truncates long text and notes the full length") {
let long = String(repeating: "x", count: 100)
let tree = MockAXNode(role: "AXStaticText", value: long)
let out = RosterReader.diagnosticTreeDump(root: tree) ?? ""
try expect(out.contains("…(100)"), "expected truncation marker with full length")
try expect(!out.contains(long), "full untruncated text should not appear")
}

test("diagnosticTreeDump bounds output to the node budget") {
let children = (0..<50).map { MockAXNode(role: "AXRow", identifier: "row-\($0)") }
let tree = MockAXNode(role: "AXList", children: children)
let out = RosterReader.diagnosticTreeDump(root: tree, maxDepth: 9, nodeBudget: 10) ?? ""
let lines = out.split(separator: "\n")
try expect(lines.count <= 11, "expected <= 11 lines (10 nodes + note), got \(lines.count)")
try expect(out.contains("budget 10 reached"), "missing truncation note")
}

test("diagnosticTreeDump stops descending at maxDepth") {
var node = MockAXNode(role: "AXLeaf", identifier: "deep")
for i in 0..<6 { node = MockAXNode(role: "AXGroup", identifier: "g\(i)", children: [node]) }
let out = RosterReader.diagnosticTreeDump(root: node, maxDepth: 2, nodeBudget: 500) ?? ""
try expect(!out.contains("id=deep"), "leaf below maxDepth should be excluded")
}
}

// MARK: - SegmentDeduplicator Tests
Expand Down
Loading
Loading