diff --git a/Sources/Meeting/BrowserCallEvidence.swift b/Sources/Meeting/BrowserCallEvidence.swift index 86a2a4244..21b904db1 100644 --- a/Sources/Meeting/BrowserCallEvidence.swift +++ b/Sources/Meeting/BrowserCallEvidence.swift @@ -18,8 +18,9 @@ // `corroboratedDelay`, as the generic browser call. // 3. The focused window is a known non-call site (ChatGPT, Claude, Loom...) // -> no prompt while it is in front. When it was in front as the browser -// took the mic, no prompt for the whole session, even after the user -// clicks to another tab; only a title from step 1 can still win. +// took the mic, no prompt for the whole session after a switch to an +// ordinary page. A title from step 1, or coming back to a call site, can +// still win. // 4. The browser is also playing audio, or the camera is on -> prompt after // `corroboratedDelay` of continuous mic use. // 5. Mic only -> prompt after `uncorroboratedDelay`. @@ -141,6 +142,16 @@ enum BrowserCallEvidence { /// one using it. Seen later (notes in Claude during a call), it only /// holds the prompt back while it stays in front. static let nonCallSiteStickyWindow: TimeInterval = 10 + /// A call shortcut (a Helium Meet PWA) opened this close to the mic + /// starting is the thing that took the mic, even if another app in the + /// family is in front. Wider than the non-call sticky window so a + /// calendar link that opens the shortcut, then leaves Calendar in front, + /// still counts. + static let callShortcutActivationSlack: TimeInterval = 15 + /// How long the first title read may hold the prompt. A hung Accessibility + /// read must not keep a real call quiet; after this the timing rules run, + /// and a late title can still upgrade or suppress before the wait ends. + static let titleReadGiveUp: TimeInterval = 1 struct Timing: Equatable { var corroboratedDelay: TimeInterval @@ -149,6 +160,7 @@ enum BrowserCallEvidence { var nonCallSiteRecheckInterval: TimeInterval = BrowserCallEvidence.nonCallSiteRecheckInterval var micReleaseGrace: TimeInterval = BrowserCallEvidence.micReleaseGrace var nonCallSiteStickyWindow: TimeInterval = BrowserCallEvidence.nonCallSiteStickyWindow + var titleReadGiveUp: TimeInterval = BrowserCallEvidence.titleReadGiveUp /// Minimum spacing between title reads within one browser session. /// Several sensor edges can land in the same second; one read answers /// all of them. @@ -215,6 +227,58 @@ enum BrowserCallEvidence { return .wait(recheckAt: min(promptAt, now.addingTimeInterval(timing.titleRecheckInterval))) } + // MARK: App shortcuts + + /// The call a Chromium app shortcut is for, from its start URL, or `nil`. + /// The URL is classified and dropped: a Meet shortcut is + /// `https://meet.google.com/...`, including the landing page. A meeting + /// code in the path is not proof a call is on, so this is a call site, + /// not a named tab. Chat, Calendar and Gmail shortcuts do not match. + static func callShortcutProvider(forURL raw: String) -> MeetingPromptProvider? { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard let host = URLComponents(string: trimmed)?.host?.lowercased() else { return nil } + let bare = host.hasPrefix("www.") ? String(host.dropFirst(4)) : host + switch bare { + case "meet.google.com": + return .googleMeet + case "teams.microsoft.com", "teams.live.com": + return .teams + case "zoom.us", "app.zoom.us": + return .zoom + default: + return nil + } + } + + /// Which call shortcut is using the shared mic helper, if either signal + /// says so. The shortcut in front wins. Otherwise a shortcut that opened + /// as the mic was taken (a calendar link) wins, so a ChatGPT tab in the + /// main browser cannot veto it. A shortcut left open from long before the + /// mic started does not: ChatGPT voice with Meet sitting in the background + /// stays a non-call. + static func claimedCallShortcut( + frontmostShortcut: MeetingPromptProvider?, + recentProvider: MeetingPromptProvider?, + recentActivationAt: Date?, + sessionStartedAt: Date, + slack: TimeInterval = callShortcutActivationSlack + ) -> MeetingPromptProvider? { + if let frontmostShortcut { return frontmostShortcut } + guard let recentProvider, let recentActivationAt else { return nil } + let delta = recentActivationAt.timeIntervalSince(sessionStartedAt) + guard abs(delta) <= slack else { return nil } + return recentProvider + } + + /// `nil` while the first title read should still hold the prompt; `.unknown` + /// once that hold has run out, so the timing rules can start. + static func verdictBeforeTitleReadLands( + pendingFor: TimeInterval, + giveUp: TimeInterval + ) -> BrowserCallTitleVerdict? { + pendingFor >= giveUp ? .unknown : nil + } + // MARK: Title classification /// Classifies the titles of every window of the browsers holding the mic. diff --git a/Sources/Meeting/BrowserWindowTitleReader.swift b/Sources/Meeting/BrowserWindowTitleReader.swift index 1f47e4ee0..d411aa684 100644 --- a/Sources/Meeting/BrowserWindowTitleReader.swift +++ b/Sources/Meeting/BrowserWindowTitleReader.swift @@ -40,29 +40,55 @@ enum BrowserWindowTitleReader { let appFamilies = Set(families.map(MeetingPromptProvider.browserAppFamily(forBundleFamily:))) // Bundle IDs are read off the main thread: that read can block on - // LaunchServices. + // LaunchServices. The frontmost pid is captured here too: a Helium + // Meet shortcut and the main browser share one mic helper, so both + // match the family, and only the app the user is in is "focused". + let frontmostPID = NSWorkspace.shared.frontmostApplication?.processIdentifier let pids: [pid_t] = await RunningApplicationsReader.applications().compactMap { app in guard app.isRegularApp, let bundleID = app.bundleIdentifier, appFamilies.contains(where: { bundleID.matchesBundleFamily($0) }) else { return nil } return app.processIdentifier } - guard !pids.isEmpty else { return [] } + let ordered = readingOrder(pids: pids, frontmostPID: frontmostPID) + guard !ordered.isEmpty else { return [] } return await withCheckedContinuation { continuation in readQueue.async { let deadline = Date().addingTimeInterval(readBudget) var titles: [BrowserWindowTitle] = [] - for pid in pids { + for pid in ordered { guard Date() < deadline else { break } - titles.append(contentsOf: windowTitles(forPID: pid, deadline: deadline)) + titles.append(contentsOf: windowTitles( + forPID: pid, + deadline: deadline, + userFocused: isUserFocused(appPID: pid, frontmostPID: frontmostPID) + )) } continuation.resume(returning: titles) } } } - private static func windowTitles(forPID pid: pid_t, deadline: Date) -> [BrowserWindowTitle] { + /// The frontmost app first. The title read has one short budget for every + /// app in the family. A Meet shortcut beside a busy main browser would + /// otherwise lose its windows to that budget. + static func readingOrder(pids: [pid_t], frontmostPID: pid_t?) -> [pid_t] { + guard let frontmostPID, let index = pids.firstIndex(of: frontmostPID) else { return pids } + var ordered = pids + ordered.remove(at: index) + ordered.insert(frontmostPID, at: 0) + return ordered + } + + /// True only for the app the user is looking at. Each browser app has its + /// own key window; that is not the focused window unless the app is frontmost. + static func isUserFocused(appPID: pid_t, frontmostPID: pid_t?) -> Bool { + guard let frontmostPID else { return false } + return appPID == frontmostPID + } + + private static func windowTitles(forPID pid: pid_t, deadline: Date, userFocused: Bool) -> [BrowserWindowTitle] { let appElement = AXUIElementCreateApplication(pid) var focusedWindow: AXUIElement? @@ -81,15 +107,15 @@ enum BrowserWindowTitleReader { // No window list (or no time left for it). The focused window // alone is still useful. guard let focusedWindow, let title = title(of: focusedWindow, deadline: deadline) else { return [] } - return [BrowserWindowTitle(title: title, isFocused: true)] + return [BrowserWindowTitle(title: title, isFocused: userFocused)] } var titles: [BrowserWindowTitle] = [] for window in windows.prefix(maxWindowsPerApp) { guard Date() < deadline else { break } guard let title = title(of: window, deadline: deadline) else { continue } - let isFocused = focusedWindow.map { CFEqual($0, window) } ?? false - titles.append(BrowserWindowTitle(title: title, isFocused: isFocused)) + let isKeyWindow = focusedWindow.map { CFEqual($0, window) } ?? false + titles.append(BrowserWindowTitle(title: title, isFocused: userFocused && isKeyWindow)) } return titles } @@ -104,6 +130,30 @@ enum BrowserWindowTitleReader { /// Bounds the next AX round trip on `element` by what is left of the /// budget. Returns false when the budget is spent. + /// The call a running Chromium app shortcut is for, or `nil`. Reads + /// `CrAppModeShortcutURL` and drops it; only the provider enum leaves. + static func callShortcutProvider(forBundleID bundleID: String) async -> MeetingPromptProvider? { + let url = await Task.detached(priority: .utility) { + shortcutURL(forBundleID: bundleID) + }.value + guard let url else { return nil } + return BrowserCallEvidence.callShortcutProvider(forURL: url) + } + + /// Off the main actor: `bundleURL` can block on LaunchServices, and the + /// URL itself is never logged. + private static func shortcutURL(forBundleID bundleID: String) -> String? { + guard let bundleURL = NSRunningApplication.runningApplications(withBundleIdentifier: bundleID).first?.bundleURL else { + return nil + } + let infoURL = bundleURL.appendingPathComponent("Contents/Info.plist") + guard let data = try? Data(contentsOf: infoURL), + let plist = try? PropertyListSerialization.propertyList(from: data, format: nil) as? [String: Any], + let url = plist["CrAppModeShortcutURL"] as? String, + !url.isEmpty else { return nil } + return url + } + private static func setTimeout(on element: AXUIElement, deadline: Date) -> Bool { let remaining = deadline.timeIntervalSinceNow guard remaining > 0.005 else { return false } diff --git a/Sources/Meeting/MeetingPromptCalendarScoring.swift b/Sources/Meeting/MeetingPromptCalendarScoring.swift index 581b35f21..c7868486e 100644 --- a/Sources/Meeting/MeetingPromptCalendarScoring.swift +++ b/Sources/Meeting/MeetingPromptCalendarScoring.swift @@ -106,9 +106,12 @@ enum MeetingPromptCalendarScoring { return "\(provider.displayName) is open" } + // A browser's own app, or a Helium web app (`net.imput.helium.app.*`, + // how a calendar link opens Meet there). Not every Chrome or Safari + // web app: a Slack shortcut in front is not the meeting tab. if provider.browserHosted, let frontmostBundleID = runtimeSnapshot.frontmostBundleID, - MeetingPromptRuntimeSnapshot.browserBundleIdentifiers.contains(frontmostBundleID) { + MeetingPromptProvider.isMeetingTabHost(frontmostBundleID) { return "meeting tab is active" } diff --git a/Sources/Meeting/MeetingPromptDetector+BrowserEvidence.swift b/Sources/Meeting/MeetingPromptDetector+BrowserEvidence.swift index be6a35485..170a5e1ed 100644 --- a/Sources/Meeting/MeetingPromptDetector+BrowserEvidence.swift +++ b/Sources/Meeting/MeetingPromptDetector+BrowserEvidence.swift @@ -24,16 +24,34 @@ extension MeetingPromptDetector { var namedCall: MeetingPromptProvider? /// A known non-call site was focused. Sticky too, so ChatGPT voice /// does not prompt when the user clicks to another tab; only a - /// call-only title can still win. + /// call-only title, or a claimed call shortcut, can still win. var sawNonCallSite = false + /// A Meet/Teams/Zoom app shortcut is the mic user for this session. + /// Beats a non-call window in a different app of the same family. + var claimedShortcut: MeetingPromptProvider? var readTask: Task? var readToken = 0 + /// When this session's first title read started. The prompt is held + /// only until `titleReadGiveUp`, not until a hung read returns. + var readStartedAt: Date? /// Browser kinds the user said Not now to in this session. var declinedKinds: [String: Date] = [:] - var verdict: BrowserCallTitleVerdict? { + func verdict(now: Date, titleReadGiveUp: TimeInterval) -> BrowserCallTitleVerdict? { if let namedCall { return .call(provider: namedCall) } + if let claimedShortcut { return .callSite(provider: claimedShortcut) } + // Coming back to a Meet tab beats an early ChatGPT glance. A + // random tab switch does not: that still needs sawNonCallSite. + if case .call = latest { return latest } + if case .callSite = latest { return latest } if sawNonCallSite { return .notCall } + if latest == nil { + guard let readStartedAt else { return nil } + return BrowserCallEvidence.verdictBeforeTitleReadLands( + pendingFor: now.timeIntervalSince(readStartedAt), + giveUp: titleReadGiveUp + ) + } return latest } } @@ -56,12 +74,24 @@ extension MeetingPromptDetector { let families = browserFamiliesForEvidence(reason: signal.reason, frontmostBundleID: frontmostBundleID) let since = browserEvidenceStart(reason: signal.reason, families: families, now: now) let playingAudio = browserIsPlayingAudio(families: families) + ensureCallShortcutLookup(frontmostBundleID) - guard let verdict = currentBrowserTitleVerdict(families: families, now: now) else { - // The first title read of this session is still running (well - // under a second); it re-evaluates when it lands. Too brief to be - // worth a suppression event. + guard let verdict = currentBrowserTitleVerdict( + families: families, + frontmostBundleID: frontmostBundleID, + now: now + ) else { + // The first title read of this session is still running, inside + // `titleReadGiveUp`. It re-evaluates when it lands; if it hangs, + // the re-check below ends the hold. Too brief to be worth a + // suppression event. browserFirstTitleReadPending = true + if let readStartedAt = browserTitles?.readStartedAt { + scheduleBrowserEvidenceRecheck( + at: readStartedAt.addingTimeInterval(browserEvidenceTiming.titleReadGiveUp), + now: now + ) + } return nil } @@ -175,7 +205,11 @@ extension MeetingPromptDetector { /// rules applied, or `nil` while the session's first read is running. /// Starts a background read when the last one is old enough; its result /// re-evaluates. - private func currentBrowserTitleVerdict(families: Set, now: Date) -> BrowserCallTitleVerdict? { + private func currentBrowserTitleVerdict( + families: Set, + frontmostBundleID: String?, + now: Date + ) -> BrowserCallTitleVerdict? { guard !families.isEmpty else { return .unknown } if var session = browserTitles, !session.families.isDisjoint(with: families) { // Same browser (another one may have joined or left): keep what @@ -187,12 +221,23 @@ extension MeetingPromptDetector { browserTitles?.readTask?.cancel() browserTitles = BrowserTitleSession(families: families, startedAt: now) } - guard let session = browserTitles else { return nil } + guard var session = browserTitles else { return nil } + // Only a shortcut of the browser holding the mic can claim it: a Meet + // shortcut in Helium says nothing about Chrome's mic. + let isMicBrowser = { (bundleID: String) in !Self.browserAppFamilies(for: [bundleID]).isDisjoint(with: families) } + let recent = recentCallShortcutActivation.flatMap { isMicBrowser($0.bundleID) ? $0 : nil } + session.claimedShortcut = BrowserCallEvidence.claimedCallShortcut( + frontmostShortcut: frontmostBundleID.flatMap { isMicBrowser($0) ? callShortcutProviders[$0] : nil }, + recentProvider: recent?.provider, + recentActivationAt: recent?.at, + sessionStartedAt: session.startedAt + ) + browserTitles = session let readIsStale = session.readAt.map { now.timeIntervalSince($0) >= browserEvidenceTiming.titleReadSpacing } ?? true if session.namedCall == nil, session.readTask == nil, readIsStale { startBrowserTitleRead(families: families) } - return session.verdict + return session.verdict(now: now, titleReadGiveUp: browserEvidenceTiming.titleReadGiveUp) } private func startBrowserTitleRead(families: Set) { @@ -200,6 +245,9 @@ extension MeetingPromptDetector { let token = browserTitleReadCounter let provider = browserWindowTitlesProvider browserTitles?.readToken = token + if browserTitles?.readStartedAt == nil { + browserTitles?.readStartedAt = Date() + } evaluationsInFlight += 1 browserTitles?.readTask = Task { @MainActor [weak self] in let titles = await provider(families) @@ -221,11 +269,16 @@ extension MeetingPromptDetector { switch verdict { case .call(let provider): session.namedCall = provider + session.sawNonCallSite = false + case .callSite: + // The user is on a call app now. An earlier non-call glance was + // not the thing holding the mic. + session.sawNonCallSite = false case .notCall: if now.timeIntervalSince(session.startedAt) <= browserEvidenceTiming.nonCallSiteStickyWindow { session.sawNonCallSite = true } - case .callSite, .unknown: + case .unknown: break } browserTitles = session @@ -272,6 +325,39 @@ extension MeetingPromptDetector { } } + /// A browser app came forward or launched. If it is a call shortcut, remember + /// it so a calendar-opened Meet PWA is not vetoed by another Helium window. + func noteCallShortcutActivation(bundleID: String, provider: MeetingPromptProvider, at: Date) { + callShortcutProviders[bundleID] = provider + callShortcutLookupDone.insert(bundleID) + recentCallShortcutActivation = (bundleID: bundleID, provider: provider, at: at) + scheduleEvaluation() + } + + /// A call shortcut quit: it can no longer be the call that is starting. + func noteCallShortcutTermination(bundleID: String) { + guard recentCallShortcutActivation?.bundleID == bundleID else { return } + recentCallShortcutActivation = nil + } + + /// Classify the frontmost app's shortcut URL once, off this pass. The next + /// evaluation uses the answer. A miss is remembered too, so a busy main + /// browser is not read again on every mic poll. + private func ensureCallShortcutLookup(_ bundleID: String?) { + guard let bundleID, MeetingPromptProvider.isBrowserBundleID(bundleID) else { return } + guard !callShortcutLookupDone.contains(bundleID) else { return } + callShortcutLookupDone.insert(bundleID) + let lookup = callShortcutProvider + Task { @MainActor [weak self] in + let site = await lookup(bundleID) + guard let self else { return } + if let site { + self.callShortcutProviders[bundleID] = site + } + self.scheduleEvaluation() + } + } + /// The browser session is over: forget the title verdict and stop /// re-checking. The next browser mic use starts its own wait. func endBrowserMicSession() { diff --git a/Sources/Meeting/MeetingPromptDetector+CalendarRuntime.swift b/Sources/Meeting/MeetingPromptDetector+CalendarRuntime.swift index ad8eeb5eb..c3253cfc8 100644 --- a/Sources/Meeting/MeetingPromptDetector+CalendarRuntime.swift +++ b/Sources/Meeting/MeetingPromptDetector+CalendarRuntime.swift @@ -44,15 +44,37 @@ extension MeetingPromptDetector { } } } + workspaceObservers.append(NSWorkspace.shared.notificationCenter.addObserver( + forName: NSWorkspace.didTerminateApplicationNotification, + object: nil, + queue: .main + ) { [weak self] notification in + let app = notification.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication + Task { @MainActor [weak self] in + guard let app, + let bundleIdentifier = await RunningApplicationsReader.bundleIdentifier(of: app) else { return } + self?.noteCallShortcutTermination(bundleID: bundleIdentifier) + } + }) } private func handleWorkspaceApplication(bundleIdentifier: String?) { - guard let bundleIdentifier, - let provider = provider(forBundleIdentifier: bundleIdentifier), - provider.supportsNativeRuntimePrompt else { return } - - recentNativeActivity[provider] = Date() - scheduleEvaluation() + guard let bundleIdentifier else { return } + if let provider = provider(forBundleIdentifier: bundleIdentifier), + provider.supportsNativeRuntimePrompt { + recentNativeActivity[provider] = Date() + scheduleEvaluation() + } + // A calendar link opens the Meet shortcut. Remember that activation + // even though the shortcut is not a native runtime app: its mic is + // the main browser's helper, and the shortcut is the call. + guard MeetingPromptProvider.isBrowserBundleID(bundleIdentifier) else { return } + let activatedAt = Date() + let lookup = callShortcutProvider + Task { @MainActor [weak self] in + guard let site = await lookup(bundleIdentifier) else { return } + self?.noteCallShortcutActivation(bundleID: bundleIdentifier, provider: site, at: activatedAt) + } } func seedNativeActivityIfNeeded(frontmostBundleID: String?, now: Date) { diff --git a/Sources/Meeting/MeetingPromptDetector.swift b/Sources/Meeting/MeetingPromptDetector.swift index 0d15f92ab..114ebcc70 100644 --- a/Sources/Meeting/MeetingPromptDetector.swift +++ b/Sources/Meeting/MeetingPromptDetector.swift @@ -67,6 +67,12 @@ final class MeetingPromptDetector { var browserWindowTitlesProvider: (Set) async -> [BrowserWindowTitle] = { families in await BrowserWindowTitleReader.titles(forBrowserFamilies: families) } + /// The call a browser app shortcut is for, from its start URL. Defaults to + /// the plist reader; unit tests inject a fixed answer. The URL is classified + /// and dropped. + var callShortcutProvider: (String) async -> MeetingPromptProvider? = { bundleID in + await BrowserWindowTitleReader.callShortcutProvider(forBundleID: bundleID) + } /// How long an unrecognized browser has to hold the mic before it prompts. /// Tests shrink these. var browserEvidenceTiming: BrowserCallEvidence.Timing = .standard @@ -142,6 +148,13 @@ final class MeetingPromptDetector { // prompt re-offers a couple of times before inheriting the full dismissal. var promptExpiryHistory: [String: (count: Int, lastExpiredAt: Date)] = [:] var browserTitleReadCounter = 0 + /// Call shortcut bundle IDs already classified. A missing key has not been + /// read; a present nil is a shortcut that is not a call app. + var callShortcutProviders: [String: MeetingPromptProvider] = [:] + var callShortcutLookupDone: Set = [] + /// The latest call shortcut that launched or came forward. Used only when + /// that was as the mic was taken; an old activation does not count. + var recentCallShortcutActivation: (bundleID: String, provider: MeetingPromptProvider, at: Date)? // Set by `adHocCandidate` during one evaluate pass when a browser session's // first title read has not landed yet. var browserFirstTitleReadPending = false diff --git a/Sources/Meeting/MeetingPromptHeuristics.swift b/Sources/Meeting/MeetingPromptHeuristics.swift index bbb22b6cc..7a53eeb8e 100644 --- a/Sources/Meeting/MeetingPromptHeuristics.swift +++ b/Sources/Meeting/MeetingPromptHeuristics.swift @@ -102,6 +102,7 @@ enum MeetingPromptProvider: String, CaseIterable, Hashable { "com.microsoft.edgemac", // Edge + helpers "com.brave.Browser", // Brave + helpers "company.thebrowser.Browser", // Arc + helpers + "net.imput.helium", // Helium (Chromium) + helpers, including .helper and .app.* shortcuts "org.mozilla.firefox", // Firefox + plugin-container children "com.apple.Safari", // Safari main process "com.apple.WebKit", // Safari / WKWebView service processes (GPU, Networking, WebContent) @@ -114,6 +115,17 @@ enum MeetingPromptProvider: String, CaseIterable, Hashable { browserBundleIDPrefixes.contains { bundleID.matchesBundleFamily($0) } } + /// Whether `bundleID` is a browser's own app (not a helper, and not a + /// Chrome, Edge or Safari web app, which may be Slack or Gmail), or a + /// Helium web app. Helium's Meet shortcut is how calendar links open + /// there; its other shortcuts can't be told apart without reading them. + static func isMeetingTabHost(_ bundleID: String) -> Bool { + if bundleID == "com.apple.WebKit" { return false } + if bundleID == "com.google.Chrome.canary" { return true } + if bundleID.matchesBundleFamily("net.imput.helium.app") { return true } + return browserBundleIDPrefixes.contains(bundleID) + } + /// The browser family prefix `bundleID` belongs to (`com.google.Chrome` /// for `com.google.Chrome.helper`), or `nil` when it is not a browser. /// Used to line a browser's mic process up with its audio-output process diff --git a/Sources/Meeting/MeetingPromptSnapshots.swift b/Sources/Meeting/MeetingPromptSnapshots.swift index fe241e7bf..2c3e13b6f 100644 --- a/Sources/Meeting/MeetingPromptSnapshots.swift +++ b/Sources/Meeting/MeetingPromptSnapshots.swift @@ -111,16 +111,6 @@ struct MeetingPromptRuntimeSnapshot: Equatable { self.isOwnCaptureActive = isOwnCaptureActive self.isMicInputPromptEnabled = isMicInputPromptEnabled } - - static let browserBundleIdentifiers: Set = [ - "com.apple.Safari", - "com.google.Chrome", - "com.google.Chrome.canary", - "com.microsoft.edgemac", - "com.brave.Browser", - "company.thebrowser.Browser", - "org.mozilla.firefox" - ] } @available(macOS 14.0, *) diff --git a/Tests/BrowserCallEvidenceTests.swift b/Tests/BrowserCallEvidenceTests.swift index e0735ebc9..ee9487933 100644 --- a/Tests/BrowserCallEvidenceTests.swift +++ b/Tests/BrowserCallEvidenceTests.swift @@ -171,6 +171,115 @@ func testBrowserCallEvidence() { ) } + runSuite("BrowserCallEvidence.callShortcutProvider — a Meet shortcut URL is Google Meet") { + assertEqual( + BrowserCallEvidence.callShortcutProvider(forURL: "https://meet.google.com/landing?lfhs=2"), + .googleMeet, + "a Helium Meet shortcut is a Google Meet call site" + ) + assertEqual( + BrowserCallEvidence.callShortcutProvider(forURL: "https://www.meet.google.com/abc-defg-hij"), + .googleMeet, + "a shortcut to one meeting is still Google Meet" + ) + assertEqual( + BrowserCallEvidence.callShortcutProvider(forURL: "https://chat.google.com/"), + nil, + "Google Chat is not a call shortcut" + ) + assertEqual( + BrowserCallEvidence.callShortcutProvider(forURL: "https://calendar.google.com/calendar/u/0/r"), + nil, + "Google Calendar is not a call shortcut" + ) + } + + runSuite("BrowserCallEvidence.claimedCallShortcut — a shortcut opened with the mic beats a background non-call window") { + let started = Date(timeIntervalSince1970: 1_000) + assertEqual( + BrowserCallEvidence.claimedCallShortcut( + frontmostShortcut: nil, + recentProvider: .googleMeet, + recentActivationAt: started.addingTimeInterval(-5), + sessionStartedAt: started + ), + .googleMeet, + "a Meet shortcut that opened as the mic was taken is the call, even if Calendar stays in front" + ) + assertEqual( + BrowserCallEvidence.claimedCallShortcut( + frontmostShortcut: .googleMeet, + recentProvider: nil, + recentActivationAt: nil, + sessionStartedAt: started + ), + .googleMeet, + "the Meet shortcut the user is in is the call even without a fresh activation" + ) + assertEqual( + BrowserCallEvidence.claimedCallShortcut( + frontmostShortcut: nil, + recentProvider: .googleMeet, + recentActivationAt: started.addingTimeInterval(-3_600), + sessionStartedAt: started + ), + nil, + "a Meet shortcut left open from an hour ago does not turn ChatGPT voice into a call" + ) + } + + runSuite("BrowserCallEvidence.verdictBeforeTitleReadLands — a hung title read does not hold the prompt") { + assertEqual( + BrowserCallEvidence.verdictBeforeTitleReadLands(pendingFor: 0.2, giveUp: 1), + nil, + "the first read still holds the prompt while it is in flight" + ) + assertEqual( + BrowserCallEvidence.verdictBeforeTitleReadLands(pendingFor: 1, giveUp: 1), + .unknown, + "once the hold runs out the timing rules start, instead of waiting on a hung read" + ) + } + + runSuite("BrowserWindowTitleReader — a frontmost Meet shortcut is the focused window") { + let meet: pid_t = 10 + let helium: pid_t = 20 + let chat: pid_t = 30 + assertEqual( + BrowserWindowTitleReader.readingOrder(pids: [helium, chat, meet], frontmostPID: meet), + [meet, helium, chat], + "the shortcut the user is in is read first so the title budget is not spent on the main browser" + ) + assertEqual( + BrowserWindowTitleReader.readingOrder(pids: [helium, chat], frontmostPID: meet), + [helium, chat], + "an unrelated frontmost app does not reorder the browsers" + ) + assertTrue( + BrowserWindowTitleReader.isUserFocused(appPID: meet, frontmostPID: meet), + "the Meet shortcut the user is in is the focused window" + ) + assertFalse( + BrowserWindowTitleReader.isUserFocused(appPID: helium, frontmostPID: meet), + "a background Helium window is not the focused window" + ) + assertFalse( + BrowserWindowTitleReader.isUserFocused(appPID: meet, frontmostPID: nil), + "with no frontmost app, no window counts as focused" + ) + } + + runSuite("BrowserCallEvidence.classify — a background non-call window does not veto a frontmost Meet shortcut") { + assertEqual( + BrowserCallEvidence.classify([ + BrowserWindowTitle(title: "ChatGPT", isFocused: false), + BrowserWindowTitle(title: "Google Meet", isFocused: true), + ]), + .callSite(provider: .googleMeet), + "the Meet shortcut in front is a call site even if another Helium window is ChatGPT" + ) + } + runSuite("BrowserCallEvidence.classify — only the focused window can say not a call") { let windows = [ BrowserWindowTitle(title: "ChatGPT", isFocused: false), diff --git a/Tests/MeetingPromptDetectorTests.swift b/Tests/MeetingPromptDetectorTests.swift index 99a338c30..32c5bc08d 100644 --- a/Tests/MeetingPromptDetectorTests.swift +++ b/Tests/MeetingPromptDetectorTests.swift @@ -1061,6 +1061,167 @@ func testMeetingPromptDetector() async { assertEqual(box.suppression?.reason, .learnedQuiet, "the skip should say it came from the learned quiet window") } + await runSuite("MeetingPromptDetector browser evidence — a Meet shortcut opened with the mic is not vetoed by ChatGPT") { + let detector = makeIsolatedDetector() + detector.isOwnCaptureActive = { false } + detector.browserEvidenceTiming = instantBrowserEvidenceTiming + let meetShortcut = "net.imput.helium.app.meet" + detector.frontmostBundleIDProvider = { "com.apple.iCal" } + showBrowserTab("ChatGPT", on: detector) + detector.noteCallShortcutActivation(bundleID: meetShortcut, provider: .googleMeet, at: Date()) + let box = CandidateBox() + detector.onPromptRequest = { candidate in + box.candidate = candidate + box.promptCount += 1 + return true + } + detector.onPromptSuppressed = { suppression in + box.suppression = suppression + } + + detector.updateMicInputUsers(["net.imput.helium.helper"]) + await waitForPromptEvaluation(detector) + + assertEqual( + box.promptCount, + 1, + "opening the Meet shortcut as the mic starts is a call, even if a ChatGPT window is focused in the main browser" + ) + assertEqual( + box.candidate?.id, + MeetingPromptDetector.browserCallSiteCandidateID, + "without an in-call title the shortcut waits as a call site" + ) + } + + await runSuite("MeetingPromptDetector browser evidence — a Meet shortcut left open does not turn ChatGPT voice into a call") { + let detector = makeIsolatedDetector() + detector.isOwnCaptureActive = { false } + detector.browserEvidenceTiming = instantBrowserEvidenceTiming + detector.frontmostBundleIDProvider = { "net.imput.helium" } + showBrowserTab("ChatGPT", on: detector) + detector.noteCallShortcutActivation( + bundleID: "net.imput.helium.app.meet", + provider: .googleMeet, + at: Date().addingTimeInterval(-3_600) + ) + let box = CandidateBox() + detector.onPromptRequest = { candidate in + box.promptCount += 1 + return true + } + detector.onPromptSuppressed = { suppression in + box.suppression = suppression + } + + detector.updateMicInputUsers(["net.imput.helium.helper"]) + await waitForPromptEvaluation(detector) + + assertEqual(box.promptCount, 0, "Meet left open in the background must not turn ChatGPT voice into a call") + assertEqual(box.suppression?.reason, .notACall, "ChatGPT in front is still not a call") + } + + await runSuite("MeetingPromptDetector browser evidence — another browser's Meet shortcut doesn't make a call") { + let detector = makeIsolatedDetector() + detector.isOwnCaptureActive = { false } + detector.browserEvidenceTiming = instantBrowserEvidenceTiming + detector.frontmostBundleIDProvider = { "com.google.Chrome" } + showBrowserTab("ChatGPT", on: detector) + detector.noteCallShortcutActivation(bundleID: "net.imput.helium.app.meet", provider: .googleMeet, at: Date()) + let box = CandidateBox() + detector.onPromptRequest = { _ in + box.promptCount += 1 + return true + } + detector.onPromptSuppressed = { suppression in + box.suppression = suppression + } + + detector.updateMicInputUsers(["com.google.Chrome.helper"]) + await waitForPromptEvaluation(detector) + + assertEqual(box.promptCount, 0, "a Helium Meet shortcut must not turn ChatGPT voice in Chrome into a call") + assertEqual(box.suppression?.reason, .notACall, "ChatGPT in front is still not a call") + } + + await runSuite("MeetingPromptDetector browser evidence — a Meet shortcut that quit doesn't make a call") { + let detector = makeIsolatedDetector() + detector.isOwnCaptureActive = { false } + detector.browserEvidenceTiming = instantBrowserEvidenceTiming + detector.frontmostBundleIDProvider = { "net.imput.helium" } + showBrowserTab("ChatGPT", on: detector) + detector.noteCallShortcutActivation(bundleID: "net.imput.helium.app.meet", provider: .googleMeet, at: Date()) + detector.noteCallShortcutTermination(bundleID: "net.imput.helium.app.meet") + let box = CandidateBox() + detector.onPromptRequest = { _ in + box.promptCount += 1 + return true + } + detector.onPromptSuppressed = { suppression in + box.suppression = suppression + } + + detector.updateMicInputUsers(["net.imput.helium.helper"]) + await waitForPromptEvaluation(detector) + + assertEqual(box.promptCount, 0, "a Meet shortcut that already quit must not turn ChatGPT voice into a call") + assertEqual(box.suppression?.reason, .notACall, "ChatGPT in front is still not a call") + } + + await runSuite("MeetingPromptDetector browser evidence — a title read that never returns stops holding the prompt") { + let detector = makeIsolatedDetector() + detector.isOwnCaptureActive = { false } + var timing = instantBrowserEvidenceTiming + timing.titleReadGiveUp = 0.2 + detector.browserEvidenceTiming = timing + // An Accessibility read that hangs until the test ends. + detector.browserWindowTitlesProvider = { _ in + try? await Task.sleep(nanoseconds: 3_600 * 1_000_000_000) + return [] + } + let box = CandidateBox() + detector.onPromptRequest = { _ in + box.promptCount += 1 + return true + } + + detector.updateMicInputUsers(["com.google.Chrome.helper"]) + // The hung read keeps an evaluation in flight, so this can't wait for + // the detector to settle; poll the outcome with a deadline instead. + for _ in 0..<100 where box.promptCount == 0 { + try? await Task.sleep(nanoseconds: 50_000_000) + } + assertEqual(box.promptCount, 1, "once the first-read hold runs out, the browser call must prompt without another event") + } + + await runSuite("MeetingPromptDetector browser evidence — coming back to Meet beats an early ChatGPT glance") { + let detector = makeIsolatedDetector() + detector.isOwnCaptureActive = { false } + detector.browserEvidenceTiming = instantBrowserEvidenceTiming + showBrowserTab("ChatGPT", on: detector) + let box = CandidateBox() + detector.onPromptRequest = { candidate in + box.candidate = candidate + box.promptCount += 1 + return true + } + detector.onPromptSuppressed = { suppression in + box.suppression = suppression + } + + detector.updateMicInputUsers(["net.imput.helium.helper"]) + await waitForPromptEvaluation(detector) + assertEqual(box.suppression?.reason, .notACall, "precondition: ChatGPT in front as the mic starts is not a call") + + showBrowserTab("Google Meet", on: detector) + await waitForPromptEvaluation(detector, extraMilliseconds: 300) + detector.updateBrowserOutputUsers(["net.imput.helium.helper"]) + await waitForPromptEvaluation(detector) + + assertEqual(box.promptCount, 1, "switching back to the Meet window should still prompt") + assertEqual(box.candidate?.id, MeetingPromptDetector.browserCallSiteCandidateID, "the Meet window is a call site, not a named tab") + } + await runSuite("MeetingPromptDetector browser evidence — ChatGPT voice stays quiet after a tab switch") { let detector = makeIsolatedDetector() detector.isOwnCaptureActive = { false } @@ -1348,6 +1509,7 @@ private func makeIsolatedDetector( detector.frontmostBundleIDProvider = { nil } detector.runningBundleIDsProvider = { [] } detector.browserWindowTitlesProvider = { _ in [] } + detector.callShortcutProvider = { _ in nil } return detector } diff --git a/Tests/MeetingPromptHeuristicsTests.swift b/Tests/MeetingPromptHeuristicsTests.swift index 05ceac7ae..6c7d7c0a1 100644 --- a/Tests/MeetingPromptHeuristicsTests.swift +++ b/Tests/MeetingPromptHeuristicsTests.swift @@ -347,6 +347,135 @@ func testMeetingPromptHeuristics() { ) } + runSuite("MeetingPromptProvider — Helium is a Chromium-family browser call") { + assertTrue( + MeetingPromptProvider.isBrowserBundleID("net.imput.helium"), + "the main Helium bundle should be recognized as a browser" + ) + assertTrue( + MeetingPromptProvider.isBrowserBundleID("net.imput.helium.helper"), + "Helium's helper (the process that holds the mic) must be recognized" + ) + assertTrue( + MeetingPromptProvider.isBrowserBundleID("net.imput.helium.helper.renderer"), + "Helium's renderer helper must be recognized so playback lines up with the mic" + ) + assertTrue( + MeetingPromptProvider.isBrowserBundleID("net.imput.helium.app.shortcut"), + "a Helium web-app shortcut shares the Helium family and must be recognized" + ) + assertFalse( + MeetingPromptProvider.isBrowserBundleID("net.imput.heliumEvil"), + "a bundle id that only shares a prefix segment should not match Helium" + ) + assertEqual( + MeetingPromptProvider.micInputProvider(forBundleID: "net.imput.helium.helper"), + .googleMeet, + "Helium holding the mic maps to the representative browser-call provider" + ) + assertEqual( + MeetingPromptProvider.cameraCallProvider(forFrontmostBundleID: "net.imput.helium"), + .googleMeet, + "Helium frontmost with the camera on maps to the generic browser call" + ) + assertNil( + MeetingPromptProvider.audioOutputProvider(forBundleID: "net.imput.helium.helper"), + "Helium output alone is browser playback and must not prompt by itself" + ) + assertEqual( + MeetingPromptProvider.browserFamily(forBundleID: "net.imput.helium.helper.renderer"), + "net.imput.helium", + "a Helium helper belongs to the Helium family so mic and playback line up" + ) + } + + runSuite("MeetingPromptCalendarScoring — a frontmost Helium window is the meeting tab on a calendar prompt") { + let now = Date(timeIntervalSince1970: 1_700_000_000) + let meet = MeetingPromptCalendarEventSnapshot( + id: "design-review", + title: "Design review", + startDate: now, + endDate: now.addingTimeInterval(30 * 60), + isAllDay: false, + url: URL(string: "https://meet.google.com/abc-defg-hij"), + location: nil, + notes: nil + ) + + func scored(frontmost: String?, event: MeetingPromptCalendarEventSnapshot = meet) -> MeetingPromptScoredCandidate? { + MeetingPromptCalendarScoring.calendarCandidates( + from: [event], + now: now, + runtimeSnapshot: MeetingPromptRuntimeSnapshot( + runningBundleIDs: [], + frontmostBundleID: frontmost, + recentNativeActivity: [:], + runtimeSuppressedUntil: [:] + ) + ).first + } + + let helium = scored(frontmost: "net.imput.helium") + assertEqual( + helium?.candidate.detail, + "Design review - meeting tab is active", + "Helium in front of a calendar Meet should say the meeting tab is active" + ) + assertEqual( + helium?.candidate.reason, + .calendarPlusRuntimeMatch, + "a frontmost browser upgrades the calendar prompt's reason" + ) + assertEqual(helium?.score, 5, "the meeting-tab boost plus the start window should outrank a plain calendar prompt") + + assertEqual( + scored(frontmost: "net.imput.helium.app.shortcut")?.candidate.detail, + "Design review - meeting tab is active", + "a Helium web-app shortcut is still the meeting tab" + ) + assertEqual( + scored(frontmost: "com.google.Chrome")?.candidate.reason, + .calendarPlusRuntimeMatch, + "Chrome still counts as the meeting tab" + ) + for webApp in ["com.google.Chrome.app.slackwebapp", "com.apple.Safari.WebApp.gmail", "com.google.Chrome.helper"] { + assertTrue( + scored(frontmost: webApp)?.candidate.reason != .calendarPlusRuntimeMatch, + "a Chrome or Safari web app or a browser helper in front is not the meeting tab" + ) + } + assertEqual( + scored(frontmost: "com.google.Chrome.canary")?.candidate.reason, + .calendarPlusRuntimeMatch, + "Chrome Canary still counts through the Chrome family prefix" + ) + + let unrelated = scored(frontmost: "com.apple.TextEdit") + assertEqual( + unrelated?.candidate.detail, + "Design review - starting now", + "an unrelated front app should not be called a meeting tab" + ) + assertEqual(unrelated?.candidate.reason, .calendarNearby, "no browser in front keeps the plain calendar reason") + assertEqual(unrelated?.score, 3, "without the meeting-tab boost the start window alone scores 3") + + let zoom = MeetingPromptCalendarEventSnapshot( + id: "zoom-review", + title: "Design review", + startDate: now, + endDate: now.addingTimeInterval(30 * 60), + isAllDay: false, + url: URL(string: "https://zoom.us/j/123456789"), + location: nil, + notes: nil + ) + assertEqual( + scored(frontmost: "net.imput.helium", event: zoom)?.candidate.detail, + "Design review - starting now", + "Helium in front does not decorate a native Zoom calendar event" + ) + } + runSuite("MeetingPromptProvider.micInputProvider — attributes mic-holding processes to a provider") { assertEqual( MeetingPromptProvider.micInputProvider(forBundleID: "us.zoom.xos"), diff --git a/Tests/MicActivityMonitorProbeFilterTests.swift b/Tests/MicActivityMonitorProbeFilterTests.swift index 638c1a03e..08e9d4d25 100644 --- a/Tests/MicActivityMonitorProbeFilterTests.swift +++ b/Tests/MicActivityMonitorProbeFilterTests.swift @@ -111,7 +111,13 @@ func testMicActivityMonitorProbeFilter() { assertFalse(MicActivityMonitor.shouldProbe(bundleID: "", ownBundleID: own), "a bundle-less daemon is dropped downstream anyway") assertFalse(MicActivityMonitor.shouldProbe(bundleID: "com.justinbetker.draft", ownBundleID: own), "our own app is never a call") assertFalse(MicActivityMonitor.shouldProbe(bundleID: "com.justinbetker.draft.helper", ownBundleID: own), "our own helpers are never a call") - for bundle in ["com.google.Chrome.helper", "com.apple.WebKit.GPU", "us.zoom.xos", "com.microsoft.teams2.helper"] { + for bundle in [ + "com.google.Chrome.helper", + "com.apple.WebKit.GPU", + "net.imput.helium.helper", + "us.zoom.xos", + "com.microsoft.teams2.helper", + ] { assertTrue(MicActivityMonitor.shouldProbe(bundleID: bundle, ownBundleID: own), "\(bundle) can be a call") } for bundle in ["com.apple.QuickTimePlayerX", "com.tinyspeck.slackmacgap", "com.spotify.client"] { diff --git a/docs/auto-call-detection-spec.md b/docs/auto-call-detection-spec.md index 9f094f1de..6049d71ad 100644 --- a/docs/auto-call-detection-spec.md +++ b/docs/auto-call-detection-spec.md @@ -146,7 +146,14 @@ results below are the provenance for the production monitor design. `com.apple.WebKit` prefix and also cover any WKWebView-based app in a web call. - So browser detection recognizes: `com.google.Chrome*`, `com.google.Chrome.canary*`, `com.microsoft.edgemac*`, `com.brave.Browser*`, `company.thebrowser.Browser*`, - `org.mozilla.firefox*`, `com.apple.Safari`, and `com.apple.WebKit*`. + `net.imput.helium*`, `org.mozilla.firefox*`, `com.apple.Safari`, and `com.apple.WebKit*`. + Helium is a Chromium fork. Its mic is held by `net.imput.helium.helper` (same + shape as Chrome's Audio Service helper), and installed web-app shortcuts use + `net.imput.helium.app.*`. The `net.imput.helium` family prefix covers both. + A calendar prompt's "meeting tab is active" line counts a browser's own app + or a Helium web app in front, not a helper and not a Chrome, Edge or Safari + web app (that may be Slack or Gmail). That line only decorates a calendar event that is already + prompting. It does not detect a call by itself. - **Self-exclusion:** our bundle ID is `com.justinbetker.draft` (legacy Draft id); exclude it by prefix so any future helper process is dropped too. - Aside: `ai.plaud.desktop.plaud` + `.helper` show up in the process list — Plaud @@ -578,11 +585,26 @@ twice. "Not now" bought 30 minutes, lived in memory, and reset on relaunch. chat, the Meet home page, a Slack huddle, WhatsApp, Discord) -> prompt after 20 s as the generic browser call (`mic:browser-call`). These only count from the focused window: a chat app left open all day says nothing. + A browser family can include app shortcuts (a Helium Meet PWA is + `net.imput.helium.app.*`, and its mic is still the main browser's helper). + Only the frontmost app's window is focused, and that app is read first. + A ChatGPT tab in the main browser must not veto a call in the shortcut. + The shortcut in front is a call site even when its window title cannot be + read. A shortcut that opened as the mic was taken (a calendar link, within + 15 s) is a call site too, so Calendar staying in front does not hide it. + A shortcut left open from earlier does not: ChatGPT voice with Meet sitting + in the background stays quiet. Either way the shortcut has to belong to the + browser holding the mic (a Helium Meet shortcut says nothing about Chrome), + and one that quit no longer counts. Known limitation: bringing the Meet + shortcut forward on its landing page while another Helium tab holds the + mic prompts, because the shortcut's title can't tell a landing page from a + call. 3. The focused window is a known non-call site (ChatGPT, Claude, Gemini, Loom, a mic test...) -> no prompt while it is in front. When it was in front within 10 s of the browser taking the mic, no prompt for the rest of - that browser session, even after a tab switch; only a title from step 1 - can still win. Without a prompt or a recording, such a session feeds + that browser session, even after a tab switch to an ordinary page; a + title from step 1, or coming back to a call site (the Meet window), can + still win. Without a prompt or a recording, such a session feeds neither the funnel event nor the missed-call nudge. Pages people keep open during calls (Google Docs, YouTube) are not on this list, and markers match whole words ("Bloomberg" is not Loom). @@ -608,7 +630,9 @@ always in front. Titles come from Accessibility (`BrowserWindowTitleReader`), which the app holds for paste-back when dictation is on: no new permission, nothing when untrusted. The round trips run on a background queue under a 0.3 s overall -budget; the first read of a session holds the prompt until it lands. Titles +budget; the first read of a session holds the prompt until it lands, or for +1 s, whichever comes first; a re-check at the 1 s mark ends the hold even if +nothing else changes. A hung read must not keep a real call quiet. Titles are classified in memory and dropped; only the coarse `call_evidence` enum reaches analytics. While a browser holds the mic and nothing is decided, the detector re-reads titles every 15 s (every 60 s once it is a non-call site),