Prompt to record Helium Google Meet calls, including the Meet shortcut

Sources/Meeting/BrowserCallEvidence.swift

@@ -18,8 +18,9 @@
 //    `corroboratedDelay`, as the generic browser call.
 // 3. The focused window is a known non-call site (ChatGPT, Claude, Loom...)
 //    -> no prompt while it is in front. When it was in front as the browser
-//    took the mic, no prompt for the whole session, even after the user
-//    clicks to another tab; only a title from step 1 can still win.
+//    took the mic, no prompt for the whole session after a switch to an
+//    ordinary page. A title from step 1, or coming back to a call site, can
+//    still win.
 // 4. The browser is also playing audio, or the camera is on -> prompt after
 //    `corroboratedDelay` of continuous mic use.
 // 5. Mic only -> prompt after `uncorroboratedDelay`.
@@ -141,6 +142,16 @@ enum BrowserCallEvidence {
     /// one using it. Seen later (notes in Claude during a call), it only
     /// holds the prompt back while it stays in front.
     static let nonCallSiteStickyWindow: TimeInterval = 10
+    /// A call shortcut (a Helium Meet PWA) opened this close to the mic
+    /// starting is the thing that took the mic, even if another app in the
+    /// family is in front. Wider than the non-call sticky window so a
+    /// calendar link that opens the shortcut, then leaves Calendar in front,
+    /// still counts.
+    static let callShortcutActivationSlack: TimeInterval = 15
+    /// How long the first title read may hold the prompt. A hung Accessibility
+    /// read must not keep a real call quiet; after this the timing rules run,
+    /// and a late title can still upgrade or suppress before the wait ends.
+    static let titleReadGiveUp: TimeInterval = 1
 
     struct Timing: Equatable {
         var corroboratedDelay: TimeInterval
@@ -149,6 +160,7 @@ enum BrowserCallEvidence {
         var nonCallSiteRecheckInterval: TimeInterval = BrowserCallEvidence.nonCallSiteRecheckInterval
         var micReleaseGrace: TimeInterval = BrowserCallEvidence.micReleaseGrace
         var nonCallSiteStickyWindow: TimeInterval = BrowserCallEvidence.nonCallSiteStickyWindow
+        var titleReadGiveUp: TimeInterval = BrowserCallEvidence.titleReadGiveUp
         /// Minimum spacing between title reads within one browser session.
         /// Several sensor edges can land in the same second; one read answers
         /// all of them.
@@ -215,6 +227,58 @@ enum BrowserCallEvidence {
         return .wait(recheckAt: min(promptAt, now.addingTimeInterval(timing.titleRecheckInterval)))
     }
 
+    // MARK: App shortcuts
+
+    /// The call a Chromium app shortcut is for, from its start URL, or `nil`.
+    /// The URL is classified and dropped: a Meet shortcut is
+    /// `https://meet.google.com/...`, including the landing page. A meeting
+    /// code in the path is not proof a call is on, so this is a call site,
+    /// not a named tab. Chat, Calendar and Gmail shortcuts do not match.
+    static func callShortcutProvider(forURL raw: String) -> MeetingPromptProvider? {
+        let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard let host = URLComponents(string: trimmed)?.host?.lowercased() else { return nil }
+        let bare = host.hasPrefix("www.") ? String(host.dropFirst(4)) : host
+        switch bare {
+        case "meet.google.com":
+            return .googleMeet
+        case "teams.microsoft.com", "teams.live.com":
+            return .teams
+        case "zoom.us", "app.zoom.us":
+            return .zoom
+        default:
+            return nil
+        }
+    }
+
+    /// Which call shortcut is using the shared mic helper, if either signal
+    /// says so. The shortcut in front wins. Otherwise a shortcut that opened
+    /// as the mic was taken (a calendar link) wins, so a ChatGPT tab in the
+    /// main browser cannot veto it. A shortcut left open from long before the
+    /// mic started does not: ChatGPT voice with Meet sitting in the background
+    /// stays a non-call.
+    static func claimedCallShortcut(
+        frontmostShortcut: MeetingPromptProvider?,
+        recentProvider: MeetingPromptProvider?,
+        recentActivationAt: Date?,
+        sessionStartedAt: Date,
+        slack: TimeInterval = callShortcutActivationSlack
+    ) -> MeetingPromptProvider? {
+        if let frontmostShortcut { return frontmostShortcut }
+        guard let recentProvider, let recentActivationAt else { return nil }
+        let delta = recentActivationAt.timeIntervalSince(sessionStartedAt)
+        guard abs(delta) <= slack else { return nil }
+        return recentProvider
+    }
+
+    /// `nil` while the first title read should still hold the prompt; `.unknown`
+    /// once that hold has run out, so the timing rules can start.
+    static func verdictBeforeTitleReadLands(
+        pendingFor: TimeInterval,
+        giveUp: TimeInterval
+    ) -> BrowserCallTitleVerdict? {
+        pendingFor >= giveUp ? .unknown : nil
+    }
+
     // MARK: Title classification
 
     /// Classifies the titles of every window of the browsers holding the mic.

Sources/Meeting/BrowserWindowTitleReader.swift

@@ -40,29 +40,55 @@ enum BrowserWindowTitleReader {
 
         let appFamilies = Set(families.map(MeetingPromptProvider.browserAppFamily(forBundleFamily:)))
         // Bundle IDs are read off the main thread: that read can block on
-        // LaunchServices.
+        // LaunchServices. The frontmost pid is captured here too: a Helium
+        // Meet shortcut and the main browser share one mic helper, so both
+        // match the family, and only the app the user is in is "focused".
+        let frontmostPID = NSWorkspace.shared.frontmostApplication?.processIdentifier
         let pids: [pid_t] = await RunningApplicationsReader.applications().compactMap { app in
             guard app.isRegularApp,
                   let bundleID = app.bundleIdentifier,
                   appFamilies.contains(where: { bundleID.matchesBundleFamily($0) }) else { return nil }
             return app.processIdentifier
         }
-        guard !pids.isEmpty else { return [] }
+        let ordered = readingOrder(pids: pids, frontmostPID: frontmostPID)
+        guard !ordered.isEmpty else { return [] }
 
         return await withCheckedContinuation { continuation in
             readQueue.async {
                 let deadline = Date().addingTimeInterval(readBudget)
                 var titles: [BrowserWindowTitle] = []
-                for pid in pids {
+                for pid in ordered {
                     guard Date() < deadline else { break }
-                    titles.append(contentsOf: windowTitles(forPID: pid, deadline: deadline))
+                    titles.append(contentsOf: windowTitles(
+                        forPID: pid,
+                        deadline: deadline,
+                        userFocused: isUserFocused(appPID: pid, frontmostPID: frontmostPID)
+                    ))
                 }
                 continuation.resume(returning: titles)
             }
         }
     }
 
-    private static func windowTitles(forPID pid: pid_t, deadline: Date) -> [BrowserWindowTitle] {
+    /// The frontmost app first. The title read has one short budget for every
+    /// app in the family. A Meet shortcut beside a busy main browser would
+    /// otherwise lose its windows to that budget.
+    static func readingOrder(pids: [pid_t], frontmostPID: pid_t?) -> [pid_t] {
+        guard let frontmostPID, let index = pids.firstIndex(of: frontmostPID) else { return pids }
+        var ordered = pids
+        ordered.remove(at: index)
+        ordered.insert(frontmostPID, at: 0)
+        return ordered
+    }
+
+    /// True only for the app the user is looking at. Each browser app has its
+    /// own key window; that is not the focused window unless the app is frontmost.
+    static func isUserFocused(appPID: pid_t, frontmostPID: pid_t?) -> Bool {
+        guard let frontmostPID else { return false }
+        return appPID == frontmostPID
+    }
+
+    private static func windowTitles(forPID pid: pid_t, deadline: Date, userFocused: Bool) -> [BrowserWindowTitle] {
         let appElement = AXUIElementCreateApplication(pid)
 
         var focusedWindow: AXUIElement?
@@ -81,15 +107,15 @@ enum BrowserWindowTitleReader {
             // No window list (or no time left for it). The focused window
             // alone is still useful.
             guard let focusedWindow, let title = title(of: focusedWindow, deadline: deadline) else { return [] }
-            return [BrowserWindowTitle(title: title, isFocused: true)]
+            return [BrowserWindowTitle(title: title, isFocused: userFocused)]
         }
 
         var titles: [BrowserWindowTitle] = []
         for window in windows.prefix(maxWindowsPerApp) {
             guard Date() < deadline else { break }
             guard let title = title(of: window, deadline: deadline) else { continue }
-            let isFocused = focusedWindow.map { CFEqual($0, window) } ?? false
-            titles.append(BrowserWindowTitle(title: title, isFocused: isFocused))
+            let isKeyWindow = focusedWindow.map { CFEqual($0, window) } ?? false
+            titles.append(BrowserWindowTitle(title: title, isFocused: userFocused && isKeyWindow))
         }
         return titles
     }
@@ -104,6 +130,30 @@ enum BrowserWindowTitleReader {
 
     /// Bounds the next AX round trip on `element` by what is left of the
     /// budget. Returns false when the budget is spent.
+    /// The call a running Chromium app shortcut is for, or `nil`. Reads
+    /// `CrAppModeShortcutURL` and drops it; only the provider enum leaves.
+    static func callShortcutProvider(forBundleID bundleID: String) async -> MeetingPromptProvider? {
+        let url = await Task.detached(priority: .utility) {
+            shortcutURL(forBundleID: bundleID)
+        }.value
+        guard let url else { return nil }
+        return BrowserCallEvidence.callShortcutProvider(forURL: url)
+    }
+
+    /// Off the main actor: `bundleURL` can block on LaunchServices, and the
+    /// URL itself is never logged.
+    private static func shortcutURL(forBundleID bundleID: String) -> String? {
+        guard let bundleURL = NSRunningApplication.runningApplications(withBundleIdentifier: bundleID).first?.bundleURL else {
+            return nil
+        }
+        let infoURL = bundleURL.appendingPathComponent("Contents/Info.plist")
+        guard let data = try? Data(contentsOf: infoURL),
+              let plist = try? PropertyListSerialization.propertyList(from: data, format: nil) as? [String: Any],
+              let url = plist["CrAppModeShortcutURL"] as? String,
+              !url.isEmpty else { return nil }
+        return url
+    }
+
     private static func setTimeout(on element: AXUIElement, deadline: Date) -> Bool {
         let remaining = deadline.timeIntervalSinceNow
         guard remaining > 0.005 else { return false }

Sources/Meeting/MeetingPromptCalendarScoring.swift

@@ -106,9 +106,12 @@ enum MeetingPromptCalendarScoring {
             return "\(provider.displayName) is open"
         }
 
+        // A browser's own app, or a Helium web app (`net.imput.helium.app.*`,
+        // how a calendar link opens Meet there). Not every Chrome or Safari
+        // web app: a Slack shortcut in front is not the meeting tab.
         if provider.browserHosted,
            let frontmostBundleID = runtimeSnapshot.frontmostBundleID,
-           MeetingPromptRuntimeSnapshot.browserBundleIdentifiers.contains(frontmostBundleID) {
+           MeetingPromptProvider.isMeetingTabHost(frontmostBundleID) {
             return "meeting tab is active"
         }
 

Sources/Meeting/MeetingPromptDetector+BrowserEvidence.swift

@@ -24,16 +24,34 @@ extension MeetingPromptDetector {
         var namedCall: MeetingPromptProvider?
         /// A known non-call site was focused. Sticky too, so ChatGPT voice
         /// does not prompt when the user clicks to another tab; only a
-        /// call-only title can still win.
+        /// call-only title, or a claimed call shortcut, can still win.
         var sawNonCallSite = false
+        /// A Meet/Teams/Zoom app shortcut is the mic user for this session.
+        /// Beats a non-call window in a different app of the same family.
+        var claimedShortcut: MeetingPromptProvider?
         var readTask: Task<Void, Never>?
         var readToken = 0
+        /// When this session's first title read started. The prompt is held
+        /// only until `titleReadGiveUp`, not until a hung read returns.
+        var readStartedAt: Date?
         /// Browser kinds the user said Not now to in this session.
         var declinedKinds: [String: Date] = [:]
 
-        var verdict: BrowserCallTitleVerdict? {
+        func verdict(now: Date, titleReadGiveUp: TimeInterval) -> BrowserCallTitleVerdict? {
             if let namedCall { return .call(provider: namedCall) }
+            if let claimedShortcut { return .callSite(provider: claimedShortcut) }
+            // Coming back to a Meet tab beats an early ChatGPT glance. A
+            // random tab switch does not: that still needs sawNonCallSite.
+            if case .call = latest { return latest }
+            if case .callSite = latest { return latest }
             if sawNonCallSite { return .notCall }
+            if latest == nil {
+                guard let readStartedAt else { return nil }
+                return BrowserCallEvidence.verdictBeforeTitleReadLands(
+                    pendingFor: now.timeIntervalSince(readStartedAt),
+                    giveUp: titleReadGiveUp
+                )
+            }
             return latest
         }
     }
@@ -56,12 +74,24 @@ extension MeetingPromptDetector {
         let families = browserFamiliesForEvidence(reason: signal.reason, frontmostBundleID: frontmostBundleID)
         let since = browserEvidenceStart(reason: signal.reason, families: families, now: now)
         let playingAudio = browserIsPlayingAudio(families: families)
+        ensureCallShortcutLookup(frontmostBundleID)
 
-        guard let verdict = currentBrowserTitleVerdict(families: families, now: now) else {
-            // The first title read of this session is still running (well
-            // under a second); it re-evaluates when it lands. Too brief to be
-            // worth a suppression event.
+        guard let verdict = currentBrowserTitleVerdict(
+            families: families,
+            frontmostBundleID: frontmostBundleID,
+            now: now
+        ) else {
+            // The first title read of this session is still running, inside
+            // `titleReadGiveUp`. It re-evaluates when it lands; if it hangs,
+            // the re-check below ends the hold. Too brief to be worth a
+            // suppression event.
             browserFirstTitleReadPending = true
+            if let readStartedAt = browserTitles?.readStartedAt {
+                scheduleBrowserEvidenceRecheck(
+                    at: readStartedAt.addingTimeInterval(browserEvidenceTiming.titleReadGiveUp),
+                    now: now
+                )
+            }
             return nil
         }
 
@@ -175,7 +205,11 @@ extension MeetingPromptDetector {
     /// rules applied, or `nil` while the session's first read is running.
     /// Starts a background read when the last one is old enough; its result
     /// re-evaluates.
-    private func currentBrowserTitleVerdict(families: Set<String>, now: Date) -> BrowserCallTitleVerdict? {
+    private func currentBrowserTitleVerdict(
+        families: Set<String>,
+        frontmostBundleID: String?,
+        now: Date
+    ) -> BrowserCallTitleVerdict? {
         guard !families.isEmpty else { return .unknown }
         if var session = browserTitles, !session.families.isDisjoint(with: families) {
             // Same browser (another one may have joined or left): keep what
@@ -187,19 +221,33 @@ extension MeetingPromptDetector {
             browserTitles?.readTask?.cancel()
             browserTitles = BrowserTitleSession(families: families, startedAt: now)
         }
-        guard let session = browserTitles else { return nil }
+        guard var session = browserTitles else { return nil }
+        // Only a shortcut of the browser holding the mic can claim it: a Meet
+        // shortcut in Helium says nothing about Chrome's mic.
+        let isMicBrowser = { (bundleID: String) in !Self.browserAppFamilies(for: [bundleID]).isDisjoint(with: families) }
+        let recent = recentCallShortcutActivation.flatMap { isMicBrowser($0.bundleID) ? $0 : nil }
+        session.claimedShortcut = BrowserCallEvidence.claimedCallShortcut(
+            frontmostShortcut: frontmostBundleID.flatMap { isMicBrowser($0) ? callShortcutProviders[$0] : nil },
+            recentProvider: recent?.provider,
+            recentActivationAt: recent?.at,
+            sessionStartedAt: session.startedAt
+        )
+        browserTitles = session
         let readIsStale = session.readAt.map { now.timeIntervalSince($0) >= browserEvidenceTiming.titleReadSpacing } ?? true
         if session.namedCall == nil, session.readTask == nil, readIsStale {
             startBrowserTitleRead(families: families)
         }
-        return session.verdict
+        return session.verdict(now: now, titleReadGiveUp: browserEvidenceTiming.titleReadGiveUp)
     }
 
     private func startBrowserTitleRead(families: Set<String>) {
         browserTitleReadCounter += 1
         let token = browserTitleReadCounter
         let provider = browserWindowTitlesProvider
         browserTitles?.readToken = token
+        if browserTitles?.readStartedAt == nil {
+            browserTitles?.readStartedAt = Date()
+        }
         evaluationsInFlight += 1
         browserTitles?.readTask = Task { @MainActor [weak self] in
             let titles = await provider(families)
@@ -221,11 +269,16 @@ extension MeetingPromptDetector {
         switch verdict {
         case .call(let provider):
             session.namedCall = provider
+            session.sawNonCallSite = false
+        case .callSite:
+            // The user is on a call app now. An earlier non-call glance was
+            // not the thing holding the mic.
+            session.sawNonCallSite = false
         case .notCall:
             if now.timeIntervalSince(session.startedAt) <= browserEvidenceTiming.nonCallSiteStickyWindow {
                 session.sawNonCallSite = true
             }
-        case .callSite, .unknown:
+        case .unknown:
             break
         }
         browserTitles = session
@@ -272,6 +325,39 @@ extension MeetingPromptDetector {
         }
     }
 
+    /// A browser app came forward or launched. If it is a call shortcut, remember
+    /// it so a calendar-opened Meet PWA is not vetoed by another Helium window.
+    func noteCallShortcutActivation(bundleID: String, provider: MeetingPromptProvider, at: Date) {
+        callShortcutProviders[bundleID] = provider
+        callShortcutLookupDone.insert(bundleID)
+        recentCallShortcutActivation = (bundleID: bundleID, provider: provider, at: at)
+        scheduleEvaluation()
+    }
+
+    /// A call shortcut quit: it can no longer be the call that is starting.
+    func noteCallShortcutTermination(bundleID: String) {
+        guard recentCallShortcutActivation?.bundleID == bundleID else { return }
+        recentCallShortcutActivation = nil
+    }
+
+    /// Classify the frontmost app's shortcut URL once, off this pass. The next
+    /// evaluation uses the answer. A miss is remembered too, so a busy main
+    /// browser is not read again on every mic poll.
+    private func ensureCallShortcutLookup(_ bundleID: String?) {
+        guard let bundleID, MeetingPromptProvider.isBrowserBundleID(bundleID) else { return }
+        guard !callShortcutLookupDone.contains(bundleID) else { return }
+        callShortcutLookupDone.insert(bundleID)
+        let lookup = callShortcutProvider
+        Task { @MainActor [weak self] in
+            let site = await lookup(bundleID)
+            guard let self else { return }
+            if let site {
+                self.callShortcutProviders[bundleID] = site
+            }
+            self.scheduleEvaluation()
+        }
+    }
+
     /// The browser session is over: forget the title verdict and stop
     /// re-checking. The next browser mic use starts its own wait.
     func endBrowserMicSession() {