feat: log each page's memory, and when it would be rebuilt

Chorus/App/AppState.swift

@@ -230,6 +230,7 @@ final class AppState {
     }
     @ObservationIgnored nonisolated(unsafe) private var quietHoursTask: Task<Void, Never>?
     @ObservationIgnored nonisolated(unsafe) private var idleHibernationTask: Task<Void, Never>?
+    @ObservationIgnored nonisolated(unsafe) private var memoryWatchTask: Task<Void, Never>?
     /// Per-service grace timers for the `.immediate` hibernation policy: a service
     /// switched away from is torn down a few seconds later unless switched back to.
     /// Keyed by service id so switching back can cancel the pending teardown.
@@ -577,6 +578,7 @@ final class AppState {
         }
         quietHoursTask?.cancel()
         idleHibernationTask?.cancel()
+        memoryWatchTask?.cancel()
     }
 
     /// Wires the WebViewPool's external-link handler so that cross-domain
@@ -1434,6 +1436,37 @@ final class AppState {
         }
     }
 
+    /// Every five minutes, logs each page's memory, and says when a page has
+    /// grown enough that it would be rebuilt (`WebViewPool.wouldRebuild`).
+    /// Nothing is rebuilt yet: this release only measures, to show whether a
+    /// background page really keeps climbing. Each pass is one
+    /// `proc_pid_rusage` call per live page. Read it back with
+    /// `log show --predicate 'subsystem == "com.nicojan.Chorus" AND eventMessage BEGINSWITH "Memory watch"'`.
+    private func startMemoryWatchTimer() {
+        memoryWatchTask?.cancel()
+        memoryWatchTask = Task { @MainActor [weak self] in
+            while !Task.isCancelled {
+                try? await Task.sleep(for: .seconds(300))
+                self?.logMemoryWatch()
+            }
+        }
+    }
+
+    private func logMemoryWatch() {
+        for reading in webViewPool.memoryReadings() {
+            // The catalog id names the service without the user's own label.
+            let service = fetchService(id: reading.id)?.catalogEntryID ?? "custom"
+            let megabytes = reading.footprint / 1_048_576
+            let baseline = reading.baseline.map { "\($0 / 1_048_576) MB" } ?? "not yet"
+            let outOfSight = Int(reading.outOfSight)
+            if reading.wouldRebuild {
+                AppLogger.webView.notice("Memory watch: would rebuild \(service, privacy: .public) at \(megabytes) MB, baseline \(baseline, privacy: .public), out of sight \(outOfSight) s")
+            } else {
+                AppLogger.webView.notice("Memory watch: \(service, privacy: .public) \(megabytes) MB, baseline \(baseline, privacy: .public), out of sight \(outOfSight) s, tabs \(reading.hasOpenTabs)")
+            }
+        }
+    }
+
     private func hibernateIdleServices() async {
         guard !isLocked else { return }
         // Wind the sweep down once nothing needs it — e.g. the only service with
@@ -2961,6 +2994,7 @@ final class AppState {
             self.refreshEffectiveDoNotDisturb()
             self.startQuietHoursTimer()
             self.startIdleHibernationTimer()
+            self.startMemoryWatchTimer()
             self.setupLockObservers()
             self.startDarkMode()
         }

Chorus/Views/WebView/WebViewPool.swift

@@ -10,6 +10,11 @@ final class WebViewPool {
     private var coordinators: [UUID: WebViewCoordinator] = [:]
     private var suspendedURLs: [UUID: String] = [:]
     private var snapshots: [UUID: NSImage] = [:]
+    /// When each live page loaded and when the user last left it, and each
+    /// page's footprint once it settled. Both feed the memory watch; see
+    /// `memoryReadings`.
+    private var visits = ServiceVisits()
+    private var baselineFootprints: [UUID: UInt64] = [:]
     /// Snapshot ids in the order they were stored, oldest first. Drives the cap
     /// below; `snapshots` alone has no order to evict by.
     private var snapshotOrder: [UUID] = []
@@ -254,6 +259,7 @@ final class WebViewPool {
             softHibernateService(previousID)
         }
         activeServiceID = instance.id
+        visits.activate(instance.id, at: Date())
 
         // Wake from full hibernation if needed
         if hibernatedServiceIDs.contains(instance.id) {
@@ -283,6 +289,7 @@ final class WebViewPool {
 
         webViews[instance.id] = webView
         lastAccessTimes[instance.id] = Date()
+        visits.loaded(instance.id, at: Date())
         observeCaptureState(webView, id: instance.id)
 
         // Restore the last-visited URL when waking from full hibernation
@@ -331,6 +338,7 @@ final class WebViewPool {
 
         webViews[instance.id] = webView
         lastAccessTimes[instance.id] = Date()
+        visits.loaded(instance.id, at: Date())
         observeCaptureState(webView, id: instance.id)
 
         // Register the hibernation-exemption flags now, not just on first
@@ -398,6 +406,7 @@ final class WebViewPool {
         // would otherwise grow unbounded across create/delete cycles.
         if activeServiceID == instanceID {
             activeServiceID = nil
+            visits.activate(nil, at: Date())
         }
         pinnedIDs.remove(instanceID)
         neverHibernateIDs.remove(instanceID)
@@ -526,6 +535,7 @@ final class WebViewPool {
     func deactivateActiveService() {
         guard let id = activeServiceID else { return }
         activeServiceID = nil
+        visits.activate(nil, at: Date())
         softHibernateService(id)
     }
 
@@ -799,6 +809,8 @@ final class WebViewPool {
         webViews.removeValue(forKey: instanceID)
         lastAccessTimes.removeValue(forKey: instanceID)
         coordinators.removeValue(forKey: instanceID)
+        visits.remove(instanceID)
+        baselineFootprints.removeValue(forKey: instanceID)
         dropSnapshot(for: instanceID)
         onServiceTornDown?(instanceID)
     }
@@ -1021,6 +1033,124 @@ final class WebViewPool {
         return true
     }
 
+    // MARK: - Memory watch (log only)
+
+    /// The floor under which no page counts as grown, however much it climbed.
+    nonisolated static let memoryWatchFloor: UInt64 = 1536 * 1_048_576
+
+    /// How many times its baseline a page must reach to count as grown. The
+    /// limit is relative so that a page which simply needs a lot of memory does
+    /// not count over and over.
+    nonisolated static let memoryWatchGrowthFactor: UInt64 = 2
+
+    /// How long after it loads a page is first measured for its baseline.
+    /// Sampled earlier, a page still loading would read small.
+    nonisolated static let memoryWatchSettleTime: TimeInterval = 5 * 60
+
+    /// How long a service must have been out of sight, counted from when the
+    /// user left it, before it may count. A page left moments ago may hold a
+    /// half-typed message.
+    nonisolated static let memoryWatchMinimumOutOfSight: TimeInterval = 10 * 60
+
+    /// Whether a page has grown enough, and is far enough from the user, that
+    /// a rebuild would be justified. Nothing acts on it yet: the memory watch
+    /// only logs it.
+    ///
+    /// The page has to be past both the floor and twice its baseline, and out
+    /// of sight for `memoryWatchMinimumOutOfSight`. Everything that makes a
+    /// teardown cost the user something rules it out: the service on screen,
+    /// one pinned, one set to Keep Loaded, one using the camera or microphone,
+    /// one playing sound, or one with tabs open, since `teardownWebView` closes
+    /// them. A footprint or baseline that could not be read rules it out too.
+    nonisolated static func wouldRebuild(
+        footprint: UInt64?,
+        baseline: UInt64?,
+        outOfSight: TimeInterval,
+        isActive: Bool,
+        isPinned: Bool,
+        keepLoaded: Bool,
+        isCapturing: Bool,
+        isPlayingAudio: Bool,
+        hasOpenTabs: Bool
+    ) -> Bool {
+        guard let footprint, let baseline else { return false }
+        let (grown, overflow) = baseline.multipliedReportingOverflow(by: memoryWatchGrowthFactor)
+        let limit = max(memoryWatchFloor, overflow ? .max : grown)
+        return footprint > limit
+            && outOfSight >= memoryWatchMinimumOutOfSight
+            && !isActive && !isPinned && !keepLoaded && !isCapturing && !isPlayingAudio && !hasOpenTabs
+    }
+
+    /// The physical footprint of the WebContent process behind a service, the
+    /// figure Activity Monitor shows as Memory. Each service has its own data
+    /// store, so the process belongs to that one service, but open tabs run in
+    /// it too and count toward it.
+    ///
+    /// The process id comes from `_webProcessIdentifier`, which is SPI, so it is
+    /// probed before use. If a macOS release drops it, this returns nil and the
+    /// watch logs nothing.
+    func webContentFootprint(for id: UUID) -> UInt64? {
+        let key = "_webProcessIdentifier"
+        guard let webView = webViews[id],
+              webView.responds(to: NSSelectorFromString(key)),
+              let pid = (webView.value(forKey: key) as? NSNumber)?.int32Value,
+              pid > 0
+        else { return nil }
+        var info = rusage_info_v4()
+        let result = withUnsafeMutablePointer(to: &info) {
+            $0.withMemoryRebound(to: rusage_info_t?.self, capacity: 1) {
+                proc_pid_rusage(pid, RUSAGE_INFO_V4, $0)
+            }
+        }
+        return result == 0 ? info.ri_phys_footprint : nil
+    }
+
+    struct MemoryReading {
+        let id: UUID
+        let footprint: UInt64
+        let baseline: UInt64?
+        let outOfSight: TimeInterval
+        let hasOpenTabs: Bool
+        let wouldRebuild: Bool
+    }
+
+    /// Measures every live page once, for the memory watch to log.
+    ///
+    /// A page gets its baseline the first time it is read after
+    /// `memoryWatchSettleTime`. When a page counts as grown, its footprint
+    /// becomes its new baseline, standing in for the rebuild that does not
+    /// happen yet. A page that settled heavy therefore counts once, not on
+    /// every pass, and the log shows how often each service would really be
+    /// rebuilt.
+    func memoryReadings(now: Date = Date()) -> [MemoryReading] {
+        webViews.keys.compactMap { id in
+            guard let footprint = webContentFootprint(for: id) else { return nil }
+            if baselineFootprints[id] == nil,
+               let age = visits.age(id, now: now), age >= Self.memoryWatchSettleTime {
+                baselineFootprints[id] = footprint
+            }
+            let baseline = baselineFootprints[id]
+            let outOfSight = visits.outOfSight(id, now: now)
+            let tabs = hasOpenTabs(id)
+            let grown = Self.wouldRebuild(
+                footprint: footprint,
+                baseline: baseline,
+                outOfSight: outOfSight,
+                isActive: id == activeServiceID,
+                isPinned: pinnedIDs.contains(id),
+                keepLoaded: neverHibernateIDs.contains(id),
+                isCapturing: mediaCaptureStates[id]?.isCapturing ?? false,
+                isPlayingAudio: audibleServiceIDs.contains(id),
+                hasOpenTabs: tabs
+            )
+            if grown { baselineFootprints[id] = footprint }
+            return MemoryReading(
+                id: id, footprint: footprint, baseline: baseline,
+                outOfSight: outOfSight, hasOpenTabs: tabs, wouldRebuild: grown
+            )
+        }
+    }
+
     /// When exceeding maxLoaded web views, fully hibernate the least recently used ones.
     /// Skips services that have an active WebRTC call, via `hibernateIfStillIdle`.
     private func evictIfNeeded() async {
@@ -1082,3 +1212,44 @@ final class PlayingAudioObserver: NSObject {
         }
     }
 }
+
+/// When each service's page loaded and when the user last switched away from
+/// it. The memory watch counts "out of sight" from the moment a service was
+/// left, not from when it was opened: half an hour in WhatsApp followed by a
+/// switch to Slack leaves WhatsApp out of sight for seconds, not half an hour.
+struct ServiceVisits {
+    private var loadedAt: [UUID: Date] = [:]
+    private var leftAt: [UUID: Date] = [:]
+    private var active: UUID?
+
+    /// A page came up. Until the user opens and leaves it, a page loaded in
+    /// the background counts as out of sight since it loaded.
+    mutating func loaded(_ id: UUID, at date: Date) {
+        loadedAt[id] = date
+        leftAt[id] = date
+    }
+
+    /// The user brought `id` on screen, or nil when nothing is. Whatever was on
+    /// screen before is left now.
+    mutating func activate(_ id: UUID?, at date: Date) {
+        if let previous = active, previous != id {
+            leftAt[previous] = date
+        }
+        active = id
+    }
+
+    mutating func remove(_ id: UUID) {
+        loadedAt.removeValue(forKey: id)
+        leftAt.removeValue(forKey: id)
+    }
+
+    func age(_ id: UUID, now: Date) -> TimeInterval? {
+        loadedAt[id].map { now.timeIntervalSince($0) }
+    }
+
+    /// Zero for the page on screen; otherwise the time since the user left it.
+    func outOfSight(_ id: UUID, now: Date) -> TimeInterval {
+        guard id != active, let left = leftAt[id] else { return 0 }
+        return now.timeIntervalSince(left)
+    }
+}

ChorusTests/ChorusTests.swift

@@ -450,6 +450,93 @@ final class ChorusTests: XCTestCase {
         XCTAssertFalse(dropped.contains(ids[4]), "the one just stored is never dropped")
     }
 
+    func testMemoryWatchCountsOnlyAPageThatKeptGrowingAndNobodyIsUsing() {
+        let mb: UInt64 = 1_048_576
+        let floor = WebViewPool.memoryWatchFloor
+        let away = WebViewPool.memoryWatchMinimumOutOfSight
+        func grown(
+            footprint: UInt64? = 2_000 * 1_048_576, baseline: UInt64? = 300 * 1_048_576,
+            outOfSight: TimeInterval = away, isActive: Bool = false,
+            isPinned: Bool = false, keepLoaded: Bool = false,
+            isCapturing: Bool = false, isPlayingAudio: Bool = false, hasOpenTabs: Bool = false
+        ) -> Bool {
+            WebViewPool.wouldRebuild(
+                footprint: footprint, baseline: baseline, outOfSight: outOfSight,
+                isActive: isActive, isPinned: isPinned, keepLoaded: keepLoaded,
+                isCapturing: isCapturing, isPlayingAudio: isPlayingAudio, hasOpenTabs: hasOpenTabs
+            )
+        }
+
+        XCTAssertTrue(grown(), "a page past the floor and twice its baseline, long out of sight, counts")
+        XCTAssertFalse(grown(footprint: floor), "at the floor is not past it")
+        XCTAssertFalse(grown(footprint: nil), "a size that cannot be read is left alone")
+        XCTAssertFalse(grown(baseline: nil), "nor a page that has not settled yet")
+        XCTAssertFalse(grown(footprint: 1_900 * mb, baseline: 1_800 * mb), "big but steady does not count")
+        XCTAssertFalse(grown(footprint: 3_600 * mb, baseline: 1_800 * mb), "exactly twice is not past it")
+        XCTAssertTrue(grown(footprint: 3_700 * mb, baseline: 1_800 * mb), "past twice its own size counts")
+        XCTAssertFalse(grown(footprint: .max, baseline: .max), "an absurd baseline cannot overflow into a count")
+
+        XCTAssertFalse(grown(outOfSight: away - 1), "a page left moments ago may hold a draft")
+        XCTAssertFalse(grown(isActive: true), "never the page on screen")
+        XCTAssertFalse(grown(isPinned: true))
+        XCTAssertFalse(grown(keepLoaded: true), "Keep Loaded means keep it as it is")
+        XCTAssertFalse(grown(isCapturing: true), "never in the middle of a call")
+        XCTAssertFalse(grown(isPlayingAudio: true))
+        XCTAssertFalse(grown(hasOpenTabs: true), "a rebuild would close the tabs")
+    }
+
+    func testMemoryWatchDoesNotCountAHeavyPageSampledEarlyOverAndOver() {
+        let mb: UInt64 = 1_048_576
+        // Sampled at 700 MB while still loading, the page settles at 1.6 GB.
+        // Under 768 MB the floor alone decides, so it counts once...
+        let settled: UInt64 = 1_600 * mb
+        func grown(_ footprint: UInt64, baseline: UInt64) -> Bool {
+            WebViewPool.wouldRebuild(
+                footprint: footprint, baseline: baseline,
+                outOfSight: WebViewPool.memoryWatchMinimumOutOfSight,
+                isActive: false, isPinned: false, keepLoaded: false,
+                isCapturing: false, isPlayingAudio: false, hasOpenTabs: false
+            )
+        }
+        XCTAssertTrue(grown(settled, baseline: 700 * mb))
+        // ...and `memoryReadings` then carries that footprint over as the new
+        // baseline, so the same size on the next pass is steady, not grown.
+        XCTAssertFalse(grown(settled, baseline: settled), "counted once, not on every pass")
+        XCTAssertTrue(grown(2 * settled + mb, baseline: settled), "only doubling again counts again")
+    }
+
+    func testOutOfSightCountsFromLeavingAServiceNotFromOpeningIt() {
+        let whatsapp = UUID(), slack = UUID()
+        let start = Date()
+        var visits = ServiceVisits()
+
+        // Open WhatsApp and stay in it for half an hour.
+        visits.loaded(whatsapp, at: start)
+        visits.activate(whatsapp, at: start)
+        XCTAssertEqual(visits.outOfSight(whatsapp, now: start.addingTimeInterval(1_800)), 0, "on screen is never out of sight")
+
+        // Switch to Slack, and sweep a few seconds later.
+        let left = start.addingTimeInterval(1_800)
+        visits.loaded(slack, at: left)
+        visits.activate(slack, at: left)
+        let sweep = left.addingTimeInterval(5)
+        XCTAssertEqual(visits.outOfSight(whatsapp, now: sweep), 5, "counted from the switch, not from opening")
+        XCTAssertFalse(WebViewPool.wouldRebuild(
+            footprint: .max / 4, baseline: 1, outOfSight: visits.outOfSight(whatsapp, now: sweep),
+            isActive: false, isPinned: false, keepLoaded: false,
+            isCapturing: false, isPlayingAudio: false, hasOpenTabs: false
+        ), "a service left seconds ago never counts, however big")
+
+        // Going to a Mac app leaves the page too.
+        visits.activate(nil, at: sweep)
+        XCTAssertEqual(visits.outOfSight(slack, now: sweep.addingTimeInterval(60)), 60)
+
+        // A page loaded in the background is out of sight since it loaded.
+        let other = UUID()
+        visits.loaded(other, at: start)
+        XCTAssertEqual(visits.outOfSight(other, now: start.addingTimeInterval(600)), 600)
+    }
+
     @MainActor
     func testSnapshotCapHandlesDegenerateCaps() {
         let ids = (0..<2).map { _ in UUID() }