From 4f6fd8f778965056a5fc53775079481195aca5a1 Mon Sep 17 00:00:00 2001 From: David Liebovitz Date: Mon, 25 May 2026 21:58:25 -0500 Subject: [PATCH 1/6] Sender: prevent post-screensaver/display-sleep stutter Adds three user-controllable options in the sender GUI to avoid the "stutter that needs an app restart" symptom after the screensaver or display sleep activates while streaming. - Settings toggle "Prevent screensaver / display sleep while streaming" (default on, persisted) gates passing `.idleDisplaySleepDisabled` to `ProcessInfo.beginActivity`, so the screensaver/display-sleep does not engage during an active session. - Settings toggle "Auto-restart capture after wake / unlock" (default on, persisted) enables observers on `screensDidWake`, `com.apple.screenIsUnlocked`, and `com.apple.screensaver.didstop`. When fired, the capture pipeline (SCStream/CGDisplayStream + VTCompressionSession + capture timers + activity) is torn down and rebuilt while keeping the network connection and the virtual display alive. The PTS sequence counter is reset so the receiver's pacing does not desync after the gap. - Per-session "Restart capture" button (enabled while streaming) runs the same soft-restart path on demand for any other stutter cause. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../TBDisplaySenderBuildInfo.swift | 2 +- .../TBDisplaySenderContentView.swift | 10 ++ .../TBDisplaySenderLocalization.swift | 24 +++ .../TBDisplaySenderManager.swift | 31 +++- .../TBDisplaySenderService.swift | 139 +++++++++++++++++- 5 files changed, 201 insertions(+), 5 deletions(-) diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift index 0a34d47..905de6d 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift @@ -1,5 +1,5 @@ enum TBDisplaySenderBuildInfo { static let marketingVersion = "2.0" - static let buildNumber = "20260521211758" + static let buildNumber = "20260525204941" static let versionDisplay = "\(marketingVersion) + build \(buildNumber)" } diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift index 39d0702..b4af181 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift @@ -163,6 +163,10 @@ struct TBDisplaySenderContentView: View { Toggle(TBDisplaySenderL10n.largeCursor(service.language), isOn: $service.largeCursor) .disabled(service.anyConnected) + + Toggle(TBDisplaySenderL10n.preventDisplaySleep(service.language), isOn: $service.preventDisplaySleep) + + Toggle(TBDisplaySenderL10n.autoRestartOnWake(service.language), isOn: $service.autoRestartOnWake) } } } @@ -336,6 +340,12 @@ private struct TBDisplaySenderSessionCard: View { .buttonStyle(.bordered) .disabled(session.isConnected || session.isStreaming || session.isCableTesting || trimmedReceiverIP.isEmpty || session.localTBIP.isEmpty) + Button(TBDisplaySenderL10n.restartCaptureButton(service.language)) { + session.restartCaptureNow() + } + .buttonStyle(.bordered) + .disabled(!session.canRestartCapture) + Button(TBDisplaySenderL10n.removeSessionButton(service.language)) { service.removeSession(session) } diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderLocalization.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderLocalization.swift index 18fa916..ae82c23 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderLocalization.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderLocalization.swift @@ -480,6 +480,30 @@ enum TBDisplaySenderL10n { } } + static func preventDisplaySleep(_ language: TBDisplaySenderLanguage) -> String { + switch language { + case .italian: return "Impedisci screensaver e sospensione display durante lo streaming" + case .english: return "Prevent screensaver / display sleep while streaming" + case .german: return "Bildschirmschoner und Display-Ruhezustand beim Streamen verhindern" + } + } + + static func autoRestartOnWake(_ language: TBDisplaySenderLanguage) -> String { + switch language { + case .italian: return "Riavvia automaticamente la cattura al risveglio" + case .english: return "Auto-restart capture after wake / unlock" + case .german: return "Aufnahme nach Aufwachen/Entsperren automatisch neu starten" + } + } + + static func restartCaptureButton(_ language: TBDisplaySenderLanguage) -> String { + switch language { + case .italian: return "Riavvia cattura" + case .english: return "Restart capture" + case .german: return "Aufnahme neu starten" + } + } + static func showMainWindow(_ language: TBDisplaySenderLanguage) -> String { switch language { case .italian: return "Mostra finestra principale" diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift index 929d4c2..c27f081 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift @@ -31,6 +31,30 @@ final class TBDisplaySenderService: ObservableObject { objectWillChange.send() } } + @Published var preventDisplaySleep: Bool = { + if UserDefaults.standard.object(forKey: "fd.tbdisplaysender.preventDisplaySleep") == nil { + return true + } + return UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.preventDisplaySleep") + }() { + didSet { + UserDefaults.standard.set(preventDisplaySleep, forKey: "fd.tbdisplaysender.preventDisplaySleep") + sessions.forEach { $0.preventDisplaySleep = preventDisplaySleep } + objectWillChange.send() + } + } + @Published var autoRestartOnWake: Bool = { + if UserDefaults.standard.object(forKey: "fd.tbdisplaysender.autoRestartOnWake") == nil { + return true + } + return UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.autoRestartOnWake") + }() { + didSet { + UserDefaults.standard.set(autoRestartOnWake, forKey: "fd.tbdisplaysender.autoRestartOnWake") + sessions.forEach { $0.autoRestartOnWake = autoRestartOnWake } + objectWillChange.send() + } + } private var sessionCancellables: [UUID: AnyCancellable] = [:] private let receiverDiscovery = TBReceiverDiscovery() @@ -70,7 +94,12 @@ final class TBDisplaySenderService: ObservableObject { } func addSession() { - let session = TBDisplaySenderSession(language: language, largeCursor: largeCursor) + let session = TBDisplaySenderSession( + language: language, + largeCursor: largeCursor, + preventDisplaySleep: preventDisplaySleep, + autoRestartOnWake: autoRestartOnWake + ) if let previous = sessions.last { session.capturePreset = previous.capturePreset session.captureSource = previous.captureSource diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift index 5238797..aebdd6c 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift @@ -394,18 +394,33 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u let id = UUID() - init(language: TBDisplaySenderLanguage, largeCursor: Bool) { + init( + language: TBDisplaySenderLanguage, + largeCursor: Bool, + preventDisplaySleep: Bool = true, + autoRestartOnWake: Bool = true + ) { self.statusText = TBDisplaySenderStatusState.ready.text(language) self.receiverPanelText = TBDisplaySenderL10n.waitingReceiverProfile(language) self.virtualDisplayText = TBDisplaySenderL10n.virtualDisplayNotCreated(language) self.language = language self.largeCursor = largeCursor + self.preventDisplaySleep = preventDisplaySleep + self.autoRestartOnWake = autoRestartOnWake self.streamResolutionText = TBDisplaySenderL10n.streamSummary( preset: .standard1440p, source: .desktopMirror, language: language ) super.init() + registerWakeObservers() + } + + deinit { + for token in wakeObservers { + NSWorkspace.shared.notificationCenter.removeObserver(token) + DistributedNotificationCenter.default().removeObserver(token) + } } @Published var isConnected = false @@ -432,6 +447,8 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u } } @Published var largeCursor: Bool + @Published var preventDisplaySleep: Bool = true + @Published var autoRestartOnWake: Bool = true @Published var capturePreset: TBDisplayCapturePreset = .standard1440p { didSet { if !isStreaming { @@ -477,6 +494,8 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u private var baselineDisplayIDs = Set() private var cursorDisplayID: CGDirectDisplayID = kCGNullDirectDisplay private var lastCursorPacket: TBMonitorCursor? + nonisolated(unsafe) private var wakeObservers: [NSObjectProtocol] = [] + private var isRestartingCaptureAfterWake = false private final class CaptureDelegate: NSObject, SCStreamOutput, SCStreamDelegate { var onFrame: ((CMSampleBuffer) -> Void)? @@ -1056,7 +1075,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u isStreaming = true if largeCursor { startCursorUpdates(displayID: display.displayID) } streamingActivity = ProcessInfo.processInfo.beginActivity( - options: [.userInitiated, .idleSystemSleepDisabled], + options: activityOptions(), reason: "TargetBridge streaming active" ) startFPSTimer() @@ -1094,13 +1113,21 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u isStreaming = true if largeCursor { startCursorUpdates(displayID: displayID) } streamingActivity = ProcessInfo.processInfo.beginActivity( - options: [.userInitiated, .idleSystemSleepDisabled], + options: activityOptions(), reason: "TargetBridge streaming active" ) startFPSTimer() return true } + private func activityOptions() -> ProcessInfo.ActivityOptions { + var options: ProcessInfo.ActivityOptions = [.userInitiated, .idleSystemSleepDisabled] + if preventDisplaySleep { + options.insert(.idleDisplaySleepDisabled) + } + return options + } + private func waitForCaptureDisplay() async throws -> SCDisplay { try await waitForVirtualDisplay( matching: session.displayID, @@ -1646,6 +1673,112 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u } } + private func registerWakeObservers() { + let handler: @Sendable (Notification) -> Void = { [weak self] _ in + Task { @MainActor [weak self] in + self?.handleSystemWake() + } + } + + wakeObservers.append( + NSWorkspace.shared.notificationCenter.addObserver( + forName: NSWorkspace.screensDidWakeNotification, + object: nil, + queue: nil, + using: handler + ) + ) + wakeObservers.append( + DistributedNotificationCenter.default().addObserver( + forName: Notification.Name("com.apple.screenIsUnlocked"), + object: nil, + queue: nil, + using: handler + ) + ) + wakeObservers.append( + DistributedNotificationCenter.default().addObserver( + forName: Notification.Name("com.apple.screensaver.didstop"), + object: nil, + queue: nil, + using: handler + ) + ) + } + + private func handleSystemWake() { + guard autoRestartOnWake else { return } + scheduleCaptureRestart(reason: "system wake", delaySeconds: 1.0) + } + + func restartCaptureNow() { + scheduleCaptureRestart(reason: "manual restart", delaySeconds: 0.0) + } + + var canRestartCapture: Bool { + isStreaming && activeProfile != nil && !isRestartingCaptureAfterWake + } + + private func scheduleCaptureRestart(reason: String, delaySeconds: Double) { + guard isStreaming, !isRestartingCaptureAfterWake, let profile = activeProfile else { return } + isRestartingCaptureAfterWake = true + NSLog("TargetBridge: \(reason) — soft restart of capture pipeline") + Task { @MainActor [weak self] in + if delaySeconds > 0 { + try? await Task.sleep(nanoseconds: UInt64(delaySeconds * 1_000_000_000)) + } + guard let self else { return } + guard self.isStreaming, self.activeProfile?.receiverName == profile.receiverName else { + self.isRestartingCaptureAfterWake = false + return + } + await self.softRestartCapture(for: profile) + self.isRestartingCaptureAfterWake = false + } + } + + private func softRestartCapture(for profile: TBMonitorDisplayProfile) async { + // Tear down only the capture pipeline — keep the network connection and virtual display. + cursorTimer?.invalidate() + cursorTimer = nil + fpsTimer?.invalidate() + fpsTimer = nil + if let directDisplayStream { + directDisplayStream.stop() + self.directDisplayStream = nil + } + if let stream = scStream { + if let delegate = captureDelegate { + try? stream.removeStreamOutput(delegate, type: .screen) + } + stream.stopCapture(completionHandler: nil) + scStream = nil + } + captureDelegate = nil + if let activity = streamingActivity { + ProcessInfo.processInfo.endActivity(activity) + streamingActivity = nil + } + if let encoder = vtEncoder { VTCompressionSessionInvalidate(encoder) } + vtEncoder = nil + vtEncoderRef?.release() + vtEncoderRef = nil + isStreaming = false + displayStreamFrameSequence = 0 + senderFPS = 0 + sentSnapshot = sentFrames + pendingVideoPackets = 0 + inFlightEncodeFrames = 0 + cursorDisplayID = kCGNullDirectDisplay + lastCursorPacket = nil + + let started = await startCapture(for: profile) + if !started { + NSLog("TargetBridge: soft restart after wake failed — falling back to full stop") + stop(resetStatusTo: .captureError("capture restart after wake failed")) + } + } + private func startFPSTimer() { fpsTimer?.invalidate() sentSnapshot = sentFrames From 46b2c79e15b74463a5c00dd9af3f2e97e3a9814b Mon Sep 17 00:00:00 2001 From: David Liebovitz Date: Tue, 26 May 2026 20:52:51 -0500 Subject: [PATCH 2/6] Sender: stable extended-display identity + diagnostic logging; default toggles off After field testing, the prior toggles did not address the long-running stutter (which is duration-driven, not screensaver-driven). Updates: - Default both "Prevent screensaver / display sleep" and "Auto-restart capture after wake" to OFF. Both remain available for users who find them useful; existing UserDefaults values are preserved. - Derive the extended-desktop virtual display productID/serialNumber deterministically from the receiver identity (name + panel dimensions) instead of randomizing each session. macOS keys window-position memory on these fields, so the random identity caused windows on the extended display to be forgotten on every reconnect. Same receiver now produces the same virtual display identity across reconnects, which lets the system restore prior window placement and matches the identity convention already used for the saved arrangement key. - Add an opt-in "Log virtual display events to Console (verbose)" toggle (default off). When enabled, registers a CGDisplayRegisterReconfigurationCallback that logs every add / remove / mirror / mode-change event affecting any display, and emits a periodic (60 s) stream snapshot to NSLog with streaming state, FPS, virtual display online status, pending packet count, in-flight encode count, and PTS sequence. Intended for diagnosing the long-duration stutter from Console.app without leaving heavy logging on for ordinary users. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../ReceiverBackedVirtualDisplaySession.swift | 21 +++- .../TBDisplaySenderBuildInfo.swift | 2 +- .../TBDisplaySenderContentView.swift | 2 + .../TBDisplaySenderLocalization.swift | 8 ++ .../TBDisplaySenderManager.swift | 24 ++-- .../TBDisplaySenderService.swift | 115 ++++++++++++++++-- 6 files changed, 147 insertions(+), 25 deletions(-) diff --git a/TargetBridge-Sender/TBDisplaySender/ReceiverBackedVirtualDisplaySession.swift b/TargetBridge-Sender/TBDisplaySender/ReceiverBackedVirtualDisplaySession.swift index fa99ada..59f488a 100644 --- a/TargetBridge-Sender/TBDisplaySender/ReceiverBackedVirtualDisplaySession.swift +++ b/TargetBridge-Sender/TBDisplaySender/ReceiverBackedVirtualDisplaySession.swift @@ -18,15 +18,28 @@ struct TBVirtualDisplayIdentity { usesDedicatedArrangementIdentity: false ) - static func extendedDesktop() -> TBVirtualDisplayIdentity { - let random = UInt32.random(in: 0x0100...0xFFFE) + static func extendedDesktop(for profile: TBMonitorDisplayProfile) -> TBVirtualDisplayIdentity { + // Deterministic identity per receiver so macOS retains window placement + // and the saved extended-desktop arrangement across reconnects. + let key = "\(profile.receiverName)|\(profile.panelWidth)x\(profile.panelHeight)" + let hash = djb2(key) + let productLow = (hash & 0x00FF) | 0x01 + let serialLow = (hash & 0xFFFE) | 0x0100 return TBVirtualDisplayIdentity( - productID: 0x6000 | (random & 0x00FF), - serialNumber: 0x2027_0000 | random, + productID: 0x6000 | productLow, + serialNumber: 0x2027_0000 | UInt32(serialLow), displayNamePrefix: "TB Extend", usesDedicatedArrangementIdentity: true ) } + + private static func djb2(_ input: String) -> UInt32 { + var hash: UInt32 = 5381 + for byte in input.utf8 { + hash = hash &* 33 &+ UInt32(byte) + } + return hash + } } @MainActor diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift index 905de6d..7e4bfe6 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift @@ -1,5 +1,5 @@ enum TBDisplaySenderBuildInfo { static let marketingVersion = "2.0" - static let buildNumber = "20260525204941" + static let buildNumber = "20260526205217" static let versionDisplay = "\(marketingVersion) + build \(buildNumber)" } diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift index b4af181..489c16c 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift @@ -167,6 +167,8 @@ struct TBDisplaySenderContentView: View { Toggle(TBDisplaySenderL10n.preventDisplaySleep(service.language), isOn: $service.preventDisplaySleep) Toggle(TBDisplaySenderL10n.autoRestartOnWake(service.language), isOn: $service.autoRestartOnWake) + + Toggle(TBDisplaySenderL10n.verboseDisplayLogging(service.language), isOn: $service.verboseDisplayLogging) } } } diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderLocalization.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderLocalization.swift index ae82c23..b2a3967 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderLocalization.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderLocalization.swift @@ -504,6 +504,14 @@ enum TBDisplaySenderL10n { } } + static func verboseDisplayLogging(_ language: TBDisplaySenderLanguage) -> String { + switch language { + case .italian: return "Diagnostica display in Console (verboso)" + case .english: return "Log virtual display events to Console (verbose)" + case .german: return "Virtuelle Display-Ereignisse in Console protokollieren (ausführlich)" + } + } + static func showMainWindow(_ language: TBDisplaySenderLanguage) -> String { switch language { case .italian: return "Mostra finestra principale" diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift index c27f081..5931c4f 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift @@ -31,30 +31,27 @@ final class TBDisplaySenderService: ObservableObject { objectWillChange.send() } } - @Published var preventDisplaySleep: Bool = { - if UserDefaults.standard.object(forKey: "fd.tbdisplaysender.preventDisplaySleep") == nil { - return true - } - return UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.preventDisplaySleep") - }() { + @Published var preventDisplaySleep: Bool = UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.preventDisplaySleep") { didSet { UserDefaults.standard.set(preventDisplaySleep, forKey: "fd.tbdisplaysender.preventDisplaySleep") sessions.forEach { $0.preventDisplaySleep = preventDisplaySleep } objectWillChange.send() } } - @Published var autoRestartOnWake: Bool = { - if UserDefaults.standard.object(forKey: "fd.tbdisplaysender.autoRestartOnWake") == nil { - return true - } - return UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.autoRestartOnWake") - }() { + @Published var autoRestartOnWake: Bool = UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.autoRestartOnWake") { didSet { UserDefaults.standard.set(autoRestartOnWake, forKey: "fd.tbdisplaysender.autoRestartOnWake") sessions.forEach { $0.autoRestartOnWake = autoRestartOnWake } objectWillChange.send() } } + @Published var verboseDisplayLogging: Bool = UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.verboseDisplayLogging") { + didSet { + UserDefaults.standard.set(verboseDisplayLogging, forKey: "fd.tbdisplaysender.verboseDisplayLogging") + sessions.forEach { $0.verboseDisplayLogging = verboseDisplayLogging } + objectWillChange.send() + } + } private var sessionCancellables: [UUID: AnyCancellable] = [:] private let receiverDiscovery = TBReceiverDiscovery() @@ -98,7 +95,8 @@ final class TBDisplaySenderService: ObservableObject { language: language, largeCursor: largeCursor, preventDisplaySleep: preventDisplaySleep, - autoRestartOnWake: autoRestartOnWake + autoRestartOnWake: autoRestartOnWake, + verboseDisplayLogging: verboseDisplayLogging ) if let previous = sessions.last { session.capturePreset = previous.capturePreset diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift index aebdd6c..a7b1218 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift @@ -238,12 +238,12 @@ enum TBDisplayCaptureSource: String, CaseIterable, Identifiable { } } - var virtualDisplayIdentity: TBVirtualDisplayIdentity { + func virtualDisplayIdentity(for profile: TBMonitorDisplayProfile) -> TBVirtualDisplayIdentity { switch self { case .desktopMirror: return .desktopMirror case .extendedDesktop: - return .extendedDesktop() + return .extendedDesktop(for: profile) } } } @@ -397,8 +397,9 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u init( language: TBDisplaySenderLanguage, largeCursor: Bool, - preventDisplaySleep: Bool = true, - autoRestartOnWake: Bool = true + preventDisplaySleep: Bool = false, + autoRestartOnWake: Bool = false, + verboseDisplayLogging: Bool = false ) { self.statusText = TBDisplaySenderStatusState.ready.text(language) self.receiverPanelText = TBDisplaySenderL10n.waitingReceiverProfile(language) @@ -407,6 +408,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u self.largeCursor = largeCursor self.preventDisplaySleep = preventDisplaySleep self.autoRestartOnWake = autoRestartOnWake + self.verboseDisplayLogging = verboseDisplayLogging self.streamResolutionText = TBDisplaySenderL10n.streamSummary( preset: .standard1440p, source: .desktopMirror, @@ -414,6 +416,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u ) super.init() registerWakeObservers() + registerDisplayReconfigurationCallback() } deinit { @@ -421,6 +424,12 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u NSWorkspace.shared.notificationCenter.removeObserver(token) DistributedNotificationCenter.default().removeObserver(token) } + if displayReconfigurationCallbackRegistered { + CGDisplayRemoveReconfigurationCallback( + Self.displayReconfigurationCallback, + Unmanaged.passUnretained(self).toOpaque() + ) + } } @Published var isConnected = false @@ -447,8 +456,17 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u } } @Published var largeCursor: Bool - @Published var preventDisplaySleep: Bool = true - @Published var autoRestartOnWake: Bool = true + @Published var preventDisplaySleep: Bool = false + @Published var autoRestartOnWake: Bool = false + @Published var verboseDisplayLogging: Bool = false { + didSet { + if verboseDisplayLogging { + startVerboseLoggingTimer() + } else { + stopVerboseLoggingTimer() + } + } + } @Published var capturePreset: TBDisplayCapturePreset = .standard1440p { didSet { if !isStreaming { @@ -496,6 +514,18 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u private var lastCursorPacket: TBMonitorCursor? nonisolated(unsafe) private var wakeObservers: [NSObjectProtocol] = [] private var isRestartingCaptureAfterWake = false + nonisolated(unsafe) private var displayReconfigurationCallbackRegistered = false + private var verboseLoggingTimer: Timer? + + nonisolated(unsafe) private static let displayReconfigurationCallback: CGDisplayReconfigurationCallBack = { displayID, flags, userInfo in + guard let userInfo else { return } + let service = Unmanaged.fromOpaque(userInfo).takeUnretainedValue() + DispatchQueue.main.async { + MainActor.assumeIsolated { + service.handleDisplayReconfiguration(displayID: displayID, flags: flags) + } + } + } private final class CaptureDelegate: NSObject, SCStreamOutput, SCStreamDelegate { var onFrame: ((CMSampleBuffer) -> Void)? @@ -976,7 +1006,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u guard self.session.create( from: profile, refreshRate: self.capturePreset.virtualDisplayRefreshRate, - identity: self.captureSource.virtualDisplayIdentity + identity: self.captureSource.virtualDisplayIdentity(for: profile) ) else { self.setStatus(.virtualDisplayCreationFailed) self.stop(resetStatusTo: nil) @@ -1706,6 +1736,77 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u ) } + private func registerDisplayReconfigurationCallback() { + guard !displayReconfigurationCallbackRegistered else { return } + let context = Unmanaged.passUnretained(self).toOpaque() + let result = CGDisplayRegisterReconfigurationCallback(Self.displayReconfigurationCallback, context) + displayReconfigurationCallbackRegistered = (result == .success) + if verboseDisplayLogging { + startVerboseLoggingTimer() + } + } + + private func handleDisplayReconfiguration(displayID: CGDirectDisplayID, flags: CGDisplayChangeSummaryFlags) { + let isOurs = session.displayID != kCGNullDirectDisplay && displayID == session.displayID + guard verboseDisplayLogging || isOurs else { return } + var parts: [String] = [] + if flags.contains(.addFlag) { parts.append("add") } + if flags.contains(.removeFlag) { parts.append("remove") } + if flags.contains(.enabledFlag) { parts.append("enabled") } + if flags.contains(.disabledFlag) { parts.append("disabled") } + if flags.contains(.mirrorFlag) { parts.append("mirror") } + if flags.contains(.unMirrorFlag) { parts.append("unMirror") } + if flags.contains(.movedFlag) { parts.append("moved") } + if flags.contains(.setMainFlag) { parts.append("setMain") } + if flags.contains(.setModeFlag) { parts.append("setMode") } + if flags.contains(.beginConfigurationFlag) { parts.append("beginConfiguration") } + if flags.contains(.desktopShapeChangedFlag) { parts.append("desktopShapeChanged") } + let flagText = parts.isEmpty ? "none" : parts.joined(separator: "|") + NSLog( + "TargetBridge: display reconfiguration displayID=%u ours=%@ flags=%@ online=[%@]", + displayID, + isOurs ? "yes" : "no", + flagText, + onlineDisplayIDs().map(String.init).joined(separator: ",") + ) + if isOurs, session.displayID != kCGNullDirectDisplay { + displayStateText = describeDisplayState(for: session.displayID) + } + } + + private func startVerboseLoggingTimer() { + stopVerboseLoggingTimer() + guard verboseDisplayLogging else { return } + let timer = Timer.scheduledTimer(withTimeInterval: 60.0, repeats: true) { [weak self] _ in + Task { @MainActor [weak self] in + self?.logStreamSnapshot() + } + } + verboseLoggingTimer = timer + logStreamSnapshot() + } + + private func stopVerboseLoggingTimer() { + verboseLoggingTimer?.invalidate() + verboseLoggingTimer = nil + } + + private func logStreamSnapshot() { + guard verboseDisplayLogging else { return } + let online = onlineDisplayIDs() + let virtualOnline = online.contains(session.displayID) + NSLog( + "TargetBridge: stream snapshot streaming=%@ fps=%d virtualID=%u online=%@ pendingPackets=%d inFlightEncode=%d ptsSeq=%lld", + isStreaming ? "yes" : "no", + senderFPS, + session.displayID, + virtualOnline ? "yes" : "no", + pendingVideoPackets, + inFlightEncodeFrames, + displayStreamFrameSequence + ) + } + private func handleSystemWake() { guard autoRestartOnWake else { return } scheduleCaptureRestart(reason: "system wake", delaySeconds: 1.0) From 63ad77c93268593899b520c754fb4538525a6116 Mon Sep 17 00:00:00 2001 From: David Liebovitz Date: Wed, 27 May 2026 20:47:06 -0500 Subject: [PATCH 3/6] Sender: capture watchdog + UI re-render storm mitigations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field diagnostics (24h run, sample + 'log show') showed the long-duration stutter is caused by macOS's replayd daemon issuing RPDaemonProxy: connection INTERRUPTED roughly every 30 minutes (sandbox extension refresh). The interruption does not reliably surface through SCStreamDelegate.didStopWithError, so the existing pipeline kept running in a degraded state until the main thread eventually wedged in recursive AppKit constraint layout. Three changes: 1) Frame-arrival watchdog New 5s repeating Timer on TBDisplaySenderSession that checks lastCaptureFrameAt (bumped on every encode() / encodeDisplaySurface() call). If isStreaming is true and no frames have arrived in >=8s, logs the gap and reuses the existing scheduleCaptureRestart() path to soft-restart only the capture pipeline. The NWConnection and the virtual display are preserved. Started after startFPSTimer() in both startCapture() paths; torn down in stop() and at the top of softRestartCapture(). This would have caught all 14 replayd interruptions observed in the diagnostic run. 2) Remove redundant objectWillChange.send() in the manager @Published fields auto-emit on every change. The didSet handlers and the discovery / addSession / removeSession / refreshBridge / applyDiscoveredReceiver paths were also calling objectWillChange .send() explicitly, producing two notifications per change. The bubble-up sink in attachSession is preserved — it is the only non-redundant path (session @Published -> manager objectWillChange). 3) Defer the discovered-receiver picker mutation .onChange(of: session.selectedReceiverID) was calling service.applyDiscoveredReceiver synchronously from within the view-update phase, producing the well-known SwiftUI runtime warning 'Publishing changes from within view updates is not allowed' (observed 16 times in the first 12 s of every launch). The mutation is now dispatched to the main queue so it runs after the current view update completes. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../TBDisplaySenderBuildInfo.swift | 2 +- .../TBDisplaySenderContentView.swift | 6 +++- .../TBDisplaySenderManager.swift | 10 ------ .../TBDisplaySenderService.swift | 31 +++++++++++++++++++ 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift index 7e4bfe6..57aa2ea 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift @@ -1,5 +1,5 @@ enum TBDisplaySenderBuildInfo { static let marketingVersion = "2.0" - static let buildNumber = "20260526205217" + static let buildNumber = "20260527204607" static let versionDisplay = "\(marketingVersion) + build \(buildNumber)" } diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift index 489c16c..e99f216 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift @@ -235,7 +235,11 @@ private struct TBDisplaySenderSessionCard: View { } .onChange(of: session.selectedReceiverID) { _, newValue in guard let receiver = service.discoveredReceivers.first(where: { $0.id == newValue }) else { return } - service.applyDiscoveredReceiver(receiver, to: session) + // Defer the mutation past SwiftUI's current view-update phase to avoid + // "Publishing changes from within view updates is not allowed". + DispatchQueue.main.async { + service.applyDiscoveredReceiver(receiver, to: session) + } } .disabled(session.isConnected || session.isStreaming) } diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift index 5931c4f..4c9ded6 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift @@ -20,7 +20,6 @@ final class TBDisplaySenderService: ObservableObject { didSet { language.persist() sessions.forEach { $0.language = language } - objectWillChange.send() } } @Published var showsMenuBarIcon = true @@ -28,28 +27,24 @@ final class TBDisplaySenderService: ObservableObject { didSet { UserDefaults.standard.set(largeCursor, forKey: "fd.tbdisplaysender.largeCursor") sessions.forEach { $0.largeCursor = largeCursor } - objectWillChange.send() } } @Published var preventDisplaySleep: Bool = UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.preventDisplaySleep") { didSet { UserDefaults.standard.set(preventDisplaySleep, forKey: "fd.tbdisplaysender.preventDisplaySleep") sessions.forEach { $0.preventDisplaySleep = preventDisplaySleep } - objectWillChange.send() } } @Published var autoRestartOnWake: Bool = UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.autoRestartOnWake") { didSet { UserDefaults.standard.set(autoRestartOnWake, forKey: "fd.tbdisplaysender.autoRestartOnWake") sessions.forEach { $0.autoRestartOnWake = autoRestartOnWake } - objectWillChange.send() } } @Published var verboseDisplayLogging: Bool = UserDefaults.standard.bool(forKey: "fd.tbdisplaysender.verboseDisplayLogging") { didSet { UserDefaults.standard.set(verboseDisplayLogging, forKey: "fd.tbdisplaysender.verboseDisplayLogging") sessions.forEach { $0.verboseDisplayLogging = verboseDisplayLogging } - objectWillChange.send() } } @@ -61,7 +56,6 @@ final class TBDisplaySenderService: ObservableObject { discoveryCancellable = receiverDiscovery.$receivers.sink { [weak self] receivers in guard let self else { return } discoveredReceivers = receivers - objectWillChange.send() } refreshBridgeInterfaces() addSession() @@ -107,7 +101,6 @@ final class TBDisplaySenderService: ObservableObject { } attachSession(session) sessions.append(session) - objectWillChange.send() } func removeSession(_ session: TBDisplaySenderSession) { @@ -116,7 +109,6 @@ final class TBDisplaySenderService: ObservableObject { sessions.removeAll { $0.id == session.id } sessionCancellables.removeValue(forKey: session.id) normalizeSessionInterfaces() - objectWillChange.send() } func stopAll() { @@ -128,7 +120,6 @@ final class TBDisplaySenderService: ObservableObject { bridgeInterfaces = detectBridgeInterfaces() receiverDiscovery.refresh() normalizeSessionInterfaces() - objectWillChange.send() } func applyDiscoveredReceiver(_ receiver: TBDiscoveredReceiver, to session: TBDisplaySenderSession) { @@ -136,7 +127,6 @@ final class TBDisplaySenderService: ObservableObject { if session.localTBIP.isEmpty { session.localTBIP = suggestedInterfaceForNewSession()?.ip ?? bridgeInterfaces.first?.ip ?? "" } - objectWillChange.send() } func sessionTitle(for session: TBDisplaySenderSession) -> String { diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift index a7b1218..0bf14f1 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift @@ -516,6 +516,8 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u private var isRestartingCaptureAfterWake = false nonisolated(unsafe) private var displayReconfigurationCallbackRegistered = false private var verboseLoggingTimer: Timer? + private var lastCaptureFrameAt: Date = Date() + private var captureHealthWatchdog: Timer? nonisolated(unsafe) private static let displayReconfigurationCallback: CGDisplayReconfigurationCallBack = { displayID, flags, userInfo in guard let userInfo else { return } @@ -793,6 +795,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u cursorTimer = nil fpsTimer?.invalidate() fpsTimer = nil + stopCaptureWatchdog() if let directDisplayStream { directDisplayStream.stop() self.directDisplayStream = nil @@ -1109,6 +1112,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u reason: "TargetBridge streaming active" ) startFPSTimer() + startCaptureWatchdog() return true } catch { if error.localizedDescription.hasPrefix("no virtual SCDisplay available") { @@ -1147,6 +1151,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u reason: "TargetBridge streaming active" ) startFPSTimer() + startCaptureWatchdog() return true } @@ -1433,6 +1438,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u } private func encode(_ sampleBuffer: CMSampleBuffer) { + lastCaptureFrameAt = Date() guard let encoder = vtEncoder, let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } @@ -1446,6 +1452,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u } fileprivate func encodeDisplaySurface(_ surface: IOSurface, displayTime: UInt64) { + lastCaptureFrameAt = Date() guard let encoder = vtEncoder else { return } if capturePreset.dropsBeforeEncodeWhenBacklogged, (pendingVideoPackets >= capturePreset.maxPendingVideoPackets || @@ -1791,6 +1798,29 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u verboseLoggingTimer = nil } + private func startCaptureWatchdog() { + captureHealthWatchdog?.invalidate() + lastCaptureFrameAt = Date() + captureHealthWatchdog = Timer.scheduledTimer(withTimeInterval: 5.0, repeats: true) { [weak self] _ in + Task { @MainActor [weak self] in + self?.checkCaptureHealth() + } + } + } + + private func stopCaptureWatchdog() { + captureHealthWatchdog?.invalidate() + captureHealthWatchdog = nil + } + + private func checkCaptureHealth() { + guard isStreaming, activeProfile != nil, !isRestartingCaptureAfterWake else { return } + let elapsed = Date().timeIntervalSince(lastCaptureFrameAt) + guard elapsed >= 8.0 else { return } + NSLog("TargetBridge: capture watchdog tripped — %.1fs since last frame, soft restart", elapsed) + scheduleCaptureRestart(reason: "watchdog (\(Int(elapsed))s without frames)", delaySeconds: 0.5) + } + private func logStreamSnapshot() { guard verboseDisplayLogging else { return } let online = onlineDisplayIDs() @@ -1844,6 +1874,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u cursorTimer = nil fpsTimer?.invalidate() fpsTimer = nil + stopCaptureWatchdog() if let directDisplayStream { directDisplayStream.stop() self.directDisplayStream = nil From a36c2142271fada8dc403bf7cda9ba2bc2a6b5e7 Mon Sep 17 00:00:00 2001 From: David Liebovitz Date: Thu, 28 May 2026 07:31:33 -0500 Subject: [PATCH 4/6] Sender: real-time PTS for CGDisplayStream capture (fix long-session stutter) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extended-desktop capture path (CGDisplayStream) stamped each frame's presentation timestamp from a frame counter scaled by the nominal frame rate: displayStreamFrameSequence += 1 let pts = CMTime(value: displayStreamFrameSequence, timescale: Int32(capturePreset.expectedFrameRate)) But CGDisplayStream delivers frames irregularly (event-driven on screen changes), so the synthetic PTS clock drifts away from real wall-clock time and the drift compounds over a session. The receiver paces playback by PTS, so after hours of streaming the cursor and video progressively stutter. It only cleared on a *sender* restart (counter resets to 0), not a receiver restart, and frames kept flowing throughout — which is exactly what was observed in the field, and why the no-frames watchdog never tripped. Fix: derive PTS from the frame's actual capture time. CGDisplayStream's displayTime is in mach-absolute units, the same host clock the SCStream path already feeds via CMSampleBufferGetPresentationTimeStamp, so the two capture paths and the receiver now agree on timestamp semantics: var pts = displayTime != 0 ? CMClockMakeHostTimeFromSystemUnits(displayTime) : CMClockGetTime(CMClockGetHostTimeClock()) A monotonic guard nudges PTS forward if two frames ever share a displayTime, since VTCompressionSession requires strictly increasing timestamps. lastEncodedDisplayPTS is reset wherever the capture pipeline restarts (stop, startDirectDisplayStream, softRestartCapture). The frame counter is retained only for the verbose diagnostic snapshot. The SCStream path was already correct and is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../TBDisplaySenderBuildInfo.swift | 2 +- .../TBDisplaySenderService.swift | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift index 57aa2ea..01d2cb5 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift @@ -1,5 +1,5 @@ enum TBDisplaySenderBuildInfo { static let marketingVersion = "2.0" - static let buildNumber = "20260527204607" + static let buildNumber = "20260528073031" static let versionDisplay = "\(marketingVersion) + build \(buildNumber)" } diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift index 0bf14f1..cc6b04a 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift @@ -518,6 +518,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u private var verboseLoggingTimer: Timer? private var lastCaptureFrameAt: Date = Date() private var captureHealthWatchdog: Timer? + private var lastEncodedDisplayPTS: CMTime? nonisolated(unsafe) private static let displayReconfigurationCallback: CGDisplayReconfigurationCallBack = { displayID, flags, userInfo in guard let userInfo else { return } @@ -839,6 +840,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u pendingVideoPackets = 0 inFlightEncodeFrames = 0 displayStreamFrameSequence = 0 + lastEncodedDisplayPTS = nil baselineDisplayIDs = [] cursorDisplayID = kCGNullDirectDisplay lastCursorPacket = nil @@ -1135,6 +1137,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u guard vtEncoder != nil else { return false } displayStreamFrameSequence = 0 + lastEncodedDisplayPTS = nil streamResolutionText = TBDisplaySenderL10n.streamSummary(preset: preset, source: captureSource, language: language) let directCapture = TBDirectDisplayStreamCapture(service: self, queue: connectionQueue) @@ -1478,7 +1481,20 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u let pixelBuffer = unmanagedPixelBuffer.takeRetainedValue() displayStreamFrameSequence += 1 - let pts = CMTime(value: displayStreamFrameSequence, timescale: Int32(capturePreset.expectedFrameRate)) + // Derive PTS from the frame's actual capture time. CGDisplayStream + // delivers frames irregularly (event-driven on screen changes), so the + // previous frame-counter PTS — which assumed an exact nominal frame + // rate — drifted away from real wall-clock time and the drift compounded + // over long sessions, pacing the receiver progressively wrong. displayTime + // is in mach-absolute units, same host clock the SCStream path already uses. + var pts = displayTime != 0 + ? CMClockMakeHostTimeFromSystemUnits(displayTime) + : CMClockGetTime(CMClockGetHostTimeClock()) + if let last = lastEncodedDisplayPTS, CMTimeCompare(pts, last) <= 0 { + // VTCompressionSession requires strictly increasing PTS. + pts = CMTimeAdd(last, CMTime(value: 1, timescale: 600)) + } + lastEncodedDisplayPTS = pts encode(pixelBuffer: pixelBuffer, presentationTimeStamp: pts, using: encoder) } @@ -1897,6 +1913,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u vtEncoderRef = nil isStreaming = false displayStreamFrameSequence = 0 + lastEncodedDisplayPTS = nil senderFPS = 0 sentSnapshot = sentFrames pendingVideoPackets = 0 From cd2fc8b30be4d3d4dbff2eb521a843ef34de9fa9 Mon Sep 17 00:00:00 2001 From: David Liebovitz Date: Fri, 29 May 2026 22:03:05 -0500 Subject: [PATCH 5/6] Sender: move video pipeline off the main thread + isolate FPS readout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field diagnostics (live `sample` of the running sender while a screensaver ran on the extended display) showed two distinct causes of the "stutters with extended use" symptom, neither addressed by the earlier PTS fix: 1) The whole capture→encode→send pipeline ran on the main thread. TBDisplaySenderSession is @MainActor, and every frame made three main-thread hops (intake, encode-complete, send-complete). When the Session Monitor window was open its SwiftUI hosting view re-laid-out continuously (~18% of the main thread in recursive AppKit constraint layout). Those layout bursts blocked the main thread, frame work queued behind them, and the pendingVideoPackets>=3 cap then dropped frames — visible as cursor/video stutter, worst during a screensaver (continuous frames). Confirmed: closing the window made it smooth and raised encoder throughput ~40%. Fix: extract TBVideoPipeline, which owns the VTCompressionSession, the pending/in-flight counters, packet building and sending, and runs entirely on a dedicated serial queue (fd.tbmonitor.sender.pipeline). Encoder setup/teardown are serialized on that queue, so a frame can never encode into an invalidated session. Both capture paths (CGDisplayStream + SCStream) feed the queue directly. The main actor keeps only UI/lifecycle state and reads the two values it polls (sentFrames for FPS, lastCaptureFrameAt for the watchdog) under a small lock — no per-frame hop to main. The real-time PTS logic is preserved unchanged. Measured: encode now runs off-main; main-thread layout dropped from ~18% to ~1% with the window open. 2) senderFPS was @Published on the session and rewritten every second by the FPS timer. Via the manager's objectWillChange bubble-up this re-rendered the entire window ~1.8x/sec (full constraint relayout) whenever the Session Monitor window was open. Fix: move senderFPS to a dedicated TBSessionLiveMetrics observable, displayed by an isolated SessionMonitorFPSRow subview, so the 1 Hz tick re-renders just that row. Measured: card/window re-renders dropped from 1.82/sec to 0/sec at idle and during a screensaver, with the window open. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../TBDisplaySenderBuildInfo.swift | 2 +- .../TBDisplaySenderContentView.swift | 24 +- .../TBDisplaySenderService.swift | 765 ++++++++++-------- 3 files changed, 452 insertions(+), 339 deletions(-) diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift index 01d2cb5..219ecbf 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift @@ -1,5 +1,5 @@ enum TBDisplaySenderBuildInfo { static let marketingVersion = "2.0" - static let buildNumber = "20260528073031" + static let buildNumber = "20260529215219" static let versionDisplay = "\(marketingVersion) + build \(buildNumber)" } diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift index e99f216..651f868 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift @@ -370,7 +370,9 @@ private struct TBDisplaySenderSessionCard: View { infoRow(TBDisplaySenderL10n.receiverLabel(service.language), session.receiverPanelText) infoRow(TBDisplaySenderL10n.virtualDisplayLabel(service.language), session.virtualDisplayText) infoRow(TBDisplaySenderL10n.streamLabel(service.language), session.streamResolutionText) - infoRow(TBDisplaySenderL10n.fpsLabel(service.language), "\(session.senderFPS)") + // Observes the dedicated metrics object so the ~1 Hz FPS tick + // re-renders only this row, not the whole session card / window. + SessionMonitorFPSRow(label: TBDisplaySenderL10n.fpsLabel(service.language), metrics: session.liveMetrics) infoRow("Capture", session.captureDisplayText) infoRow("State", session.displayStateText) } @@ -521,6 +523,26 @@ private struct TBDisplaySenderSessionCard: View { } } +/// FPS readout that observes only `TBSessionLiveMetrics`. Isolating it here means +/// the once-per-second FPS update invalidates just this small row instead of the +/// entire session card (and, via the manager bubble-up, the whole window). +private struct SessionMonitorFPSRow: View { + let label: String + @ObservedObject var metrics: TBSessionLiveMetrics + + var body: some View { + HStack(alignment: .firstTextBaseline, spacing: 14) { + Text(label) + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + .frame(width: 138, alignment: .leading) + Text("\(metrics.senderFPS)") + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) + } + } +} + private struct SurfaceCard: View { @ViewBuilder let content: Content diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift index cc6b04a..d58fd58 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift @@ -249,12 +249,12 @@ enum TBDisplayCaptureSource: String, CaseIterable, Identifiable { } private final class TBDirectDisplayStreamCapture { - private let serviceRef: UnsafeMutableRawPointer + private let pipelineRefValue: UInt private let queue: DispatchQueue private var stream: CGDisplayStream? - init(service: TBDisplaySenderSession, queue: DispatchQueue) { - self.serviceRef = Unmanaged.passUnretained(service).toOpaque() + init(pipeline: TBVideoPipeline, queue: DispatchQueue) { + self.pipelineRefValue = UInt(bitPattern: Unmanaged.passUnretained(pipeline).toOpaque()) self.queue = queue } @@ -265,7 +265,7 @@ private final class TBDirectDisplayStreamCapture { CGDisplayStream.minimumFrameTime: 1.0 / Double(preset.expectedFrameRate) ] - let serviceRefValue = UInt(bitPattern: serviceRef) + let pipelineRefValue = self.pipelineRefValue let displayStream = CGDisplayStream( dispatchQueueDisplay: displayID, outputWidth: preset.width, @@ -274,19 +274,13 @@ private final class TBDirectDisplayStreamCapture { properties: properties, queue: queue ) { status, displayTime, surface, _ in + // This handler is delivered on `queue`, which is the pipeline's own + // serial queue — so we run the encode synchronously here, off the + // main thread, with no extra hop. guard status == .frameComplete, let surface else { return } - let surfaceRefValue = UInt(bitPattern: Unmanaged.passRetained(surface).toOpaque()) - DispatchQueue.main.async { - guard let serviceRef = UnsafeRawPointer(bitPattern: serviceRefValue), - let surfaceRef = UnsafeRawPointer(bitPattern: surfaceRefValue) else { - return - } - let service = Unmanaged.fromOpaque(serviceRef).takeUnretainedValue() - let surface = Unmanaged.fromOpaque(surfaceRef).takeRetainedValue() - MainActor.assumeIsolated { - service.encodeDisplaySurface(surface, displayTime: displayTime) - } - } + guard let pipelineRef = UnsafeRawPointer(bitPattern: pipelineRefValue) else { return } + let pipeline = Unmanaged.fromOpaque(pipelineRef).takeUnretainedValue() + pipeline.encodeDisplaySurface(surface, displayTime: displayTime) } guard let displayStream, displayStream.start() == .success else { @@ -303,6 +297,369 @@ private final class TBDirectDisplayStreamCapture { } } +/// Owns the capture→encode→send video pipeline and runs it entirely on a +/// dedicated serial queue, off the main thread. SwiftUI layout (or any other +/// main-thread work) therefore cannot stall frame delivery. All mutable encode +/// state is confined to `queue`; the two values the main thread polls +/// (`sentFrames`, `lastCaptureFrameAt`) are guarded by a small lock instead of +/// a per-frame hop back to main. +private final class TBVideoPipeline: @unchecked Sendable { + let queue = DispatchQueue(label: "fd.tbmonitor.sender.pipeline", qos: .userInteractive) + + private let preset: TBDisplayCapturePreset + private let connection: NWConnection + private let displayName: String + private let displayID: CGDirectDisplayID + private let onFirstFrame: @Sendable () -> Void + + // Confined to `queue`. + private var vtEncoder: VTCompressionSession? + private var vtEncoderRef: Unmanaged? + private var pendingVideoPackets = 0 + private var inFlightEncodeFrames = 0 + private var displayStreamFrameSequence: CMTimeValue = 0 + private var lastEncodedDisplayPTS: CMTime? + private var ackSent: Bool + private var running = false + + // Read from the main thread (fps timer / watchdog); guarded by `lock`. + private let lock = NSLock() + private var _sentFrames = 0 + private var _lastCaptureFrameAt = Date() + + init(preset: TBDisplayCapturePreset, + connection: NWConnection, + displayName: String, + displayID: CGDirectDisplayID, + ackAlreadySent: Bool, + onFirstFrame: @escaping @Sendable () -> Void) { + self.preset = preset + self.connection = connection + self.displayName = displayName + self.displayID = displayID + self.ackSent = ackAlreadySent + self.onFirstFrame = onFirstFrame + } + + // MARK: - Lifecycle (called from the main actor) + + /// Sets up the encoder on `queue`. Returns false if the hardware encoder + /// could not be created. + func start() -> Bool { + queue.sync { + setupEncoder() + running = vtEncoder != nil + return running + } + } + + /// Tears the encoder down on `queue`. Because the queue is serial, any + /// in-flight `encode` completes before `VTCompressionSessionInvalidate`, + /// so a frame can never encode into an invalidated session. + func stop() { + queue.sync { + running = false + if let encoder = vtEncoder { VTCompressionSessionInvalidate(encoder) } + vtEncoder = nil + vtEncoderRef?.release() + vtEncoderRef = nil + } + } + + // MARK: - Snapshots for the main thread + + var sentFramesSnapshot: Int { + lock.lock(); defer { lock.unlock() } + return _sentFrames + } + + var lastCaptureFrameAtSnapshot: Date { + lock.lock(); defer { lock.unlock() } + return _lastCaptureFrameAt + } + + func diagnosticsSnapshot() -> (pending: Int, inFlight: Int, ptsSeq: CMTimeValue) { + queue.sync { (pending: pendingVideoPackets, inFlight: inFlightEncodeFrames, ptsSeq: displayStreamFrameSequence) } + } + + private func markCaptureFrame() { + lock.lock(); _lastCaptureFrameAt = Date(); lock.unlock() + } + + // MARK: - Encoder setup (on `queue`) + + private func setupEncoder() { + if let encoder = vtEncoder { VTCompressionSessionInvalidate(encoder) } + vtEncoder = nil + vtEncoderRef?.release() + vtEncoderRef = nil + + let spec: NSDictionary = [ + kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder: true, + kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder: true + ] + let retained = Unmanaged.passRetained(self) + vtEncoderRef = retained + + let callback: VTCompressionOutputCallback = { ref, _, status, _, sampleBuffer in + guard let ref else { return } + let pipeline = Unmanaged.fromOpaque(ref).takeUnretainedValue() + pipeline.queue.async { + pipeline.inFlightEncodeFrames = max(0, pipeline.inFlightEncodeFrames - 1) + guard status == noErr, let sampleBuffer else { return } + pipeline.handleEncoded(sampleBuffer) + } + } + + var session: VTCompressionSession? + guard VTCompressionSessionCreate( + allocator: nil, + width: Int32(preset.width), + height: Int32(preset.height), + codecType: preset.codecType, + encoderSpecification: spec, + imageBufferAttributes: nil, + compressedDataAllocator: nil, + outputCallback: callback, + refcon: retained.toOpaque(), + compressionSessionOut: &session + ) == noErr, let session else { + retained.release() + vtEncoderRef = nil + return + } + + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue) + if preset.codecType == kCMVideoCodecType_HEVC { + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_HEVC_Main_AutoLevel) + } else { + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_H264_Main_AutoLevel) + } + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse) + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate, value: NSNumber(value: preset.expectedFrameRate)) + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval, value: NSNumber(value: preset.maxKeyFrameInterval)) + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, value: NSNumber(value: preset.maxKeyFrameIntervalDuration)) + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxFrameDelayCount, value: NSNumber(value: preset.maxFrameDelayCount)) + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate, value: NSNumber(value: preset.averageBitRate)) + if preset.prioritizeSpeed { + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, value: kCFBooleanTrue) + } + VTCompressionSessionPrepareToEncodeFrames(session) + vtEncoder = session + } + + // MARK: - Encode paths (on `queue`) + + /// SCStream capture path. Must be dispatched onto `queue` by the caller. + func encode(_ sampleBuffer: CMSampleBuffer) { + markCaptureFrame() + guard running, let encoder = vtEncoder, + let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) + else { return } + if preset.dropsBeforeEncodeWhenBacklogged, + (pendingVideoPackets >= preset.maxPendingVideoPackets || + inFlightEncodeFrames >= preset.maxInFlightEncodeFrames) { + return + } + let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + encode(pixelBuffer: pixelBuffer, presentationTimeStamp: pts, using: encoder) + } + + /// CGDisplayStream capture path. Delivered directly on `queue` by + /// `TBDirectDisplayStreamCapture`. + func encodeDisplaySurface(_ surface: IOSurfaceRef, displayTime: UInt64) { + markCaptureFrame() + guard running, let encoder = vtEncoder else { return } + if preset.dropsBeforeEncodeWhenBacklogged, + (pendingVideoPackets >= preset.maxPendingVideoPackets || + inFlightEncodeFrames >= preset.maxInFlightEncodeFrames) { + return + } + + let attrs: NSDictionary = [ + kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey: preset.width, + kCVPixelBufferHeightKey: preset.height, + kCVPixelBufferIOSurfacePropertiesKey: NSDictionary() + ] + var unmanagedPixelBuffer: Unmanaged? + guard CVPixelBufferCreateWithIOSurface( + kCFAllocatorDefault, + surface, + attrs, + &unmanagedPixelBuffer + ) == kCVReturnSuccess, let unmanagedPixelBuffer else { + return + } + let pixelBuffer = unmanagedPixelBuffer.takeRetainedValue() + + displayStreamFrameSequence += 1 + // Derive PTS from the frame's actual capture time. CGDisplayStream + // delivers frames irregularly (event-driven on screen changes), so a + // frame-counter PTS would drift away from real wall-clock time over a + // long session and pace the receiver progressively wrong. displayTime is + // in mach-absolute units, the same host clock the SCStream path uses. + var pts = displayTime != 0 + ? CMClockMakeHostTimeFromSystemUnits(displayTime) + : CMClockGetTime(CMClockGetHostTimeClock()) + if let last = lastEncodedDisplayPTS, CMTimeCompare(pts, last) <= 0 { + // VTCompressionSession requires strictly increasing PTS. + pts = CMTimeAdd(last, CMTime(value: 1, timescale: 600)) + } + lastEncodedDisplayPTS = pts + encode(pixelBuffer: pixelBuffer, presentationTimeStamp: pts, using: encoder) + } + + private func encode(pixelBuffer: CVPixelBuffer, presentationTimeStamp pts: CMTime, using encoder: VTCompressionSession) { + inFlightEncodeFrames += 1 + let status = VTCompressionSessionEncodeFrame( + encoder, + imageBuffer: pixelBuffer, + presentationTimeStamp: pts, + duration: .invalid, + frameProperties: nil, + sourceFrameRefcon: nil, + infoFlagsOut: nil + ) + if status != noErr { + inFlightEncodeFrames = max(0, inFlightEncodeFrames - 1) + } + } + + private func handleEncoded(_ sampleBuffer: CMSampleBuffer) { + guard running else { return } + + if !ackSent { + ackSent = true + let ack = TBMonitorCreateSessionAck( + accepted: true, + displayName: displayName, + displayID: displayID + ) + if let packet = TBMonitorProtocol.makeJSONPacket(type: .createSessionAck, value: ack) { + connection.send(content: packet, completion: .contentProcessed({ _ in })) + } + onFirstFrame() + } + + let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[CFString: Any]] + let notSync = attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool ?? false + let isKeyframe = !notSync + + if !isKeyframe, pendingVideoPackets >= preset.maxPendingVideoPackets { + return + } + + if isKeyframe, + let format = CMSampleBufferGetFormatDescription(sampleBuffer), + let packet = buildParamSetsPacket(from: format, codecType: preset.codecType) { + connection.send(content: packet, completion: .contentProcessed({ _ in })) + } + + if let packet = buildFramePacket(from: sampleBuffer) { + pendingVideoPackets += 1 + connection.send(content: packet, completion: .contentProcessed({ [weak self] _ in + guard let self else { return } + self.queue.async { + self.pendingVideoPackets = max(0, self.pendingVideoPackets - 1) + } + })) + lock.lock(); _sentFrames += 1; lock.unlock() + } + } + + private func buildParamSetsPacket(from format: CMVideoFormatDescription, codecType: CMVideoCodecType) -> Data? { + if codecType == kCMVideoCodecType_HEVC { + var count = 0 + CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + format, + parameterSetIndex: 0, + parameterSetPointerOut: nil, + parameterSetSizeOut: nil, + parameterSetCountOut: &count, + nalUnitHeaderLengthOut: nil + ) + guard count > 0 else { return nil } + + var payload = Data([2, UInt8(count)]) + for index in 0..? + var size = 0 + CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + format, + parameterSetIndex: index, + parameterSetPointerOut: &pointer, + parameterSetSizeOut: &size, + parameterSetCountOut: nil, + nalUnitHeaderLengthOut: nil + ) + guard let pointer else { continue } + TBMonitorProtocol.appendBE32(&payload, UInt32(size)) + payload.append(UnsafeBufferPointer(start: pointer, count: size)) + } + return TBMonitorProtocol.makePacket(type: .paramSets, payload: payload) + } else { + var count = 0 + CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + format, + parameterSetIndex: 0, + parameterSetPointerOut: nil, + parameterSetSizeOut: nil, + parameterSetCountOut: &count, + nalUnitHeaderLengthOut: nil + ) + guard count > 0 else { return nil } + + var payload = Data([1, UInt8(count)]) + for index in 0..? + var size = 0 + CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + format, + parameterSetIndex: index, + parameterSetPointerOut: &pointer, + parameterSetSizeOut: &size, + parameterSetCountOut: nil, + nalUnitHeaderLengthOut: nil + ) + guard let pointer else { continue } + TBMonitorProtocol.appendBE32(&payload, UInt32(size)) + payload.append(UnsafeBufferPointer(start: pointer, count: size)) + } + return TBMonitorProtocol.makePacket(type: .paramSets, payload: payload) + } + } + + private func buildFramePacket(from sampleBuffer: CMSampleBuffer) -> Data? { + guard let blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else { return nil } + let totalLength = CMBlockBufferGetDataLength(blockBuffer) + guard totalLength > 0 else { return nil } + + var payload = Data(count: totalLength) + let status = payload.withUnsafeMutableBytes { rawBuffer in + guard let baseAddress = rawBuffer.baseAddress else { + return kCMBlockBufferBadCustomBlockSourceErr + } + return CMBlockBufferCopyDataBytes( + blockBuffer, + atOffset: 0, + dataLength: totalLength, + destination: baseAddress + ) + } + guard status == kCMBlockBufferNoErr else { return nil } + return TBMonitorProtocol.makePacket(type: .frame, payload: payload) + } +} + +/// Live, frequently-updating session readouts (currently just the FPS counter), +/// split out of `TBDisplaySenderSession` so their ~1 Hz changes only invalidate +/// the small subview that displays them rather than the whole session card. +@MainActor +final class TBSessionLiveMetrics: ObservableObject { + @Published var senderFPS = 0 +} + @MainActor final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @unchecked Sendable { private static let receiverIPDefaultsKey = "fd.tbdisplaysender.receiverIP" @@ -445,7 +802,10 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u UserDefaults.standard.set(receiverIP, forKey: Self.receiverIPDefaultsKey) } } - @Published var senderFPS = 0 + // Live FPS readout. Kept on a dedicated observable so its once-per-second + // update only re-renders the small FPS subview — not the whole session card + // or (via the manager's objectWillChange bubble-up) the entire window. + let liveMetrics = TBSessionLiveMetrics() @Published var receiverPanelText: String @Published var virtualDisplayText: String @Published var captureDisplayText = "Capture display: n/a" @@ -493,10 +853,8 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u private var captureDelegate: CaptureDelegate? private var scStream: SCStream? private var directDisplayStream: TBDirectDisplayStreamCapture? - private var vtEncoder: VTCompressionSession? - private var vtEncoderRef: Unmanaged? + private var pipeline: TBVideoPipeline? - private var sentFrames = 0 private var sentSnapshot = 0 private var sessionAckSent = false private var fpsTimer: Timer? @@ -506,9 +864,6 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u private var heartbeatSequence: UInt64 = 0 private var statusState: TBDisplaySenderStatusState = .ready private var streamingActivity: NSObjectProtocol? - private var pendingVideoPackets = 0 - private var inFlightEncodeFrames = 0 - private var displayStreamFrameSequence: CMTimeValue = 0 private var baselineDisplayIDs = Set() private var cursorDisplayID: CGDirectDisplayID = kCGNullDirectDisplay private var lastCursorPacket: TBMonitorCursor? @@ -516,9 +871,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u private var isRestartingCaptureAfterWake = false nonisolated(unsafe) private var displayReconfigurationCallbackRegistered = false private var verboseLoggingTimer: Timer? - private var lastCaptureFrameAt: Date = Date() private var captureHealthWatchdog: Timer? - private var lastEncodedDisplayPTS: CMTime? nonisolated(unsafe) private static let displayReconfigurationCallback: CGDisplayReconfigurationCallBack = { displayID, flags, userInfo in guard let userInfo else { return } @@ -813,10 +1166,8 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u ProcessInfo.processInfo.endActivity(activity) streamingActivity = nil } - if let encoder = vtEncoder { VTCompressionSessionInvalidate(encoder) } - vtEncoder = nil - vtEncoderRef?.release() - vtEncoderRef = nil + pipeline?.stop() + pipeline = nil connection?.stateUpdateHandler = nil connection?.cancel() connection = nil @@ -833,14 +1184,9 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u setStatus(status) } refreshLocalizedText() - senderFPS = 0 - sentFrames = 0 + liveMetrics.senderFPS = 0 sentSnapshot = 0 sessionAckSent = false - pendingVideoPackets = 0 - inFlightEncodeFrames = 0 - displayStreamFrameSequence = 0 - lastEncodedDisplayPTS = nil baselineDisplayIDs = [] cursorDisplayID = kCGNullDirectDisplay lastCursorPacket = nil @@ -1053,6 +1399,24 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u private func startCapture(for profile: TBMonitorDisplayProfile) async -> Bool { do { let preset = capturePreset + guard let connection else { return false } + + // The encode/send pipeline runs entirely on its own serial queue, + // off the main thread, so SwiftUI layout can never stall frame + // delivery. Preset/dimensions/codec are immutable for a session + // (the pickers are disabled while streaming), so we capture them once. + let pipeline = TBVideoPipeline( + preset: preset, + connection: connection, + displayName: session.displayName, + displayID: session.displayID, + ackAlreadySent: sessionAckSent, + onFirstFrame: { [weak self] in + Task { @MainActor in self?.handleFirstEncodedFrame() } + } + ) + guard pipeline.start() else { return false } + self.pipeline = pipeline if captureSource == .extendedDesktop, session.displayID != kCGNullDirectDisplay { if startDirectDisplayStream(displayID: session.displayID, preset: preset) { @@ -1075,18 +1439,11 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u configuration.scalesToFit = true configuration.captureResolution = preset.captureResolution - setupEncoder( - width: preset.width, - height: preset.height, - preset: preset, - codecType: preset.codecType, - averageBitRate: preset.averageBitRate - ) streamResolutionText = TBDisplaySenderL10n.streamSummary(preset: preset, source: captureSource, language: language) let delegate = CaptureDelegate() - delegate.onFrame = { [weak self] sampleBuffer in - self?.encode(sampleBuffer) + delegate.onFrame = { sampleBuffer in + pipeline.queue.async { pipeline.encode(sampleBuffer) } } delegate.onError = { [weak self] error in Task { @MainActor [weak self] in @@ -1127,20 +1484,13 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u } private func startDirectDisplayStream(displayID: CGDirectDisplayID, preset: TBDisplayCapturePreset) -> Bool { - setupEncoder( - width: preset.width, - height: preset.height, - preset: preset, - codecType: preset.codecType, - averageBitRate: preset.averageBitRate - ) - guard vtEncoder != nil else { return false } + guard let pipeline else { return false } - displayStreamFrameSequence = 0 - lastEncodedDisplayPTS = nil streamResolutionText = TBDisplaySenderL10n.streamSummary(preset: preset, source: captureSource, language: language) - let directCapture = TBDirectDisplayStreamCapture(service: self, queue: connectionQueue) + // Deliver frames straight onto the pipeline's own queue — the handler + // runs there, so encode happens off the main thread with no extra hop. + let directCapture = TBDirectDisplayStreamCapture(pipeline: pipeline, queue: pipeline.queue) guard directCapture.start(displayID: displayID, preset: preset, showCursor: !largeCursor) else { return false } @@ -1380,268 +1730,6 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u return Array(displays.prefix(Int(count))) } - private func setupEncoder(width: Int, height: Int, preset: TBDisplayCapturePreset, codecType: CMVideoCodecType, averageBitRate: Int) { - if let encoder = vtEncoder { VTCompressionSessionInvalidate(encoder) } - vtEncoder = nil - vtEncoderRef?.release() - vtEncoderRef = nil - - let spec: NSDictionary = [ - kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder: true, - kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder: true - ] - let retained = Unmanaged.passRetained(self) - vtEncoderRef = retained - - let callback: VTCompressionOutputCallback = { ref, _, status, _, sampleBuffer in - guard let ref else { return } - let service = Unmanaged.fromOpaque(ref).takeUnretainedValue() - DispatchQueue.main.async { - service.inFlightEncodeFrames = max(0, service.inFlightEncodeFrames - 1) - guard status == noErr, let sampleBuffer else { return } - service.handleEncoded(sampleBuffer) - } - } - - var session: VTCompressionSession? - guard VTCompressionSessionCreate( - allocator: nil, - width: Int32(width), - height: Int32(height), - codecType: codecType, - encoderSpecification: spec, - imageBufferAttributes: nil, - compressedDataAllocator: nil, - outputCallback: callback, - refcon: retained.toOpaque(), - compressionSessionOut: &session - ) == noErr, let session else { - retained.release() - vtEncoderRef = nil - return - } - - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue) - if codecType == kCMVideoCodecType_HEVC { - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_HEVC_Main_AutoLevel) - } else { - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_H264_Main_AutoLevel) - } - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse) - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate, value: NSNumber(value: preset.expectedFrameRate)) - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval, value: NSNumber(value: preset.maxKeyFrameInterval)) - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, value: NSNumber(value: preset.maxKeyFrameIntervalDuration)) - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxFrameDelayCount, value: NSNumber(value: preset.maxFrameDelayCount)) - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate, value: NSNumber(value: averageBitRate)) - if preset.prioritizeSpeed { - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, value: kCFBooleanTrue) - } - VTCompressionSessionPrepareToEncodeFrames(session) - vtEncoder = session - } - - private func encode(_ sampleBuffer: CMSampleBuffer) { - lastCaptureFrameAt = Date() - guard let encoder = vtEncoder, - let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) - else { return } - if capturePreset.dropsBeforeEncodeWhenBacklogged, - (pendingVideoPackets >= capturePreset.maxPendingVideoPackets || - inFlightEncodeFrames >= capturePreset.maxInFlightEncodeFrames) { - return - } - let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - encode(pixelBuffer: pixelBuffer, presentationTimeStamp: pts, using: encoder) - } - - fileprivate func encodeDisplaySurface(_ surface: IOSurface, displayTime: UInt64) { - lastCaptureFrameAt = Date() - guard let encoder = vtEncoder else { return } - if capturePreset.dropsBeforeEncodeWhenBacklogged, - (pendingVideoPackets >= capturePreset.maxPendingVideoPackets || - inFlightEncodeFrames >= capturePreset.maxInFlightEncodeFrames) { - return - } - - let attrs: NSDictionary = [ - kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA, - kCVPixelBufferWidthKey: capturePreset.width, - kCVPixelBufferHeightKey: capturePreset.height, - kCVPixelBufferIOSurfacePropertiesKey: NSDictionary() - ] - var unmanagedPixelBuffer: Unmanaged? - guard CVPixelBufferCreateWithIOSurface( - kCFAllocatorDefault, - surface, - attrs, - &unmanagedPixelBuffer - ) == kCVReturnSuccess, let unmanagedPixelBuffer else { - return - } - let pixelBuffer = unmanagedPixelBuffer.takeRetainedValue() - - displayStreamFrameSequence += 1 - // Derive PTS from the frame's actual capture time. CGDisplayStream - // delivers frames irregularly (event-driven on screen changes), so the - // previous frame-counter PTS — which assumed an exact nominal frame - // rate — drifted away from real wall-clock time and the drift compounded - // over long sessions, pacing the receiver progressively wrong. displayTime - // is in mach-absolute units, same host clock the SCStream path already uses. - var pts = displayTime != 0 - ? CMClockMakeHostTimeFromSystemUnits(displayTime) - : CMClockGetTime(CMClockGetHostTimeClock()) - if let last = lastEncodedDisplayPTS, CMTimeCompare(pts, last) <= 0 { - // VTCompressionSession requires strictly increasing PTS. - pts = CMTimeAdd(last, CMTime(value: 1, timescale: 600)) - } - lastEncodedDisplayPTS = pts - encode(pixelBuffer: pixelBuffer, presentationTimeStamp: pts, using: encoder) - } - - private func encode(pixelBuffer: CVPixelBuffer, presentationTimeStamp pts: CMTime, using encoder: VTCompressionSession) { - inFlightEncodeFrames += 1 - let status = VTCompressionSessionEncodeFrame( - encoder, - imageBuffer: pixelBuffer, - presentationTimeStamp: pts, - duration: .invalid, - frameProperties: nil, - sourceFrameRefcon: nil, - infoFlagsOut: nil - ) - if status != noErr { - inFlightEncodeFrames = max(0, inFlightEncodeFrames - 1) - } - } - - private func handleEncoded(_ sampleBuffer: CMSampleBuffer) { - guard let connection, isConnected else { return } - - if !sessionAckSent { - sessionAckSent = true - firstFrameTimer?.invalidate() - firstFrameTimer = nil - let ack = TBMonitorCreateSessionAck( - accepted: true, - displayName: session.displayName, - displayID: session.displayID - ) - if let packet = TBMonitorProtocol.makeJSONPacket(type: .createSessionAck, value: ack) { - send(packet) - } - setStatus(.captureActive(capturePreset.description, capturePreset.codecName, captureSource)) - } - - let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[CFString: Any]] - let notSync = attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool ?? false - let isKeyframe = !notSync - - if !isKeyframe, pendingVideoPackets >= capturePreset.maxPendingVideoPackets { - return - } - - if isKeyframe, - let format = CMSampleBufferGetFormatDescription(sampleBuffer), - let packet = buildParamSetsPacket(from: format, codecType: capturePreset.codecType) { - send(packet) - } - - if let packet = buildFramePacket(from: sampleBuffer) { - pendingVideoPackets += 1 - connection.send(content: packet, completion: .contentProcessed({ [weak self] _ in - guard let self else { return } - Task { @MainActor [weak self] in - guard let self else { return } - pendingVideoPackets = max(0, pendingVideoPackets - 1) - } - })) - sentFrames += 1 - } - } - - private func buildParamSetsPacket(from format: CMVideoFormatDescription, codecType: CMVideoCodecType) -> Data? { - if codecType == kCMVideoCodecType_HEVC { - var count = 0 - CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( - format, - parameterSetIndex: 0, - parameterSetPointerOut: nil, - parameterSetSizeOut: nil, - parameterSetCountOut: &count, - nalUnitHeaderLengthOut: nil - ) - guard count > 0 else { return nil } - - var payload = Data([2, UInt8(count)]) - for index in 0..? - var size = 0 - CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( - format, - parameterSetIndex: index, - parameterSetPointerOut: &pointer, - parameterSetSizeOut: &size, - parameterSetCountOut: nil, - nalUnitHeaderLengthOut: nil - ) - guard let pointer else { continue } - TBMonitorProtocol.appendBE32(&payload, UInt32(size)) - payload.append(UnsafeBufferPointer(start: pointer, count: size)) - } - return TBMonitorProtocol.makePacket(type: .paramSets, payload: payload) - } else { - var count = 0 - CMVideoFormatDescriptionGetH264ParameterSetAtIndex( - format, - parameterSetIndex: 0, - parameterSetPointerOut: nil, - parameterSetSizeOut: nil, - parameterSetCountOut: &count, - nalUnitHeaderLengthOut: nil - ) - guard count > 0 else { return nil } - - var payload = Data([1, UInt8(count)]) - for index in 0..? - var size = 0 - CMVideoFormatDescriptionGetH264ParameterSetAtIndex( - format, - parameterSetIndex: index, - parameterSetPointerOut: &pointer, - parameterSetSizeOut: &size, - parameterSetCountOut: nil, - nalUnitHeaderLengthOut: nil - ) - guard let pointer else { continue } - TBMonitorProtocol.appendBE32(&payload, UInt32(size)) - payload.append(UnsafeBufferPointer(start: pointer, count: size)) - } - return TBMonitorProtocol.makePacket(type: .paramSets, payload: payload) - } - } - - private func buildFramePacket(from sampleBuffer: CMSampleBuffer) -> Data? { - guard let blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else { return nil } - let totalLength = CMBlockBufferGetDataLength(blockBuffer) - guard totalLength > 0 else { return nil } - - var payload = Data(count: totalLength) - let status = payload.withUnsafeMutableBytes { rawBuffer in - guard let baseAddress = rawBuffer.baseAddress else { - return kCMBlockBufferBadCustomBlockSourceErr - } - return CMBlockBufferCopyDataBytes( - blockBuffer, - atOffset: 0, - dataLength: totalLength, - destination: baseAddress - ) - } - guard status == kCMBlockBufferNoErr else { return nil } - return TBMonitorProtocol.makePacket(type: .frame, payload: payload) - } - private func startCursorUpdates(displayID: CGDirectDisplayID) { cursorTimer?.invalidate() cursorDisplayID = displayID @@ -1816,7 +1904,6 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u private func startCaptureWatchdog() { captureHealthWatchdog?.invalidate() - lastCaptureFrameAt = Date() captureHealthWatchdog = Timer.scheduledTimer(withTimeInterval: 5.0, repeats: true) { [weak self] _ in Task { @MainActor [weak self] in self?.checkCaptureHealth() @@ -1830,8 +1917,8 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u } private func checkCaptureHealth() { - guard isStreaming, activeProfile != nil, !isRestartingCaptureAfterWake else { return } - let elapsed = Date().timeIntervalSince(lastCaptureFrameAt) + guard isStreaming, activeProfile != nil, !isRestartingCaptureAfterWake, let pipeline else { return } + let elapsed = Date().timeIntervalSince(pipeline.lastCaptureFrameAtSnapshot) guard elapsed >= 8.0 else { return } NSLog("TargetBridge: capture watchdog tripped — %.1fs since last frame, soft restart", elapsed) scheduleCaptureRestart(reason: "watchdog (\(Int(elapsed))s without frames)", delaySeconds: 0.5) @@ -1841,15 +1928,16 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u guard verboseDisplayLogging else { return } let online = onlineDisplayIDs() let virtualOnline = online.contains(session.displayID) + let diag = pipeline?.diagnosticsSnapshot() ?? (pending: 0, inFlight: 0, ptsSeq: 0) NSLog( "TargetBridge: stream snapshot streaming=%@ fps=%d virtualID=%u online=%@ pendingPackets=%d inFlightEncode=%d ptsSeq=%lld", isStreaming ? "yes" : "no", - senderFPS, + liveMetrics.senderFPS, session.displayID, virtualOnline ? "yes" : "no", - pendingVideoPackets, - inFlightEncodeFrames, - displayStreamFrameSequence + diag.pending, + diag.inFlight, + diag.ptsSeq ) } @@ -1907,17 +1995,11 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u ProcessInfo.processInfo.endActivity(activity) streamingActivity = nil } - if let encoder = vtEncoder { VTCompressionSessionInvalidate(encoder) } - vtEncoder = nil - vtEncoderRef?.release() - vtEncoderRef = nil + pipeline?.stop() + pipeline = nil isStreaming = false - displayStreamFrameSequence = 0 - lastEncodedDisplayPTS = nil - senderFPS = 0 - sentSnapshot = sentFrames - pendingVideoPackets = 0 - inFlightEncodeFrames = 0 + liveMetrics.senderFPS = 0 + sentSnapshot = 0 cursorDisplayID = kCGNullDirectDisplay lastCursorPacket = nil @@ -1928,14 +2010,23 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u } } + private func handleFirstEncodedFrame() { + guard !sessionAckSent else { return } + sessionAckSent = true + firstFrameTimer?.invalidate() + firstFrameTimer = nil + setStatus(.captureActive(capturePreset.description, capturePreset.codecName, captureSource)) + } + private func startFPSTimer() { fpsTimer?.invalidate() - sentSnapshot = sentFrames + sentSnapshot = pipeline?.sentFramesSnapshot ?? 0 fpsTimer = Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { [weak self] _ in guard let self else { return } MainActor.assumeIsolated { - senderFPS = sentFrames - sentSnapshot - sentSnapshot = sentFrames + let total = pipeline?.sentFramesSnapshot ?? 0 + liveMetrics.senderFPS = total - sentSnapshot + sentSnapshot = total } } } From f9d41c73bb9e15257d78d7339ca1ca6ffe60f619 Mon Sep 17 00:00:00 2001 From: David Liebovitz Date: Sun, 31 May 2026 00:22:13 -0500 Subject: [PATCH 6/6] Sender: fix CGDisplayStream teardown use-after-free in the pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crash report (24h session, EXC_BAD_ACCESS at 0x10) faulted on the fd.tbmonitor.sender.pipeline queue inside SkyLight's _CGYDisplayStreamFrameAvailable, while the main thread was concurrently in softRestartCapture(for:) tearing the pipeline down. The screensaver's autoRestartOnWake fires a soft restart on every display-sleep/wake, so the teardown path ran many times over the session and eventually raced a frame callback. Two lifetime bugs in TBDirectDisplayStreamCapture, both introduced when the encode pipeline moved off the main thread: 1) stop() called CGDisplayStream.stop() then immediately released the stream (`stream = nil`). CGDisplayStreamStop is asynchronous — frames already in flight keep arriving until the stream delivers a final `.stopped` frame — so releasing it early let a queued frame event fire into a freed stream, crashing inside SkyLight. 2) The frame handler dereferenced the TBVideoPipeline via an *unretained* pointer, while softRestartCapture/stop set `pipeline = nil` concurrently — a second use-after-free window. Fix: - Hold the pipeline with a strong reference so it (and its delivery queue) outlives every frame callback; combined with the existing `running` guard in encodeDisplaySurface, a late in-flight frame after pipeline.stop() simply no-ops. - Keep the capture object and its CGDisplayStream alive from stop() until the stream delivers `.stopped` (a self-retain released in the handler), honoring CGDisplayStream's documented async-stop contract so the stream is never freed with frame events still queued. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../TBDisplaySenderBuildInfo.swift | 2 +- .../TBDisplaySenderService.swift | 40 ++++++++++++++----- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift index 219ecbf..f05c437 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderBuildInfo.swift @@ -1,5 +1,5 @@ enum TBDisplaySenderBuildInfo { static let marketingVersion = "2.0" - static let buildNumber = "20260529215219" + static let buildNumber = "20260531002036" static let versionDisplay = "\(marketingVersion) + build \(buildNumber)" } diff --git a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift index d58fd58..150f661 100644 --- a/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift +++ b/TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift @@ -249,12 +249,20 @@ enum TBDisplayCaptureSource: String, CaseIterable, Identifiable { } private final class TBDirectDisplayStreamCapture { - private let pipelineRefValue: UInt + // Strong reference so the pipeline (and its delivery queue) outlives every + // frame callback — a stray frame must never deref a freed pipeline. + private let pipeline: TBVideoPipeline private let queue: DispatchQueue private var stream: CGDisplayStream? + // CGDisplayStreamStop is asynchronous: frames already in flight keep arriving + // until the stream delivers a final `.stopped` frame, and releasing the + // CGDisplayStream before then crashes inside SkyLight's + // `_CGYDisplayStreamFrameAvailable`. This self-reference keeps the capture + // object (and the stream) alive from stop() until that `.stopped` frame. + private var pendingStopRetain: TBDirectDisplayStreamCapture? init(pipeline: TBVideoPipeline, queue: DispatchQueue) { - self.pipelineRefValue = UInt(bitPattern: Unmanaged.passUnretained(pipeline).toOpaque()) + self.pipeline = pipeline self.queue = queue } @@ -265,7 +273,6 @@ private final class TBDirectDisplayStreamCapture { CGDisplayStream.minimumFrameTime: 1.0 / Double(preset.expectedFrameRate) ] - let pipelineRefValue = self.pipelineRefValue let displayStream = CGDisplayStream( dispatchQueueDisplay: displayID, outputWidth: preset.width, @@ -273,14 +280,21 @@ private final class TBDirectDisplayStreamCapture { pixelFormat: Int32(kCVPixelFormatType_32BGRA), properties: properties, queue: queue - ) { status, displayTime, surface, _ in - // This handler is delivered on `queue`, which is the pipeline's own - // serial queue — so we run the encode synchronously here, off the - // main thread, with no extra hop. + ) { [weak self] status, displayTime, surface, _ in + // Delivered on `queue` — the pipeline's own serial queue — so encode + // runs here, off the main thread, with no extra hop. + guard let self else { return } + if status == .stopped { + // The stream has fully drained; no further frames will arrive, so + // it is now safe to release the stream and drop the self-retain. + self.stream = nil + self.pendingStopRetain = nil + return + } guard status == .frameComplete, let surface else { return } - guard let pipelineRef = UnsafeRawPointer(bitPattern: pipelineRefValue) else { return } - let pipeline = Unmanaged.fromOpaque(pipelineRef).takeUnretainedValue() - pipeline.encodeDisplaySurface(surface, displayTime: displayTime) + // After pipeline.stop(), encodeDisplaySurface() no-ops on its `running` + // guard, so a late in-flight frame here is harmless. + self.pipeline.encodeDisplaySurface(surface, displayTime: displayTime) } guard let displayStream, displayStream.start() == .success else { @@ -292,8 +306,12 @@ private final class TBDirectDisplayStreamCapture { } func stop() { + guard stream != nil else { return } + // Stay alive until the `.stopped` frame arrives (see pendingStopRetain); + // the stream is released in the handler, never here, so it is never freed + // with frame events still queued on `queue`. + pendingStopRetain = self stream?.stop() - stream = nil } }