diff --git a/.gitignore b/.gitignore index b3a0594..e01123f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ Windows/test-output/ # Linux client Linux/build/ LinuxHost/build/ + +# Mac client +dist/ diff --git a/Client/ConnectView.swift b/Client/ConnectView.swift index a6c44a6..55b74fa 100644 --- a/Client/ConnectView.swift +++ b/Client/ConnectView.swift @@ -4,12 +4,6 @@ import SwiftUI // Visual language shared with Pinry Saver: one centered column (max 480 pt) on the system // background, a big mark up top, labeled rounded fields, and a single pill-shaped action. -extension Color { - static let iframeTeal = Color(red: 0x2E / 255, green: 0xE6 / 255, blue: 0xD6 / 255) // #2EE6D6 - /// Text on teal: the accent is light, so dark text reads better than white. - static let iframeInk = Color(red: 0x04 / 255, green: 0x16 / 255, blue: 0x1A / 255) -} - struct ConnectView: View { @EnvironmentObject private var session: StreamSession @StateObject private var browser = HostBrowser() @@ -17,6 +11,8 @@ struct ConnectView: View { @AppStorage("manualHost") private var manualHost = "" @AppStorage("uiScale") private var uiScale = 2.0 @AppStorage("lastHostName") private var lastHostName = "" + @AppStorage("savedHosts") private var savedHostsJSON = "[]" + @State private var pendingSave: SavedHost? @State private var selectedHostID: String? @State private var showingError = false @State private var errorMessage = "" @@ -27,6 +23,13 @@ struct ConnectView: View { private var trimmedAddress: String { manualHost.trimmingCharacters(in: .whitespaces) } + private var savedHosts: [SavedHost] { + get { (try? JSONDecoder().decode([SavedHost].self, from: Data(savedHostsJSON.utf8))) ?? [] } + nonmutating set { + savedHostsJSON = (try? String(decoding: JSONEncoder().encode(newValue), as: UTF8.self)) ?? "[]" + } + } + private var targetName: String? { if !trimmedAddress.isEmpty { return trimmedAddress } return selectedHost?.name @@ -69,6 +72,11 @@ struct ConnectView: View { selectedHostID = (hosts.first { $0.name == lastHostName } ?? (hosts.count == 1 ? hosts.first : nil))?.id } .onChange(of: session.phase) { _, phase in + // Only bookmark addresses that actually worked, most recent first. + if phase == .streaming, let saved = pendingSave { + savedHosts = [saved] + savedHosts.filter { $0.address != saved.address } + } + if phase != .connecting { pendingSave = nil } if case .failed(let message) = phase { errorMessage = message showingError = true @@ -105,7 +113,7 @@ struct ConnectView: View { VStack(alignment: .leading, spacing: 24) { VStack(alignment: .leading, spacing: 8) { FieldLabel("Mac:") - if browser.hosts.isEmpty { + if browser.hosts.isEmpty && savedHosts.isEmpty { HStack(spacing: 10) { ProgressView() Text("Looking for Macs running iframe-host…") @@ -123,6 +131,19 @@ struct ConnectView: View { manualHost = "" } } + ForEach(savedHosts) { saved in + HostRow(name: saved.address, symbol: "bookmark", + selected: saved.address == trimmedAddress) { + manualHost = saved.address + pin = saved.pin + } + .contextMenu { + Button("Forget \(saved.address)", systemImage: "trash", role: .destructive) { + savedHosts = savedHosts.filter { $0.address != saved.address } + if trimmedAddress == saved.address { manualHost = "" } + } + } + } } } } @@ -220,15 +241,9 @@ struct ConnectView: View { private func connect() { guard canConnect else { return } if !trimmedAddress.isEmpty { - var host = trimmedAddress - var port = IFrame.defaultPort - if let colon = host.lastIndex(of: ":"), !host.contains("::"), - let p = UInt16(host[host.index(after: colon)...]) { - port = p - host = String(host[.. Void var body: some View { Button(action: action) { HStack(spacing: 12) { - Image(systemName: "macmini") + Image(systemName: symbol) .font(.system(size: 17)) .foregroundColor(selected ? .iframeTeal : Color(uiColor: .secondaryLabel)) Text(name) @@ -308,33 +324,3 @@ private extension View { ) } } - -/// The cursor mark streaking up and to the right, with a short motion trail -/// (iFrame's take on Pinry's falling-pin loader). -struct IFrameLoadingView: View { - let size: CGFloat - @State private var progress: CGFloat = -1 - - var body: some View { - ZStack { - ForEach(0..<3, id: \.self) { index in - Image("IFrameMark") - .resizable() - .aspectRatio(contentMode: .fit) - .frame(width: size * 0.7, height: size * 0.7) - .opacity(1.0 - CGFloat(index) * 0.3) - .offset(x: (progress - CGFloat(index) * 0.18) * size, - y: -(progress - CGFloat(index) * 0.18) * size) - } - } - .frame(width: size, height: size) - .clipped() - .drawingGroup() - .onAppear { - progress = -1 - withAnimation(.easeInOut(duration: 0.6).repeatForever(autoreverses: false)) { - progress = 1 - } - } - } -} diff --git a/Client/IFrameApp.swift b/Client/IFrameApp.swift index a090ec1..f28dad4 100644 --- a/Client/IFrameApp.swift +++ b/Client/IFrameApp.swift @@ -1,5 +1,6 @@ import Network import SwiftUI +import UIKit @main struct IFrameApp: App { @@ -24,32 +25,15 @@ struct IFrameApp: App { } } -final class HostBrowser: ObservableObject { - struct Found: Identifiable, Hashable { - let id: String - let name: String - let endpoint: NWEndpoint +extension StreamSession { + /// The iPad's screen in pixels, in the current orientation. + static func screenPixels() -> CGSize { + let screen = UIScreen.main + return CGSize(width: screen.bounds.width * screen.scale, height: screen.bounds.height * screen.scale) } - @Published private(set) var hosts: [Found] = [] - private var browser: NWBrowser? - - func start() { - guard browser == nil else { return } - let browser = NWBrowser(for: .bonjour(type: IFrame.serviceType, domain: nil), using: .tcp) - browser.browseResultsChangedHandler = { [weak self] results, _ in - let found = results.compactMap { result -> Found? in - guard case let .service(name, _, _, _) = result.endpoint else { return nil } - return Found(id: "\(result.endpoint)", name: name, endpoint: result.endpoint) - } - self?.hosts = found.sorted { $0.name < $1.name } - } - browser.start(queue: .main) - self.browser = browser - } - - func stop() { - browser?.cancel() - browser = nil + func connect(to endpoint: NWEndpoint, pin: String, label: String, uiScale: Double) { + connect(to: endpoint, pin: pin, label: label, deviceName: UIDevice.current.name, + maxFPS: UIScreen.main.maximumFramesPerSecond, pixels: Self.screenPixels(), uiScale: uiScale) } } diff --git a/Client/StreamView.swift b/Client/StreamView.swift index f599e70..8d2391f 100644 --- a/Client/StreamView.swift +++ b/Client/StreamView.swift @@ -9,12 +9,18 @@ import UIKit /// scrolling map 1:1 to the Mac. Touch: tap = click, two-finger tap = right click, /// drag = move pointer, long-press then drag = click-drag, two-finger drag = scroll, /// three-finger tap = toolbar. +/// +/// Fingers work in one of two modes. Direct: the pointer jumps to your finger. Trackpad (like +/// Microsoft Remote Desktop's mouse mode): drags move the pointer relatively with acceleration, +/// a flick lets it glide to a stop, and taps click wherever the pointer is. Pencil is always direct. final class StreamUIView: UIView, UIPointerInteractionDelegate { override class var layerClass: AnyClass { AVSampleBufferDisplayLayer.self } private var displayLayer: AVSampleBufferDisplayLayer { layer as! AVSampleBufferDisplayLayer } let session: StreamSession - var welcome: Welcome? + var welcome: Welcome? { + didSet { if welcome != oldValue { layoutCursor() } } + } var onThreeFingerTap: (() -> Void)? private let keyboardProxy = KeyboardProxy() @@ -27,6 +33,19 @@ final class StreamUIView: UIView, UIPointerInteractionDelegate { private var focusRect: CGRect? // normalized, while a Mac text field is focused private var keyboardTop: CGFloat? // in view coordinates, while the on-screen keyboard is up + var trackpadMode = true + private var cursor = CGPoint(x: 0.5, y: 0.5) { // normalized, where we last put the Mac pointer + didSet { positionCursor() } + } + /// The Mac's pointer, drawn here so it moves the instant you do; capture leaves it out of the video. + private let cursorLayer = CALayer() + private var macCursor: MacCursor? + private var touchIsPencil = false + private var lastDragLocation: CGPoint? + private var lastDragTime: CFTimeInterval = 0 + private var glideVelocity = CGPoint.zero // view points per second + private var glideLink: CADisplayLink? + init(session: StreamSession) { self.session = session super.init(frame: .zero) @@ -34,11 +53,15 @@ final class StreamUIView: UIView, UIPointerInteractionDelegate { isMultipleTouchEnabled = true displayLayer.videoGravity = .resizeAspect addInteraction(UIPointerInteraction(delegate: self)) + cursorLayer.isHidden = true // until the Mac sends a shape (older hosts draw it in the video) + cursorLayer.zPosition = 1 + layer.addSublayer(cursorLayer) keyboardProxy.session = session addSubview(keyboardProxy) setUpGestures() session.attach(renderer: displayLayer.sampleBufferRenderer) session.onTextFocus = { [weak self] editable, rect in self?.textFocusChanged(editable, rect) } + session.onCursor = { [weak self] cursor in self?.cursorChanged(cursor) } NotificationCenter.default.addObserver(self, selector: #selector(keyboardFrameChanged(_:)), name: UIResponder.keyboardWillChangeFrameNotification, object: nil) NotificationCenter.default.addObserver(self, selector: #selector(keyboardWillHide(_:)), @@ -102,6 +125,7 @@ final class StreamUIView: UIView, UIPointerInteractionDelegate { DispatchQueue.main.async { self.becomeFirstResponder() } } else { releaseKeys() + stopGlide() // the display link retains us } } @@ -112,6 +136,7 @@ final class StreamUIView: UIView, UIPointerInteractionDelegate { /// When the iPad rotates, reshape the Mac's virtual display to match instead of letterboxing. override func layoutSubviews() { super.layoutSubviews() + layoutCursor() guard let welcome, welcome.isVirtual, bounds.width > 0, bounds.height > 0 else { return } let viewIsLandscape = bounds.width > bounds.height let streamIsLandscape = welcome.width > welcome.height @@ -190,7 +215,113 @@ final class StreamUIView: UIView, UIPointerInteractionDelegate { @objc private func handleHover(_ g: UIHoverGestureRecognizer) { guard g.state == .began || g.state == .changed else { return } - session.mouseMove(normalized(g.location(in: self))) + moveCursor(to: normalized(g.location(in: self))) + } + + // MARK: Local cursor + + private func cursorChanged(_ cursor: MacCursor) { + macCursor = cursor + CATransaction.begin() + CATransaction.setDisableActions(true) + cursorLayer.contents = cursor.image + cursorLayer.contentsScale = 2 + cursorLayer.isHidden = false + CATransaction.commit() + layoutCursor() + } + + /// Sizes the pointer like it would be on the Mac: Mac points scaled to the video on screen. + private func layoutCursor() { + guard let macCursor, let welcome, welcome.pointWidth > 0 else { return } + let scale = videoRect.width / CGFloat(welcome.pointWidth) // view points per Mac point + CATransaction.begin() + CATransaction.setDisableActions(true) + cursorLayer.bounds = CGRect(x: 0, y: 0, width: macCursor.size.width * scale, height: macCursor.size.height * scale) + cursorLayer.anchorPoint = CGPoint(x: macCursor.size.width > 0 ? macCursor.hotspot.x / macCursor.size.width : 0, + y: macCursor.size.height > 0 ? macCursor.hotspot.y / macCursor.size.height : 0) + CATransaction.commit() + positionCursor() + } + + private func positionCursor() { + guard macCursor != nil else { return } + let rect = videoRect + CATransaction.begin() + CATransaction.setDisableActions(true) + cursorLayer.position = CGPoint(x: rect.minX + cursor.x * rect.width, y: rect.minY + cursor.y * rect.height) + CATransaction.commit() + } + + /// Fingers in trackpad mode act on the pointer, not on the spot they touch. + private var relative: Bool { trackpadMode && !touchIsPencil } + + private func moveCursor(to p: CGPoint) { + cursor = p + session.mouseMove(p) + } + + /// Moves the pointer by a distance in view points. + private func moveCursor(by d: CGPoint) { + let rect = videoRect + guard rect.width > 0, rect.height > 0 else { return } + moveCursor(to: CGPoint(x: min(max(cursor.x + d.x / rect.width, 0), 1), + y: min(max(cursor.y + d.y / rect.height, 0), 1))) + } + + /// Pointer acceleration: slow drags stay precise, fast ones cover the screen. + private func gain(forSpeed speed: CGFloat) -> CGFloat { + let t = min(max((speed - 80) / 1400, 0), 1) + return 1 + 2.2 * t * t * (3 - 2 * t) + } + + private func clickTarget(_ g: UIGestureRecognizer) -> CGPoint { + relative ? cursor : normalized(g.location(in: self)) + } + + // MARK: Flick glide + + private func startGlide(_ velocity: CGPoint) { + let speed = hypot(velocity.x, velocity.y) + guard speed > 350 else { return } + let g = gain(forSpeed: speed) + glideVelocity = CGPoint(x: velocity.x * g, y: velocity.y * g) + if glideLink == nil { + let link = CADisplayLink(target: self, selector: #selector(glideStep(_:))) + link.preferredFrameRateRange = CAFrameRateRange(minimum: 60, maximum: 120, preferred: 120) + link.add(to: .main, forMode: .common) + glideLink = link + } + } + + @objc private func glideStep(_ link: CADisplayLink) { + let dt = CGFloat(min(link.targetTimestamp - link.timestamp, 1.0 / 30)) + moveCursor(by: CGPoint(x: glideVelocity.x * dt, y: glideVelocity.y * dt)) + let decay = exp(-dt * 4.5) // ~0.22 s time constant: a quick, smooth ease-out + glideVelocity = CGPoint(x: glideVelocity.x * decay, y: glideVelocity.y * decay) + // Stop when it's crawling or pinned against an edge. + let pinnedX = (cursor.x <= 0 && glideVelocity.x < 0) || (cursor.x >= 1 && glideVelocity.x > 0) + let pinnedY = (cursor.y <= 0 && glideVelocity.y < 0) || (cursor.y >= 1 && glideVelocity.y > 0) + if pinnedX { glideVelocity.x = 0 } + if pinnedY { glideVelocity.y = 0 } + if hypot(glideVelocity.x, glideVelocity.y) < 25 { stopGlide() } + } + + private func stopGlide() { + glideLink?.invalidate() + glideLink = nil + glideVelocity = .zero + } + + /// Relative drag step with acceleration, shared by plain drags and click-drags. + private func relativeDrag(to location: CGPoint, began: Bool) { + let now = CACurrentMediaTime() + defer { lastDragLocation = location; lastDragTime = now } + guard !began, let last = lastDragLocation else { return } + let d = CGPoint(x: location.x - last.x, y: location.y - last.y) + let dt = max(now - lastDragTime, 1.0 / 240) + let g = gain(forSpeed: hypot(d.x, d.y) / CGFloat(dt)) + moveCursor(by: CGPoint(x: d.x * g, y: d.y * g)) } @objc private func handleScroll(_ g: UIPanGestureRecognizer) { @@ -201,11 +332,15 @@ final class StreamUIView: UIView, UIPointerInteractionDelegate { } @objc private func handleTap(_ g: UITapGestureRecognizer) { - session.click(0, at: normalized(g.location(in: self))) + let p = clickTarget(g) + cursor = p + session.click(0, at: p) } @objc private func handleTwoFingerTap(_ g: UITapGestureRecognizer) { - session.click(1, at: normalized(g.location(in: self))) + let p = clickTarget(g) + cursor = p + session.click(1, at: p) } @objc private func handleThreeFingerTap(_ g: UITapGestureRecognizer) { @@ -213,39 +348,65 @@ final class StreamUIView: UIView, UIPointerInteractionDelegate { } @objc private func handleLongPress(_ g: UILongPressGestureRecognizer) { - let p = normalized(g.location(in: self)) + let location = g.location(in: self) switch g.state { case .began: - session.mouseMove(p) - session.mouseButton(0, down: true, at: p) + if relative { + relativeDrag(to: location, began: true) + } else { + moveCursor(to: normalized(location)) + } + session.mouseButton(0, down: true, at: cursor) case .changed: - session.mouseMove(p) + if relative { + relativeDrag(to: location, began: false) + } else { + moveCursor(to: normalized(location)) + } case .ended, .cancelled, .failed: - session.mouseButton(0, down: false, at: p) + session.mouseButton(0, down: false, at: cursor) default: break } } @objc private func handleDrag(_ g: UIPanGestureRecognizer) { - guard g.state == .began || g.state == .changed else { return } - session.mouseMove(normalized(g.location(in: self))) + let location = g.location(in: self) + switch g.state { + case .began, .changed: + if relative { + relativeDrag(to: location, began: g.state == .began) + } else { + moveCursor(to: normalized(location)) + } + case .ended: + if relative { startGlide(g.velocity(in: self)) } + default: + break + } } // MARK: Trackpad / mouse buttons (needs UIApplicationSupportsIndirectInputEvents) override func touchesBegan(_ touches: Set, with event: UIEvent?) { + if let touch = touches.first(where: { $0.type != .indirectPointer }) { + touchIsPencil = touch.type == .pencil + stopGlide() // touching down catches a gliding pointer, like a trackpad + } for touch in touches where touch.type == .indirectPointer { + stopGlide() let button: UInt8 = event?.buttonMask.contains(.secondary) == true ? 1 : 0 pointerButton = button - session.mouseButton(button, down: true, at: normalized(touch.location(in: self))) + let p = normalized(touch.location(in: self)) + cursor = p + session.mouseButton(button, down: true, at: p) } super.touchesBegan(touches, with: event) } override func touchesMoved(_ touches: Set, with event: UIEvent?) { for touch in touches where touch.type == .indirectPointer { - session.mouseMove(normalized(touch.location(in: self))) + moveCursor(to: normalized(touch.location(in: self))) } super.touchesMoved(touches, with: event) } @@ -267,7 +428,7 @@ final class StreamUIView: UIView, UIPointerInteractionDelegate { } } - /// Hide the iPad pointer; the Mac's own cursor is in the video. + /// Hide the iPad pointer; we draw the Mac's own cursor (or it's in the video). func pointerInteraction(_ interaction: UIPointerInteraction, styleFor region: UIPointerRegion) -> UIPointerStyle? { .hidden() } @@ -410,6 +571,7 @@ struct StreamViewRepresentable: UIViewRepresentable { let session: StreamSession let welcome: Welcome? let handle: StreamViewHandle + let trackpadMode: Bool let onThreeFingerTap: () -> Void func makeUIView(context: Context) -> StreamUIView { @@ -424,18 +586,21 @@ struct StreamViewRepresentable: UIViewRepresentable { view.setNeedsLayout() } view.onThreeFingerTap = onThreeFingerTap + view.trackpadMode = trackpadMode } } struct StreamScreen: View { @EnvironmentObject private var session: StreamSession @StateObject private var handle = StreamViewHandle() + @AppStorage("trackpadMode") private var trackpadMode = true @State private var showToolbar = true @State private var showStats = false var body: some View { ZStack(alignment: .top) { - StreamViewRepresentable(session: session, welcome: session.welcome, handle: handle) { + StreamViewRepresentable(session: session, welcome: session.welcome, handle: handle, + trackpadMode: trackpadMode) { withAnimation(.easeOut(duration: 0.15)) { showToolbar.toggle() } } .ignoresSafeArea() @@ -483,6 +648,8 @@ struct StreamScreen: View { private var toolbar: some View { HStack(spacing: 12) { ToolbarCircle(symbol: "keyboard") { handle.view?.toggleSoftwareKeyboard() } + // Lit = trackpad mode (relative pointer with flick glide); off = pointer follows finger. + ToolbarCircle(symbol: "rectangle.and.hand.point.up.left", active: trackpadMode) { trackpadMode.toggle() } ToolbarCircle(symbol: "gauge.with.dots.needle.50percent", active: showStats) { showStats.toggle() } ToolbarCircle(symbol: "xmark") { session.disconnect() } ToolbarCircle(symbol: "chevron.up") { @@ -509,28 +676,3 @@ private struct ToolbarCircle: View { .buttonStyle(.plain) } } - -struct StatsView: View { - let stats: ClientStats - let welcome: Welcome? - - var body: some View { - VStack(alignment: .leading, spacing: 2) { - if let w = welcome { - Text("\(w.hostName) · \(w.width)×\(w.height) \(w.codec.name) @ \(w.fps)") - } - if let h = stats.host { - Text(String(format: "%.0f fps · %.1f Mbps (target %.0f)", h.fps, h.mbps, h.targetMbps)) - Text(String(format: "encode %.1f ms · decode %.1f ms", h.encodeMs, stats.decodeMs)) - Text(String(format: "net RTT %.1f ms · frame ack %.1f ms", stats.rttMs, h.latencyMs)) - if h.dropped > 0 { Text("skipped \(h.dropped) frames (congestion)") } - } else { - Text("waiting for stats…") - } - } - .font(.system(size: 11, design: .monospaced)) - .foregroundStyle(.white) - .padding(8) - .background(.black.opacity(0.6), in: RoundedRectangle(cornerRadius: 8)) - } -} diff --git a/ClientShared/HostBrowser.swift b/ClientShared/HostBrowser.swift new file mode 100644 index 0000000..ddd2997 --- /dev/null +++ b/ClientShared/HostBrowser.swift @@ -0,0 +1,33 @@ +import Combine +import Network + +/// Finds Macs advertising iframe-host over Bonjour. +final class HostBrowser: ObservableObject { + struct Found: Identifiable, Hashable { + let id: String + let name: String + let endpoint: NWEndpoint + } + + @Published private(set) var hosts: [Found] = [] + private var browser: NWBrowser? + + func start() { + guard browser == nil else { return } + let browser = NWBrowser(for: .bonjour(type: IFrame.serviceType, domain: nil), using: .tcp) + browser.browseResultsChangedHandler = { [weak self] results, _ in + let found = results.compactMap { result -> Found? in + guard case let .service(name, _, _, _) = result.endpoint else { return nil } + return Found(id: "\(result.endpoint)", name: name, endpoint: result.endpoint) + } + self?.hosts = found.sorted { $0.name < $1.name } + } + browser.start(queue: .main) + self.browser = browser + } + + func stop() { + browser?.cancel() + browser = nil + } +} diff --git a/ClientShared/SavedHost.swift b/ClientShared/SavedHost.swift new file mode 100644 index 0000000..a10ec1f --- /dev/null +++ b/ClientShared/SavedHost.swift @@ -0,0 +1,22 @@ +import Foundation +import Network + +/// A Mac reached by address rather than Bonjour, remembered after it first connects. +struct SavedHost: Codable, Identifiable, Equatable { + var address: String + var pin: String + var id: String { address } + + /// Parses "host" or "host:port" (default port otherwise). Returns the endpoint and the bare host. + static func endpoint(for address: String) -> (NWEndpoint, String)? { + var host = address + var port = IFrame.defaultPort + if let colon = host.lastIndex(of: ":"), !host.contains("::"), + let p = UInt16(host[host.index(after: colon)...]) { + port = p + host = String(host[.. 0 { Text("skipped \(h.dropped) frames (congestion)") } + } else { + Text("waiting for stats…") + } + } + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.white) + .padding(8) + .background(.black.opacity(0.6), in: RoundedRectangle(cornerRadius: 8)) + } +} diff --git a/Client/StreamSession.swift b/ClientShared/StreamSession.swift similarity index 86% rename from Client/StreamSession.swift rename to ClientShared/StreamSession.swift index 1fbd176..0f69e7b 100644 --- a/Client/StreamSession.swift +++ b/ClientShared/StreamSession.swift @@ -1,9 +1,16 @@ import AVFoundation import Combine +import ImageIO import Network -import UIKit import VideoToolbox +/// The Mac's cursor shape. Hotspot and size are in Mac points; the image is 2x. +struct MacCursor { + var image: CGImage + var hotspot: CGPoint + var size: CGSize +} + struct ClientStats: Equatable { var host: HostStats? var decodeFPS = 0 @@ -12,7 +19,7 @@ struct ClientStats: Equatable { } /// Owns the connection to iframe-host: receives and decodes video straight into the display -/// layer's renderer on one high-priority queue, and sends input. +/// layer's renderer on one high-priority queue, and sends input. Shared by the iPad and Mac apps. final class StreamSession: ObservableObject { enum Phase: Equatable { case idle @@ -30,6 +37,9 @@ final class StreamSession: ObservableObject { /// The rect is normalized over the stream. var onTextFocus: ((Bool, CGRect) -> Void)? + /// Called on the main thread with the Mac's cursor shape whenever it changes. + var onCursor: ((MacCursor) -> Void)? + // Everything below is confined to `queue`. private let queue = DispatchQueue(label: "iframe.client", qos: .userInteractive) private var channel: MessageChannel? @@ -55,18 +65,13 @@ final class StreamSession: ObservableObject { // MARK: Connection - /// Mac points per iPad pixel choice (see DisplayRequest). 0 = use the Mac's own display. + /// Mac points per client pixel choice (see DisplayRequest). 0 = use the Mac's own display. private var uiScale: Double = 2 - /// The current screen in pixels, in the current orientation. - static func screenPixels() -> CGSize { - let screen = UIScreen.main - return CGSize(width: screen.bounds.width * screen.scale, height: screen.bounds.height * screen.scale) - } - - func connect(to endpoint: NWEndpoint, pin: String, label: String, uiScale: Double) { + /// `pixels` is the client's screen (or window) size in pixels. + func connect(to endpoint: NWEndpoint, pin: String, label: String, deviceName: String, maxFPS: Int, + pixels: CGSize, uiScale: Double) { self.uiScale = uiScale - let pixels = Self.screenPixels() hostLabel = label phase = .connecting stats = ClientStats() @@ -74,10 +79,11 @@ final class StreamSession: ObservableObject { let hello = Hello( version: IFrame.protocolVersion, pin: pin.trimmingCharacters(in: .whitespaces), - name: UIDevice.current.name, + name: deviceName, supportsHEVC: VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC), - maxFPS: UIScreen.main.maximumFramesPerSecond, - display: DisplayRequest(width: Int(pixels.width), height: Int(pixels.height), uiScale: uiScale)) + maxFPS: maxFPS, + display: DisplayRequest(width: Int(pixels.width), height: Int(pixels.height), uiScale: uiScale), + localCursor: true) queue.async { [self] in teardown() @@ -107,8 +113,10 @@ final class StreamSession: ObservableObject { } } - /// Asks the host to reshape its virtual display, e.g. after the iPad rotates. - func requestDisplay(pixels: CGSize) { + /// Asks the host to reshape its virtual display, e.g. after the iPad rotates or the Mac window resizes. + /// Pass `uiScale` to change density too. + func requestDisplay(pixels: CGSize, uiScale newScale: Double? = nil) { + if let newScale { uiScale = newScale } guard uiScale > 0, let data = try? JSONEncoder().encode( DisplayRequest(width: Int(pixels.width), height: Int(pixels.height), uiScale: uiScale)) else { return } send(.display, data) @@ -167,6 +175,14 @@ final class StreamSession: ObservableObject { guard let editable = r.u8(), let x = r.f32(), let y = r.f32(), let w = r.f32(), let h = r.f32() else { return } let rect = CGRect(x: CGFloat(x), y: CGFloat(y), width: CGFloat(w), height: CGFloat(h)) DispatchQueue.main.async { self.onTextFocus?(editable != 0, rect) } + case .cursor: + var r = ByteReader(data) + guard let hx = r.u16(), let hy = r.u16(), let w = r.u16(), let h = r.u16(), + let source = CGImageSourceCreateWithData(r.rest() as CFData, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else { return } + let cursor = MacCursor(image: image, hotspot: CGPoint(x: Int(hx), y: Int(hy)), + size: CGSize(width: Int(w), height: Int(h))) + DispatchQueue.main.async { self.onCursor?(cursor) } case .stats: hostStats = try? JSONDecoder().decode(HostStats.self, from: data) case .pong: @@ -274,6 +290,10 @@ final class StreamSession: ObservableObject { send(.key, w.data) } + func requestKeyframe() { + send(.requestKeyframe) + } + func tapKey(_ code: UInt16) { key(code, action: .down, mods: []) key(code, action: .up, mods: []) diff --git a/ClientShared/Theme.swift b/ClientShared/Theme.swift new file mode 100644 index 0000000..37d3f19 --- /dev/null +++ b/ClientShared/Theme.swift @@ -0,0 +1,37 @@ +import SwiftUI + +extension Color { + static let iframeTeal = Color(red: 0x2E / 255, green: 0xE6 / 255, blue: 0xD6 / 255) // #2EE6D6 + /// Text on teal: the accent is light, so dark text reads better than white. + static let iframeInk = Color(red: 0x04 / 255, green: 0x16 / 255, blue: 0x1A / 255) +} + +/// The cursor mark streaking up and to the right, with a short motion trail +/// (iFrame's take on Pinry's falling-pin loader). +struct IFrameLoadingView: View { + let size: CGFloat + @State private var progress: CGFloat = -1 + + var body: some View { + ZStack { + ForEach(0..<3, id: \.self) { index in + Image("IFrameMark") + .resizable() + .aspectRatio(contentMode: .fit) + .frame(width: size * 0.7, height: size * 0.7) + .opacity(1.0 - CGFloat(index) * 0.3) + .offset(x: (progress - CGFloat(index) * 0.18) * size, + y: -(progress - CGFloat(index) * 0.18) * size) + } + } + .frame(width: size, height: size) + .clipped() + .drawingGroup() + .onAppear { + progress = -1 + withAnimation(.easeInOut(duration: 0.6).repeatForever(autoreverses: false)) { + progress = 1 + } + } + } +} diff --git a/Host/CursorWatcher.swift b/Host/CursorWatcher.swift new file mode 100644 index 0000000..2b4b539 --- /dev/null +++ b/Host/CursorWatcher.swift @@ -0,0 +1,64 @@ +import AppKit + +/// Tracks the Mac's current cursor shape so clients can draw the pointer locally: it then +/// moves with zero latency and never gets lost in the video. (Capture runs without the +/// cursor for those clients.) +final class CursorWatcher { + struct Shape { + var png: Data + var pointWidth: Int + var pointHeight: Int + var hotspotX: Int + var hotspotY: Int + } + + var onChange: ((Shape) -> Void)? + private var timer: DispatchSourceTimer? + private var lastPixels: Data? + private var lastHotspot = CGPoint(x: -1, y: -1) + + func start() { + let timer = DispatchSource.makeTimerSource(queue: .main) + timer.schedule(deadline: .now(), repeating: .milliseconds(33)) + timer.setEventHandler { [weak self] in self?.poll() } + timer.resume() + self.timer = timer + } + + func stop() { + timer?.cancel() + timer = nil + } + + /// Forces the next poll to report the shape again (e.g. after the stream restarts). + func resend() { + DispatchQueue.main.async { self.lastPixels = nil } + } + + private func poll() { + guard let cursor = NSCursor.currentSystem else { return } + let image = cursor.image + let size = image.size + guard size.width > 0, size.height > 0, size.width <= 256, size.height <= 256 else { return } + + // Render at 2x so it stays crisp on high-density client screens. + guard let rep = NSBitmapImageRep(bitmapDataPlanes: nil, pixelsWide: Int(size.width * 2), pixelsHigh: Int(size.height * 2), + bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false, + colorSpaceName: .deviceRGB, bytesPerRow: 0, bitsPerPixel: 0) else { return } + rep.size = size + NSGraphicsContext.saveGraphicsState() + NSGraphicsContext.current = NSGraphicsContext(bitmapImageRep: rep) + image.draw(in: NSRect(origin: .zero, size: size)) + NSGraphicsContext.restoreGraphicsState() + guard let raw = rep.bitmapData else { return } + let pixels = Data(bytes: raw, count: rep.bytesPerRow * rep.pixelsHigh) + let hotspot = cursor.hotSpot + guard pixels != lastPixels || hotspot != lastHotspot else { return } + lastPixels = pixels + lastHotspot = hotspot + + guard let png = rep.representation(using: .png, properties: [:]) else { return } + onChange?(Shape(png: png, pointWidth: Int(size.width.rounded()), pointHeight: Int(size.height.rounded()), + hotspotX: Int(hotspot.x.rounded()), hotspotY: Int(hotspot.y.rounded()))) + } +} diff --git a/Host/H264SPS.swift b/Host/H264SPS.swift new file mode 100644 index 0000000..a5b469a --- /dev/null +++ b/Host/H264SPS.swift @@ -0,0 +1,221 @@ +import Foundation + +/// Rewrites an H.264 SPS so decoders know frames are never reordered. +/// +/// VideoToolbox leaves the VUI `bitstream_restriction_flag` at 0, so a spec-following decoder +/// (Windows/Chromium hardware decoders, for one) assumes it may need to buffer a full DPB of +/// frames before output. Holding 4+ frames deadlocks against the host's `maxInflight` ack +/// window. We encode with `AllowFrameReordering` off, so declaring `max_num_reorder_frames = 0` +/// is accurate and lets those decoders output each frame immediately. +enum H264SPS { + /// Returns the SPS NAL unit (header byte included, no start code) with the bitstream + /// restriction set, or nil if it couldn't be parsed. + static func withoutReordering(_ nal: Data) -> Data? { + let bytes = [UInt8](nal) + guard bytes.count > 4, bytes[0] & 0x1F == 7 else { return nil } + var r = BitReader(removeEmulationPrevention(Array(bytes[1...]))) + var w = BitWriter() + + // Copies one field through unchanged, returning its value. + func bits(_ n: Int) -> UInt32? { + guard let v = r.bits(n) else { return nil } + w.bits(v, n) + return v + } + func ue() -> UInt32? { + guard let v = r.ue() else { return nil } + w.ue(v) + return v + } + func se() -> Bool { + guard let v = r.ue() else { return false } + w.ue(v) // se(v) shares ue(v)'s bit pattern, so copy it as-is + return true + } + func hrd() -> Bool { + guard let count = ue(), count < 32, bits(4) != nil, bits(4) != nil else { return false } + for _ in 0...count { + guard ue() != nil, ue() != nil, bits(1) != nil else { return false } + } + return bits(20) != nil // four 5-bit delay/offset lengths + } + + guard let profile = bits(8), bits(8) != nil, bits(8) != nil, ue() != nil else { return nil } + if [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135].contains(profile) { + guard let chroma = ue() else { return nil } + if chroma == 3 { guard bits(1) != nil else { return nil } } + guard ue() != nil, ue() != nil, bits(1) != nil, let scaling = bits(1) else { return nil } + if scaling == 1 { + for i in 0..<(chroma == 3 ? 12 : 8) { + guard let present = bits(1) else { return nil } + guard present == 1 else { continue } + var last: Int32 = 8, next: Int32 = 8 + for _ in 0..<(i < 6 ? 16 : 64) where next != 0 { + guard let delta = r.se() else { return nil } + w.se(delta) + next = (last + delta + 256) % 256 + if next != 0 { last = next } + } + } + } + } + guard ue() != nil, let pocType = ue() else { return nil } + if pocType == 0 { + guard ue() != nil else { return nil } + } else if pocType == 1 { + guard bits(1) != nil, se(), se(), let cycle = ue(), cycle < 256 else { return nil } + for _ in 0.. [UInt8] { + var out: [UInt8] = [] + out.reserveCapacity(bytes.count) + var zeros = 0 + for byte in bytes { + if zeros >= 2 && byte == 3 { + zeros = 0 + continue + } + zeros = byte == 0 ? zeros + 1 : 0 + out.append(byte) + } + return out + } + + /// Inserts 0x03 wherever 0x00 0x00 would be followed by a byte <= 0x03. + static func addEmulationPrevention(_ bytes: [UInt8]) -> [UInt8] { + var out: [UInt8] = [] + out.reserveCapacity(bytes.count + 4) + var zeros = 0 + for byte in bytes { + if zeros >= 2 && byte <= 3 { + out.append(3) + zeros = 0 + } + zeros = byte == 0 ? zeros + 1 : 0 + out.append(byte) + } + return out + } +} + +private struct BitReader { + let bytes: [UInt8] + var pos = 0 // in bits + + init(_ bytes: [UInt8]) { self.bytes = bytes } + + mutating func bits(_ n: Int) -> UInt32? { + guard pos + n <= bytes.count * 8 else { return nil } + var v: UInt32 = 0 + for _ in 0..> 3] >> (7 - UInt8(pos & 7)) & 1) + pos += 1 + } + return v + } + + mutating func ue() -> UInt32? { + var zeros = 0 + while true { + guard let bit = bits(1) else { return nil } + if bit == 1 { break } + zeros += 1 + guard zeros < 32 else { return nil } + } + guard let rest = bits(zeros) else { return nil } + return (1 << zeros) - 1 + rest + } + + mutating func se() -> Int32? { + guard let k = ue() else { return nil } + return k & 1 == 1 ? Int32((k + 1) / 2) : -Int32(k / 2) + } +} + +private struct BitWriter { + var bytes: [UInt8] = [] + private var count = 0 // bits written + + mutating func bits(_ value: UInt32, _ n: Int) { + for i in stride(from: n - 1, through: 0, by: -1) { + if count & 7 == 0 { bytes.append(0) } + if value >> i & 1 == 1 { bytes[bytes.count - 1] |= 0x80 >> UInt8(count & 7) } + count += 1 + } + } + + mutating func ue(_ value: UInt32) { + let v = UInt64(value) + 1 + let length = 64 - v.leadingZeroBitCount + bits(0, length - 1) + for i in stride(from: length - 1, through: 0, by: -1) { bits(UInt32(v >> i & 1), 1) } + } + + mutating func se(_ value: Int32) { + ue(value > 0 ? UInt32(value) * 2 - 1 : UInt32(-value) * 2) + } + + mutating func trailingBits() { + bits(1, 1) + while count & 7 != 0 { bits(0, 1) } + } +} diff --git a/Host/HostServer.swift b/Host/HostServer.swift index db52498..c865e1e 100644 --- a/Host/HostServer.swift +++ b/Host/HostServer.swift @@ -1,3 +1,4 @@ +import ApplicationServices import Foundation import IOKit.pwr_mgt import Network @@ -102,6 +103,9 @@ final class ClientSession { private var focusTimer: DispatchSourceTimer? private var lastFocus = FocusWatcher.Focus.none private var streamDisplay: CGDirectDisplayID? + private var cursorWatcher: CursorWatcher? + private var inputCounts: [String: Int] = [:] + private var lastInputLog = Date() init(connection: NWConnection, server: HostServer, queue: DispatchQueue) { self.channel = MessageChannel(connection: connection, queue: queue) @@ -137,6 +141,8 @@ final class ClientSession { streamer = nil focusTimer?.cancel() focusTimer = nil + cursorWatcher?.stop() + cursorWatcher = nil if sleepAssertion != 0 { IOPMAssertionRelease(sleepAssertion) sleepAssertion = 0 @@ -156,6 +162,7 @@ final class ClientSession { } var r = ByteReader(data) let input = server.injector + countInput(type, data) switch type { case .mouseMove: if let x = r.f32(), let y = r.f32() { input.move(x: x, y: y) } @@ -211,6 +218,43 @@ final class ClientSession { self.hello = hello restartStream(display: hello.display) startFocusWatcher() + if hello.localCursor == true { + let watcher = CursorWatcher() + watcher.onChange = { [weak self] shape in + var w = ByteWriter(capacity: shape.png.count + 8) + w.u16(UInt16(clamping: shape.hotspotX)) + w.u16(UInt16(clamping: shape.hotspotY)) + w.u16(UInt16(clamping: shape.pointWidth)) + w.u16(UInt16(clamping: shape.pointHeight)) + w.bytes(shape.png) + self?.channel.send(.cursor, w.data) + } + watcher.start() + cursorWatcher = watcher + } + } + + /// Periodic summary of input received, so input problems show up in the log. + private func countInput(_ type: MsgType, _ data: Data) { + let label: String + switch type { + case .mouseMove: label = "moves" + case .mouseButton: label = "clicks" + case .scroll: label = "scrolls" + case .key: label = "keys" + case .text: label = "text" + default: return + } + inputCounts[label, default: 0] += 1 + if type == .key, data.count >= 3 { + inputCounts["last key 0x" + String(Int(data[data.startIndex]) << 8 | Int(data[data.startIndex + 1]), radix: 16)] = 0 + } + guard Date().timeIntervalSince(lastInputLog) > 5 else { return } + let summary = inputCounts.sorted { $0.key < $1.key } + .map { $0.value > 0 ? "\($0.key) \($0.value)" : $0.key }.joined(separator: ", ") + hostLog("input from \(name): \(summary)\(AXIsProcessTrusted() ? "" : " — NOT INJECTED: Accessibility permission missing")") + inputCounts = [:] + lastInputLog = Date() } private var hello: Hello? @@ -235,7 +279,8 @@ final class ClientSession { if virtual == nil { hostLog("falling back to the existing display") } } - let display = try await findDisplay(id: virtual?.displayID) + let expected = virtual.map { VirtualScreen.pointSize(for: $0.request) } + let display = try await findDisplay(id: virtual?.displayID, expectedPoints: expected) let mode = CGDisplayCopyDisplayMode(display.displayID) let refresh = mode?.refreshRate ?? 60 let fps = min(fpsLimit, refresh > 0 ? Int(refresh.rounded()) : 60) @@ -256,7 +301,8 @@ final class ClientSession { maxBitrate: Int(startMbps * 1.5 * 1_000_000), maxInflight: server.config.maxInflight) - let streamer = try Streamer(display: display, codec: codec, config: config, captureSize: capture) + let streamer = try Streamer(display: display, codec: codec, config: config, captureSize: capture, + showsCursor: hello.localCursor != true) streamer.onFormat = { [weak self] codec, sets in self?.channel.send(.format, Wire.format(codec: codec, parameterSets: sets)) } @@ -282,6 +328,7 @@ final class ClientSession { codec: streamer.codec, fps: fps, hostName: Host.current().localizedName ?? "Mac", isVirtual: virtual != nil)) + self.cursorWatcher?.resend() // the client rebuilds its cursor for the new scale continuation.resume(returning: true) } } @@ -324,11 +371,16 @@ final class ClientSession { } /// A freshly created virtual display can take a moment to show up in ScreenCaptureKit. - private func findDisplay(id: CGDirectDisplayID?) async throws -> SCDisplay { + /// ScreenCaptureKit can briefly report a stale display (old size) after a reshape, so for + /// virtual displays wait until it reports the expected point size too. + private func findDisplay(id: CGDirectDisplayID?, expectedPoints: (width: Int, height: Int)? = nil) async throws -> SCDisplay { for attempt in 0..<20 { let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) if let id { - if let match = content.displays.first(where: { $0.displayID == id }) { return match } + if let match = content.displays.first(where: { $0.displayID == id }), + expectedPoints.map({ abs(match.width - $0.width) <= 1 && abs(match.height - $0.height) <= 1 }) ?? true { + return match + } } else { let main = CGMainDisplayID() let displays = content.displays.sorted { diff --git a/Host/Probe.swift b/Host/Probe.swift index 905e0b1..75d2390 100644 --- a/Host/Probe.swift +++ b/Host/Probe.swift @@ -11,6 +11,8 @@ func runProbe() -> Never { let pin = option("--pin") ?? "" let seconds = option("--seconds").flatMap(Double.init) ?? 10 let skipDecode = arguments.contains("--no-decode") + let forceH264 = arguments.contains("--h264") // what Windows clients get + let localCursor = arguments.contains("--local-cursor") // e.g. --screen 2388x1668@2 to exercise the virtual display like an 11" iPad Pro would. let screen: DisplayRequest? = option("--screen").flatMap { spec in let parts = spec.split(separator: "@") @@ -48,8 +50,8 @@ func runProbe() -> Never { print("probe: connected to \(host):\(port)") channel.send(.hello, json: Hello( version: IFrame.protocolVersion, pin: pin, name: "iframe-probe", - supportsHEVC: VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC), maxFPS: 120, - display: screen)) + supportsHEVC: !forceH264 && VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC), maxFPS: 120, + display: screen, localCursor: localCursor)) let timer = DispatchSource.makeTimerSource(queue: queue) timer.schedule(deadline: .now() + 0.25, repeating: 0.25) timer.setEventHandler { @@ -80,9 +82,18 @@ func runProbe() -> Never { print("probe: wrong PIN") exit(2) case .format: - if let (codec, sets) = Wire.parseFormat(data), !decoder.setFormat(codec: codec, parameterSets: sets) { + guard let (codec, sets) = Wire.parseFormat(data) else { return } + if codec == .h264, let sps = sets.first(where: { $0.first.map { $0 & 0x1F == 7 } == true }) { + print("probe: SPS \(sps.map { String(format: "%02x", $0) }.joined())") + } + if !decoder.setFormat(codec: codec, parameterSets: sets) { print("probe: could not create decoder") } + case .cursor: + var r = ByteReader(data) + if let hx = r.u16(), let hy = r.u16(), let w = r.u16(), let h = r.u16() { + print("probe: cursor \(w)x\(h) pt, hotspot \(hx),\(hy), PNG \(r.rest().count) bytes") + } case .frame: guard let frame = Wire.parseFrame(data) else { return } if firstFrameAt == nil { firstFrameAt = nowNanos() } diff --git a/Host/Streamer.swift b/Host/Streamer.swift index d0efe92..dfebe8e 100644 --- a/Host/Streamer.swift +++ b/Host/Streamer.swift @@ -70,7 +70,11 @@ final class Streamer: NSObject, SCStreamOutput, SCStreamDelegate { private var statsTimer: DispatchSourceTimer? private var stopped = false - init(display: SCDisplay, codec preferred: VideoCodec, config: Config, captureSize: (width: Int, height: Int)? = nil) throws { + private let showsCursor: Bool + + init(display: SCDisplay, codec preferred: VideoCodec, config: Config, captureSize: (width: Int, height: Int)? = nil, + showsCursor: Bool = true) throws { + self.showsCursor = showsCursor self.display = display self.config = config self.fps = config.fps @@ -158,7 +162,7 @@ final class Streamer: NSObject, SCStreamOutput, SCStreamDelegate { cfg.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange cfg.colorMatrix = CGDisplayStream.yCbCrMatrix_ITU_R_709_2 cfg.colorSpaceName = CGColorSpace.sRGB - cfg.showsCursor = true + cfg.showsCursor = showsCursor cfg.queueDepth = 5 cfg.capturesAudio = false if #available(macOS 14.0, *) { cfg.captureResolution = .best } @@ -398,7 +402,11 @@ final class Streamer: NSObject, SCStreamOutput, SCStreamDelegate { var pointer: UnsafePointer? var size = 0 guard get(index, &pointer, &size, nil) == noErr, let pointer else { return nil } - return Data(bytes: pointer, count: size) + let set = Data(bytes: pointer, count: size) + if codec == .h264, set.first.map({ $0 & 0x1F == 7 }) == true { + return H264SPS.withoutReordering(set) ?? set + } + return set } } } diff --git a/Host/VirtualScreen.swift b/Host/VirtualScreen.swift index 18c28a5..9e49ad4 100644 --- a/Host/VirtualScreen.swift +++ b/Host/VirtualScreen.swift @@ -29,7 +29,19 @@ final class VirtualScreen { self.refreshRate = refreshRate } + /// Largest backing size (pixels per side) a display may use. The display is created at this + /// size up front so any client shape can be applied in place: tearing a virtual display down + /// and recreating it races with macOS reusing the same display ID. + static let maxBackingPixels = 8192 + + /// Mac point size the display will have for `request` (what ScreenCaptureKit reports). + static func pointSize(for request: DisplayRequest) -> (width: Int, height: Int) { + let size = backingPixels(for: request) + return (size.pointWidth, size.pointHeight) + } + /// Returns a virtual display matching `request`, reusing and reshaping the existing one when possible. + /// Returns nil if the display didn't come online in the requested shape. static func obtain(for request: DisplayRequest, refreshRate: Double) -> VirtualScreen? { let screen = DispatchQueue.main.sync { () -> VirtualScreen? in let backing = backingPixels(for: request) @@ -38,39 +50,56 @@ final class VirtualScreen { if current.request == request, current.refreshRate == refreshRate { return current } if current.apply(request, refreshRate: refreshRate) { return current } } - shared = nil // releasing the old display removes it + if let old = shared { + let oldID = old.displayID + shared = nil // releasing the old display removes it + waitUntilGone(oldID) + } shared = create(for: request, refreshRate: refreshRate) return shared } - screen?.waitUntilReady() + // Check readiness on the main thread too: CoreGraphics only refreshes its view of a + // reshaped display there; background-thread queries keep returning the old mode. + guard let screen, DispatchQueue.main.sync(execute: { screen.waitUntilReady() }) else { return nil } return screen } + private static func waitUntilGone(_ id: CGDirectDisplayID) { + for _ in 0..<60 where CGDisplayIsOnline(id) != 0 { + Thread.sleep(forTimeInterval: 0.05) + } + } + /// A new or reshaped display takes a moment to come online with its mode; capture and - /// frame-rate decisions must wait for it. Called off the main thread. - private func waitUntilReady() { + /// frame-rate decisions must wait for it. Main thread only. + private func waitUntilReady() -> Bool { let size = Self.backingPixels(for: request) - for _ in 0..<60 { - if let mode = CGDisplayCopyDisplayMode(displayID), mode.width == size.pointWidth, mode.refreshRate > 0 { break } - selectMode(pointWidth: size.pointWidth, pixelWidth: size.width, refreshRate: refreshRate) + var ready = false + for _ in 0..<100 { + if let mode = CGDisplayCopyDisplayMode(displayID), mode.width == size.pointWidth, + mode.height == size.pointHeight, mode.pixelWidth == size.width, mode.refreshRate > 0 { + ready = true + break + } Thread.sleep(forTimeInterval: 0.05) } let mode = CGDisplayCopyDisplayMode(displayID) - hostLog("virtual display: \(mode?.width ?? 0)x\(mode?.height ?? 0) pt, \(mode?.pixelWidth ?? 0)x\(mode?.pixelHeight ?? 0) px @ \(Int(mode?.refreshRate ?? 0)) Hz (display \(displayID))") + hostLog("virtual display\(ready ? "" : " NOT READY"): \(mode?.width ?? 0)x\(mode?.height ?? 0) pt, \(mode?.pixelWidth ?? 0)x\(mode?.pixelHeight ?? 0) px @ \(Int(mode?.refreshRate ?? 0)) Hz (display \(displayID))") + return ready } private static func create(for request: DisplayRequest, refreshRate: Double) -> VirtualScreen? { - let backing = backingPixels(for: request) - // Square max size so the same display can rotate between landscape and portrait. - let maxPixels = max(backing.width, backing.height) + // Square, generous max size: any orientation or window shape can then be applied in place. + let maxPixels = maxBackingPixels let descriptor = CGVirtualDisplayDescriptor() descriptor.setDispatchQueue(queue) descriptor.name = "iFrame Display" descriptor.maxPixelsWide = UInt32(maxPixels) descriptor.maxPixelsHigh = UInt32(maxPixels) - // iPad Pro panels are 264 ppi; physical size drives macOS's default scaling choices. - let mm = Double(maxPixels) * 25.4 / 264 - descriptor.sizeInMillimeters = CGSize(width: mm, height: mm) + // A small physical size (very high ppi) makes macOS default to the HiDPI variant of each + // mode. We must never select modes ourselves: after any CGDisplaySetDisplayMode or + // CGConfigureDisplayWithDisplayMode, macOS ignores later applySettings reshapes. + descriptor.sizeInMillimeters = CGSize(width: 263, height: 263) descriptor.vendorID = 0x676C // "gl" descriptor.productID = 0x6964 // "id" descriptor.serialNum = 1 @@ -88,7 +117,10 @@ final class VirtualScreen { /// Mac points and backing pixels for a request. HiDPI modes always render at 2x points. private static func backingPixels(for request: DisplayRequest) -> (width: Int, height: Int, pointWidth: Int, pointHeight: Int) { - let scale = min(max(request.uiScale, 1), 2) + var scale = min(max(request.uiScale, 1), 2) + // HiDPI renders at 2x points; keep that within the display's maximum. + let longest = Double(max(request.width, request.height)) + scale = max(scale, longest * 2 / Double(maxBackingPixels)) let pw = Int((Double(request.width) / scale).rounded()) & ~1 let ph = Int((Double(request.height) / scale).rounded()) & ~1 return (pw * 2, ph * 2, pw, ph) @@ -106,18 +138,6 @@ final class VirtualScreen { guard display.apply(settings) else { return false } self.request = request self.refreshRate = refreshRate - selectMode(pointWidth: size.pointWidth, pixelWidth: size.width, refreshRate: refreshRate) return true } - - /// macOS usually picks the right mode itself; make sure it's the HiDPI one at full refresh. - private func selectMode(pointWidth: Int, pixelWidth: Int, refreshRate: Double) { - let options = [kCGDisplayShowDuplicateLowResolutionModes: true] as CFDictionary - if let current = CGDisplayCopyDisplayMode(displayID), - current.width == pointWidth, current.pixelWidth == pixelWidth, current.refreshRate == refreshRate { return } - let modes = CGDisplayCopyAllDisplayModes(displayID, options) as? [CGDisplayMode] ?? [] - if let match = modes.first(where: { $0.width == pointWidth && $0.pixelWidth == pixelWidth && $0.refreshRate == refreshRate }) { - CGDisplaySetDisplayMode(displayID, match, nil) - } - } } diff --git a/Host/main.swift b/Host/main.swift index dcb7846..77b6695 100644 --- a/Host/main.swift +++ b/Host/main.swift @@ -28,7 +28,7 @@ HOST OPTIONS --pin fixed PIN (default: random each launch) PROBE OPTIONS - --pin --port --seconds --screen x[@scale] + --pin --port --seconds --screen x[@scale] --h264 --local-cursor """ var arguments = Array(CommandLine.arguments.dropFirst()) @@ -98,7 +98,10 @@ signalSource.setEventHandler { } signalSource.resume() -dispatchMain() +// A real run loop, not dispatchMain(): CoreGraphics delivers display reconfiguration +// notifications through the main run loop. Without it, this process never sees a virtual +// display change shape after its first mode, and capture targets a stale display. +RunLoop.main.run() func localIPv4Addresses() -> [String] { var result: [String] = [] diff --git a/Linux/README.md b/Linux/README.md index b8f9748..c3982ed 100644 --- a/Linux/README.md +++ b/Linux/README.md @@ -87,7 +87,8 @@ The window title shows the host, resolution, decoder, fps, bitrate, round trip a ### Other options `--scroll-speed X`, `--invert-scroll`, `--local-cursor` (show the Linux pointer over the stream -as well; the host's cursor is part of the video), `--view-only` (watch without sending mouse or +even when the host draws its cursor into the video; a Mac sends its cursor shape instead, which +becomes the pointer), `--view-only` (watch without sending mouse or keyboard), `--no-hw`, `--h264`, `--vsync`, `--stats`. Colours follow the stream's own YUV matrix: BT.709 from the Mac, BT.601 from NVENC. diff --git a/Linux/src/cursor.c b/Linux/src/cursor.c new file mode 100644 index 0000000..052de40 --- /dev/null +++ b/Linux/src/cursor.c @@ -0,0 +1,62 @@ +#include +#include + +#include + +#include "iframe.h" + +// The host sends the Mac's cursor as a PNG. FFmpeg (already linked for video) decodes PNG, +// so this needs no image library. + +bool cursor_decode_png(const uint8_t *png, size_t len, uint8_t **rgba, int *width, int *height) { + *rgba = NULL; + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_PNG); + if (!codec) return false; + AVCodecContext *ctx = avcodec_alloc_context3(codec); + AVPacket *pkt = av_packet_alloc(); + AVFrame *frame = av_frame_alloc(); + bool ok = false; + if (!ctx || !pkt || !frame || avcodec_open2(ctx, codec, NULL) < 0) goto done; + if (av_new_packet(pkt, (int)len) < 0) goto done; + memcpy(pkt->data, png, len); + if (avcodec_send_packet(ctx, pkt) < 0 || avcodec_receive_frame(ctx, frame) < 0) goto done; + + int w = frame->width, h = frame->height; + if (w <= 0 || h <= 0 || w > 512 || h > 512) goto done; + if (frame->format != AV_PIX_FMT_RGBA && frame->format != AV_PIX_FMT_RGB24 && frame->format != AV_PIX_FMT_PAL8) + goto done; + uint8_t *out = malloc((size_t)w * h * 4); + if (!out) goto done; + for (int y = 0; y < h; y++) { + const uint8_t *src = frame->data[0] + (size_t)y * frame->linesize[0]; + uint8_t *dst = out + (size_t)y * w * 4; + if (frame->format == AV_PIX_FMT_RGBA) { + memcpy(dst, src, (size_t)w * 4); + } else if (frame->format == AV_PIX_FMT_PAL8) { + const uint32_t *palette = (const uint32_t *)frame->data[1]; // native-endian ARGB + for (int x = 0; x < w; x++) { + uint32_t c = palette[src[x]]; + dst[4 * x] = c >> 16; + dst[4 * x + 1] = c >> 8; + dst[4 * x + 2] = c; + dst[4 * x + 3] = c >> 24; + } + } else { + for (int x = 0; x < w; x++) { + dst[4 * x] = src[3 * x]; + dst[4 * x + 1] = src[3 * x + 1]; + dst[4 * x + 2] = src[3 * x + 2]; + dst[4 * x + 3] = 255; + } + } + } + *rgba = out; + *width = w; + *height = h; + ok = true; +done: + av_frame_free(&frame); + av_packet_free(&pkt); + avcodec_free_context(&ctx); + return ok; +} diff --git a/Linux/src/iframe.h b/Linux/src/iframe.h index c23a0cf..72c246c 100644 --- a/Linux/src/iframe.h +++ b/Linux/src/iframe.h @@ -22,6 +22,7 @@ enum { MSG_PONG = 0x05, MSG_AUTH_FAILED = 0x06, MSG_TEXT_FOCUS = 0x07, + MSG_CURSOR = 0x08, // hotspot x, y, size w, h (u16, Mac points), then a PNG at 2x // client -> host MSG_HELLO = 0x10, MSG_MOUSE_MOVE = 0x11, @@ -78,6 +79,9 @@ bool json_string(const char *json, size_t len, const char *key, char *out, size_ bool json_bool(const char *json, size_t len, const char *key, bool *out); void json_escape(const char *in, char *out, size_t cap); +// cursor.c — the Mac's cursor shape arrives as a PNG; decodes it to RGBA (caller frees). +bool cursor_decode_png(const uint8_t *png, size_t len, uint8_t **rgba, int *width, int *height); + // discover.c — Bonjour browsing via avahi-browse. typedef struct { char name[128]; diff --git a/Linux/src/main.c b/Linux/src/main.c index 738743b..5d97344 100644 --- a/Linux/src/main.c +++ b/Linux/src/main.c @@ -59,7 +59,7 @@ static struct { .scroll_speed = 1, }; -static Uint32 EV_FRAME, EV_WELCOME, EV_STATUS, EV_STATS, EV_FATAL; +static Uint32 EV_FRAME, EV_WELCOME, EV_STATUS, EV_STATS, EV_FATAL, EV_CURSOR; static atomic_bool quitting; @@ -90,6 +90,15 @@ static char decoder_name[96]; static atomic_int request_w, request_h; // display size to ask for (drawable pixels) static int max_fps = 60; +// The Mac's current cursor shape (under state_lock). Hosts that send it capture without the +// cursor, so we draw it as the system pointer here: it then moves with no network delay. +static struct { + uint8_t *rgba; // PNG decoded, at 2x Mac points; NULL until the host sends one + int px_w, px_h; + int hot_x, hot_y; // Mac points + int pt_w, pt_h; +} mac_cursor; + // MARK: - Sending static bool send_msg(uint8_t type, const void *payload, uint32_t len) { @@ -148,10 +157,11 @@ static void send_hello(void) { gethostname(host, sizeof host - 1); json_escape(opt.pin, pin, sizeof pin); json_escape(host, name, sizeof name); - char json[512]; + char json[640]; + // localCursor: the host sends cursor shapes and leaves the pointer out of the video. int n = snprintf(json, sizeof json, "{\"version\":%d,\"pin\":\"%s\",\"name\":\"%s\",\"os\":\"linux\",\"supportsHEVC\":%s,\"maxFPS\":%d," - "\"display\":{\"width\":%d,\"height\":%d,\"uiScale\":%g}}", + "\"localCursor\":true,\"display\":{\"width\":%d,\"height\":%d,\"uiScale\":%g}}", IFRAME_PROTOCOL_VERSION, pin, name, opt.h264_only ? "false" : "true", max_fps, atomic_load(&request_w), atomic_load(&request_h), opt.scale); send_msg(MSG_HELLO, json, n); @@ -262,6 +272,24 @@ static void handle_welcome(const uint8_t *p, uint32_t len) { push_event(EV_WELCOME); } +static void handle_cursor(const uint8_t *p, uint32_t len) { + if (len <= 8) return; + uint8_t *rgba; + int w, h; + if (!cursor_decode_png(p + 8, len - 8, &rgba, &w, &h)) return; + SDL_LockMutex(state_lock); + free(mac_cursor.rgba); + mac_cursor.rgba = rgba; + mac_cursor.px_w = w; + mac_cursor.px_h = h; + mac_cursor.hot_x = p[0] << 8 | p[1]; + mac_cursor.hot_y = p[2] << 8 | p[3]; + mac_cursor.pt_w = p[4] << 8 | p[5]; + mac_cursor.pt_h = p[6] << 8 | p[7]; + SDL_UnlockMutex(state_lock); + push_event(EV_CURSOR); +} + static void handle_stats(const uint8_t *p, uint32_t len) { const char *j = (const char *)p; HostStats s = {0}; @@ -335,6 +363,7 @@ static int network_thread(void *unused) { case MSG_FORMAT: handle_format(dec, body, len); break; case MSG_FRAME: handle_frame(dec, body, len, &last_keyframe_request); break; case MSG_STATS: handle_stats(body, len); break; + case MSG_CURSOR: handle_cursor(body, len); break; case MSG_PONG: if (len >= 8) { SDL_LockMutex(state_lock); @@ -354,6 +383,8 @@ static int network_thread(void *unused) { decoder_free(dec); SDL_LockMutex(state_lock); have_welcome = false; + free(mac_cursor.rgba); // the next host may draw the cursor into the video itself + mac_cursor.rgba = NULL; SDL_UnlockMutex(state_lock); SDL_LockMutex(frame_lock); av_frame_unref(pending_frame); @@ -585,6 +616,76 @@ static void render(void) { SDL_RenderPresent(renderer); } +// MARK: - Cursor + +static SDL_Cursor *shown_cursor; + +/// Shows the Mac's cursor shape as our own pointer, sized to how big the Mac's screen appears +/// in the window. Without a shape from the host (older host, Linux host, or not connected) the +/// pointer stays hidden, because the cursor is in the video. +static void apply_cursor(void) { + SDL_LockMutex(state_lock); + double point_w = have_welcome && welcome.width > 0 ? welcome.point_width : 0; + int stream_w = welcome.width, stream_h = welcome.height; + SDL_Surface *src = NULL; + int hot_x = 0, hot_y = 0, pt_w = 0, pt_h = 0; + if (mac_cursor.rgba && point_w > 0 && stream_h > 0) { + src = SDL_CreateRGBSurfaceWithFormat(0, mac_cursor.px_w, mac_cursor.px_h, 32, SDL_PIXELFORMAT_RGBA32); + if (src) { + for (int y = 0; y < src->h; y++) + memcpy((uint8_t *)src->pixels + (size_t)y * src->pitch, mac_cursor.rgba + (size_t)y * src->w * 4, + (size_t)src->w * 4); + } + hot_x = mac_cursor.hot_x; + hot_y = mac_cursor.hot_y; + pt_w = mac_cursor.pt_w; + pt_h = mac_cursor.pt_h; + } + SDL_UnlockMutex(state_lock); + + if (!src) { + if (shown_cursor) { + SDL_SetCursor(SDL_GetDefaultCursor()); + SDL_FreeCursor(shown_cursor); + shown_cursor = NULL; + } + SDL_ShowCursor(opt.local_cursor ? SDL_ENABLE : SDL_DISABLE); + return; + } + + // X11 cursors are in device pixels; Wayland scales cursor surfaces like window content. + int w, h; + const char *driver = SDL_GetCurrentVideoDriver(); + if (driver && !strcmp(driver, "wayland")) SDL_GetWindowSize(window, &w, &h); + else SDL_GetRendererOutputSize(renderer, &w, &h); + float fit = fminf((float)w / stream_w, (float)h / stream_h); + float scale = stream_w * fit / (float)point_w; // cursor pixels per Mac point + int cw = (int)lroundf(fminf(pt_w * scale, 256)), ch = (int)lroundf(fminf(pt_h * scale, 256)); + if (cw < 1) cw = 1; + if (ch < 1) ch = 1; + + SDL_Surface *scaled = SDL_CreateRGBSurfaceWithFormat(0, cw, ch, 32, SDL_PIXELFORMAT_RGBA32); + SDL_Cursor *cursor = NULL; + if (scaled) { +#if SDL_VERSION_ATLEAST(2, 0, 16) + if (SDL_SoftStretchLinear(src, NULL, scaled, NULL) != 0) +#endif + { + SDL_SetSurfaceBlendMode(src, SDL_BLENDMODE_NONE); + SDL_BlitScaled(src, NULL, scaled, NULL); + } + int hx = (int)lroundf(hot_x * scale), hy = (int)lroundf(hot_y * scale); + cursor = SDL_CreateColorCursor(scaled, hx < cw ? hx : cw - 1, hy < ch ? hy : ch - 1); + SDL_FreeSurface(scaled); + } + SDL_FreeSurface(src); + if (!cursor) return; + SDL_SetCursor(cursor); + if (shown_cursor) SDL_FreeCursor(shown_cursor); + shown_cursor = cursor; + SDL_ShowCursor(SDL_ENABLE); +} + static void show_pending_frame(void) { SDL_LockMutex(frame_lock); bool got = frame_pending; @@ -680,7 +781,8 @@ static void usage(FILE *f) { " (on a Linux host every key maps 1:1 unless this is given)\n" " --scroll-speed X wheel multiplier (default 1)\n" " --invert-scroll reverse wheel direction\n" - " --local-cursor show the Linux cursor over the stream too\n" + " --local-cursor show the Linux pointer even when the host draws the cursor into the\n" + " video (Macs send their cursor shape and don't need this)\n" " --no-hw software decoding only\n" " --h264 ask for H.264 instead of HEVC\n" " --vsync sync presentation to the monitor (smoother, adds latency)\n" @@ -815,12 +917,13 @@ int main(int argc, char **argv) { return 1; } SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709); - Uint32 base = SDL_RegisterEvents(5); + Uint32 base = SDL_RegisterEvents(6); EV_FRAME = base; EV_WELCOME = base + 1; EV_STATUS = base + 2; EV_STATS = base + 3; EV_FATAL = base + 4; + EV_CURSOR = base + 5; Uint32 flags = SDL_WINDOW_RESIZABLE | SDL_WINDOW_ALLOW_HIGHDPI; if (opt.fullscreen) flags |= SDL_WINDOW_FULLSCREEN_DESKTOP; @@ -882,6 +985,9 @@ int main(int argc, char **argv) { av_frame_unref(shown_frame); render(); } + apply_cursor(); + } else if (e.type == EV_CURSOR) { + apply_cursor(); update_title(false); } else if (e.type == EV_STATUS) { update_title(false); @@ -892,7 +998,7 @@ int main(int argc, char **argv) { running = false; } else if (e.type == SDL_WINDOWEVENT) { switch (e.window.event) { - case SDL_WINDOWEVENT_SIZE_CHANGED: resize_at = SDL_GetTicks(); render(); break; + case SDL_WINDOWEVENT_SIZE_CHANGED: resize_at = SDL_GetTicks(); render(); apply_cursor(); break; case SDL_WINDOWEVENT_EXPOSED: render(); break; case SDL_WINDOWEVENT_FOCUS_LOST: release_all(); break; case SDL_WINDOWEVENT_FOCUS_GAINED: if (keyboard_grab) set_keyboard_grab(true); break; @@ -946,6 +1052,7 @@ int main(int argc, char **argv) { av_frame_free(&pending_frame); av_frame_free(&shown_frame); if (texture) SDL_DestroyTexture(texture); + if (shown_cursor) SDL_FreeCursor(shown_cursor); SDL_DestroyRenderer(renderer); SDL_DestroyWindow(window); SDL_Quit(); diff --git a/MacClient/Assets.xcassets/AccentColor.colorset/Contents.json b/MacClient/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 0000000..c1b71b9 --- /dev/null +++ b/MacClient/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,3 @@ +{ "colors" : [ { "idiom" : "universal", "color" : { "color-space" : "srgb", + "components" : { "red" : "0x2E", "green" : "0xE6", "blue" : "0xD6", "alpha" : "1.000" } } } ], + "info" : { "author" : "xcode", "version" : 1 } } diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/Contents.json b/MacClient/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 0000000..396aee9 --- /dev/null +++ b/MacClient/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,68 @@ +{ + "images": [ + { + "filename": "icon_16x16.png", + "idiom": "mac", + "scale": "1x", + "size": "16x16" + }, + { + "filename": "icon_16x16@2x.png", + "idiom": "mac", + "scale": "2x", + "size": "16x16" + }, + { + "filename": "icon_32x32.png", + "idiom": "mac", + "scale": "1x", + "size": "32x32" + }, + { + "filename": "icon_32x32@2x.png", + "idiom": "mac", + "scale": "2x", + "size": "32x32" + }, + { + "filename": "icon_128x128.png", + "idiom": "mac", + "scale": "1x", + "size": "128x128" + }, + { + "filename": "icon_128x128@2x.png", + "idiom": "mac", + "scale": "2x", + "size": "128x128" + }, + { + "filename": "icon_256x256.png", + "idiom": "mac", + "scale": "1x", + "size": "256x256" + }, + { + "filename": "icon_256x256@2x.png", + "idiom": "mac", + "scale": "2x", + "size": "256x256" + }, + { + "filename": "icon_512x512.png", + "idiom": "mac", + "scale": "1x", + "size": "512x512" + }, + { + "filename": "icon_512x512@2x.png", + "idiom": "mac", + "scale": "2x", + "size": "512x512" + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} \ No newline at end of file diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_128x128.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_128x128.png new file mode 100644 index 0000000..7d32245 Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_128x128.png differ diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_128x128@2x.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_128x128@2x.png new file mode 100644 index 0000000..d7a20d7 Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_128x128@2x.png differ diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_16x16.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_16x16.png new file mode 100644 index 0000000..f7dc361 Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_16x16.png differ diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_16x16@2x.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_16x16@2x.png new file mode 100644 index 0000000..3ae3456 Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_16x16@2x.png differ diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_256x256.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_256x256.png new file mode 100644 index 0000000..d7a20d7 Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_256x256.png differ diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_256x256@2x.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_256x256@2x.png new file mode 100644 index 0000000..1f64797 Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_256x256@2x.png differ diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_32x32.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_32x32.png new file mode 100644 index 0000000..3ae3456 Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_32x32.png differ diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_32x32@2x.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_32x32@2x.png new file mode 100644 index 0000000..b6bb866 Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_32x32@2x.png differ diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_512x512.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_512x512.png new file mode 100644 index 0000000..1f64797 Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_512x512.png differ diff --git a/MacClient/Assets.xcassets/AppIcon.appiconset/icon_512x512@2x.png b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_512x512@2x.png new file mode 100644 index 0000000..1cbea3e Binary files /dev/null and b/MacClient/Assets.xcassets/AppIcon.appiconset/icon_512x512@2x.png differ diff --git a/MacClient/Assets.xcassets/Contents.json b/MacClient/Assets.xcassets/Contents.json new file mode 100644 index 0000000..46cba7d --- /dev/null +++ b/MacClient/Assets.xcassets/Contents.json @@ -0,0 +1 @@ +{ "info" : { "author" : "xcode", "version" : 1 } } diff --git a/MacClient/Assets.xcassets/IFrameMark.imageset/Contents.json b/MacClient/Assets.xcassets/IFrameMark.imageset/Contents.json new file mode 100644 index 0000000..5e68fcf --- /dev/null +++ b/MacClient/Assets.xcassets/IFrameMark.imageset/Contents.json @@ -0,0 +1,2 @@ +{ "images" : [ { "filename" : "iframe-mark.png", "idiom" : "universal" } ], + "info" : { "author" : "xcode", "version" : 1 } } diff --git a/MacClient/Assets.xcassets/IFrameMark.imageset/iframe-mark.png b/MacClient/Assets.xcassets/IFrameMark.imageset/iframe-mark.png new file mode 100644 index 0000000..945976e Binary files /dev/null and b/MacClient/Assets.xcassets/IFrameMark.imageset/iframe-mark.png differ diff --git a/MacClient/IFrameMacApp.swift b/MacClient/IFrameMacApp.swift new file mode 100644 index 0000000..f19d5e6 --- /dev/null +++ b/MacClient/IFrameMacApp.swift @@ -0,0 +1,213 @@ +import AppKit +import Network +import SwiftUI + +@main +struct IFrameMacApp: App { + @NSApplicationDelegateAdaptor private var appDelegate: AppDelegate + @StateObject private var session = StreamSession() + @StateObject private var window = MainWindow() + @AppStorage("showStats") private var showStats = false + @AppStorage("macDensity") private var density = 1.0 + + var body: some Scene { + Window("iFrame", id: "main") { + Group { + if session.phase == .streaming { + MacStreamScreen() + } else { + MacConnectView() + } + } + .environmentObject(session) + .environmentObject(window) + .background(WindowReader(window: window)) + .onChange(of: session.phase) { old, phase in + window.phaseChanged(from: old, to: phase, session: session) + } + .onChange(of: session.welcome) { _, welcome in + if let welcome { window.welcomeChanged(welcome) } + } + } + .windowResizability(.contentMinSize) + .defaultSize(width: MainWindow.connectSize.width, height: MainWindow.connectSize.height) + .commands { + CommandGroup(replacing: .newItem) {} + // ⌃⌥⌘ shortcuts stay with this app; every other key combo goes to the remote Mac. + CommandMenu("Stream") { + Button("Enter or Exit Full Screen") { window.window?.toggleFullScreen(nil) } + .keyboardShortcut("f", modifiers: [.control, .option, .command]) + Toggle("Show Stats", isOn: $showStats) + .keyboardShortcut("s", modifiers: [.control, .option, .command]) + Button("Refresh Picture") { session.requestKeyframe() } + .keyboardShortcut("r", modifiers: [.control, .option, .command]) + .disabled(session.phase != .streaming) + Divider() + Picker("Mac Display", selection: $density) { + ForEach(MacDensity.virtualChoices) { choice in + Text(choice.label).tag(choice.rawValue) + } + } + .disabled(session.phase == .streaming && session.welcome?.isVirtual != true) + Divider() + Button("Disconnect") { session.disconnect() } + .keyboardShortcut("d", modifiers: [.control, .option, .command]) + .disabled(session.phase == .idle) + } + } + } +} + +final class AppDelegate: NSObject, NSApplicationDelegate { + func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true } +} + +/// How dense the Mac's virtual display is, relative to this Mac's own screen. +/// 1 = the same size things are here (sharpest); lower = more desktop space; 0 = mirror the Mac's display. +enum MacDensity: Double, CaseIterable, Identifiable { + case match = 1.0 + case more = 0.8 + case most = 0.667 + case mirror = 0 + + var id: Double { rawValue } + static let virtualChoices: [MacDensity] = [.match, .more, .most] + + var label: String { + switch self { + case .match: return "Match this Mac · sharpest" + case .more: return "More space" + case .most: return "Most space" + case .mirror: return "Use the Mac's own display" + } + } + + /// The host's uiScale (pixels per Mac point) for a screen with this backing scale. + func uiScale(backing: CGFloat) -> Double { + rawValue <= 0 ? 0 : max(1, Double(backing) * rawValue) + } +} + +/// The app's single window: sized small for the connect form and large for the stream. +final class MainWindow: ObservableObject { + static let connectSize = CGSize(width: 520, height: 720) + + weak var window: NSWindow? + /// Content size (points) the stream will use, decided when connecting. + private var streamSize: CGSize? + private var connectFrame: NSRect? + + var backingScale: CGFloat { window?.backingScaleFactor ?? NSScreen.main?.backingScaleFactor ?? 2 } + var maxFPS: Int { (window?.screen ?? NSScreen.main)?.maximumFramesPerSecond ?? 60 } + + /// Picks the window size for streaming and returns it in pixels for the host's virtual display. + func prepareStream() -> CGSize { + let size: CGSize + if let window, window.styleMask.contains(.fullScreen) { + size = window.contentLayoutRect.size + } else { + let visible = (window?.screen ?? NSScreen.main)?.visibleFrame.size ?? CGSize(width: 1440, height: 900) + let titleBar = chrome.height + let saved = CGSize(width: UserDefaults.standard.double(forKey: "streamWidth"), + height: UserDefaults.standard.double(forKey: "streamHeight")) + let wanted = saved.width >= 640 && saved.height >= 480 + ? saved + : CGSize(width: visible.width * 0.85, height: visible.height * 0.85 - titleBar) + size = CGSize(width: min(wanted.width, visible.width).rounded(), + height: min(wanted.height, visible.height - titleBar).rounded()) + } + streamSize = size + return CGSize(width: size.width * backingScale, height: size.height * backingScale) + } + + func phaseChanged(from old: StreamSession.Phase, to phase: StreamSession.Phase, session: StreamSession) { + guard let window else { return } + let fullScreen = window.styleMask.contains(.fullScreen) + if phase == .streaming, old != .streaming { + window.contentMinSize = CGSize(width: 640, height: 480) + window.title = session.welcome?.hostName ?? session.hostLabel + guard !fullScreen, let size = streamSize else { return } + connectFrame = window.frame + setContentSize(size, centered: true) + } else if old == .streaming, phase != .streaming { + window.title = "iFrame" + window.contentMinSize = CGSize(width: 420, height: 520) + guard !fullScreen else { return } + let content = window.contentLayoutRect.size + UserDefaults.standard.set(content.width, forKey: "streamWidth") + UserDefaults.standard.set(content.height, forKey: "streamHeight") + if let connectFrame { + window.setFrame(connectFrame, display: true, animate: true) + } else { + setContentSize(Self.connectSize, centered: true) + } + } + } + + /// Mirroring the Mac's own display: shape the window like it. + func welcomeChanged(_ welcome: Welcome) { + guard let window else { return } + window.title = welcome.hostName + guard !welcome.isVirtual, welcome.width > 0, welcome.height > 0 else { + return + } + let aspect = CGSize(width: welcome.width, height: welcome.height) + guard !window.styleMask.contains(.fullScreen) else { return } + let bounds = window.contentLayoutRect.size + let scale = min(bounds.width / aspect.width, bounds.height / aspect.height) + setContentSize(CGSize(width: (aspect.width * scale).rounded(), height: (aspect.height * scale).rounded()), + centered: false) + } + + /// Window frame minus the area SwiftUI lays out in. SwiftUI windows use a full-size content + /// view, so this is the title bar (plus toolbar), not what frameRect(forContentRect:) reports. + private var chrome: CGSize { + guard let window else { return CGSize(width: 0, height: 28) } + return CGSize(width: window.frame.width - window.contentLayoutRect.width, + height: window.frame.height - window.contentLayoutRect.height) + } + + /// Sizes the window so SwiftUI's layout area (below the title bar) is exactly `size`. + private func setContentSize(_ size: CGSize, centered: Bool) { + guard let window else { return } + var frame = NSRect(origin: .zero, size: CGSize(width: size.width + chrome.width, height: size.height + chrome.height)) + let visible = (window.screen ?? NSScreen.main)?.visibleFrame ?? .zero + if centered { + frame.origin = CGPoint(x: visible.midX - frame.width / 2, y: visible.midY - frame.height / 2) + } else { + // Keep the top-left corner where it is. + frame.origin = CGPoint(x: window.frame.minX, y: window.frame.maxY - frame.height) + } + frame.origin.x = max(visible.minX, min(frame.origin.x, visible.maxX - frame.width)) + frame.origin.y = max(visible.minY, min(frame.origin.y, visible.maxY - frame.height)) + window.setFrame(frame, display: true, animate: true) + } +} + +/// Hands the hosting NSWindow to MainWindow. +private struct WindowReader: NSViewRepresentable { + let window: MainWindow + + func makeNSView(context: Context) -> NSView { + let view = NSView() + DispatchQueue.main.async { [weak view] in + window.window = view?.window + view?.window?.tabbingMode = .disallowed + } + return view + } + + func updateNSView(_ view: NSView, context: Context) { + if window.window == nil, let w = view.window { window.window = w } + } +} + +extension StreamSession { + func connect(to endpoint: NWEndpoint, pin: String, label: String, window: MainWindow, density: MacDensity) { + let pixels = window.prepareStream() + connect(to: endpoint, pin: pin, label: label, + deviceName: Host.current().localizedName ?? "Mac", + maxFPS: window.maxFPS, pixels: pixels, + uiScale: density.uiScale(backing: window.backingScale)) + } +} diff --git a/MacClient/Info.plist b/MacClient/Info.plist new file mode 100644 index 0000000..4989e59 --- /dev/null +++ b/MacClient/Info.plist @@ -0,0 +1,34 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleDisplayName + iFrame + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + $(MARKETING_VERSION) + CFBundleVersion + $(CURRENT_PROJECT_VERSION) + LSApplicationCategoryType + public.app-category.utilities + NSBonjourServices + + _iframe._tcp + + NSHumanReadableCopyright + MIT License + NSLocalNetworkUsageDescription + iFrame finds and connects to Macs running iframe-host on your network. + + diff --git a/MacClient/MacConnectView.swift b/MacClient/MacConnectView.swift new file mode 100644 index 0000000..9c60beb --- /dev/null +++ b/MacClient/MacConnectView.swift @@ -0,0 +1,325 @@ +import Network +import SwiftUI + +// The iPad connect screen, adapted for a Mac window: one centered column, the mark up top, +// labeled rounded fields, and a single pill-shaped action. + +struct MacConnectView: View { + @EnvironmentObject private var session: StreamSession + @EnvironmentObject private var window: MainWindow + @StateObject private var browser = HostBrowser() + @AppStorage("pin") private var pin = "" + @AppStorage("manualHost") private var manualHost = "" + @AppStorage("macDensity") private var density = MacDensity.match.rawValue + @AppStorage("lastHostName") private var lastHostName = "" + @AppStorage("savedHosts") private var savedHostsJSON = "[]" + @State private var pendingSave: SavedHost? + @State private var selectedHostID: String? + @State private var showingError = false + @State private var errorMessage = "" + + private var selectedHost: HostBrowser.Found? { + browser.hosts.first { $0.id == selectedHostID } + } + + private var trimmedAddress: String { manualHost.trimmingCharacters(in: .whitespaces) } + + private var savedHosts: [SavedHost] { + get { (try? JSONDecoder().decode([SavedHost].self, from: Data(savedHostsJSON.utf8))) ?? [] } + nonmutating set { + savedHostsJSON = (try? String(decoding: JSONEncoder().encode(newValue), as: UTF8.self)) ?? "[]" + } + } + + private var targetName: String? { + if !trimmedAddress.isEmpty { return trimmedAddress } + return selectedHost?.name + } + + private var canConnect: Bool { + !pin.isEmpty && targetName != nil && session.phase != .connecting + } + + var body: some View { + ZStack { + Color(nsColor: .windowBackgroundColor).ignoresSafeArea() + + ScrollView { + VStack(alignment: .leading, spacing: 0) { + header + fields + connectButton + footer + } + .frame(maxWidth: 440) + .padding(.horizontal, 32) + .frame(maxWidth: .infinity) + } + + if session.phase == .connecting { + connectingCard + } + } + .frame(minWidth: 420, minHeight: 520) + .onAppear { + browser.start() + // `open iFrame.app --args -manualHost -pin -autoConnect YES` connects at launch + // (launch arguments override the saved settings for that run). + if UserDefaults.standard.bool(forKey: "autoConnect"), session.phase == .idle { + UserDefaults.standard.removeObject(forKey: "autoConnect") + DispatchQueue.main.async(execute: connect) + } + } + .onDisappear { browser.stop() } + .onChange(of: browser.hosts) { _, hosts in + // Preselect the Mac used last time, or the only one around. + guard selectedHostID == nil || !hosts.contains(where: { $0.id == selectedHostID }) else { return } + selectedHostID = (hosts.first { $0.name == lastHostName } ?? (hosts.count == 1 ? hosts.first : nil))?.id + } + .onChange(of: session.phase) { _, phase in + // Only bookmark addresses that actually worked, most recent first. + if phase == .streaming, let saved = pendingSave { + savedHosts = [saved] + savedHosts.filter { $0.address != saved.address } + } + if phase != .connecting { pendingSave = nil } + if case .failed(let message) = phase { + errorMessage = message + showingError = true + } + } + .alert("Couldn't connect", isPresented: $showingError) { + Button("OK") {} + } message: { + Text(errorMessage) + } + } + + // MARK: Sections + + private var header: some View { + VStack(spacing: 12) { + Image("IFrameMark") + .resizable() + .aspectRatio(contentMode: .fit) + .frame(width: 112) + Text("iFrame") + .font(.system(size: 30, weight: .bold)) + Text("Another Mac, in a window. Run iframe-host on it to get started.") + .font(.system(size: 11)) + .foregroundColor(Color(nsColor: .tertiaryLabelColor)) + .multilineTextAlignment(.center) + } + .frame(maxWidth: .infinity) + .padding(.top, 36) + .padding(.bottom, 28) + } + + private var fields: some View { + VStack(alignment: .leading, spacing: 20) { + VStack(alignment: .leading, spacing: 8) { + FieldLabel("Mac:") + if browser.hosts.isEmpty && savedHosts.isEmpty { + HStack(spacing: 10) { + ProgressView().controlSize(.small) + Text("Looking for Macs running iframe-host…") + .foregroundColor(Color(nsColor: .secondaryLabelColor)) + } + .font(.system(size: 13)) + .frame(maxWidth: .infinity, alignment: .leading) + .frameFieldBackground() + } else { + VStack(spacing: 6) { + ForEach(browser.hosts) { host in + HostRow(name: host.name, + selected: host.id == selectedHostID && trimmedAddress.isEmpty) { + selectedHostID = host.id + manualHost = "" + } + } + ForEach(savedHosts) { saved in + HostRow(name: saved.address, symbol: "bookmark", + selected: saved.address == trimmedAddress) { + manualHost = saved.address + pin = saved.pin + } + .contextMenu { + Button("Forget \(saved.address)", systemImage: "trash", role: .destructive) { + savedHosts = savedHosts.filter { $0.address != saved.address } + if trimmedAddress == saved.address { manualHost = "" } + } + } + } + } + } + } + + FrameInputField(label: "PIN:", placeholder: "Shown by iframe-host", text: $pin, monospaced: true) + .onSubmit(connect) + + VStack(alignment: .leading, spacing: 8) { + FieldLabel("Mac display:") + Menu { + Picker("Mac display", selection: $density) { + ForEach(MacDensity.allCases) { choice in + Text(choice.label).tag(choice.rawValue) + } + } + .pickerStyle(.inline) + } label: { + Text(MacDensity(rawValue: density)?.label ?? MacDensity.match.label) + .font(.system(size: 13)) + } + .menuStyle(.borderlessButton) + .frameFieldBackground() + Text(density > 0 + ? "The Mac gets a virtual display shaped like this window. Resize or go full screen and it follows." + : "Shows the Mac's own screen, scaled to fit this window.") + .font(.system(size: 11)) + .foregroundColor(Color(nsColor: .tertiaryLabelColor)) + } + + FrameInputField(label: "Or connect by address (optional):", + placeholder: "192.168.1.20, mac-mini.local, Tailscale name", + text: $manualHost) + .onSubmit(connect) + } + } + + private var connectButton: some View { + Button(action: connect) { + Text(targetName.map { "Connect to \($0)" } ?? "Connect") + .lineLimit(1) + .font(.system(size: 15, weight: .semibold)) + .foregroundColor(canConnect ? .iframeInk : Color(nsColor: .secondaryLabelColor)) + .frame(maxWidth: .infinity) + .frame(height: 44) + .background(canConnect ? Color.iframeTeal : Color.gray.opacity(0.3)) + .cornerRadius(22) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .keyboardShortcut(.defaultAction) + .disabled(!canConnect) + .padding(.top, 28) + } + + private var footer: some View { + VStack(spacing: 0) { + Divider().padding(.top, 24) + Text("Default port \(String(IFrame.defaultPort)). Use host:port to override. While streaming, ⌃⌥⌘D disconnects and ⌃⌥⌘F toggles full screen; every other shortcut goes to the remote Mac.") + .font(.system(size: 11)) + .foregroundColor(Color(nsColor: .tertiaryLabelColor)) + .multilineTextAlignment(.center) + .padding(.top, 12) + .padding(.bottom, 20) + } + .frame(maxWidth: .infinity) + } + + private var connectingCard: some View { + ZStack { + Color.black.opacity(0.4).ignoresSafeArea() + VStack(spacing: 20) { + IFrameLoadingView(size: 64) + Text("Connecting to \(session.hostLabel)…") + .font(.system(size: 15, weight: .semibold)) + Button("Cancel") { session.disconnect() } + .keyboardShortcut(.cancelAction) + } + .padding(32) + .background(Color(nsColor: .windowBackgroundColor)) + .cornerRadius(16) + .shadow(radius: 20) + } + .transition(.opacity) + } + + // MARK: Actions + + private func connect() { + guard canConnect else { return } + let choice = MacDensity(rawValue: density) ?? .match + if !trimmedAddress.isEmpty { + guard let (endpoint, host) = SavedHost.endpoint(for: trimmedAddress) else { return } + pendingSave = SavedHost(address: trimmedAddress, pin: pin) + session.connect(to: endpoint, pin: pin, label: host, window: window, density: choice) + } else if let host = selectedHost { + lastHostName = host.name + session.connect(to: host.endpoint, pin: pin, label: host.name, window: window, density: choice) + } + } +} + +// MARK: - Components + +private struct FieldLabel: View { + let text: String + init(_ text: String) { self.text = text } + + var body: some View { + Text(text) + .font(.system(size: 13)) + .foregroundColor(Color(nsColor: .secondaryLabelColor)) + } +} + +private struct FrameInputField: View { + let label: String + let placeholder: String + @Binding var text: String + var monospaced = false + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + FieldLabel(label) + TextField(placeholder, text: $text) + .textFieldStyle(.plain) + .autocorrectionDisabled() + .font(monospaced ? .system(size: 13).monospacedDigit() : .system(size: 13)) + .frameFieldBackground() + } + } +} + +private struct HostRow: View { + let name: String + var symbol = "macmini" + let selected: Bool + let action: () -> Void + + var body: some View { + Button(action: action) { + HStack(spacing: 10) { + Image(systemName: symbol) + .font(.system(size: 15)) + .foregroundColor(selected ? .iframeTeal : Color(nsColor: .secondaryLabelColor)) + Text(name) + .font(.system(size: 13)) + .foregroundColor(.primary) + Spacer() + Image(systemName: selected ? "checkmark.circle.fill" : "circle") + .font(.system(size: 16)) + .foregroundColor(selected ? .iframeTeal : Color(nsColor: .tertiaryLabelColor)) + } + .frameFieldBackground(highlighted: selected) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + } +} + +private extension View { + /// The rounded, hairline-bordered field box used throughout the iPad app. + func frameFieldBackground(highlighted: Bool = false) -> some View { + padding(.horizontal, 12) + .padding(.vertical, 10) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(nsColor: .controlBackgroundColor)) + .cornerRadius(10) + .overlay( + RoundedRectangle(cornerRadius: 10) + .stroke(highlighted ? Color.iframeTeal : Color(nsColor: .separatorColor), + lineWidth: highlighted ? 2 : 1) + ) + } +} diff --git a/MacClient/MacStreamView.swift b/MacClient/MacStreamView.swift new file mode 100644 index 0000000..05a8478 --- /dev/null +++ b/MacClient/MacStreamView.swift @@ -0,0 +1,358 @@ +import AppKit +import AVFoundation +import SwiftUI + +/// The video surface plus all input handling. +/// +/// Mouse, trackpad and keyboard map 1:1 to the remote Mac. NSEvent key codes are already macOS +/// virtual key codes, so keys pass straight through. While this view has focus, every key combo +/// (⌘Q, ⌘W, ⌘Space…) goes to the remote Mac except ⌃⌥⌘ ones, which drive this app's Stream menu. +/// The system still keeps ⌘Tab and other shortcuts it handles before apps see them. +/// +/// The Mac's pointer shape arrives as a picture and becomes this window's NSCursor, so the pointer +/// is the real hardware cursor and moves with zero lag. Capture leaves it out of the video. +final class StreamNSView: NSView { + let session: StreamSession + private let displayLayer = AVSampleBufferDisplayLayer() + + var welcome: Welcome? { + didSet { + guard welcome != oldValue else { return } + updateCursor() + scheduleDisplayRequest() + } + } + + /// See MacDensity. Changing it reshapes the virtual display. + var density: Double = 1 { + didSet { if density != oldValue { scheduleDisplayRequest(force: true) } } + } + + private var keyMonitor: Any? + private var heldKeys = Set() + private var heldModifiers = Set() + private var heldButtons = Set() + private var lastPoint = CGPoint(x: 0.5, y: 0.5) + private var macCursor: MacCursor? + private var nsCursor: NSCursor = .arrow + + init(session: StreamSession) { + self.session = session + super.init(frame: .zero) + wantsLayer = true + layerContentsRedrawPolicy = .never + displayLayer.videoGravity = .resizeAspect + displayLayer.backgroundColor = NSColor.black.cgColor + session.attach(renderer: displayLayer.sampleBufferRenderer) + session.onCursor = { [weak self] cursor in + self?.macCursor = cursor + self?.updateCursor() + } + } + + required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") } + + override func makeBackingLayer() -> CALayer { displayLayer } + + override var acceptsFirstResponder: Bool { true } + override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true } + + override func viewDidMoveToWindow() { + super.viewDidMoveToWindow() + let center = NotificationCenter.default + center.removeObserver(self) + if let window { + DispatchQueue.main.async { window.makeFirstResponder(self) } + installKeyMonitor() + center.addObserver(self, selector: #selector(windowResignedKey), + name: NSWindow.didResignKeyNotification, object: window) + } else { + releaseAll() + removeKeyMonitor() + pendingRequest?.cancel() + } + } + + override func resignFirstResponder() -> Bool { + releaseAll() + return super.resignFirstResponder() + } + + @objc private func windowResignedKey() { + releaseAll() + } + + // MARK: Geometry + + private var videoRect: CGRect { + guard let welcome, welcome.width > 0, welcome.height > 0 else { return bounds } + return AVMakeRect(aspectRatio: CGSize(width: welcome.width, height: welcome.height), insideRect: bounds) + } + + /// View point -> 0...1 over the video, top-left origin like the Mac's screen. + private func normalized(_ event: NSEvent) -> CGPoint { + let p = convert(event.locationInWindow, from: nil) + let rect = videoRect + guard rect.width > 0, rect.height > 0 else { return lastPoint } + lastPoint = CGPoint(x: min(max((p.x - rect.minX) / rect.width, 0), 1), + y: min(max((rect.maxY - p.y) / rect.height, 0), 1)) + return lastPoint + } + + /// Mac points per view point, so trackpad scrolling moves content 1:1 under your fingers. + private var scrollScale: CGFloat { + guard let welcome, videoRect.width > 0 else { return 1 } + return CGFloat(welcome.pointWidth) / videoRect.width + } + + override func setFrameSize(_ newSize: NSSize) { + super.setFrameSize(newSize) + updateCursor() + scheduleDisplayRequest() + } + + override func viewDidEndLiveResize() { + super.viewDidEndLiveResize() + scheduleDisplayRequest() + } + + override func viewDidChangeBackingProperties() { + super.viewDidChangeBackingProperties() + scheduleDisplayRequest(force: true) + } + + // MARK: Virtual display follows the window + + private var pendingRequest: DispatchWorkItem? + private var requestedPixels: CGSize? + private var requestedScale: Double? + + /// The host's virtual display is reshaped in place, so resizing the window (or going full screen) + /// gives the Mac a desktop exactly this size instead of letterboxing a fixed one. + private func scheduleDisplayRequest(force: Bool = false) { + guard let welcome, welcome.isVirtual else { return } + if requestedPixels == nil { + // The stream we connected with already matches; only react to changes from here on. + requestedPixels = targetPixels + requestedScale = targetScale + if !force { return } + } + pendingRequest?.cancel() + let work = DispatchWorkItem { [weak self] in self?.sendDisplayRequest() } + pendingRequest = work + DispatchQueue.main.asyncAfter(deadline: .now() + 0.4, execute: work) + } + + private var backing: CGFloat { window?.backingScaleFactor ?? 2 } + + private var targetPixels: CGSize { + // The host needs at least 640x480 for a virtual display. + CGSize(width: max(640, (bounds.width * backing).rounded()), height: max(480, (bounds.height * backing).rounded())) + } + + private var targetScale: Double { + MacDensity(rawValue: density)?.uiScale(backing: backing) ?? Double(backing) + } + + private func sendDisplayRequest() { + guard !inLiveResize, window != nil, bounds.width > 0, welcome?.isVirtual == true else { return } + let pixels = targetPixels + let scale = targetScale + if let current = requestedPixels, abs(current.width - pixels.width) <= 2, + abs(current.height - pixels.height) <= 2, requestedScale == scale { return } + requestedPixels = pixels + requestedScale = scale + session.requestDisplay(pixels: pixels, uiScale: scale) + } + + // MARK: Cursor + + /// Sizes the Mac's pointer like it is on the Mac: Mac points scaled to the video on screen. + private func updateCursor() { + if let macCursor, let welcome, welcome.pointWidth > 0 { + let scale = videoRect.width / CGFloat(welcome.pointWidth) // view points per Mac point + let size = NSSize(width: max(1, macCursor.size.width * scale), height: max(1, macCursor.size.height * scale)) + nsCursor = NSCursor(image: NSImage(cgImage: macCursor.image, size: size), + hotSpot: NSPoint(x: macCursor.hotspot.x * scale, y: macCursor.hotspot.y * scale)) + } else { + nsCursor = .arrow + } + window?.invalidateCursorRects(for: self) + // Cursor rects only apply on the next mouse move; show a new shape right away. + if let window, window.isKeyWindow, videoRect.contains(convert(window.mouseLocationOutsideOfEventStream, from: nil)) { + nsCursor.set() + } + } + + override func resetCursorRects() { + addCursorRect(videoRect, cursor: nsCursor) + } + + // MARK: Mouse + + private var trackingArea: NSTrackingArea? + + override func updateTrackingAreas() { + super.updateTrackingAreas() + if let trackingArea { removeTrackingArea(trackingArea) } + let area = NSTrackingArea(rect: .zero, options: [.mouseMoved, .activeInKeyWindow, .inVisibleRect], + owner: self, userInfo: nil) + addTrackingArea(area) + trackingArea = area + } + + override func mouseMoved(with event: NSEvent) { session.mouseMove(normalized(event)) } + override func mouseDragged(with event: NSEvent) { session.mouseMove(normalized(event)) } + override func rightMouseDragged(with event: NSEvent) { session.mouseMove(normalized(event)) } + override func otherMouseDragged(with event: NSEvent) { session.mouseMove(normalized(event)) } + + override func mouseDown(with event: NSEvent) { button(0, down: true, event) } + override func mouseUp(with event: NSEvent) { button(0, down: false, event) } + override func rightMouseDown(with event: NSEvent) { button(1, down: true, event) } + override func rightMouseUp(with event: NSEvent) { button(1, down: false, event) } + override func otherMouseDown(with event: NSEvent) { button(UInt8(clamping: event.buttonNumber), down: true, event) } + override func otherMouseUp(with event: NSEvent) { button(UInt8(clamping: event.buttonNumber), down: false, event) } + + private func button(_ button: UInt8, down: Bool, _ event: NSEvent) { + if down { + window?.makeFirstResponder(self) + heldButtons.insert(button) + } else { + heldButtons.remove(button) + } + let p = normalized(event) + session.mouseMove(p) + session.mouseButton(button, down: down, at: p) + } + + override func scrollWheel(with event: NSEvent) { + var dx = event.scrollingDeltaX + var dy = event.scrollingDeltaY + if event.hasPreciseScrollingDeltas { + dx *= scrollScale + dy *= scrollScale + } else { + // Wheel mice report lines. + dx *= 16 + dy *= 16 + } + guard dx != 0 || dy != 0 else { return } + session.scroll(dx: dx, dy: dy) + } + + // MARK: Keyboard + + /// Device-dependent modifier bits (NX_DEVICE*KEYMASK), to tell a left from a right modifier. + private static let modifierBits: [UInt16: UInt] = [ + 0x3B: 0x0001, 0x3E: 0x2000, // left, right control + 0x38: 0x0002, 0x3C: 0x0004, // left, right shift + 0x37: 0x0008, 0x36: 0x0010, // left, right command + 0x3A: 0x0020, 0x3D: 0x0040, // left, right option + ] + private static let capsLock: UInt16 = 0x39 + private static let appShortcut: NSEvent.ModifierFlags = [.control, .option, .command] + + /// A local monitor sees every key event before menus do, including key-ups with ⌘ held + /// (which never reach keyUp(with:)), so shortcuts can go to the remote Mac. + private func installKeyMonitor() { + guard keyMonitor == nil else { return } + keyMonitor = NSEvent.addLocalMonitorForEvents(matching: [.keyDown, .keyUp, .flagsChanged]) { [weak self] event in + guard let self, let window = self.window, event.window === window, window.isKeyWindow, + window.firstResponder === self else { return event } + if event.type == .keyDown, + event.modifierFlags.intersection(.deviceIndependentFlagsMask).isSuperset(of: Self.appShortcut) { + return event // the Stream menu + } + self.forward(event) + return nil + } + } + + private func removeKeyMonitor() { + if let keyMonitor { NSEvent.removeMonitor(keyMonitor) } + keyMonitor = nil + } + + private static func mods(_ flags: NSEvent.ModifierFlags) -> KeyMods { + var mods = KeyMods() + if flags.contains(.shift) { mods.insert(.shift) } + if flags.contains(.control) { mods.insert(.control) } + if flags.contains(.option) { mods.insert(.option) } + if flags.contains(.command) { mods.insert(.command) } + if flags.contains(.capsLock) { mods.insert(.capsLock) } + return mods + } + + private func forward(_ event: NSEvent) { + let code = event.keyCode + let mods = Self.mods(event.modifierFlags) + switch event.type { + case .keyDown: + heldKeys.insert(code) + session.key(code, action: event.isARepeat ? .repeatDown : .down, mods: mods) + case .keyUp: + heldKeys.remove(code) + session.key(code, action: .up, mods: mods) + case .flagsChanged: + if code == Self.capsLock { + // Each press toggles caps lock; there's no separate release event. + session.key(code, action: .down, mods: mods) + session.key(code, action: .up, mods: mods) + } else if let bit = Self.modifierBits[code] { + let down = event.modifierFlags.rawValue & bit != 0 + if down { heldModifiers.insert(code) } else { heldModifiers.remove(code) } + session.key(code, action: down ? .down : .up, mods: mods) + } + default: + break + } + } + + /// Lets go of everything on the remote Mac when focus leaves, so nothing stays stuck down. + func releaseAll() { + for code in heldKeys.union(heldModifiers) { session.key(code, action: .up, mods: []) } + heldKeys.removeAll() + heldModifiers.removeAll() + for button in heldButtons { session.mouseButton(button, down: false, at: lastPoint) } + heldButtons.removeAll() + } +} + +private struct StreamViewRepresentable: NSViewRepresentable { + let session: StreamSession + let welcome: Welcome? + let density: Double + + func makeNSView(context: Context) -> StreamNSView { + StreamNSView(session: session) + } + + func updateNSView(_ view: StreamNSView, context: Context) { + view.welcome = welcome + view.density = density + } + + static func dismantleNSView(_ view: StreamNSView, coordinator: ()) { + view.releaseAll() + } +} + +struct MacStreamScreen: View { + @EnvironmentObject private var session: StreamSession + @AppStorage("showStats") private var showStats = false + @AppStorage("macDensity") private var density = MacDensity.match.rawValue + + var body: some View { + ZStack(alignment: .topLeading) { + StreamViewRepresentable(session: session, welcome: session.welcome, + density: density > 0 ? density : MacDensity.match.rawValue) + if showStats { + StatsView(stats: session.stats, welcome: session.welcome) + .padding(10) + .allowsHitTesting(false) + } + } + .background(Color.black) + .frame(minWidth: 640, minHeight: 480) + } +} diff --git a/README.md b/README.md index 6ea2408..9d80430 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # iFrame -Low-latency Mac → iPad or Windows remote desktop for Apple silicon. It sends a video stream instead of pixel tiles. +Low-latency Mac → iPad, Mac, Windows or Linux remote desktop for Apple silicon. It sends a video stream instead of pixel tiles. **Host requirements:** a Mac with Apple silicon running macOS 14 or later (developed on an M4 Mac mini, macOS 15) and Xcode command-line tools. -**Clients:** an iPad on iPadOS 17 or later, or a Windows 10/11 x64 PC. Building the iPad app requires Xcode and [XcodeGen](https://github.com/yonaskolb/XcodeGen), plus `rsvg-convert` and ImageMagick if you regenerate icons (`brew install xcodegen librsvg imagemagick`). The [Windows client](Windows/README.md) uses Node.js to build and includes its runtime when packaged. +**Clients:** an iPad on iPadOS 17 or later, a Mac on macOS 14 or later (Apple silicon or Intel), or a Windows 10/11 x64 PC. Building the iPad app requires Xcode and [XcodeGen](https://github.com/yonaskolb/XcodeGen), plus `rsvg-convert` and ImageMagick if you regenerate icons (`brew install xcodegen librsvg imagemagick`). The [Windows client](Windows/README.md) uses Node.js to build and includes its runtime when packaged. ``` Mac (iframe-host) iPad (iFrame app) @@ -18,6 +18,7 @@ CGEvent input injection ``` What makes it fast: +- **Local cursor.** The Mac sends its cursor shape (arrow, I-beam, resize…) and leaves the pointer out of the video. Every client draws it at the local pointer position, so it moves with no network delay. - **Zero-copy pipeline.** Capture hands IOSurfaces straight to the media engine. No color conversion and no CPU copies. - **Low-latency rate control.** One frame in, one frame out, no reordering. Keyframes only on connect or after an error, so there are no periodic spikes. - **Ack-based flow control.** At most 3 frames can be unacknowledged. When Wi-Fi backs up, frames are skipped *before* encoding instead of piling up in socket buffers, and the newest screen goes out as soon as the link frees up. Lag never builds up. @@ -56,6 +57,30 @@ Building it yourself? Change `DEVELOPMENT_TEAM` and the `com.toddfaulls` bundle Your Mac appears under **Mac** on the connect screen. Enter the PIN, pick a display density, and tap **Connect**. +## Client (Mac) + +A native Universal app (Apple silicon + Intel, macOS 14+), target `iFrameMac` in the same Xcode project. + +```sh +scripts/build-mac-client.sh # → dist/iFrame.app and dist/iFrame--macOS-Universal.zip +``` + +It signs with your Apple Development / Developer ID identity if you have one (`security unlock-keychain` +first over SSH), otherwise ad hoc; `IFRAME_SIGN_IDENTITY=-` forces ad hoc. The app isn't notarized, so on +another Mac open it once with right-click → **Open** (or System Settings → Privacy & Security → **Open Anyway**). + +- **Display:** the host makes a virtual display shaped like the window. Resize the window or go full + screen and the remote desktop reshapes to fit, instead of being letterboxed. *More space* / *Most space* + trade sharpness for room; *Use the Mac's own display* mirrors the host's screen instead. +- **Input:** mouse (all buttons), trackpad scrolling 1:1, and the keyboard pass straight through. Key codes are + already Mac key codes, so every layout and shortcut works. While the stream has focus, **every** shortcut + goes to the remote Mac (⌘Q, ⌘W, ⌘Space…) except ⌃⌥⌘ ones: ⌃⌥⌘F full screen, ⌃⌥⌘S stats, + ⌃⌥⌘R refresh, ⌃⌥⌘D disconnect. macOS still keeps ⌘Tab and Mission Control for itself. +- **Cursor:** the remote Mac's pointer shape becomes this window's real cursor, so it moves with no lag. +- Don't run it on the host Mac itself against its own virtual display. The window would sit on the display + it reshapes. +- Testing: `open dist/iFrame.app --args -manualHost -pin -autoConnect YES` connects at launch. + ## Client (Linux) `Linux/` has a C client (SDL2 + FFmpeg, NVDEC/VAAPI hardware decoding) that speaks the same @@ -116,7 +141,6 @@ the host PIN. The packaged app includes its runtime. See [Windows controls, test ## Known limits / next steps - **Video isn't encrypted; there's only a PIN.** Use it on a trusted LAN or over Tailscale. -- **The cursor is part of the video.** A local cursor overlay would make pointing feel instant. - **Transport is TCP.** That's fine on a LAN. UDP with FEC would handle lossy networks better. - **No audio and no clipboard sync yet.** diff --git a/Shared/Protocol.swift b/Shared/Protocol.swift index b3cf491..740462a 100644 --- a/Shared/Protocol.swift +++ b/Shared/Protocol.swift @@ -22,6 +22,7 @@ enum MsgType: UInt8 { case pong = 0x05 // echo of ping payload case authFailed = 0x06 case textFocus = 0x07 // editable u8, x f32, y f32, w f32, h f32 (normalized over the stream) + case cursor = 0x08 // hotspot x u16, y u16, size w u16, h u16 (Mac points), PNG at 2x // client -> host case hello = 0x10 // JSON Hello case mouseMove = 0x11 // x f32, y f32 (normalized 0...1) @@ -49,6 +50,8 @@ struct Hello: Codable { var supportsHEVC: Bool var maxFPS: Int var display: DisplayRequest? + /// The client draws the pointer itself from `cursor` messages; capture omits it. + var localCursor: Bool? } /// The client's screen in pixels (current orientation). The host creates a virtual display diff --git a/Windows/README.md b/Windows/README.md index a7b8ee0..24d1ebc 100644 --- a/Windows/README.md +++ b/Windows/README.md @@ -62,7 +62,7 @@ FFmpeg is only needed to regenerate that fixture. Real Mac capture, permission b - Converts four-byte AVCC NAL lengths to Annex B and injects SPS/PPS on keyframes. Frames are drawn directly to a canvas and acknowledged after decode and drawing, preserving the host's flow control. Malformed frames trigger throttled keyframe recovery; repeated decoder errors close the session. - The renderer is sandboxed with context isolation, no Node integration, a restrictive content security policy, and a narrow validated IPC bridge. Socket framing is bounded at the host protocol's 32 MiB maximum. - The host protocol is **not encrypted**. Use a trusted LAN or Tailscale; do not expose the host directly to the Internet. -- No audio, automatic clipboard synchronization, HEVC, or local remote-cursor overlay. The Mac cursor is part of the video. The Send text button is an explicit text transfer, not clipboard sync. +- No audio, automatic clipboard synchronization, or HEVC. (The Mac's cursor is drawn locally as the Windows pointer, in the Mac's current shape.) The Send text button is an explicit text transfer, not clipboard sync. - This is an Electron client, not a native WinUI application. It includes Chromium and therefore has a larger distribution and memory footprint. MIT, matching the repository license. Bundled Electron and dependency licenses are included in the distribution. diff --git a/Windows/src/protocol.cjs b/Windows/src/protocol.cjs index 2425e16..96cfc4d 100644 --- a/Windows/src/protocol.cjs +++ b/Windows/src/protocol.cjs @@ -1,7 +1,7 @@ 'use strict'; const MAX_MESSAGE = 32 << 20; const T = Object.freeze({ welcome: 1, format: 2, frame: 3, stats: 4, pong: 5, - authFailed: 6, textFocus: 7, hello: 16, mouseMove: 17, mouseButton: 18, + authFailed: 6, textFocus: 7, cursor: 8, hello: 16, mouseMove: 17, mouseButton: 18, scroll: 19, key: 20, text: 21, requestKeyframe: 22, ping: 23, ack: 24, display: 25 }); function packet(type, payload = Buffer.alloc(0)) { diff --git a/Windows/src/session.cjs b/Windows/src/session.cjs index 6986b0f..78fecbd 100644 --- a/Windows/src/session.cjs +++ b/Windows/src/session.cjs @@ -24,7 +24,9 @@ class Session extends EventEmitter { this.emit('message', { type, data }); this.emit('state', { phase: 'streaming', welcome: w }); } else if (!welcomed) { - throw new Error('Unexpected message before welcome'); + // The Mac's cursor shape can arrive first; anything else is a protocol error. + if (type !== T.cursor) throw new Error('Unexpected message before welcome'); + this.emit('message', { type, data }); } else if (type === T.pong) { if (data.length !== 8) throw new Error('Invalid ping response'); const elapsed = process.hrtime.bigint() - data.readBigUInt64BE(); @@ -45,7 +47,8 @@ class Session extends EventEmitter { socket.on('close', () => fail('The Mac closed the connection. Check its screen recording permission and host log.')); socket.on('connect', () => { this.send(T.hello, Buffer.from(JSON.stringify({ version: 1, pin: o.pin, name: os.hostname(), supportsHEVC: false, - maxFPS: o.fps, display: { width: o.width, height: o.height, uiScale: o.scale } }))); + // localCursor: the Mac sends its cursor shape and leaves the pointer out of the video. + localCursor: true, maxFPS: o.fps, display: { width: o.width, height: o.height, uiScale: o.scale } }))); this.timer = setInterval(() => { if (!this.ready) return; if ([...this.pending.values()].some(time => Date.now() - time > 10000)) diff --git a/Windows/test/protocol.test.cjs b/Windows/test/protocol.test.cjs index 2d09bd7..392f312 100644 --- a/Windows/test/protocol.test.cjs +++ b/Windows/test/protocol.test.cjs @@ -65,7 +65,11 @@ test('loopback handshake, decode acknowledgements, ping and input', async t => { }; await wait(() => messages.length > 0); const hello = JSON.parse(messages[0].data); assert.equal(hello.pin,'1234'); assert.equal(hello.supportsHEVC,false); - assert.equal(hello.display.uiScale,2); + assert.equal(hello.display.uiScale,2); assert.equal(hello.localCursor,true); + // The Mac sends its cursor shape before the welcome; that must not end the session. + const cursor = once(session,'message'); + peer.write(packet(T.cursor,Buffer.from([0,4,0,2,0,17,0,23,0x89,0x50,0x4e,0x47]))); + assert.equal((await cursor)[0].type,T.cursor); const streaming = once(session,'state'); peer.write(packet(T.welcome,Buffer.from(JSON.stringify({width:1920,height:1080,codec:0})))); assert.equal((await streaming)[0].phase,'streaming'); diff --git a/Windows/ui/app.mjs b/Windows/ui/app.mjs index ecf0a6c..49097e6 100644 --- a/Windows/ui/app.mjs +++ b/Windows/ui/app.mjs @@ -1,5 +1,6 @@ import { Video } from './video.mjs'; import { attachInput } from './input.mjs'; +import { MacCursor } from './cursor.mjs'; const $ = id => document.getElementById(id), api = window.iframe; let active = false, welcome, rtt = 0, hostStats = {}, frames = 0, decodeTotal = 0; const settings = ['host','port','resolution','fps','scale']; @@ -15,6 +16,8 @@ function saveSettings() { async function fail(message) { await api.disconnect(); status(message); } const video = new Video($('video'), api, fail, ms => { frames++; decodeTotal += ms; $('waiting').hidden = true; }, message => { $('waiting').textContent = message; }); +const macCursor = new MacCursor($('video')); +window.addEventListener('resize', () => macCursor.apply()); const releaseInput = attachInput($('video'), api, { active: () => active, swap: () => $('swap').checked, shortcut: code => { if (code === 'KeyF') toggleFullscreen(); else if (code === 'KeyQ') disconnect(); else $('fullscreen').focus(); } @@ -60,16 +63,20 @@ api.onState(state => { $('host-name').textContent = welcome.hostName; $('stream-info').textContent = `${welcome.width} × ${welcome.height} · H.264 · ${welcome.fps} fps`; $('keyboard-mode').textContent = $('swap').checked ? 'Ctrl → ⌘ Command' : 'Windows → ⌘ Command'; - $('pin').value = ''; $('video').focus(); + $('pin').value = ''; $('video').focus(); macCursor.setWelcome(welcome); } else if (!connecting) { - releaseInput(); active = false; video.close(); welcome = null; frames = 0; decodeTotal = 0; hostStats = {}; rtt = 0; + releaseInput(); active = false; video.close(); macCursor.reset(); welcome = null; frames = 0; decodeTotal = 0; hostStats = {}; rtt = 0; $('text-dialog').close('cancel'); $('connect-page').hidden = false; $('stream-page').hidden = true; status(state.phase === 'failed' ? state.message : ''); $('connect').focus(); } else status('Connecting to your Mac…'); }); api.onMessage(({type, data}) => { try { - if (type === 1) { video.close(); $('waiting').hidden = false; } + if (type === 1) { // also re-sent when the Mac's display is reshaped + video.close(); $('waiting').hidden = false; + welcome = JSON.parse(new TextDecoder().decode(data)); macCursor.setWelcome(welcome); + } + else if (type === 8) macCursor.update(data); else if (type === 2) video.configure(data); else if (type === 3) video.frame(data); else if (type === 4) hostStats = JSON.parse(new TextDecoder().decode(data)); diff --git a/Windows/ui/cursor.mjs b/Windows/ui/cursor.mjs new file mode 100644 index 0000000..d97dd9f --- /dev/null +++ b/Windows/ui/cursor.mjs @@ -0,0 +1,49 @@ +// The Mac's cursor, drawn as our own pointer. Hosts that send cursor shapes leave the cursor out +// of the video, so the pointer moves with Windows' own zero-latency mouse instead of a frame late. +export class MacCursor { + constructor(canvas) { this.canvas = canvas; this.shape = null; this.welcome = null; this.version = 0; } + + /** Payload: hotspot x, y, size w, h (u16 big endian, Mac points), then a PNG at 2x. */ + update(bytes) { + const data = new Uint8Array(bytes); + if (data.length <= 8) return; + const view = new DataView(data.buffer, data.byteOffset, data.byteLength); + const shape = {hotspotX: view.getUint16(0), hotspotY: view.getUint16(2), width: view.getUint16(4), height: view.getUint16(6)}; + const version = ++this.version; + createImageBitmap(new Blob([data.subarray(8)], {type: 'image/png'})).then(bitmap => { + if (version !== this.version) { bitmap.close(); return; } // a newer shape already arrived + this.shape?.bitmap.close(); + this.shape = {...shape, bitmap}; + this.apply(); + }).catch(() => {}); + } + + setWelcome(welcome) { this.welcome = welcome; this.apply(); } + + reset() { + this.version++; this.shape?.bitmap.close(); this.shape = null; this.welcome = null; + this.canvas.style.cursor = ''; // back to the stylesheet: hidden while the cursor is in the video + } + + /** Scales the shape to how big the Mac's screen appears in the window, as on the Mac itself. */ + apply() { + const {shape, welcome, canvas} = this; + if (!shape || !welcome?.pointWidth || !welcome.width || !welcome.height) return; + const rect = canvas.getBoundingClientRect(); + if (!rect.width || !rect.height) return; + const shownWidth = welcome.width * Math.min(rect.width / welcome.width, rect.height / welcome.height); + const scale = shownWidth / welcome.pointWidth; // CSS px per Mac point + // Chromium ignores cursor images over 128 CSS px. + const cssW = Math.max(1, Math.min(shape.width * scale, 128)), cssH = Math.max(1, Math.min(shape.height * scale, 128)); + const render = ratio => { + const c = document.createElement('canvas'); + c.width = Math.max(1, Math.round(cssW * ratio)); c.height = Math.max(1, Math.round(cssH * ratio)); + const g = c.getContext('2d'); g.imageSmoothingQuality = 'high'; g.drawImage(shape.bitmap, 0, 0, c.width, c.height); + return c.toDataURL('image/png'); + }; + const hx = Math.min(Math.round(shape.hotspotX * scale), Math.round(cssW) - 1); + const hy = Math.min(Math.round(shape.hotspotY * scale), Math.round(cssH) - 1); + const ratio = window.devicePixelRatio || 1; + canvas.style.cursor = `image-set(url("${render(ratio)}") ${ratio}x) ${hx} ${hy}, url("${render(1)}") ${hx} ${hy}, default`; + } +} diff --git a/project.yml b/project.yml index 843f8b3..274a54b 100644 --- a/project.yml +++ b/project.yml @@ -3,6 +3,7 @@ options: bundleIdPrefix: com.toddfaulls deploymentTarget: iOS: "17.0" + macOS: "14.0" createIntermediateGroups: true settings: base: @@ -13,6 +14,7 @@ targets: platform: iOS sources: - path: Client + - path: ClientShared - path: Shared settings: base: @@ -48,3 +50,37 @@ targets: - UIInterfaceOrientationPortraitUpsideDown - UIInterfaceOrientationLandscapeLeft - UIInterfaceOrientationLandscapeRight + iFrameMac: + type: application + platform: macOS + sources: + - path: MacClient + - path: ClientShared + - path: Shared + settings: + base: + PRODUCT_NAME: iFrame + PRODUCT_BUNDLE_IDENTIFIER: com.toddfaulls.iframe.mac + CODE_SIGN_STYLE: Automatic + DEVELOPMENT_TEAM: 6PB44SHJS5 + ENABLE_HARDENED_RUNTIME: YES + ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon + MARKETING_VERSION: "0.1" + CURRENT_PROJECT_VERSION: "1" + # Universal: Apple silicon and Intel in one app. + ARCHS: "$(ARCHS_STANDARD)" + ONLY_ACTIVE_ARCH: NO + configs: + Release: + SWIFT_OPTIMIZATION_LEVEL: "-O" + info: + path: MacClient/Info.plist + properties: + CFBundleDisplayName: iFrame + CFBundleShortVersionString: $(MARKETING_VERSION) + CFBundleVersion: $(CURRENT_PROJECT_VERSION) + LSApplicationCategoryType: public.app-category.utilities + NSHumanReadableCopyright: MIT License + NSLocalNetworkUsageDescription: iFrame finds and connects to Macs running iframe-host on your network. + NSBonjourServices: + - _iframe._tcp diff --git a/scripts/build-icons.sh b/scripts/build-icons.sh index e116846..02f8979 100755 --- a/scripts/build-icons.sh +++ b/scripts/build-icons.sh @@ -1,6 +1,6 @@ #!/bin/bash # Regenerates every icon from scripts/make-icon.py: iPad app icon set (light/dark/tinted) -# and the macOS iFrame.icns for iFrame Host. Needs rsvg-convert and ImageMagick (brew). +# the macOS iFrame.icns for iFrame Host, and the Mac client's icon set. Needs rsvg-convert and ImageMagick (brew). set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" cd "$ROOT" @@ -44,4 +44,19 @@ for sz in 16 32 128 256 512; do magick Design/icon-mac-1024.png -resize $((sz*2))x$((sz*2)) "$TMP/iFrame.iconset/icon_${sz}x${sz}@2x.png" done iconutil -c icns "$TMP/iFrame.iconset" -o Design/iFrame.icns + +# Mac client: same tile as an asset catalog icon set, plus the logo and accent color. +MACSET=MacClient/Assets.xcassets/AppIcon.appiconset; mkdir -p "$MACSET" +cp "$TMP"/iFrame.iconset/*.png "$MACSET/" +python3 - "$MACSET" <<'PY' +import json, sys +images = [] +for sz in (16, 32, 128, 256, 512): + for scale in (1, 2): + name = f"icon_{sz}x{sz}{'@2x' if scale == 2 else ''}.png" + images.append({"filename": name, "idiom": "mac", "scale": f"{scale}x", "size": f"{sz}x{sz}"}) +json.dump({"images": images, "info": {"author": "xcode", "version": 1}}, open(sys.argv[1] + "/Contents.json", "w"), indent=2) +PY +rm -rf MacClient/Assets.xcassets/IFrameMark.imageset MacClient/Assets.xcassets/AccentColor.colorset +cp -R "$MARK" "$COLOR" MacClient/Assets.xcassets/ echo "icons rebuilt" diff --git a/scripts/build-mac-client.sh b/scripts/build-mac-client.sh new file mode 100755 index 0000000..b601d95 --- /dev/null +++ b/scripts/build-mac-client.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Builds the iFrame Mac client as one Universal app (Apple silicon + Intel) and zips it for +# copying to other Macs: dist/iFrame.app and dist/iFrame--macOS-Universal.zip. +# +# Signs with your Apple Development / Developer ID identity when there is one, otherwise ad hoc. +# Set IFRAME_SIGN_IDENTITY=- to force ad hoc. Over SSH, run `security unlock-keychain` first. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +DERIVED="$ROOT/.build/mac-client" +DIST="$ROOT/dist" + +IDENTITY="${IFRAME_SIGN_IDENTITY:-$(security find-identity -v -p codesigning | awk -F'"' '/Developer ID Application|Apple Development/ {print $2; exit}')}" +IDENTITY="${IDENTITY:--}" +SIGNING=() +if [ "$IDENTITY" = "-" ]; then + SIGNING=(CODE_SIGN_STYLE=Manual CODE_SIGN_IDENTITY=- DEVELOPMENT_TEAM=) +fi + +echo "==> generating project" +xcodegen generate --quiet + +echo "==> building (arm64 + x86_64, signing: $IDENTITY)" +# The generic destination is what makes xcodebuild build every architecture, not just this Mac's. +APP="$DERIVED/Build/Products/Release/iFrame.app" +rm -rf "$APP" +set +e +xcodebuild -project iFrame.xcodeproj -scheme iFrameMac -configuration Release \ + -destination 'generic/platform=macOS' -derivedDataPath "$DERIVED" ${SIGNING[@]+"${SIGNING[@]}"} build \ + | grep -E "error:|warning: .*\.swift|BUILD (SUCCEEDED|FAILED)" +BUILD_STATUS=${PIPESTATUS[0]} +set -e +if [ "$BUILD_STATUS" -ne 0 ] || [ ! -d "$APP" ] || ! codesign --verify --strict "$APP" 2>/dev/null; then + echo "build or signing failed (over SSH? run: security unlock-keychain)" + exit 1 +fi + +VERSION="$(/usr/libexec/PlistBuddy -c 'Print CFBundleShortVersionString' "$APP/Contents/Info.plist")" +ZIP="$DIST/iFrame-$VERSION-macOS-Universal.zip" +mkdir -p "$DIST" +rm -rf "$DIST/iFrame.app" "$ZIP" +ditto "$APP" "$DIST/iFrame.app" +ditto -c -k --keepParent "$DIST/iFrame.app" "$ZIP" + +SIGNER="$(codesign -dv "$DIST/iFrame.app" 2>&1 | awk -F= '/^Authority/ {print $2; exit}')" +echo "==> $(lipo -archs "$DIST/iFrame.app/Contents/MacOS/iFrame") · signed ${SIGNER:-ad hoc}" +echo " $DIST/iFrame.app" +echo " $ZIP"